Forward the location policy on the empty-rank path (#611)

src/underworld3/function/_dminterp_wrapper.pyx

@@ -178,7 +178,18 @@ cdef class CachedDMInterpolationInfo:
                                            <size_t*> &cells_view[0],
                                            1 if hint_authoritative else 0)
         else:
-            ierr = DMInterpolationSetUp_UW(self._ipInfo, dm, 0, 1, NULL, 0)
+            # No hint array to pass (no points, or no cells) — but the POLICY
+            # still has to be forwarded. Hardcoding 0 here made a rank with zero
+            # local points disagree with its peers about whether the hint is
+            # authoritative, and petsc_tools.c takes the DMLocatePoints branch
+            # when it is not. DMLocatePoints is COLLECTIVE on the mesh DM, so
+            # that rank blocked inside DMGetBoundingBox -> MPI_Allreduce while
+            # the others bypassed and ran on to DMSwarmMigrate -> MPI_Comm_dup:
+            # a deadlock whenever a query set leaves some rank empty (#611).
+            # The policy is a mesh capability and already agrees across ranks;
+            # it just has to survive the trip.
+            ierr = DMInterpolationSetUp_UW(self._ipInfo, dm, 0, 1, NULL,
+                                           1 if hint_authoritative else 0)
         if ierr != 0:
             DMInterpolationDestroy(&self._ipInfo)
             raise RuntimeError(f"DMInterpolationSetUp_UW failed with error {ierr}")

src/underworld3/function/petsc_tools.c

@@ -69,7 +69,17 @@ PetscErrorCode DMInterpolationSetUp_UW(DMInterpolationInfo ctx, DM dm, PetscBool
   PetscCall(PetscMalloc2(N, &foundProcs, N, &globalProcs));
   for (p = 0; p < N; ++p) foundProcs[p] = size;
   cellSF = NULL;
-  if (owning_cell && hintAuthoritative) {
+  /* N == 0 is included deliberately. An empty hint array reaches C as a NULL
+     `owning_cell`, which flipped this test and sent a rank with no local points
+     down the DMLocatePoints branch ALONE -- and DMLocatePoints is collective on
+     the mesh DM communicator (the comment below is right that the
+     Allreduce(foundProcs) is a COMM_SELF no-op, but DMLocatePoints itself is
+     not). Three ranks bypassed while the empty one blocked inside
+     DMGetBoundingBox -> MPI_Allreduce, deadlocking the job (#611). With no
+     points there is nothing to locate, so bypassing is trivially correct and
+     the branch now depends only on `hintAuthoritative`, which is a mesh
+     capability and agrees across ranks. */
+  if ((owning_cell || N == 0) && hintAuthoritative) {
     /*
       Bypass DMLocatePoints when the caller supplies an AUTHORITATIVE hint
       (ported from feature/dminterp-bypass-element-check, 17a5a8d).

tests/parallel/test_1076_global_evaluate_empty_rank.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""global_evaluate must not deadlock when a rank receives no query points (#611).
+
+The cell-location policy (`mesh._hint_is_authoritative`) decides whether the
+barycentric hint may bypass PETSc's `DMLocatePoints`. `DMLocatePoints` is
+COLLECTIVE on the mesh DM communicator, so every rank has to reach the same
+verdict — and the policy is a mesh capability, so they do.
+
+It was then thrown away in transit. `CachedDMInterpolationInfo.create_structure`
+had no hint array to pass when a rank held zero points, and passed
+`hintAuthoritative = 0` hardcoded along with the NULL. That rank alone took the
+`DMLocatePoints` branch and blocked inside `DMGetBoundingBox -> MPI_Allreduce`,
+while its peers bypassed and ran on to `DMSwarmMigrate -> MPI_Comm_dup`.
+
+Measured before the fix, at np=4: three ranks in `MPI_Comm_dup`, one in
+`MPI_Allreduce`, no progress in 900 s on a 300-point query.
+
+The trigger is a query set that leaves some rank empty. Biasing every point into
+x > 0.5 does it at np=4 — one rank owns only x < 0.5 — but not at np=2, where
+the coarser partition leaves both ranks straddling the split. That is exactly
+why this hid at np=2 and appeared at np=4.
+"""
+import numpy as np
+import pytest
+
+import underworld3 as uw
+
+pytestmark = [pytest.mark.level_1, pytest.mark.tier_a]
+
+
+def _mesh_and_field(tag):
+    mesh = uw.meshing.UnstructuredSimplexBox(
+        minCoords=(0.0, 0.0), maxCoords=(1.0, 1.0))
+    field = uw.discretisation.MeshVariable(tag, mesh, mesh.dim, degree=2)
+    return mesh, field
+
+
+def test_global_evaluate_with_points_only_on_some_ranks():
+    """The regression. Every point in x > 0.5, so some rank owns none of them."""
+    _mesh, field = _mesh_and_field("u1076a")
+    rng = np.random.default_rng(42)
+    coords = rng.random((300, 2))
+    coords[:, 0] = 0.5 + 0.5 * coords[:, 0]
+
+    result = uw.function.global_evaluate(field.sym, coords)
+    assert result.shape[0] == 300, (
+        f"rank {uw.mpi.rank}: expected 300 results, got {result.shape[0]}")
+
+
+def test_global_evaluate_with_points_everywhere_still_works():
+    """Negative control.
+
+    The unbiased query already passed before the fix, so if this ever fails the
+    change has broken the ordinary path rather than repaired the empty-rank one.
+    """
+    _mesh, field = _mesh_and_field("u1076b")
+    rng = np.random.default_rng(42)
+    coords = rng.random((300, 2))
+
+    result = uw.function.global_evaluate(field.sym, coords)
+    assert result.shape[0] == 300, (
+        f"rank {uw.mpi.rank}: expected 300 results, got {result.shape[0]}")