diff --git a/src/underworld3/function/_dminterp_wrapper.pyx b/src/underworld3/function/_dminterp_wrapper.pyx index 6c10c7ae..966dc0e0 100644 --- a/src/underworld3/function/_dminterp_wrapper.pyx +++ b/src/underworld3/function/_dminterp_wrapper.pyx @@ -178,7 +178,18 @@ cdef class CachedDMInterpolationInfo: &cells_view[0], 1 if hint_authoritative else 0) else: - ierr = DMInterpolationSetUp_UW(self._ipInfo, dm, 0, 1, NULL, 0) + # No hint array to pass (no points, or no cells) — but the POLICY + # still has to be forwarded. Hardcoding 0 here made a rank with zero + # local points disagree with its peers about whether the hint is + # authoritative, and petsc_tools.c takes the DMLocatePoints branch + # when it is not. DMLocatePoints is COLLECTIVE on the mesh DM, so + # that rank blocked inside DMGetBoundingBox -> MPI_Allreduce while + # the others bypassed and ran on to DMSwarmMigrate -> MPI_Comm_dup: + # a deadlock whenever a query set leaves some rank empty (#611). + # The policy is a mesh capability and already agrees across ranks; + # it just has to survive the trip. + ierr = DMInterpolationSetUp_UW(self._ipInfo, dm, 0, 1, NULL, + 1 if hint_authoritative else 0) if ierr != 0: DMInterpolationDestroy(&self._ipInfo) raise RuntimeError(f"DMInterpolationSetUp_UW failed with error {ierr}") diff --git a/src/underworld3/function/petsc_tools.c b/src/underworld3/function/petsc_tools.c index 96936d38..7bec52e7 100644 --- a/src/underworld3/function/petsc_tools.c +++ b/src/underworld3/function/petsc_tools.c @@ -69,7 +69,17 @@ PetscErrorCode DMInterpolationSetUp_UW(DMInterpolationInfo ctx, DM dm, PetscBool PetscCall(PetscMalloc2(N, &foundProcs, N, &globalProcs)); for (p = 0; p < N; ++p) foundProcs[p] = size; cellSF = NULL; - if (owning_cell && hintAuthoritative) { + /* N == 0 is included deliberately. An empty hint array reaches C as a NULL + `owning_cell`, which flipped this test and sent a rank with no local points + down the DMLocatePoints branch ALONE -- and DMLocatePoints is collective on + the mesh DM communicator (the comment below is right that the + Allreduce(foundProcs) is a COMM_SELF no-op, but DMLocatePoints itself is + not). Three ranks bypassed while the empty one blocked inside + DMGetBoundingBox -> MPI_Allreduce, deadlocking the job (#611). With no + points there is nothing to locate, so bypassing is trivially correct and + the branch now depends only on `hintAuthoritative`, which is a mesh + capability and agrees across ranks. */ + if ((owning_cell || N == 0) && hintAuthoritative) { /* Bypass DMLocatePoints when the caller supplies an AUTHORITATIVE hint (ported from feature/dminterp-bypass-element-check, 17a5a8d). diff --git a/tests/parallel/test_1076_global_evaluate_empty_rank.py b/tests/parallel/test_1076_global_evaluate_empty_rank.py new file mode 100644 index 00000000..e8ec8a41 --- /dev/null +++ b/tests/parallel/test_1076_global_evaluate_empty_rank.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""global_evaluate must not deadlock when a rank receives no query points (#611). + +The cell-location policy (`mesh._hint_is_authoritative`) decides whether the +barycentric hint may bypass PETSc's `DMLocatePoints`. `DMLocatePoints` is +COLLECTIVE on the mesh DM communicator, so every rank has to reach the same +verdict — and the policy is a mesh capability, so they do. + +It was then thrown away in transit. `CachedDMInterpolationInfo.create_structure` +had no hint array to pass when a rank held zero points, and passed +`hintAuthoritative = 0` hardcoded along with the NULL. That rank alone took the +`DMLocatePoints` branch and blocked inside `DMGetBoundingBox -> MPI_Allreduce`, +while its peers bypassed and ran on to `DMSwarmMigrate -> MPI_Comm_dup`. + +Measured before the fix, at np=4: three ranks in `MPI_Comm_dup`, one in +`MPI_Allreduce`, no progress in 900 s on a 300-point query. + +The trigger is a query set that leaves some rank empty. Biasing every point into +x > 0.5 does it at np=4 — one rank owns only x < 0.5 — but not at np=2, where +the coarser partition leaves both ranks straddling the split. That is exactly +why this hid at np=2 and appeared at np=4. +""" +import numpy as np +import pytest + +import underworld3 as uw + +pytestmark = [pytest.mark.level_1, pytest.mark.tier_a] + + +def _mesh_and_field(tag): + mesh = uw.meshing.UnstructuredSimplexBox( + minCoords=(0.0, 0.0), maxCoords=(1.0, 1.0)) + field = uw.discretisation.MeshVariable(tag, mesh, mesh.dim, degree=2) + return mesh, field + + +def test_global_evaluate_with_points_only_on_some_ranks(): + """The regression. Every point in x > 0.5, so some rank owns none of them.""" + _mesh, field = _mesh_and_field("u1076a") + rng = np.random.default_rng(42) + coords = rng.random((300, 2)) + coords[:, 0] = 0.5 + 0.5 * coords[:, 0] + + result = uw.function.global_evaluate(field.sym, coords) + assert result.shape[0] == 300, ( + f"rank {uw.mpi.rank}: expected 300 results, got {result.shape[0]}") + + +def test_global_evaluate_with_points_everywhere_still_works(): + """Negative control. + + The unbiased query already passed before the fix, so if this ever fails the + change has broken the ordinary path rather than repaired the empty-rank one. + """ + _mesh, field = _mesh_and_field("u1076b") + rng = np.random.default_rng(42) + coords = rng.random((300, 2)) + + result = uw.function.global_evaluate(field.sym, coords) + assert result.shape[0] == 300, ( + f"rank {uw.mpi.rank}: expected 300 results, got {result.shape[0]}")