This is actual an issue with openmpi. I’ve pushed some updates that should fix this issue.
Thanks to @minrk for discussing these issues.
Minimal failing example with openmpi (works with mpich)
from mpi4py import MPI
import numpy as np
comm = MPI.COMM_WORLD
assert (comm.size == 2)
dtype = np.int64
if comm.rank == 0:
outgoing_edges = np.array([], dtype=dtype)
elif comm.rank == 1:
outgoing_edges = np.array([0], dtype=dtype)
sub_comm = comm.Create_dist_graph(
list([comm.rank]), [len(np.unique(outgoing_edges))], outgoing_edges, reorder=False)
source, dest, _ = sub_comm.Get_dist_neighbors()
send_data = np.full(len(dest), comm.rank, dtype=dtype)
recv_data = np.full(len(source), -1, dtype=dtype)
print(comm.rank, source, dest,
send_data, recv_data, flush=True)
sub_comm.Neighbor_alltoall(send_data, recv_data)
comm.Barrier()
print(comm.rank, source, dest,
send_data, recv_data, flush=True)
Corrected code to work with openmpi
from mpi4py import MPI
import numpy as np
comm = MPI.COMM_WORLD
assert (comm.size == 2)
dtype = np.int64
if comm.rank == 0:
outgoing_edges = np.array([], dtype=dtype)
elif comm.rank == 1:
outgoing_edges = np.array([0], dtype=dtype)
sub_comm = comm.Create_dist_graph(
list([comm.rank]), [len(np.unique(outgoing_edges))], outgoing_edges, reorder=False)
source, dest, _ = sub_comm.Get_dist_neighbors()
send_data = np.full(max(len(dest), 1), comm.rank, dtype=dtype)
recv_data = np.full(max(len(source), 1), -1, dtype=dtype)
print(comm.rank, source, dest,
send_data, recv_data, flush=True)
sub_comm.Neighbor_alltoall(send_data, recv_data)
send_data = send_data[:len(dest)]
recv_data = recv_data[:len(source)]
comm.Barrier()
print(comm.rank, source, dest,
send_data, recv_data, flush=True)