Parallel function saving and reading it back using PETSc or adios4dolfinx

dolfinx
22

I changed it to

and it still gives;

Mesh written to checkpoint_0.bp on comm 1/3

Mesh written to checkpoint_0.bp on comm 2/3

Mesh written to checkpoint_0.bp on comm 3/3

Mesh read from checkpoint_0.bp on comm 1/2
Function written to checkpoint_1.bp on comm 1/2

Mesh read from checkpoint_0.bp on comm 2/2
Function written to checkpoint_1.bp on comm 2/2

Traceback (most recent call last):
  File "demo.py", line 114, in <module>
    assert query_3.successful(), query_3.error
AssertionError: [<RemoteError[ ]:Exception(MPI_ERR_TRUNCATE: message truncated)>]
Stopping cluster <Cluster(cluster_id='1707334019-xzjr', profile='default', controller=<running>, engine_sets=['1707334020'])>

there is some problem during function reading. I did trying my tests again for v0.7.2. When I run mesh with;

# adios_mesh.py
from dolfinx.mesh import create_unit_square
import adios4dolfinx
from mpi4py import MPI
import shutil

mesh = create_unit_square(MPI.COMM_WORLD, 30, 30)
shutil.rmtree("mesh", ignore_errors=True)
adios4dolfinx.write_mesh(mesh, "mesh")

and write function with;

# adios_write.py
import numba
import numpy as np
from dolfinx.fem import (Function, FunctionSpace, locate_dofs_geometrical)
from dolfinx.mesh import locate_entities
from ufl import FiniteElement
from mpi4py import MPI
import dolfinx.fem
import dolfinx.io
import adios4dolfinx
import os
import pathlib
import shutil

def export_function(function: dolfinx.fem.Function, directory: str, filename: str) -> None:

    comm = function.function_space.mesh.comm
    visualization_directory = os.path.join(directory, filename + ".bp")
    checkpointing_directory = os.path.join(
        directory, filename + "_checkpoint.bp")

    shutil.rmtree(visualization_directory, ignore_errors=True)
    os.makedirs(visualization_directory, exist_ok=True)
    print(visualization_directory)
    # Export for visualization
    with dolfinx.io.VTXWriter(comm, visualization_directory, function, "bp4") as vtx_file:
        vtx_file.write(0)
    # Export for checkpointing
    adios4dolfinx.write_function(function, pathlib.Path(visualization_directory), "bp4")

mesh = adios4dolfinx.read_mesh(MPI.COMM_WORLD, "mesh", engine='BP4', ghost_mode=dolfinx.mesh.GhostMode.shared_facet)

v_cg = FiniteElement("CG", mesh.ufl_cell(), 2)
Q = FunctionSpace(mesh, v_cg)

def Omega_0(x):
    return x[1] <= 0.3

def Omega_1(x):
    return x[1] >= 0.7

kappa = Function(Q)
cells_0 = locate_entities(mesh, mesh.topology.dim, Omega_0)
cells_1 = locate_entities(mesh, mesh.topology.dim, Omega_1)

@numba.njit
def unroll_dofmap(dofs, bs):
    dofs_unrolled = np.zeros(bs*len(dofs), dtype=np.int32)
    for i, dof in enumerate(dofs):
        for b in range(bs):
            dofs_unrolled[i*bs+b] = dof*bs+b
    return dofs_unrolled

dofs_0 = locate_dofs_geometrical(Q, Omega_0)
dofs_1 = locate_dofs_geometrical(Q, Omega_1)
dofmap_bs = Q.dofmap.bs
kappa.x.array[unroll_dofmap(dofs_0, dofmap_bs)] = 1
kappa.x.array[unroll_dofmap(dofs_1, dofmap_bs)] = 0.1

export_function(kappa, "adios_dir", "test_export")

and finally testing the export with;

# assert.py
import numba
import numpy as np
from dolfinx.fem import (Function, FunctionSpace, locate_dofs_geometrical)
from dolfinx.mesh import create_unit_square, locate_entities
from ufl import FiniteElement
from mpi4py import MPI
import dolfinx.fem
import dolfinx.io
import adios4dolfinx
import os
import pathlib

mesh = adios4dolfinx.read_mesh(MPI.COMM_WORLD, "mesh", engine='BP4', ghost_mode=dolfinx.mesh.GhostMode.shared_facet)

v_cg = FiniteElement("CG", mesh.ufl_cell(), 2)
Q = FunctionSpace(mesh, v_cg)

def Omega_0(x):
    return x[1] <= 0.3

def Omega_1(x):
    return x[1] >= 0.7

kappa = Function(Q)
cells_0 = locate_entities(mesh, mesh.topology.dim, Omega_0)
cells_1 = locate_entities(mesh, mesh.topology.dim, Omega_1)

@numba.njit
def unroll_dofmap(dofs, bs):
    dofs_unrolled = np.zeros(bs*len(dofs), dtype=np.int32)
    for i, dof in enumerate(dofs):
        for b in range(bs):
            dofs_unrolled[i*bs+b] = dof*bs+b
    return dofs_unrolled

dofs_0 = locate_dofs_geometrical(Q, Omega_0)
dofs_1 = locate_dofs_geometrical(Q, Omega_1)
dofmap_bs = Q.dofmap.bs
kappa.x.array[unroll_dofmap(dofs_0, dofmap_bs)] = 1
kappa.x.array[unroll_dofmap(dofs_1, dofmap_bs)] = 0.1

def import_function(function_space: dolfinx.fem.FunctionSpace, directory: str, filename: str) -> dolfinx.fem.Function:

    function = dolfinx.fem.Function(function_space)
    checkpointing_directory = os.path.join(directory, filename + ".bp")
    adios4dolfinx.read_function(function, pathlib.Path(checkpointing_directory), "bp4")
    return function

kappa_import = import_function(Q, "adios_dir", "test_export")

print("original", np.count_nonzero(kappa.vector.array))
print("imported", np.count_nonzero(kappa_import.vector.array))

assert np.allclose(kappa.vector.array, kappa_import.vector.array)

gives;

Traceback (most recent call last):
  File "assert.py", line 50, in <module>
    kappa_import = import_function(Q, "adios_dir", "test_export")
  File "assert.py", line 47, in import_function
    adios4dolfinx.read_function(function, pathlib.Path(checkpointing_directory), "bp4")
  File "/home/ee331/Dev/Venvs/v073complex/lib/python3.8/site-packages/adios4dolfinx/checkpointing.py", line 337, in read_function
    inc_cells, inc_perms = send_and_recv_cell_perm(
  File "/home/ee331/Dev/Venvs/v073complex/lib/python3.8/site-packages/adios4dolfinx/comm_helpers.py", line 203, in send_and_recv_cell_perm
    mesh_to_data.Neighbor_alltoall(out_size, recv_size)
  File "mpi4py/MPI/Comm.pyx", line 2154, in mpi4py.MPI.Topocomm.Neighbor_alltoall
mpi4py.MPI.Exception: MPI_ERR_TRUNCATE: message truncated

what does
mesh_to_data.Neighbor_alltoall(out_size, recv_size)
do?