DofMap and ghost values

dolfinx
3

Here is an example that creates the local_to_global map containing both owned and unowned dofs (unrolled for block size).

import dolfinx
from mpi4py import MPI
import ufl
import numpy as np
domain = dolfinx.mesh.create_unit_square(MPI.COMM_WORLD, 2, 2)
el0 = ufl.FiniteElement("Lagrange", domain.ufl_cell(), 1)
el2 = ufl.VectorElement("Lagrange", domain.ufl_cell(), 2)
el = ufl.MixedElement([el0, el2])
V = dolfinx.fem.FunctionSpace(domain, el)

u = dolfinx.fem.Function(V)

# Create local map
local_range = V.dofmap.index_map.local_range
num_dofs_local = V.dofmap.index_map.size_local
local_dof_map = np.arange(num_dofs_local, dtype=np.int64)
local_dof_map += local_range[0]


# Get ghost indices
num_ghosts = V.dofmap.index_map.num_ghosts
ghosts = V.dofmap.index_map.ghosts
ghost_ownership = V.dofmap.index_map.owners

# Unroll for block size
bs = V.dofmap.index_map_bs
local_to_global = np.zeros((num_dofs_local+num_ghosts)*bs, dtype=np.int64)
owners = np.zeros_like(local_to_global)
for i in range(bs):
    for j in range(num_dofs_local):
        local_to_global[j*bs+i] = local_dof_map[j]*bs+i
        owners[j*bs+i] = domain.comm.rank

    for j in range(num_ghosts):
        local_to_global[num_dofs_local*bs+ j*bs + i] = ghosts[j]*bs+i
        owners[num_dofs_local*bs + j*bs+i] = ghost_ownership[j]

import time
time.sleep(domain.comm.rank)
print(f"Rank {domain.comm.rank}, Block size {bs} Num local dofs {num_dofs_local*bs} Num ghosts {num_ghosts*bs}", local_to_global, owners)

yielding:

mpirun -n 3 python3 script.py 
Rank 0, Block size 1 Num local dofs 19 Num ghosts 17 [ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 48 49 50 24 25
 26 27 28 31 32 19 20 21 22 23 57 58] [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 2 2]
Rank 1, Block size 1 Num local dofs 14 Num ghosts 31 [19 20 21 22 23 24 25 26 27 28 29 30 31 32 48  1 49 50  4  5 57 58  6  7
 33 36 37 53 54  0  2  3 10 11 35 40 41 51 52 44 45  8  9 55 56] [1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 0 2 2 0 0 2 2 0 0 2 2 2 2 2 0 0 0 0 0 2 2 2
 2 2 2 2 0 0 2 2]
Rank 2, Block size 1 Num local dofs 26 Num ghosts 17 [33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
 57 58 19 20 21 24 25 26 27 28 29 30  1  4  5 22 23  6  7] [2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 0
 0 0 1 1 0 0]