Hello, I have a question about loading mesh and allocating boundary conditions using MPI. It works totally fine when I run using a single processor, but it has some problems when I use multiple MPI processors.
Have you ever experienced one rank mistakenly recognizes inner cells as wall boundary, and solution explodes?
So, what I am trying to solve is basically solving LES for flows around 2D backward facing step. When I load the .xdmf mesh file and allocate boundary conditions, I am using a code as follows:
def load_mesh(filename):
with XDMFFile(MPI.COMM_WORLD, 'grid/bfsc.xdmf', "r") as xdmf:
mesh = xdmf.read_mesh(ghost_mode=GhostMode.shared_vertex, name="Grid")
# mesh = xdmf.read_mesh(name="Grid")
inflow_x = -1.651
outflow_x = 0.635
top_y = 0.1143
bott_y = 0.0
step_h = 0.0127
fdim = mesh.topology.dim - 1
tdim = mesh.topology.dim
mesh.topology.create_connectivity(tdim - 1, 0)
mesh.topology.create_connectivity(tdim - 1, tdim)
# Define boundaries
inflow = dmesh.locate_entities_boundary(
mesh,
fdim,
lambda x: np.isclose(x[0], inflow_x)
)
outflow = dmesh.locate_entities_boundary(
mesh,
fdim,
lambda x: np.isclose(x[0], outflow_x)
)
def walls_boundary(x, top_y, bott_y, step_h, inflow_x, outflow_x):
b1 = np.isclose(x[1], step_h)
b2 = np.isclose(x[1], bott_y)
b3 = np.isclose(x[1], step_h)
b4 = (x[0] > inflow_x) & (x[0] < outflow_x)
return b1 | b2 | b3 | b4
walls = dmesh.locate_entities_boundary(
mesh,
fdim,
lambda X: walls_boundary(X, top_y, bott_y, step_h, inflow_x, outflow_x)
)
return mesh, inflow, outflow, walls, step_h, fdim
When I run this simulation on a single rank and two ranks, the flowfields looks like this:
It seemed that simulation with two ranks(below) has some problem with boundary conditions allocations. So, I tried to print the coordinates of facets which were recognized as the bottom of the wall using this:
def load_mesh(filename):
with XDMFFile(MPI.COMM_WORLD, 'grid/bfsc.xdmf', "r") as xdmf:
mesh = xdmf.read_mesh(ghost_mode=GhostMode.shared_vertex, name="Grid")
# mesh = xdmf.read_mesh(name="Grid")
inflow_x = -1.651
outflow_x = 0.635
top_y = 0.1143
bott_y = 0.0
step_h = 0.0127
fdim = mesh.topology.dim - 1
tdim = mesh.topology.dim
mesh.topology.create_connectivity(tdim - 1, 0)
mesh.topology.create_connectivity(tdim - 1, tdim)
# Define boundaries
inflow = dmesh.locate_entities_boundary(
mesh,
fdim,
lambda x: np.isclose(x[0], inflow_x)
)
outflow = dmesh.locate_entities_boundary(
mesh,
fdim,
lambda x: np.isclose(x[0], outflow_x)
)
def walls_boundary(x, top_y, bott_y, step_h, inflow_x, outflow_x):
b1 = np.isclose(x[1], step_h)
return b1
walls = dmesh.locate_entities_boundary(
mesh,
fdim,
lambda X: walls_boundary(X, top_y, bott_y, step_h, inflow_x, outflow_x)
)
wall_facets = walls
geometry = mesh.geometry
connectivity = mesh.topology.connectivity(fdim, 0)
wall_vertex_indices = set()
for facet in wall_facets:
vertex_indices = connectivity.links(facet)
wall_vertex_indices.update(vertex_indices)
wall_vertex_indices = sorted(list(wall_vertex_indices))
wall_vertex_coords = geometry.x[wall_vertex_indices]
if mesh.comm.rank == 0:
for coord in wall_vertex_coords:
print(coord)
return mesh, inflow, outflow, walls, step_h, fdim
And when I use one rank, the print result is as follows:
[-1.64601486 0.0127 0. ]
[-1.651 0.0127 0. ]
[-1.64102972 0.0127 0. ]
[-1.63604458 0.0127 0. ]
[-1.63105944 0.0127 0. ]
[-1.6260743 0.0127 0. ]
[-1.62108916 0.0127 0. ]
[-1.61610402 0.0127 0. ]
[-1.59616347 0.0127 0. ]
[-1.60114861 0.0127 0. ]
[-1.60613375 0.0127 0. ]
[-1.61111889 0.0127 0. ]
[-1.59117833 0.0127 0. ]
[-1.58619319 0.0127 0. ]
[-1.58120805 0.0127 0. ]
[-1.57622291 0.0127 0. ]
[-1.57123777 0.0127 0. ]
[-1.56625263 0.0127 0. ]
[-1.55628235 0.0127 0. ]
[-1.56126749 0.0127 0. ]
[-1.55129721 0.0127 0. ]
[-1.54132693 0.0127 0. ]
[-1.54631207 0.0127 0. ]
[-1.5363418 0.0127 0. ]
[-1.53135666 0.0127 0. ]
[-1.52637152 0.0127 0. ]
[-1.52138638 0.0127 0. ]
[-1.51640124 0.0127 0. ]
[-1.5114161 0.0127 0. ]
[-1.50643096 0.0127 0. ]
[-1.50144582 0.0127 0. ]
[-1.49646068 0.0127 0. ]
[-1.49147554 0.0127 0. ]
[-1.48150526 0.0127 0. ]
[-1.4864904 0.0127 0. ]
[-1.47652012 0.0127 0. ]
[-1.47153498 0.0127 0. ]
[-1.46654984 0.0127 0. ]
[-1.45657957 0.0127 0. ]
[-1.46156471 0.0127 0. ]
[-1.45159443 0.0127 0. ]
[-1.44660929 0.0127 0. ]
[-1.44162415 0.0127 0. ]
[-1.43663901 0.0127 0. ]
[-1.43165387 0.0127 0. ]
[-1.42666873 0.0127 0. ]
[-1.42168359 0.0127 0. ]
[-1.41669845 0.0127 0. ]
[-1.41171331 0.0127 0. ]
[-1.40672817 0.0127 0. ]
[-1.40174303 0.0127 0. ]
[-1.39675789 0.0127 0. ]
[-1.39177275 0.0127 0. ]
[-1.38678762 0.0127 0. ]
[-1.38180248 0.0127 0. ]
[-1.37681734 0.0127 0. ]
[-1.35687678 0.0127 0. ]
[-1.36186192 0.0127 0. ]
[-1.36684706 0.0127 0. ]
[-1.3718322 0.0127 0. ]
[-1.35189164 0.0127 0. ]
[-1.33195108 0.0127 0. ]
[-1.33693622 0.0127 0. ]
[-1.34192136 0.0127 0. ]
[-1.3469065 0.0127 0. ]
[-1.32696594 0.0127 0. ]
[-1.3219808 0.0127 0. ]
[-1.31699567 0.0127 0. ]
[-1.31201053 0.0127 0. ]
[-1.30702539 0.0127 0. ]
[-1.30204025 0.0127 0. ]
[-1.29705511 0.0127 0. ]
[-1.29206997 0.0127 0. ]
[-1.28708483 0.0127 0. ]
[-1.28209969 0.0127 0. ]
[-1.27711455 0.0127 0. ]
[-1.27212941 0.0127 0. ]
[-1.26714427 0.0127 0. ]
[-1.26215913 0.0127 0. ]
[-1.25717399 0.0127 0. ]
[-1.25218885 0.0127 0. ]
[-1.24720371 0.0127 0. ]
[-1.24221858 0.0127 0. ]
[-1.22227802 0.0127 0. ]
[-1.22726316 0.0127 0. ]
[-1.2322483 0.0127 0. ]
[-1.23723344 0.0127 0. ]
[-1.21729288 0.0127 0. ]
[-1.21230774 0.0127 0. ]
[-1.2073226 0.0127 0. ]
[-1.20233746 0.0127 0. ]
[-1.19236718 0.0127 0. ]
[-1.19735232 0.0127 0. ]
[-1.18738204 0.0127 0. ]
[-1.1823969 0.0127 0. ]
[-1.17741176 0.0127 0. ]
[-1.17242662 0.0127 0. ]
[-1.16744149 0.0127 0. ]
[-1.16245635 0.0127 0. ]
[-1.15747121 0.0127 0. ]
[-1.15248607 0.0127 0. ]
[-1.14750093 0.0127 0. ]
[-1.14251579 0.0127 0. ]
[-1.13753065 0.0127 0. ]
[-1.13254551 0.0127 0. ]
[-1.12756037 0.0127 0. ]
[-1.12257523 0.0127 0. ]
[-1.11759009 0.0127 0. ]
[-1.11260495 0.0127 0. ]
[-1.10761981 0.0127 0. ]
[-1.10263467 0.0127 0. ]
[-1.09764953 0.0127 0. ]
[-1.0926644 0.0127 0. ]
[-1.08767926 0.0127 0. ]
[-1.08269412 0.0127 0. ]
[-1.07770898 0.0127 0. ]
[-1.07272384 0.0127 0. ]
[-1.0677387 0.0127 0. ]
[-1.05776842 0.0127 0. ]
[-1.06275356 0.0127 0. ]
[-1.05278328 0.0127 0. ]
[-1.04779814 0.0127 0. ]
[-1.042813 0.0127 0. ]
[-1.03782786 0.0127 0. ]
[-1.03284272 0.0127 0. ]
[-1.02785758 0.0127 0. ]
[-1.02287244 0.0127 0. ]
[-1.01788731 0.0127 0. ]
[-1.01290217 0.0127 0. ]
[-1.00791703 0.0127 0. ]
[-1.00293189 0.0127 0. ]
[-0.99296161 0.0127 0. ]
[-0.99794675 0.0127 0. ]
[-0.98797647 0.0127 0. ]
[-0.98299133 0.0127 0. ]
[-0.97800619 0.0127 0. ]
[-0.97302105 0.0127 0. ]
[-0.96803591 0.0127 0. ]
[-0.95806563 0.0127 0. ]
[-0.96305077 0.0127 0. ]
[-0.95308049 0.0127 0. ]
[-0.94809535 0.0127 0. ]
[-0.94311022 0.0127 0. ]
[-0.93313994 0.0127 0. ]
[-0.93812508 0.0127 0. ]
[-0.9281548 0.0127 0. ]
[-0.92316966 0.0127 0. ]
[-0.91818452 0.0127 0. ]
[-0.91319938 0.0127 0. ]
[-0.90821424 0.0127 0. ]
[-0.9032291 0.0127 0. ]
[-0.89325882 0.0127 0. ]
[-0.89824396 0.0127 0. ]
[-0.88827368 0.0127 0. ]
[-0.88328854 0.0127 0. ]
[-0.8783034 0.0127 0. ]
[-0.87331826 0.0127 0. ]
[-0.86833313 0.0127 0. ]
[-0.84340743 0.0127 0. ]
[-0.84839257 0.0127 0. ]
[-0.85337771 0.0127 0. ]
[-0.85836285 0.0127 0. ]
[-0.86334799 0.0127 0. ]
[-0.83842229 0.0127 0. ]
[-0.83343715 0.0127 0. ]
[-0.82845201 0.0127 0. ]
[-0.82346687 0.0127 0. ]
[-0.81848173 0.0127 0. ]
[-0.81349659 0.0127 0. ]
[-0.80851145 0.0127 0. ]
[-0.80352631 0.0127 0. ]
[-0.79854117 0.0127 0. ]
[-0.79355604 0.0127 0. ]
[-0.78358576 0.0127 0. ]
[-0.7885709 0.0127 0. ]
[-0.77860062 0.0127 0. ]
[-0.76863034 0.0127 0. ]
[-0.77361548 0.0127 0. ]
[-0.7636452 0.0127 0. ]
[-0.75866006 0.0127 0. ]
[-0.75367492 0.0127 0. ]
[-0.74370464 0.0127 0. ]
[-0.74868978 0.0127 0. ]
[-0.7387195 0.0127 0. ]
[-0.73373436 0.0127 0. ]
[-0.72874922 0.0127 0. ]
[-0.71877895 0.0127 0. ]
[-0.72376408 0.0127 0. ]
[-0.71379381 0.0127 0. ]
[-0.70880867 0.0127 0. ]
[-0.70382353 0.0127 0. ]
[-0.69883839 0.0127 0. ]
[-0.69385325 0.0127 0. ]
[-0.68886811 0.0127 0. ]
[-0.68388297 0.0127 0. ]
[-0.67889783 0.0127 0. ]
[-0.66892755 0.0127 0. ]
[-0.67391269 0.0127 0. ]
[-0.66394241 0.0127 0. ]
[-0.65895727 0.0127 0. ]
[-0.65397213 0.0127 0. ]
[-0.648987 0.0127 0. ]
[-0.64400186 0.0127 0. ]
[-0.63901672 0.0127 0. ]
[-0.63403158 0.0127 0. ]
[-0.62904644 0.0127 0. ]
[-0.6240613 0.0127 0. ]
[-0.61907616 0.0127 0. ]
[-0.61409102 0.0127 0. ]
[-0.60910588 0.0127 0. ]
[-0.60412074 0.0127 0. ]
[-0.5991356 0.0127 0. ]
[-0.59415046 0.0127 0. ]
[-0.58418018 0.0127 0. ]
[-0.58916532 0.0127 0. ]
[-0.57919504 0.0127 0. ]
[-0.57420991 0.0127 0. ]
[-0.56922477 0.0127 0. ]
[-0.55925449 0.0127 0. ]
[-0.56423963 0.0127 0. ]
[-0.55426935 0.0127 0. ]
[-0.54928421 0.0127 0. ]
[-0.54429907 0.0127 0. ]
[-0.53432879 0.0127 0. ]
[-0.53931393 0.0127 0. ]
[-0.52934365 0.0127 0. ]
[-0.52435851 0.0127 0. ]
[-0.51937337 0.0127 0. ]
[-0.51438823 0.0127 0. ]
[-0.50940309 0.0127 0. ]
[-0.50441795 0.0127 0. ]
[-0.49444768 0.0127 0. ]
[-0.49943282 0.0127 0. ]
[-0.48946254 0.0127 0. ]
[-0.4844774 0.0127 0. ]
[-0.47949226 0.0127 0. ]
[-0.46952198 0.0127 0. ]
[-0.47450712 0.0127 0. ]
[-0.46453684 0.0127 0. ]
[-0.4595517 0.0127 0. ]
[-0.45456656 0.0127 0. ]
[-0.44459628 0.0127 0. ]
[-0.44958142 0.0127 0. ]
[-0.43961114 0.0127 0. ]
[-0.434626 0.0127 0. ]
[-0.42964086 0.0127 0. ]
[-0.40970031 0.0127 0. ]
[-0.41468545 0.0127 0. ]
[-0.41967059 0.0127 0. ]
[-0.42465573 0.0127 0. ]
[-0.40471517 0.0127 0. ]
[-0.39973003 0.0127 0. ]
[-0.39474489 0.0127 0. ]
[-0.38477461 0.0127 0. ]
[-0.38975975 0.0127 0. ]
[-0.37978947 0.0127 0. ]
[-0.37480433 0.0127 0. ]
[-0.36981919 0.0127 0. ]
[-0.36483405 0.0127 0. ]
[-0.35984891 0.0127 0. ]
[-0.35486377 0.0127 0. ]
[-0.34987864 0.0127 0. ]
[-0.3448935 0.0127 0. ]
[-0.32495294 0.0127 0. ]
[-0.32993808 0.0127 0. ]
[-0.33492322 0.0127 0. ]
[-0.33990836 0.0127 0. ]
[-0.3199678 0.0127 0. ]
[-0.31498266 0.0127 0. ]
[-0.30999752 0.0127 0. ]
[-0.30501238 0.0127 0. ]
[-0.2950421 0.0127 0. ]
[-0.30002724 0.0127 0. ]
[-0.29005696 0.0127 0. ]
[-0.28507182 0.0127 0. ]
[-0.28008668 0.0127 0. ]
[-0.27510155 0.0127 0. ]
[-0.27011641 0.0127 0. ]
[-0.26513127 0.0127 0. ]
[-0.26014613 0.0127 0. ]
[-0.25516099 0.0127 0. ]
[-0.25017585 0.0127 0. ]
[-0.24519071 0.0127 0. ]
[-0.24020557 0.0127 0. ]
[-0.23522043 0.0127 0. ]
[-0.23023529 0.0127 0. ]
[-0.21029473 0.0127 0. ]
[-0.21527987 0.0127 0. ]
[-0.22026501 0.0127 0. ]
[-0.22525015 0.0127 0. ]
[-0.20530959 0.0127 0. ]
[-0.20032446 0.0127 0. ]
[-0.19533932 0.0127 0. ]
[-0.19035418 0.0127 0. ]
[-0.18536904 0.0127 0. ]
[-0.1803839 0.0127 0. ]
[-0.17041362 0.0127 0. ]
[-0.17539876 0.0127 0. ]
[-0.16542848 0.0127 0. ]
[-0.16044334 0.0127 0. ]
[-0.1554582 0.0127 0. ]
[-0.14548792 0.0127 0. ]
[-0.15047306 0.0127 0. ]
[-0.14050278 0.0127 0. ]
[-0.13551764 0.0127 0. ]
[-0.1305325 0.0127 0. ]
[-0.12554737 0.0127 0. ]
[-0.12056223 0.0127 0. ]
[-0.11557709 0.0127 0. ]
[-0.11059195 0.0127 0. ]
[-0.10560681 0.0127 0. ]
[-0.09563653 0.0127 0. ]
[-0.10062167 0.0127 0. ]
[-0.09065139 0.0127 0. ]
[-0.08566625 0.0127 0. ]
[-0.08068111 0.0127 0. ]
[-0.07569597 0.0127 0. ]
[-0.07071083 0.0127 0. ]
[0. 0.0127 0. ]
[-0.06572569 0.0127 0. ]
[-0.06074055 0.0127 0. ]
[-0.05575542 0.0127 0. ]
[-0.0408 0.0127 0. ]
[-0.04578514 0.0127 0. ]
[-0.05077028 0.0127 0. ]
[-0.03581486 0.0127 0. ]
[-0.00281242 0.0127 0. ]
[-0.02584458 0.0127 0. ]
[-0.03082972 0.0127 0. ]
[-0.00637627 0.0127 0. ]
[-0.02085944 0.0127 0. ]
[-0.01088916 0.0127 0. ]
[-0.0158743 0.0127 0. ]
As you can see the y-coordinate of all points is equal to 0.0127.
However, when I use two MPI ranks, the result is as follows:
[-0.02085944 0.0127 0. ]
[-0.0158743 0.0127 0. ]
[-0.01088916 0.0127 0. ]
[-0.00637627 0.0127 0. ]
[-0.02584458 0.0127 0. ]
[-0.03581486 0.0127 0. ]
[-0.00281242 0.0127 0. ]
[-0.03082972 0.0127 0. ]
[-0.0408 0.0127 0. ]
[0. 0.0127 0. ]
[-0.05538574 0.09041807 0. ]
It is recognizing [-0.05538574 0.09041807 0. ] as wall facet, and this point is actually a point of fluid cells.
This also happens when I try to print the points, which are located at the top of the wall.
[0.615 0.1143 0. ]
[0.605 0.1143 0. ]
[0.6 0.1143 0. ]
[0.595 0.1143 0. ]
[0.635 0.1143 0. ]
[0.62 0.1143 0. ]
[0.625 0.1143 0. ]
[0.61 0.1143 0. ]
[0.58 0.1143 0. ]
[0.59 0.1143 0. ]
[0.63 0.1143 0. ]
[0.575 0.1143 0. ]
[0.585 0.1143 0. ]
[0.57 0.1143 0. ]
[0.565 0.1143 0. ]
[0.56 0.1143 0. ]
[0.55 0.1143 0. ]
[0.555 0.1143 0. ]
[0.545 0.1143 0. ]
[0.54 0.1143 0. ]
[0.53 0.1143 0. ]
[0.535 0.1143 0. ]
[0.525 0.1143 0. ]
[0.52 0.1143 0. ]
[0.51 0.1143 0. ]
[0.515 0.1143 0. ]
[0.505 0.1143 0. ]
[0.5 0.1143 0. ]
[0.49 0.1143 0. ]
[0.495 0.1143 0. ]
[0.485 0.1143 0. ]
[0.48 0.1143 0. ]
[0.47 0.1143 0. ]
[0.475 0.1143 0. ]
[0.465 0.1143 0. ]
[0.46 0.1143 0. ]
[0.45 0.1143 0. ]
[0.455 0.1143 0. ]
[0.445 0.1143 0. ]
[0.44 0.1143 0. ]
[0.435 0.1143 0. ]
[0.43 0.1143 0. ]
[0.42 0.1143 0. ]
[0.425 0.1143 0. ]
[0.415 0.1143 0. ]
[0.41 0.1143 0. ]
[0.405 0.1143 0. ]
[0.395 0.1143 0. ]
[0.4 0.1143 0. ]
[0.39 0.1143 0. ]
[0.385 0.1143 0. ]
[0.38 0.1143 0. ]
[0.375 0.1143 0. ]
[0.37 0.1143 0. ]
[0.36 0.1143 0. ]
[0.365 0.1143 0. ]
[0.355 0.1143 0. ]
[0.35 0.1143 0. ]
[0.345 0.1143 0. ]
[0.34 0.1143 0. ]
[0.335 0.1143 0. ]
[0.33 0.1143 0. ]
[0.325 0.1143 0. ]
[0.32 0.1143 0. ]
[0.315 0.1143 0. ]
[0.31 0.1143 0. ]
[0.305 0.1143 0. ]
[0.3 0.1143 0. ]
[0.295 0.1143 0. ]
[0.29 0.1143 0. ]
[0.285 0.1143 0. ]
[0.28 0.1143 0. ]
[0.275 0.1143 0. ]
[0.265 0.1143 0. ]
[0.27 0.1143 0. ]
[0.26 0.1143 0. ]
[0.255 0.1143 0. ]
[0.25 0.1143 0. ]
[0.245 0.1143 0. ]
[0.24 0.1143 0. ]
[0.235 0.1143 0. ]
[0.23 0.1143 0. ]
[0.22 0.1143 0. ]
[0.225 0.1143 0. ]
[0.215 0.1143 0. ]
[0.21 0.1143 0. ]
[0.205 0.1143 0. ]
[0.2 0.1143 0. ]
[0.195 0.1143 0. ]
[0.19 0.1143 0. ]
[0.18 0.1143 0. ]
[0.185 0.1143 0. ]
[0.175 0.1143 0. ]
[0.17 0.1143 0. ]
[0.165 0.1143 0. ]
[0.16 0.1143 0. ]
[0.145 0.1143 0. ]
[0.155 0.1143 0. ]
[0.15 0.1143 0. ]
[0.14 0.1143 0. ]
[0.135 0.1143 0. ]
[0.13 0.1143 0. ]
[0.125 0.1143 0. ]
[0.12 0.1143 0. ]
[0.115 0.1143 0. ]
[0.11 0.1143 0. ]
[0.105 0.1143 0. ]
[0.1 0.1143 0. ]
[0.095 0.1143 0. ]
[0.085 0.1143 0. ]
[0.09 0.1143 0. ]
[0.08 0.1143 0. ]
[0.075 0.1143 0. ]
[0.07 0.1143 0. ]
[0.065 0.1143 0. ]
[0.06 0.1143 0. ]
[0.055 0.1143 0. ]
[0.045 0.1143 0. ]
[0.05 0.1143 0. ]
[0.04 0.1143 0. ]
[0.035 0.1143 0. ]
[0.03 0.1143 0. ]
[0.025 0.1143 0. ]
[0.02 0.1143 0. ]
[0.015 0.1143 0. ]
[0.005 0.1143 0. ]
[0.01 0.1143 0. ]
[0. 0.1143 0. ]
[-0.00997583 0.1143 0. ]
[-0.00498792 0.1143 0. ]
[-0.01496375 0.1143 0. ]
[-0.01995166 0.1143 0. ]
[-0.02493958 0.1143 0. ]
[-0.02992749 0.1143 0. ]
[-0.03491541 0.1143 0. ]
[-0.04489124 0.1143 0. ]
[-0.03990332 0.1143 0. ]
[-0.04987915 0.1143 0. ]
[-0.05486707 0.1143 0. ]
[-0.05985498 0.1143 0. ]
[-0.0648429 0.1143 0. ]
[-0.04568185 0.05373779 0. ]