I have two problems after the update:
- Running the mesh converter routine used to run OK, but running on single processor of cluster crashes.
- Running model on MPI cluster crashes but ran fine before.
Before update, MPI cluster worked fine. What is happening?
First, mesh conversion (which should not even use MPI). The following lines are in a file MeshConvert.py.
import meshio
msh = meshio.read("SIW.msh")
for cell in msh.cells:
if cell.type == "tetra":
tetra_cells = cell.data
for key in msh.cell_data_dict["gmsh:physical"].keys():
if key == "tetra":
tetra_data = msh.cell_data_dict["gmsh:physical"][key]
tetra_mesh = meshio.Mesh(points=msh.points, cells={"tetra": tetra_cells},
cell_data={"VolumeRegions":[tetra_data]})
meshio.write("mesh.xdmf", tetra_mesh)
from dolfin import *
mesh = Mesh()
with XDMFFile("mesh.xdmf") as infile:
infile.read(mesh)
mvc = MeshValueCollection("size_t", mesh, 3)
with XDMFFile("mesh.xdmf") as infile:
infile.read(mvc, "VolumeRegions")
cf = cpp.mesh.MeshFunctionSizet(mesh, mvc)
info(mesh)
# Volume domains
File("VolSubDomains.pvd").write(cf)
File("Mesh.pvd").write(mesh)This generates the following error when running “python3 MeshConvert.py”
--------------------------------------------------------------------------
[[17532,1],0]: A high-performance Open MPI point-to-point messaging module
was unable to find any relevant network interfaces:
Module: OpenFabrics (openib)
Host: Oliver
Another transport will be used instead, although this may result in
lower performance.
NOTE: You can disable this warning by setting the MCA parameter
btl_base_warn_component_unused to 0.
--------------------------------------------------------------------------
Traceback (most recent call last):
File "MeshConvert.py", line 20, in <module>
from dolfin import *
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/__init__.py", line 144, in <module>
from .fem.assembling import (assemble, assemble_system, assemble_multimesh, assemble_mixed,
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/fem/assembling.py", line 38, in <module>
from ufl.form import sub_forms_by_domain
ImportError: cannot import name 'sub_forms_by_domain'
Why does this even reference MPI? It should run locally on a single processor. Also, dolfin seems confused. No assembly is taking place.
The mesh file is found at this link.
Second, the model itself runs fine on a single PC, but MPI runs die (but used to run fine before update). The code is
import meshio
from dolfin import *
import numpy as np
import cmath as cm
import matplotlib.pyplot as plt
import os, sys, traceback
a = 3.0;
b = 0.5;
d = 0.3;
s = 0.5;
l = 10.0;
w = 3.0;
pi = 3.1415926536
tol = 1.0e-12
eta = 377.0
Dk = 3.55 # R4350
k0 = 0.65
beta = sqrt(k0 * k0 * Dk - pow((pi * 4.0 /(6.0 * a)),2.0))
class PEC(SubDomain):
def inside(self, x, on_boundary):
return on_boundary
class InputBC(SubDomain):
def inside(self, x, on_boundary):
return on_boundary and near(x[2], 0.0, tol) and (x[0] < 3.0 * a / 4.0+tol)
class OutputBC(SubDomain):
def inside(self, x, on_boundary):
return on_boundary and (near(x[2], l, tol) or near(x[0], a, tol) or (near(x[2], 0, tol) and (x[0] > 3.0*a/4-tol)))
class PMC(SubDomain):
def inside(self, x, on_boundary):
return on_boundary and near(x[0], 0.0, tol)
mesh = Mesh()
with XDMFFile("mesh.xdmf") as infile:
infile.read(mesh)
mvc = MeshValueCollection("size_t", mesh, 3)
info(mesh)
# Mark boundaries
sub_domains = MeshFunction("size_t", mesh, mesh.topology().dim() - 1)
sub_domains.set_all(4)
pec = PEC()
pec.mark(sub_domains, 0)
in_port = InputBC()
in_port.mark(sub_domains, 1)
out_port = OutputBC()
out_port.mark(sub_domains, 2)
pmc = PMC()
pmc.mark(sub_domains, 3)
File("BoxSubDomains.pvd").write(sub_domains)
# Set up function spaces
# For low order problem
cell = tetrahedron
ele_type = FiniteElement('N1curl', cell, 2) # H(curl) element for EM
V2 = FunctionSpace(mesh, MixedElement([ele_type, ele_type]))
V = FunctionSpace(mesh, ele_type)
u_r, u_i = TrialFunctions(V2)
v_r, v_i = TestFunctions(V2)
#surface integral definitions from boundaries
ds = Measure('ds', domain = mesh, subdomain_data = sub_domains)
# with source and sink terms
u0 = Constant((0.0, 0.0, 0.0)) #PEC definition
h_src = Expression(('cos(pi*x[0] * 4.0 / (6.0 * a))', 0.0, 0.0), degree = 2, a = a, pi = pi)
e_src = Expression((0.0, 'cos(pi*x[0]*4.0/(6.0*a))', 0.0), degree = 2, a = a, pi = pi)
#Boundary condition dictionary
boundary_conditions = {0: {'PEC' : u0},
1: {'InputBC': (h_src, Dk)},
2: {'OutputBC': 1.0},
3: {'PMC': 0.0}}
n = FacetNormal(mesh)
#Build PEC boundary conditions for real and imaginary parts
bcs = []
for i in boundary_conditions:
if 'PEC' in boundary_conditions[i]:
bc = DirichletBC(V2.sub(0), boundary_conditions[i]['PEC'], sub_domains, i)
bcs.append(bc)
bc = DirichletBC(V2.sub(1), boundary_conditions[i]['PEC'], sub_domains, i)
bcs.append(bc)
# Build input BC source term and loading term
integral_source = []
integrals_load =[]
for i in boundary_conditions:
if 'InputBC' in boundary_conditions[i]:
r, s = boundary_conditions[i]['InputBC']
bb1 = 2.0 * beta * inner(v_i, cross(n, r)) * ds(i) #Factor of two from field equivalence principle
integral_source.append(bb1)
bb2 = inner(cross(n, v_i), cross(n, u_r)) * beta * ds(i)
integrals_load.append(bb2)
bb2 = inner(-cross(n, v_r), cross(n, u_i)) * beta * ds(i)
integrals_load.append(bb2)
for i in boundary_conditions:
if 'OutputBC' in boundary_conditions[i]:
r = boundary_conditions[i]['OutputBC']
bb2 = inner(cross(n, v_i), cross(n, u_r)) * beta * ds(i)
integrals_load.append(bb2)
bb2 = inner(-cross(n, v_r), cross(n, u_i)) * beta * ds(i)
integrals_load.append(bb2)
# for PMC, do nothing. Natural BC.
a = (inner(curl(v_r), curl(u_r)) + inner(curl(v_i), curl(u_i)) - Dk * k0 * k0 * (inner(v_r, u_r) + inner(v_i, u_i))) * dx + sum(integrals_load)
L = sum(integral_source)
u1 = Function(V2)
vdim = u1.vector().size()
print("Vdim = ", vdim)
u1 = Function(V2)
vdim = u1.vector().size()
print ('Solution vector size = ',vdim)
solve(a == L, u1, bcs, solver_parameters = {'linear_solver' : 'mumps'})
u1_r, u1_i = u1.split(True)
fp = File("SIWEField_r.pvd")
fp << u1_r
fp = File("SIWEField_i.pvd")
fp << u1_i
fp = File('SIWWaveFile.pvd')
ut = u1_r.copy(deepcopy=True)
for i in range(50):
ut.vector().zero()
ut.vector().axpy(-sin(pi * i / 25.0), u1_i.vector())
ut.vector().axpy(cos(pi * i / 25.0), u1_r.vector())
fp << (ut, i)
H = interpolate(h_src, V) # Get input field
P = assemble((-dot(u1_r,cross(curl(u1_i),n))+dot(u1_i,cross(curl(u1_r),n))) * ds(2))
P_refl = assemble((-dot(u1_i,cross(curl(u1_r), n)) + dot(u1_r, cross(curl(u1_i), n))) * ds(1))
P_inc = assemble((dot(H, H) * eta / (2.0 * sqrt(Dk))) * ds(1))
print("Integrated power on rad boundary:", P/(2.0 * k0 * eta))
print("Incident power at port 1:", P_inc)
print("Integrated reflected power on port 1:", P_inc - P_refl / (2.0 * k0 * eta))
E = interpolate(e_src, V) # Incident E field
ccr = assemble(-dot(u1_r - E * (eta / sqrt(Dk)), E * (eta / sqrt(Dk))) * ds(1))
cci = assemble(dot(u1_i, E) * ds(1)) * eta / sqrt(Dk)
cc = assemble(dot(E, E) * ds(1)) * eta * eta / Dk
sys.exit(0)Running this code using the command line below produces the following error stream.
bill@Oliver:~/Cluster$ mpirun --mca orte_base_help_aggregate 0 --mca btl tcp,self -np 32 --host Oliver:8,node0:8,node1:8,node2:8 python3 SIW.py
Traceback (most recent call last):
File "SIW.py", line 2, in <module>
from dolfin import *
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/__init__.py", line 144, in <module>
from .fem.assembling import (assemble, assemble_system, assemble_multimesh, assemble_mixed,
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/fem/assembling.py", line 38, in <module>
from ufl.form import sub_forms_by_domain
ImportError: cannot import name 'sub_forms_by_domain'
Traceback (most recent call last):
File "SIW.py", line 2, in <module>
from dolfin import *
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/__init__.py", line 144, in <module>
from .fem.assembling import (assemble, assemble_system, assemble_multimesh, assemble_mixed,
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/fem/assembling.py", line 38, in <module>
from ufl.form import sub_forms_by_domain
ImportError: cannot import name 'sub_forms_by_domain'
Traceback (most recent call last):
File "SIW.py", line 2, in <module>
from dolfin import *
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/__init__.py", line 144, in <module>
from .fem.assembling import (assemble, assemble_system, assemble_multimesh, assemble_mixed,
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/fem/assembling.py", line 38, in <module>
from ufl.form import sub_forms_by_domain
ImportError: cannot import name 'sub_forms_by_domain'
Traceback (most recent call last):
File "SIW.py", line 2, in <module>
from dolfin import *
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/__init__.py", line 144, in <module>
from .fem.assembling import (assemble, assemble_system, assemble_multimesh, assemble_mixed,
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/fem/assembling.py", line 38, in <module>
from ufl.form import sub_forms_by_domain
ImportError: cannot import name 'sub_forms_by_domain'
Traceback (most recent call last):
File "SIW.py", line 2, in <module>
from dolfin import *
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/__init__.py", line 144, in <module>
from .fem.assembling import (assemble, assemble_system, assemble_multimesh, assemble_mixed,
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/fem/assembling.py", line 38, in <module>
from ufl.form import sub_forms_by_domain
ImportError: cannot import name 'sub_forms_by_domain'
Traceback (most recent call last):
File "SIW.py", line 2, in <module>
from dolfin import *
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/__init__.py", line 144, in <module>
from .fem.assembling import (assemble, assemble_system, assemble_multimesh, assemble_mixed,
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/fem/assembling.py", line 38, in <module>
from ufl.form import sub_forms_by_domain
ImportError: cannot import name 'sub_forms_by_domain'
Traceback (most recent call last):
File "SIW.py", line 2, in <module>
from dolfin import *
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/__init__.py", line 144, in <module>
from .fem.assembling import (assemble, assemble_system, assemble_multimesh, assemble_mixed,
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/fem/assembling.py", line 38, in <module>
from ufl.form import sub_forms_by_domain
ImportError: cannot import name 'sub_forms_by_domain'
Traceback (most recent call last):
File "SIW.py", line 2, in <module>
from dolfin import *
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/__init__.py", line 144, in <module>
from .fem.assembling import (assemble, assemble_system, assemble_multimesh, assemble_mixed,
File "/usr/lib/petsc/lib/python3/dist-packages/dolfin/fem/assembling.py", line 38, in <module>
from ufl.form import sub_forms_by_domain
ImportError: cannot import name 'sub_forms_by_domain'Any idea what is happening? The assembling error appears again. Something in Version 2019.2.0 is different.