Ffcx compilation error

Yuxiang_Chen · December 16, 2024, 1:21pm

My current goal is to iterate through all the units which involves using assemble_scalar to perform boundary integration . I have encountered some problems. Here is MWE

import dolfinx
from dolfinx import *
# from dolfinx import default_scalar_type, default_real_type
from dolfinx.fem import (Constant,  Function, functionspace, assemble_scalar, 
                         dirichletbc, form, locate_dofs_topological)
# from dolfinx.fem.petsc import LinearProblem
# from dolfinx.io import XDMFFile
from dolfinx.mesh import create_unit_square, locate_entities, meshtags
# from dolfinx.plot import vtk_mesh
# import basix
# from mpi4py import MPI
from joblib import Parallel, delayed
import ufl
from ufl import (FacetNormal, Measure, SpatialCoordinate, TestFunction, TrialFunction, 
                 div, dot, dx, dS, grad, inner, lhs, rhs,jump, avg)
from pathlib import Path
import numpy as np
import time
from time import sleep
from mpi4py import MPI
N = 100
L = 100
# Generating mesh
msh = mesh.create_rectangle(comm = MPI.COMM_WORLD, points = ((0, 0), (L, L)), n = (N, N),
                            cell_type = mesh.CellType.triangle)

DG1 = fem.FunctionSpace(msh, ("Discontinuous Lagrange", 0))
RT  =  fem.FunctionSpace(msh, ("RT", 1))
u_f = fem.Function(RT)
def f(x):
    return (np.sin(np.pi*x[0]), np.sin(np.pi*x[1]))

u_f.interpolate(f)

u = fem.Function(DG1)

for i in range(0, u.vector.size):
    u.vector[i] = i


num_cells = msh.topology.index_map(msh.topology.dim).size_local+ msh.topology.index_map(msh.topology.dim).num_ghosts
msh.topology.create_connectivity(msh.topology.dim-1, msh.topology.dim)
num_edges = msh.topology.index_map(1).size_local + msh.topology.index_map(1).num_ghosts

cell_list = np.arange(num_cells, dtype=np.int32)
edge_list = np.arange(num_edges, dtype=np.int32)


tdim = msh.topology.dim # Topology dimension
fdim = tdim - 1            # Facet dimension 
msh.topology.create_connectivity(fdim, tdim)
e4c_list = msh.topology.connectivity(tdim, fdim)
boundary_facets = dolfinx.mesh.exterior_facet_indices(msh.topology)
edge_list1 = np.delete(edge_list, boundary_facets)
sort_index = np.argsort(edge_list1)
cell_tag = meshtags(msh, msh.topology.dim, cell_list, cell_list)
edge_tag = meshtags(msh, 1, edge_list1[sort_index], edge_list1[sort_index])
de = Measure("dS", domain=msh, subdomain_data = edge_tag)
dx = Measure("dx", domain=msh, subdomain_data = cell_tag)
n = FacetNormal(msh)
# for i in range(num_cells):
h = ufl.CellDiameter(msh)
h_avg = avg(h)
lmbda = ufl.conditional(ufl.gt(dot(u_f, n), 0), 1, 0)
epsilons = 1e-16
c_uw = lmbda("+") * u("+") + lmbda("-") * u("-")
h = ufl.CellDiameter(msh)
h_avg = avg(h)
# start_time = time.time()
cache_dir = f"{str(Path.cwd())}/.11cache_test"
cffi_options = ["-Ofast", "-march=native"]
jit_options = {"cffi_extra_compile_args": cffi_options, "cache_dir": cache_dir, 
                    "cffi_libraries": ["m"]}

t_DG0 = []
for i in range(num_cells):
    ap = 0
    an = 0
    cp = 0
    cn = 0
    # t_DG0 = []
    for j in e4c_list.links(i):
        if j in boundary_facets: 
            continue
        a = assemble_scalar(form(inner(c_uw, dot(u_f, n)("+")) * de(j), jit_options=jit_options))
        c1 = assemble_scalar(form(1/h_avg * jump(u) * de(j), jit_options=jit_options))
        if a > 0:
            ap += a
        else: 
            an += a  
        c1 += 1
        if  c1 > 0:  
            cp += c1
        else:    
            cn += c1                   
    t_DG0.append(min(ap, cp, 100))

Below is the problem I encountered

Traceback (most recent call last):
  File "/export/home/chenyux/Phi_Stable/111.py", line 85, in <module>
    a = assemble_scalar(form(inner(c_uw, dot(u_f, n)("+")) * de(j), jit_options=jit_options))
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/dolfinx/fem/forms.py", line 188, in form
    return _create_form(form)
           ^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/dolfinx/fem/forms.py", line 183, in _create_form
    return _form(form)
           ^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/dolfinx/fem/forms.py", line 141, in _form
    ufcx_form, module, code = jit.ffcx_jit(mesh.comm, form,
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/dolfinx/jit.py", line 56, in mpi_jit
    return local_jit(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/dolfinx/jit.py", line 204, in ffcx_jit
    r = ffcx.codegeneration.jit.compile_forms([ufl_object], options=p_ffcx, **p_jit)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/ffcx/codegeneration/jit.py", line 201, in compile_forms
    obj, module = _load_objects(cache_dir, module_name, form_names)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/ffcx/codegeneration/jit.py", line 319, in _load_objects
    compiled_module = importlib.util.module_from_spec(spec)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<frozen importlib._bootstrap>", line 573, in module_from_spec
  File "<frozen importlib._bootstrap_external>", line 1233, in create_module
  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
ImportError: /export/home/chenyux/Phi_Stable/.11cache_test/libffcx_forms_68d2a57526bb8713ce83b3d0b0538ee916ae33cc.cpython-311-x86_64-linux-gnu.so: failed to map segment from shared object
double free or corruption (!prev)

When I used MPI parallelism, I encountered the following error. Following the prompts, I deleted the failed file and increased the compilation time, but the error persisted.

Traceback (most recent call last):
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/ffcx/codegeneration/jit.py", line 68, in get_cached_module
    open(c_filename, "x")
FileExistsError: [Errno 17] File exists: '/export/home/chenyux/Phi_Stable/test1/.cachetest_1/libffcx_forms_e4f18e2aaed336cdfddd78aa7d4f1674fe3d757d.c'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/export/home/chenyux/Phi_Stable/test1/2test1.py", line 532, in <module>
    dt.value = get_time_step_adaptive(msh, DG0, u_f_l, c_n, phi_l, phi_n, mu_f, gamma2, delta1, delta2, beta, 10000)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/Phi_Stable/test1/Time_step_mpi.py", line 89, in get_time_step_adaptive
    c1 = assemble_scalar(form(gamma/h_avg * jump(mu_np) * de(j), jit_options=jit_options))                                                                                                        
                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/dolfinx/fem/forms.py", line 188, in form
    return _create_form(form)
           ^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/dolfinx/fem/forms.py", line 183, in _create_form
    return _form(form)
           ^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/dolfinx/fem/forms.py", line 141, in _form
    ufcx_form, module, code = jit.ffcx_jit(mesh.comm, form,
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/dolfinx/jit.py", line 85, in mpi_jit
    output = local_jit(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/dolfinx/jit.py", line 204, in ffcx_jit
    r = ffcx.codegeneration.jit.compile_forms([ufl_object], options=p_ffcx, **p_jit)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/ffcx/codegeneration/jit.py", line 176, in compile_forms
    obj, mod = get_cached_module(module_name, form_names, cache_dir, timeout)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/export/home/chenyux/.conda/envs/fenicsx-env/lib/python3.11/site-packages/ffcx/codegeneration/jit.py", line 90, in get_cached_module
    raise TimeoutError(f"""JIT compilation timed out, probably due to a failed previous compile.
TimeoutError: JIT compilation timed out, probably due to a failed previous compile.
        Try cleaning cache (e.g. remove /export/home/chenyux/Phi_Stable/test1/.cachetest_1/libffcx_forms_e4f18e2aaed336cdfddd78aa7d4f1674fe3d757d.c) or increase timeout option.

dokken · December 16, 2024, 2:09pm

Please note that

You are using an old version of DOLFINx (please specify what version)
Your implementation is very inefficient. You could for instance use:
compile_form and create_form from v0.9.0, which would then only compile your form once: FEniCS release notes | FEniCS Project

i.e. a minimal code is shown below for parts of your problem:

from dolfinx import mesh, fem

from dolfinx.mesh import meshtags

import ufl

from pathlib import Path
import numpy as np
from mpi4py import MPI

N = 100
L = 100
# Generating mesh
msh = mesh.create_rectangle(
    comm=MPI.COMM_WORLD,
    points=((0, 0), (L, L)),
    n=(N, N),
    cell_type=mesh.CellType.triangle,
)

# Define mesh independent UFL formulation
import basix.ufl

c_el = ufl.Mesh(msh.ufl_domain().ufl_coordinate_element())
el0 = basix.ufl.element("Discontinuous Lagrange", msh.basix_cell(), 0)
el1 = basix.ufl.element("RT", msh.basix_cell(), 1)
DG0_symbolic = ufl.FunctionSpace(c_el, el0)
RT1_symbolic = ufl.FunctionSpace(c_el, el1)

u_f_symb = ufl.Coefficient(RT1_symbolic)
u_symb = ufl.Coefficient(DG0_symbolic)

n = ufl.FacetNormal(c_el)
h = ufl.CellDiameter(c_el)
h_avg = ufl.avg(h)
lmbda = ufl.conditional(ufl.gt(ufl.dot(u_f_symb, n), 0), 1, 0)
epsilons = 1e-16
c_uw = lmbda("+") * u_symb("+") + lmbda("-") * u_symb("-")

integration_id = 397
de = ufl.Measure("dS", subdomain_id=integration_id)
form_0 = ufl.inner(c_uw, ufl.dot(u_f_symb, n)("+")) * de
form_1 = 1 / h_avg * ufl.jump(u_symb) * de

# Compile forms once
cache_dir = f"{str(Path.cwd())}/.11cache_test"
cffi_options = ["-Ofast", "-march=native"]
jit_options = {
    "cffi_extra_compile_args": cffi_options,
    "cache_dir": cache_dir,
    "cffi_libraries": ["m"],
}
compiled_form0 = fem.compile_form(MPI.COMM_WORLD, form_0, jit_options=jit_options)
compiled_form1 = fem.compile_form(MPI.COMM_WORLD, form_1, jit_options=jit_options)

DG1 = fem.functionspace(msh, DG0_symbolic.ufl_element())
RT = fem.functionspace(msh, RT1_symbolic.ufl_element())
u_f = fem.Function(RT)


def f(x):
    return (np.sin(np.pi * x[0]), np.sin(np.pi * x[1]))


u_f.interpolate(f)

u = fem.Function(DG1)

for i in range(0, u.x.array.size):
    u.x.array[i] = i


num_cells = (
    msh.topology.index_map(msh.topology.dim).size_local
    + msh.topology.index_map(msh.topology.dim).num_ghosts
)
msh.topology.create_connectivity(msh.topology.dim - 1, msh.topology.dim)
num_edges = msh.topology.index_map(1).size_local + msh.topology.index_map(1).num_ghosts

cell_list = np.arange(num_cells, dtype=np.int32)
edge_list = np.arange(num_edges, dtype=np.int32)


tdim = msh.topology.dim  # Topology dimension
fdim = tdim - 1  # Facet dimension
msh.topology.create_connectivity(fdim, tdim)
e4c_list = msh.topology.connectivity(tdim, fdim)
boundary_facets = mesh.exterior_facet_indices(msh.topology)
edge_list1 = np.delete(edge_list, boundary_facets)
sort_index = np.argsort(edge_list1)
cell_tag = meshtags(msh, msh.topology.dim, cell_list, cell_list)
edge_tag = meshtags(msh, 1, edge_list1[sort_index], edge_list1[sort_index])

# start_time = time.time()


t_DG0 = []
for i in range(num_cells):
    for j in e4c_list.links(i):
        if j in boundary_facets:
            continue
        facet_entities = fem.compute_integration_domains(
            fem.IntegralType.interior_facet,
            msh.topology,
            np.array([j], dtype=np.int32),
        )
        subdomains = {
            fem.IntegralType.interior_facet: [(integration_id, facet_entities)]
        }
        form0 = fem.create_form(
            compiled_form0, [], msh, subdomains, {u_symb: u, u_f_symb: u_f}, {}
        )
        print(i, j, msh.comm.allreduce(fem.assemble_scalar(form0), op=MPI.SUM))

Yuxiang_Chen · December 18, 2024, 12:45am

Thank you very much！ I have updated to the latest version. Now the compilation speed is much faster. It is effective.

Topic		Replies	Views
Regarding the use of dolfinx.fem.assemble_scalar General	2	767	January 23, 2023
Incompatible function arguments for assemble_scalar Errors	1	1115	April 29, 2022
Asemble_scalar error "LiteralFloat- no array attribute " dolfinx error , bug	11	150	June 4, 2024
Assembling Process General	1	143	May 21, 2024
Error and I can't follow up with new fixes Errors	5	42	October 22, 2024

Ffcx compilation error

Related topics