Compute the sensitivity in parallel with DOLFINx

Hi all, I am trying to compute the sensitivity with DOLFINx based on the adjoint method (lecture_12_sensitivities.pdf (fenicsproject.org)).

Given:
(1) PDE: \mathbf{R}(\mathbf{u})=\mathbf{0}
(2) Variable m
(3) Objective functional g(m,\mathbf{u}(m))

Compute the sensitivity:
\dfrac{\text{d}g(m,\mathbf{u}(m))}{\text{d}m} = \dfrac{\partial g}{\partial m}+\left(\dfrac{\partial \mathbf{R}}{\partial m}\right)^\text{T} \cdot \boldsymbol{\lambda}

where the adjoint vector \boldsymbol{\lambda} can be solved from
\left(\dfrac{\partial \mathbf{R}}{\partial \mathbf{u}}\right)^\text{T} \cdot \boldsymbol{\lambda} = -\dfrac{\partial g}{\partial \mathbf{u}}

I guess this is very similar to the dolfin-adjoint package. But it seems we do not have the corresponding package in DOLFINx.

I try to implement the adjoint method and it works well when running in serial. The MWE is as follows:

import dolfinx, ufl
from dolfinx.generation import RectangleMesh
from dolfinx.mesh import CellType
from dolfinx.fem import VectorFunctionSpace, FunctionSpace, Function, DirichletBC
from dolfinx.nls import NewtonSolver
from mpi4py import MPI
import numpy as np
import scipy.sparse

# mesh and function space
mesh = RectangleMesh(MPI.COMM_WORLD, [[0, 0, 0], [1, 1, 0]], \
    [3, 3], CellType.quadrilateral)
V = VectorFunctionSpace(mesh, ("CG", 1))
S = FunctionSpace(mesh, ("DG", 0))
u = Function(V) # test function (displacment field)
v = ufl.TestFunction(V) # trial function

# kinematics
lambda_ = 0.5769
mu = Function(S) # in this case, we set the mu as a design variable
mu.vector.array = 0.3846
P = 2.0*mu*ufl.sym(ufl.grad(u)) + lambda_*ufl.tr(ufl.sym(ufl.grad(u)))*ufl.Identity(len(u))
dx = ufl.Measure("dx", metadata={"quadrature_degree": 4})
R = ufl.inner(ufl.grad(v), P)*dx

# boundary conditions
left = lambda x: np.isclose(x[0], 0)
right = lambda x: np.isclose(x[0], 1)
fdim = mesh.topology.dim - 1

u1 = Function(V)
boundary_facets = dolfinx.mesh.locate_entities_boundary(mesh, fdim, left)
with u1.vector.localForm() as loc:
    loc.set(0)
bc1 = DirichletBC(u1, dolfinx.fem.locate_dofs_topological(V, fdim, boundary_facets))
u2 = Function(V)
boundary_facets = dolfinx.mesh.locate_entities_boundary(mesh, fdim, right)
with u2.vector.localForm() as loc:
    loc.set(1)
bc2 = DirichletBC(u2, dolfinx.fem.locate_dofs_topological(V, fdim, boundary_facets))
bcs = [bc1, bc2]

# solve the problem
problem = dolfinx.fem.NonlinearProblem(R, u, bcs)
solver = NewtonSolver(MPI.COMM_WORLD, problem)
solver.solve(u)
# print(u.vector.array)

# set an arbitrary functinal
J = mu*ufl.dot(u, u)*dx
J_old = dolfinx.fem.assemble_scalar(J)

# partial derivative of J w.r.t. mu
dJdmu = dolfinx.fem.assemble_vector(ufl.derivative(J, mu)) 
dJdmu.assemble()
dJdmu = dJdmu.getArray()

# partial derivative of R w.r.t. mu
dRdmu = dolfinx.fem.assemble_matrix(ufl.derivative(R, mu)) # partial derivative
dRdmu.assemble()
ai, aj, av = dRdmu.getValuesCSR()
dRdmu = scipy.sparse.csr_matrix((av, aj, ai))

# reset the boundary condition
with u2.vector.localForm() as loc:
    loc.set(0.0)

# solve the adjoint vector
lhs = ufl.adjoint(ufl.derivative(R, u))
rhs = -ufl.derivative(J, u)
problem = dolfinx.fem.LinearProblem(lhs, rhs, bcs=bcs)
lambda_vec = problem.solve().vector.array

# compute the sensitivity
dJdmu += dRdmu.T@lambda_vec

However, when running in parallel, I have to collect the displacement field to root 0 (based on Gather solutions in parallel in FEniCSX - FEniCS Project) and then evaluate the sensitivity in a global mesh.

So, my question is, is there a way to make the above code also support the parallel computation?

The problem with your code is that you start using scipy matrices to do the operations. If you continue using ‘pure’ dolfinx/ufl/petsc operations, everything works as expected.
Consider the minimal code below:

import dolfinx
import ufl
from dolfinx.generation import RectangleMesh
from dolfinx.mesh import CellType
from dolfinx.fem import VectorFunctionSpace, FunctionSpace, Function, DirichletBC
from dolfinx.nls import NewtonSolver
from mpi4py import MPI
from petsc4py import PETSc
import numpy as np

# mesh and function space
mesh = RectangleMesh(MPI.COMM_WORLD, [[0, 0, 0], [1, 1, 0]],
                     [3, 3], CellType.quadrilateral)
V = VectorFunctionSpace(mesh, ("CG", 1))
S = FunctionSpace(mesh, ("DG", 0))
u = Function(V)  # test function (displacment field)
v = ufl.TestFunction(V)  # trial function

# kinematics
lambda_ = 0.5769
mu = Function(S)  # in this case, we set the mu as a design variable
mu.vector.array = 0.3846
P = 2.0*mu*ufl.sym(ufl.grad(u)) + lambda_ * \
    ufl.tr(ufl.sym(ufl.grad(u)))*ufl.Identity(len(u))
dx = ufl.Measure("dx", metadata={"quadrature_degree": 4})
R = ufl.inner(ufl.grad(v), P)*dx

# boundary conditions


def left(x): return np.isclose(x[0], 0)
def right(x): return np.isclose(x[0], 1)


fdim = mesh.topology.dim - 1

u1 = Function(V)
boundary_facets = dolfinx.mesh.locate_entities_boundary(mesh, fdim, left)
with u1.vector.localForm() as loc:
    loc.set(0)
bc1 = DirichletBC(u1, dolfinx.fem.locate_dofs_topological(
    V, fdim, boundary_facets))
u2 = Function(V)
boundary_facets = dolfinx.mesh.locate_entities_boundary(mesh, fdim, right)
with u2.vector.localForm() as loc:
    loc.set(1)
bc2 = DirichletBC(u2, dolfinx.fem.locate_dofs_topological(
    V, fdim, boundary_facets))
bcs = [bc1, bc2]

# solve the problem
problem = dolfinx.fem.NonlinearProblem(R, u, bcs)
solver = NewtonSolver(MPI.COMM_WORLD, problem)
solver.solve(u)
# print(u.vector.array)

# set an arbitrary functinal
J = mu*ufl.dot(u, u)*dx
J_old = dolfinx.fem.assemble_scalar(J)

# partial derivative of J w.r.t. mu
dJdmu = dolfinx.fem.assemble_vector(ufl.derivative(J, mu))
dJdmu.assemble()

# partial derivative of R w.r.t. mu
dRdmu = dolfinx.fem.assemble_matrix(
    ufl.adjoint(ufl.derivative(R, mu)))  # partial derivative
dRdmu.assemble()

# reset the boundary condition
with u2.vector.localForm() as loc:
    loc.set(0.0)

# solve the adjoint vector
lhs = ufl.adjoint(ufl.derivative(R, u))
rhs = -ufl.derivative(J, u)
problem = dolfinx.fem.LinearProblem(lhs, rhs, bcs=bcs)
lmbda = problem.solve()

dJdmu += dRdmu*lmbda.vector
dJdmu.ghostUpdate(addv=PETSc.InsertMode.INSERT, mode=PETSc.ScatterMode.FORWARD)
print(dJdmu.array)
4 Likes

Hello,

I was wondering if it is possible to compute the sensitivity when the design parameter is a boundary condition.

To keep the explanation simple, we can consider the same problem as @Yingqi_Jia proposed above in this post. This would be the updated dolfinx code:

import dolfinx
import ufl
from mpi4py import MPI
from petsc4py import PETSc
import numpy as np

# mesh and function space
mesh = dolfinx.mesh.create_rectangle(MPI.COMM_WORLD, [np.array([0, 0]), np.array([1, 1])], [10, 10], dolfinx.mesh.CellType.quadrilateral)
V = dolfinx.fem.VectorFunctionSpace(mesh, ("CG", 1))
S = dolfinx.fem.FunctionSpace(mesh, ("DG", 0))
u = dolfinx.fem.Function(V)  # test function (displacment field)
v = ufl.TestFunction(V)  # trial function

# kinematics
lambda_ = 0.5769
mu = dolfinx.fem.Function(S)  # in this case, we set the mu as a design variable
mu.vector.array = 0.3846
P = 2.0*mu*ufl.sym(ufl.grad(u)) + lambda_ * ufl.tr(ufl.sym(ufl.grad(u)))*ufl.Identity(len(u))
dx = ufl.Measure("dx", metadata={"quadrature_degree": 4})
R = ufl.inner(P, ufl.grad(v))*dx

# boundary conditions
def left(x): 
    return np.isclose(x[0], 0)
def right(x): 
    return np.isclose(x[0], 1)

fdim = mesh.topology.dim - 1

u1 = dolfinx.fem.Function(V)
boundary_facets = dolfinx.mesh.locate_entities_boundary(mesh, fdim, left)
with u1.vector.localForm() as loc:
    loc.set(0)
bc1 = dolfinx.fem.dirichletbc(u1, dolfinx.fem.locate_dofs_topological(V, fdim, boundary_facets))
u2 = dolfinx.fem.Function(V)
boundary_facets = dolfinx.mesh.locate_entities_boundary(mesh, fdim, right)
with u2.vector.localForm() as loc:
    loc.set(1)
bc2 = dolfinx.fem.dirichletbc(u2, dolfinx.fem.locate_dofs_topological(V, fdim, boundary_facets))
bcs = [bc1, bc2]

# solve the problem
problem = dolfinx.fem.petsc.NonlinearProblem(R, u, bcs)
solver = dolfinx.nls.petsc.NewtonSolver(MPI.COMM_WORLD, problem)
solver.solve(u)
# print(u.vector.array)

# set an arbitrary functinal
J = mu*ufl.dot(u, u)*dx
J_old = dolfinx.fem.assemble_scalar(dolfinx.fem.form(J))
print(J_old)

# partial derivative of J w.r.t. mu
dJdmu = dolfinx.fem.petsc.assemble_vector(dolfinx.fem.form(ufl.derivative(J, mu)))
dJdmu.assemble()

# partial derivative of R w.r.t. mu
dRdmu = dolfinx.fem.petsc.assemble_matrix(dolfinx.fem.form(ufl.adjoint(ufl.derivative(R, mu))))  # partial derivative
dRdmu.assemble()

# reset the boundary condition
with u2.vector.localForm() as loc:
    loc.set(0.0)

# solve the adjoint vector
lhs = ufl.adjoint(ufl.derivative(R, u))
rhs = -ufl.derivative(J, u)
problem = dolfinx.fem.petsc.LinearProblem(lhs, rhs, bcs=bcs)
lmbda = problem.solve()

dJdmu += dRdmu*lmbda.vector
dJdmu.ghostUpdate(addv=PETSc.InsertMode.INSERT, mode=PETSc.ScatterMode.FORWARD)
print(dJdmu.array)

Using this code, it is possible to compute the sensitivity of a cost function with respect to a parameter mu that appears in the PDE explicitly. The derivative of the residual of the PDE with respect to that parameter is easy to compute using ufl.derivative, i.e.: dRdmu = dolfinx.fem.petsc.assemble_matrix(dolfinx.fem.form(ufl.adjoint(ufl.derivative(R, mu)))) .

However, I am not sure if it is possible to use this approach when we want to compute the derivative of the residual of the PDE with respect to a boundary condition (for example, u1: the Dirichlet boundary condition in the left boundary).

Does anybody know if this would be possible?

Thank you very much in advance.

See chapter 4.4 of: https://www.duo.uio.no/bitstream/handle/10852/63505/SebastianMitusch-thesis.pdf?sequence=1&isAllowed=y#page35

1 Like

Thank you very much. It is very useful.

Hi,

why it’s necessary to reset the boundary condition, i.e., why

# reset the boundary condition
with u2.vector.localForm() as loc:
    loc.set(0.0)

is necessary? Thank you.

The bc of the adjoint eq is 0 on Dirichlet boundaries.

1 Like

I am trying to make the solution work for dolfinx 0.8.0 but it is leading me to error with the assemble_matrix() step. I would appreciate if you could kindly fix it.

My adaptation is below:

import dolfinx
import dolfinx.fem.petsc
import ufl
from dolfinx.fem import form
from dolfinx.mesh import CellType
from dolfinx.fem import functionspace, Function, DirichletBC
from dolfinx.nls.petsc import NewtonSolver
from dolfinx.fem.petsc import NonlinearProblem, LinearProblem
from mpi4py import MPI
from petsc4py import PETSc
import numpy as np

# mesh and function space
mesh = dolfinx.mesh.create_rectangle(MPI.COMM_WORLD, np.array([[0, 0], [1, 1]]), [3, 3], cell_type=CellType.triangle)

V = functionspace(mesh, ("CG", 1, (2,)))
S = functionspace(mesh, ("DG", 0))
u = Function(V)  # test function (displacment field)
v = ufl.TestFunction(V)  # trial function

# kinematics
lambda_ = 0.5769
mu = Function(S)  # in this case, we set the mu as a design variable
mu.vector.array = 0.3846
P = 2.0*mu*ufl.sym(ufl.grad(u)) + lambda_ * \
    ufl.tr(ufl.sym(ufl.grad(u)))*ufl.Identity(len(u))
dx = ufl.Measure("dx", metadata={"quadrature_degree": 4})
R = ufl.inner(P, ufl.grad(v))*dx

# boundary conditions
def left(x): return np.isclose(x[0], 0)
def right(x): return np.isclose(x[0], 1)

fdim = mesh.topology.dim - 1

u1 = Function(V)
with u1.vector.localForm() as loc:
    loc.set(0)

boundary_facets = dolfinx.mesh.locate_entities_boundary(mesh, fdim, left)
bc1 = dolfinx.fem.dirichletbc(u1, dolfinx.fem.locate_dofs_topological(
    V, fdim, boundary_facets))
u2 = Function(V)
boundary_facets = dolfinx.mesh.locate_entities_boundary(mesh, fdim, right)
with u2.vector.localForm() as loc:
    loc.set(1)
bc2 = dolfinx.fem.dirichletbc(u2, dolfinx.fem.locate_dofs_topological(
    V, fdim, boundary_facets))
bcs = [bc1, bc2]

# solve the problem
problem = NonlinearProblem(R, u, bcs)
solver = NewtonSolver(MPI.COMM_WORLD, problem)
solver.solve(u)
# print(u.vector.array)

# set an arbitrary functinal
J = mu*ufl.dot(u, u)*dx
J_old = dolfinx.fem.assemble_scalar(form(J))

# partial derivative of J w.r.t. mu
dJdmu = dolfinx.fem.assemble_vector(form(ufl.derivative(J, mu)))
# dJdmu.assemble()

# partial derivative of R w.r.t. mu
dRdmu_form = form(ufl.adjoint(ufl.derivative(R, mu)))
dRdmu = dolfinx.fem.assemble_matrix(dRdmu_form)  # partial derivative
# dRdmu.assemble()

# reset the boundary condition
with u2.vector.localForm() as loc:
    loc.set(0.0)

# solve the adjoint vector
lhs = ufl.adjoint(ufl.derivative(R, u))
rhs = -ufl.derivative(J, u)
problem = LinearProblem(lhs, rhs, bcs=bcs)
lmbda = problem.solve()

dJdmu += dRdmu*lmbda.vector
dJdmu.ghostUpdate(addv=PETSc.InsertMode.INSERT, mode=PETSc.ScatterMode.FORWARD)
print(dJdmu.array)

This leads to the following error:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In[31], line 67
     63 # dJdmu.assemble()
     64 
     65 # partial derivative of R w.r.t. mu
     66 dRdmu_form = form(ufl.adjoint(ufl.derivative(R, mu)))
---> 67 dRdmu = dolfinx.fem.assemble_matrix(dRdmu_form)  # partial derivative
     68 # dRdmu.assemble()
     69 
     70 # reset the boundary condition
     71 with u2.vector.localForm() as loc:

File ~/miniconda3/envs/fenics/lib/python3.12/functools.py:909, in singledispatch.<locals>.wrapper(*args, **kw)
    905 if not args:
    906     raise TypeError(f'{funcname} requires at least '
    907                     '1 positional argument')
--> 909 return dispatch(args[0].__class__)(*args, **kw)

File ~/miniconda3/envs/fenics/lib/python3.12/site-packages/dolfinx/fem/assemble.py:255, in assemble_matrix(a, bcs, diagonal, constants, coeffs, block_mode)
    253 bcs = [] if bcs is None else bcs
    254 A: la.MatrixCSR = create_matrix(a, block_mode)
--> 255 _assemble_matrix_csr(A, a, bcs, diagonal, constants, coeffs)
    256 return A

File ~/miniconda3/envs/fenics/lib/python3.12/site-packages/dolfinx/fem/assemble.py:292, in _assemble_matrix_csr(A, a, bcs, diagonal, constants, coeffs)
    290 constants = _pack_constants(a._cpp_object) if constants is None else constants
    291 coeffs = _pack_coefficients(a._cpp_object) if coeffs is None else coeffs
--> 292 _cpp.fem.assemble_matrix(A._cpp_object, a._cpp_object, constants, coeffs, bcs)
    294 # If matrix is a 'diagonal'block, set diagonal entry for constrained
    295 # dofs
    296 if a.function_spaces[0] is a.function_spaces[1]:

RuntimeError: Non-square blocksize unsupported in Python

Use dolfinx.fem.petsc.assemble_ instead of dolfinx.fem.assemble_ resolves the issue.
MWE:

import dolfinx
import dolfinx.fem.petsc
import ufl
from dolfinx.fem import form
from dolfinx.mesh import CellType
from dolfinx.fem import functionspace, Function, DirichletBC
from dolfinx.nls.petsc import NewtonSolver
from dolfinx.fem.petsc import NonlinearProblem, LinearProblem
from mpi4py import MPI
from petsc4py import PETSc
import numpy as np

# mesh and function space
mesh = dolfinx.mesh.create_rectangle(MPI.COMM_WORLD, np.array([[0, 0], [1, 1]]), [3, 3], cell_type=CellType.triangle)

V = functionspace(mesh, ("CG", 1, (2,)))
S = functionspace(mesh, ("DG", 0))
u = Function(V)  # test function (displacment field)
v = ufl.TestFunction(V)  # trial function

# kinematics
lambda_ = 0.5769
mu = Function(S)  # in this case, we set the mu as a design variable
mu.vector.array = 0.3846
P = 2.0*mu*ufl.sym(ufl.grad(u)) + lambda_ * \
    ufl.tr(ufl.sym(ufl.grad(u)))*ufl.Identity(len(u))
dx = ufl.Measure("dx", metadata={"quadrature_degree": 4})
R = ufl.inner(P, ufl.grad(v))*dx

# boundary conditions
def left(x): return np.isclose(x[0], 0)
def right(x): return np.isclose(x[0], 1)

fdim = mesh.topology.dim - 1

u1 = Function(V)
with u1.vector.localForm() as loc:
    loc.set(0)

boundary_facets = dolfinx.mesh.locate_entities_boundary(mesh, fdim, left)
bc1 = dolfinx.fem.dirichletbc(u1, dolfinx.fem.locate_dofs_topological(
    V, fdim, boundary_facets))
u2 = Function(V)
boundary_facets = dolfinx.mesh.locate_entities_boundary(mesh, fdim, right)
with u2.vector.localForm() as loc:
    loc.set(1)
bc2 = dolfinx.fem.dirichletbc(u2, dolfinx.fem.locate_dofs_topological(
    V, fdim, boundary_facets))
bcs = [bc1, bc2]

# solve the problem
problem = NonlinearProblem(R, u, bcs)
solver = NewtonSolver(MPI.COMM_WORLD, problem)
solver.solve(u)
# print(u.vector.array)

# set an arbitrary functinal
J = mu*ufl.dot(u, u)*dx
J_old = dolfinx.fem.assemble_scalar(form(J))

# partial derivative of J w.r.t. mu
dJdmu = dolfinx.fem.petsc.assemble_vector(form(ufl.derivative(J, mu)))
# dJdmu.assemble()

# partial derivative of R w.r.t. mu
dRdmu_form = form(ufl.adjoint(ufl.derivative(R, mu)))
dRdmu = dolfinx.fem.petsc.assemble_matrix(dRdmu_form)  # partial derivative
dRdmu.assemble()

# reset the boundary condition
with u2.vector.localForm() as loc:
    loc.set(0.0)

# solve the adjoint vector
lhs = ufl.adjoint(ufl.derivative(R, u))
rhs = -ufl.derivative(J, u)
problem = LinearProblem(lhs, rhs, bcs=bcs)
lmbda = problem.solve()

dJdmu += dRdmu*lmbda.vector
dJdmu.ghostUpdate(addv=PETSc.InsertMode.INSERT, mode=PETSc.ScatterMode.FORWARD)
print(dJdmu.array)
1 Like