DataFrame collecting the data is only shown the result in last iteration

hello, I am writing topology optimization code and want to collect data using DataFrame but the result as shown in DataFrame is only the data in the last iteration , how can DataFrame collect the result in every iteration.

import dolfinx 
import numpy as np
import pandas as pd
import dolfinx.nls.petsc
import ufl
import dolfinx.fem.petsc
import pyvista
import matplotlib.pyplot as plt
from dolfinx.mesh import CellType, create_rectangle
from mpi4py import MPI
from dolfinx import fem, nls, io, default_scalar_type
from ufl import Measure, inner, grad, div, TestFunction, TrialFunction, adjoint, derivative, SpatialCoordinate, TestFunction, TrialFunction, dot, ds, dx
from dolfinx.fem import (Constant, Function, FunctionSpace,
                         assemble_scalar, dirichletbc, form, locate_dofs_geometrical)
from petsc4py import PETSc
from dolfinx.plot import vtk_mesh
import pyvista as pv
import time
#Library for ML
import seaborn as sns
import tensorflow as tf
import tensorflow.compat.v1 as tf
from sklearn.model_selection import train_test_split
from tensorflow.keras.layers import Dense

from tensorflow.keras.models import Sequential
from sklearn.preprocessing import LabelEncoder
import ast

start_time = time.time()

#Parameter
D0 = 1;
k0 = -1.0;
eta = 2.0;
gamma = 2.0;
alpha = 1e-16
Rfl = 0.0018

# mesh and function space
mesh = create_rectangle(MPI.COMM_WORLD, [[0, 0], [1, 1]], [100, 100], CellType.quadrilateral)
V = fem.FunctionSpace(mesh, ("CG", 1))
c = fem.Function(V)
phi = TestFunction(V)
lambdh = fem.Function(V)
lamda_real,sen_real,eps_real,c_real = fem.Function(V),fem.Function(V),fem.Function(V),fem.Function(V)
# The eps is the design variable
eps = fem.Function(V)
eps.x.array[:] = 0.5

data_sen = []
data_lam = []
data_eps = []
data_c = []
data_ite = []

loop = 5
for i in range(loop):
    eps_real = eps.x.array.real
    data_eps.append(eps_real)
    df_eps = pd.DataFrame(data_eps)
    df_eps_final = pd.DataFrame(df_eps)
    print("iteration ",i)
    print("eps",eps_real)
    
    #boundary_markers = FacetFunction('size_t',mesh)
    ds = Measure('ds')
    dx = Measure("dx") ##integration over different parts of the domain
    x = SpatialCoordinate(mesh)
    gu = Constant(mesh, default_scalar_type(0)) ##Neauman = 0
    
    # Residual of the variational form of Poisson problem
    R = inner(D0*eps**eta*grad(c), grad(phi))*dx - inner(k0*(1-eps)**gamma*c,phi)*dx - inner(gu,phi) *ds     # " - inner(gu,phi) *ds " is boundary neauman 
    
    ##Dirichlet 
    def on_boundary(x):
        return np.isclose(x[0], 0)
    dofs_L = fem.locate_dofs_geometrical(V , on_boundary)
    bc_L = fem.dirichletbc(default_scalar_type(1), dofs_L, V) ##Dirichlet = 1
    bcs = [bc_L]

    #Forming and solving the linear system
    problem_u = fem.petsc.NonlinearProblem(R, c, bcs)
    solver = nls.petsc.NewtonSolver(MPI.COMM_WORLD,problem_u)
    solver.rtol = 1e-16
    solver.solve(c)
    
    #how big is the solution
    J = -ufl.inner((k0*(1-eps)**gamma),c)*dx ##Objective function
    J_form = fem.form(J)
    J_old = fem.assemble_scalar(J_form)
    
    #solve the adjoint vector
    lhs = ufl.adjoint(ufl.derivative(R,c))
    rhs = -ufl.derivative(J,c) 
    
    def on_boundary_1(x):
        return np.isclose(x[0], 0)
    dofs_L_adj = fem.locate_dofs_geometrical(V , on_boundary_1)
    bc_L_adj = fem.dirichletbc(default_scalar_type(0), dofs_L_adj, V) ##Dirichlet = 1
    bcs_adjoint = [bc_L_adj]
    
    #Adjective problem
    problem_adj = dolfinx.fem.petsc.LinearProblem(lhs, rhs, bcs_adjoint, petsc_options={"ksp_type": "preonly", "pc_type": "lu","pc_factor_mat_solver_type": "mumps"})
    
    #solve adj problem in lamda (lamda is adjoint variable)
    lamda = problem_adj.solve()

    dJde = dolfinx.fem.Expression(k0*gamma*((1-eps)**(gamma-1))*c+dot(D0*eta*(eps**(eta-1))*grad(c),grad(lamda))+inner(k0*gamma*((1-eps)**(gamma-1))*c,lamda), V.element.interpolation_points())
    sens = fem.Function(V)
    sens.interpolate(dJde)
    
    # steepest descent with constant beta as stepsize
    beta = 0.02
    delta_eps = beta*sens
    eps_new = dolfinx.fem.Expression(eps + delta_eps, V.element.interpolation_points())
        
    # determine new value for the design variables 
    eps.interpolate(eps_new)

    eps1 = fem.Function(V)
    phi_3 = TestFunction(V)

    filter = inner((Rfl**2)*grad(eps1), grad(phi_3))*dx + inner(eps1,phi_3)*dx - inner(eps,phi_3)*dx
    filter_eps = fem.petsc.NonlinearProblem(filter, eps1)
    solver = nls.petsc.NewtonSolver(MPI.COMM_WORLD,filter_eps)
    solver.rtol = 1e-16
    solver.solve(eps1)
    eps.interpolate(eps1)
    eps.x.array[eps.x.array > 1] = 1
    eps.x.array[eps.x.array < 0] = 0 
    
    import math
    lamda_real,sen_real,eps_real,c_real = lamda.x.array.real, sens.x.array.real, eps.x.array.real, c.x.array.real

    ###collect data###
    data_c.append(c_real)
    df_c = pd.DataFrame(data_c)
    df_c_final = pd.DataFrame(df_c)
    
    print("c",c_real)
    print(" ")
    
    #data = eps
    #df = pd.DataFrame(data)
   
end_time = time.time()
execution_time = end_time - start_time
print("Execution time:", execution_time, "seconds")

the result from printed c and eps

iteration  0
eps [0.5 0.5 0.5 ... 0.5 0.5 0.5]
c [1.         1.         0.99243388 ... 0.64808462 0.64805222 0.64805222]
 
iteration  1
eps [0.49144952 0.49144952 0.49145228 ... 0.49159995 0.4916007  0.4916007 ]
c [1.         1.         0.99202673 ... 0.63116817 0.63113441 0.63113441]
 
iteration  2
eps [0.4836042  0.4836042  0.48359976 ... 0.48349888 0.48350033 0.48350033]
c [1.         1.         0.99163397 ... 0.61468617 0.61465109 0.61465109]
 
iteration  3
eps [0.47648518 0.47648518 0.47646429 ... 0.47569298 0.47569508 0.47569508]
c [1.         1.         0.99126011 ... 0.59872791 0.59869154 0.59869154]
 
iteration  4
eps [0.47009843 0.47009843 0.47005283 ... 0.46817523 0.46817795 0.46817795]
c [1.         1.         0.990909   ... 0.58336524 0.5833276  0.5833276 ]

show dataframe collecting ‘c’

show dataframe collecting ‘eps’

You need to copy the data contained in c_real into a new numpy array, for instance using data_c.append(c_real.copy()).

The reason is that you have defined c = fem.Function(V) outside of the for loop, hence at every solve the content of c.array gets overwritten.

thank you so much

I have another question , now I’m trying to add machine learning model to my topology optimization code , I got sensitivity (sens) predicted from ML and it has to be use for updating porosity(eps) with this code

sens = y_pred_flattened
print("sens from ml",type(sens)) 
print(sens)

beta = 0.02
delta_eps = beta*sens
eps_new = dolfinx.fem.Expression(eps + delta_eps, V.element.interpolation_points())
eps.interpolate(eps_new)
sens from ml <class 'numpy.ndarray'>
[-0.4347316  -0.4347316  -0.43203345 ... -0.3439212  -0.34392124
 -0.03649079]
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[10], line 6
      4 beta = 0.02
      5 delta_eps = beta*sens
----> 6 eps_new = dolfinx.fem.Expression(eps + delta_eps, V.element.interpolation_points())
      7 eps.interpolate(eps_new)
      9 '''
     10 sen3 = fem.Function(V)
     11 #sen3 = sens
   (...)
     24 eps.interpolate(eps_new)
     25 '''

File /usr/local/dolfinx-complex/lib/python3.10/dist-packages/dolfinx/fem/function.py:122, in Expression.__init__(self, e, X, comm, form_compiler_options, jit_options, dtype)
    120 if comm is None:
    121     try:
--> 122         mesh = ufl.domain.extract_unique_domain(e).ufl_cargo()
    123         comm = mesh.comm
    124     except AttributeError:

File /usr/local/lib/python3.10/dist-packages/ufl/domain.py:372, in extract_unique_domain(expr)
    370 def extract_unique_domain(expr):
    371     """Return the single unique domain expression is defined on or throw an error."""
--> 372     domains = extract_domains(expr)
    373     if len(domains) == 1:
    374         return domains[0]

File /usr/local/lib/python3.10/dist-packages/ufl/domain.py:365, in extract_domains(expr)
    363 """Return all domains expression is defined on."""
    364 domainlist = []
--> 365 for t in traverse_unique_terminals(expr):
    366     domainlist.extend(t.ufl_domains())
    367 return sorted(join_domains(domainlist))

File /usr/local/lib/python3.10/dist-packages/ufl/corealg/traversal.py:136, in traverse_unique_terminals(expr, visited)
    134 def traverse_unique_terminals(expr, visited=None):
    135     """Traverse unique terminals."""
--> 136     for op in unique_pre_traversal(expr, visited=visited):
    137         if op._ufl_is_terminal_:
    138             yield op

File /usr/local/lib/python3.10/dist-packages/ufl/corealg/traversal.py:68, in unique_pre_traversal(expr, visited)
     66     visited = set()
     67 lifo = [expr]
---> 68 visited.add(expr)
     70 while lifo:
     71     expr = lifo.pop()

TypeError: unhashable type: 'numpy.ndarray'

normally sensitivity (sens) in topology optimization has to be look like this form but the Machine learning cannot learn data with an imaginary number (this is the reason why I cut the imaginary part out ) so I cannot updating porosity(eps) because it has different type of data

sens from topo <class 'dolfinx.fem.function.Function'>
[-0.12749074+0.j -0.12749074+0.j -0.1309794 +0.j ... -0.30200866+0.j
 -0.30195467+0.j -0.30195467+0.j]

the question is how can i convert <class ‘numpy.ndarray’> to < class ‘dolfinx.fem.function.Function’>

Are you running a complex build of dolfinx? If you don’t need complex numbers, just use a real build and it will not add 0j to the array content.

What does using real build exactly mean and how do I do it?? I’m quit confused. I tried running everthing on python3-ipykernel not dolfinx-complex but I checked the type with ‘type()’ command, the value didn’t contain 0j anymore and it was still <class ‘dolfinx.fem.function.Function’>

from petsc4py import PETSc
print(PETSc.ScalarType)

will print <class 'numpy.float64'> in real numbers mode, and <class 'numpy.complex128'> if in complex mode.

So It means i’m in real mode now? However, I still cannot use sen_new from Machine learning model in the equation below, how do I fix it.

from petsc4py import PETSc
print(PETSc.ScalarType)

y_pred_flattened = y_pred.flatten()
sen_new = y_pred_flattened
print("sen_new",sen_new)
#print("sen_new",type(sen_new))

beta = 0.02
delta_eps = beta*sen_new
eps_new = dolfinx.fem.Expression(eps + delta_eps, V.element.interpolation_points())
eps.interpolate(eps_new)

print(eps.x.array)

<class ‘numpy.float64’>
sen_new [-0.15050066 -0.15050066 -0.15220425 … -0.2803556 -0.2803979
-0.28039792]

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[41], line 10
      8 beta = 0.02
      9 delta_eps = beta*sen_new
---> 10 eps_new = dolfinx.fem.Expression(eps + delta_eps, V.element.interpolation_points())
     11 eps.interpolate(eps_new)
     13 print(eps.x.array)

File /usr/local/dolfinx-real/lib/python3.10/dist-packages/dolfinx/fem/function.py:122, in Expression.__init__(self, e, X, comm, form_compiler_options, jit_options, dtype)
    120 if comm is None:
    121     try:
--> 122         mesh = ufl.domain.extract_unique_domain(e).ufl_cargo()
    123         comm = mesh.comm
    124     except AttributeError:

File /usr/local/lib/python3.10/dist-packages/ufl/domain.py:372, in extract_unique_domain(expr)
    370 def extract_unique_domain(expr):
    371     """Return the single unique domain expression is defined on or throw an error."""
--> 372     domains = extract_domains(expr)
    373     if len(domains) == 1:
    374         return domains[0]

File /usr/local/lib/python3.10/dist-packages/ufl/domain.py:365, in extract_domains(expr)
    363 """Return all domains expression is defined on."""
    364 domainlist = []
--> 365 for t in traverse_unique_terminals(expr):
    366     domainlist.extend(t.ufl_domains())
    367 return sorted(join_domains(domainlist))

File /usr/local/lib/python3.10/dist-packages/ufl/corealg/traversal.py:136, in traverse_unique_terminals(expr, visited)
    134 def traverse_unique_terminals(expr, visited=None):
    135     """Traverse unique terminals."""
--> 136     for op in unique_pre_traversal(expr, visited=visited):
    137         if op._ufl_is_terminal_:
    138             yield op

File /usr/local/lib/python3.10/dist-packages/ufl/corealg/traversal.py:68, in unique_pre_traversal(expr, visited)
     66     visited = set()
     67 lifo = [expr]
---> 68 visited.add(expr)
     70 while lifo:
     71     expr = lifo.pop()

TypeError: unhashable type: 'numpy.ndarray'

I am not sure what you are trying to do here, but I assume that the issue is that delta_eps is a numpy array. It is unclear how a numpy.ndarray should be added to a dolfinx.fem.Function.

Please provide more details as to what you are trying to do, and preferably a minimal reproducible example.

This is the minimal framework to explain what I am trying to, I checked the type of all value in optimization loop and it was <class ‘dolfinx.fem.function.Function’> but then sensitivity(sens) predicted from Machine learning was <class ‘numpy.ndarray’>

I don’t know how to add that sensitivity(sens) from ML model into those equation I asked before.

delta_eps = beta*sens
eps_new = dolfinx.fem.Expression(eps + delta_eps, V.element.interpolation_points())

minimal framework

eps.x.array[:] = 0.5
for i in range
         #solving concentration
         R = inner(D0*eps**eta*grad(c), grad(phi))*dx - inner(k0*(1-eps)**gamma*c,phi)*dx - inner(gu,phi) *ds
         solver.solve(c)

         #solving  lamda from Objective function
         J = -ufl.inner((k0*(1-eps)**gamma),c)*dx ##Objective function

         lamda = problem_adj.solve()

         
        remainder = i % 30
        if remainder < 10
             #solving sensitivity(dJde=sens)
             
             #collecting data (eps,c,lamda,sensitivity) from previous iteration 

        elif remainder < 30:  
             #using Machine learning model to predict sensitivity(sens)
       
       ####OUT of if else####
       # updating porosity (eps) using sensitivity(sens) predicted from Machine learning model
       beta = 0.02
       delta_eps = beta*sens
       eps_new = dolfinx.fem.Expression(eps + delta_eps, V.element.interpolation_points())

#then replacing the 'eps_new' into eps and recalculate the values ​​from #solving concentration

and this is sensitivity(sens) predicted from Machine learning model look like

sens from ML : [-0.16931154 -0.16931154 -0.17171979 … -0.28193933 -0.28196728
-0.2819672 ]

Are these clear or you want some minimal code showing how I arranged data.

I also check the size of function space V and size of sensitivity(sens = y_pred) predicted from Machine learning model, the size of both is matching.

num_dofs = V.dofmap.index_map.size_global
print("num_dofs of function space (V):", num_dofs)

size_y_pred = np.shape(y_pred)
print("Size of sens_predict:", size_y_pred)
num_dofs of function space (V): 10201
Size of sens_predict: (10201, 1)

If the sensitivity array corresponds to each degree of freedom in the mesh, then eps.x.array[:] += delta_eps.