Merge remote-tracking branch 'origin/eigensolvers-dev' into arpack-dev

# Conflicts:
#	linalg/eigensolvers.hpp

.gitignore

@@ -451,8 +451,8 @@ miniapps/plasma/pic/*.csv
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
-tests/unit/cunit_tests
-tests/unit/pcunit_tests
+tests/unit/gpu_unit_tests
+tests/unit/pgpu_unit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*

CMakeLists.txt

@@ -592,13 +592,6 @@ if (MFEM_USE_ENZYME)
     set(ENZYME_INCLUDE_DIRS ${ENZYME_DIR}/include)
 endif()
 
-if (MFEM_USE_PROTEUS)
-    enable_language(C)
-    find_package(proteus REQUIRED PATHS "${PROTEUS_DIR}")
-    message(STATUS "${PROTEUS_DIR}/include")
-    include_directories("${PROTEUS_DIR}/include")
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -735,16 +728,6 @@ mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
-
-if (MFEM_USE_PROTEUS)
-    add_library(ClangProteusFlags INTERFACE IMPORTED)
-    set_target_properties(ClangProteusFlags PROPERTIES
-        INTERFACE_COMPILE_OPTIONS "-fpass-plugin=$<TARGET_FILE:ProteusPass>"
-    )
-    target_link_libraries(mfem PUBLIC ClangProteusFlags)
-    target_link_libraries(mfem PUBLIC proteus)
-endif()
-
 if (TPL_TARGETS)
   add_dependencies(mfem ${TPL_TARGETS})
 endif()

config/config.hpp

@@ -157,22 +157,4 @@ constexpr real_t operator""_r(unsigned long long v)
 #endif
 #endif // MFEM_USE_MPI not defined
 
-#ifdef NVTX_DBG_HPP
-#include NVTX_DBG_HPP
-#else
-#define db1(...)
-#define dbg(...)
-#define dbl(...)
-#define dba(...)
-#define dbc(...)
-#define NVTX_MARK_FUNCTION
-#define NVTX_MARK_BEGIN(...)
-#define NVTX_INI(...)
-#define NVTX_END(...)
-#define NVTX_MARK_INI(...)
-#define NVTX_MARK_END(...)
-#define NVTX_MARK(...)
-#define NVTX(...)
-#endif
-
 #endif // MFEM_CONFIG_HPP

config/config.hpp.in

@@ -97,6 +97,9 @@
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE
 
+// Enable MFEM functionality based on the ARPACK library.
+// #define MFEM_USE_ARPACK
+
 // Enable MFEM functionality based on the SuperLU_DIST library.
 // #define MFEM_USE_SUPERLU
 // #define MFEM_USE_SUPERLU5

config/config.mk.in

@@ -32,6 +32,7 @@ MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
+MFEM_USE_ARPACK        = @MFEM_USE_ARPACK@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
 MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@

config/defaults.mk

@@ -178,6 +178,7 @@ MFEM_USE_ALGOIM        = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
+MFEM_USE_ARPACK        = NO
 MFEM_USE_MKL_CPARDISO  = NO
 MFEM_USE_MKL_PARDISO   = NO
 MFEM_USE_MOONOLITH     = NO
@@ -427,6 +428,14 @@ NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
  $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
  -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB)
 
+# ARPACK library configuration
+ARPACK_DIR = @MFEM_DIR@/../ARPACK
+ifeq ($(MFEM_USE_MPI),YES)
+ARPACK_LIB = -L$(ARPACK_DIR) -lparpack -larpack
+else
+ARPACK_LIB = -L$(ARPACK_DIR) -larpack
+endif
+
 # PETSc library configuration (version greater or equal to 3.8 or the dev branch)
 PETSC_ARCH := arch-linux2-c-debug
 PETSC_DIR  := $(MFEM_DIR)/../petsc/$(PETSC_ARCH)

examples/CMakeLists.txt

@@ -47,9 +47,15 @@ list(APPEND ALL_EXE_SRCS
   ex39.cpp
   ex40.cpp
   ex41.cpp
-  jitplayground.cpp
   )
 
+if (MFEM_USE_ARPACK)
+list(APPEND ALL_EXE_SRCS
+  ex11.pp
+  ex13.pp
+)
+endif()
+
 if (MFEM_USE_MPI)
   list(APPEND ALL_EXE_SRCS
     ex0p.cpp

examples/ex11.cpp

@@ -0,0 +1,298 @@
+//                       MFEM Example 11 - Serial Version
+//
+// Compile with: make ex11
+//
+// Sample runs:  ex11 -m ../data/square-disc.mesh
+//               ex11 -m ../data/star.mesh
+//               ex11 -m ../data/star-mixed.mesh
+//               ex11 -m ../data/periodic-annulus-sector.msh
+//               ex11 -m ../data/square-disc-p2.vtk -o 2
+//               ex11 -m ../data/square-disc-p3.mesh -o 3
+//               ex11 -m ../data/square-disc-nurbs.mesh -o -1
+//               ex11 -m ../data/disc-nurbs.mesh -o -1 -n 20
+//               ex11 -m ../data/star-surf.mesh
+//               ex11 -m ../data/square-disc-surf.mesh
+//               ex11 -m ../data/inline-segment.mesh
+//               ex11 -m ../data/inline-quad.mesh
+//               ex11 -m ../data/inline-tri.mesh
+//               ex11 -m ../data/amr-quad.mesh
+//               ex11 -m ../data/amr-hex.mesh
+//               ex11 -m ../data/mobius-strip.mesh -n 8
+//
+// Description:  This example code demonstrates the use of MFEM to solve the
+//               eigenvalue problem -Delta u = lambda u with homogeneous
+//               Dirichlet boundary conditions.
+//
+//               We compute a number of the lowest eigenmodes by discretizing
+//               the Laplacian and Mass operators using a FE space of the
+//               specified order, or an isoparametric/isogeometric space if
+//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of the ARPACK eigenvalue solver
+//               (regular inverse mode). Reusing a single GLVis visualization
+//               window for multiple eigenfunctions is also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+#ifdef MFEM_USE_ARPACK
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int ser_ref_levels = 3;
+   int order = 1;
+   int nev = 5;
+   double dbc_eig = 1e3;
+   bool visualization = 1;
+   bool arp_solver = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nev, "-n", "--num-eigs",
+                  "Number of desired eigenmodes.");
+   args.AddOption(&dbc_eig, "-d", "--dbc-eig",
+                  "Eigenvalues associated with Dirichlet BC "
+                  "(should be larger than the maximum desired eigenvalue).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+   int dim = mesh->Dimension();
+
+   // 3. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
+   //    specified on the command line with -rs).
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 4. Define a finite element space on the mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   int size = fespace->GetVSize();
+
+   cout << "Number of unknowns: " << size << endl;
+
+   // 5. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   //    element space. The first corresponds to the Laplacian operator -Delta,
+   //    while the second is a simple mass matrix needed on the right hand side
+   //    of the generalized eigenvalue problem below. The boundary conditions
+   //    are implemented by elimination with special values on the diagonal to
+   //    shift the Dirichlet eigenvalues out of the computational range. After
+   //    serial and parallel assembly we extract the corresponding parallel
+   //    matrices A and M.
+   ConstantCoefficient one(1.0);
+   Array<int> ess_bdr;
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   if (mesh->bdr_attributes.Size() == 0)
+   {
+      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
+      // closed surface.
+      a->AddDomainIntegrator(new MassIntegrator(one));
+   }
+   a->Assemble();
+   if (mesh->bdr_attributes.Size() != 0)
+   {
+      a->EliminateEssentialBCDiag(ess_bdr, dbc_eig);
+   }
+   a->Finalize();
+
+   BilinearForm *m = new BilinearForm(fespace);
+   m->AddDomainIntegrator(new MassIntegrator(one));
+   m->Assemble();
+   if (mesh->bdr_attributes.Size() != 0)
+   {
+      // shift the eigenvalue corresponding to eliminated dofs to a large value
+      m->EliminateEssentialBCDiag(ess_bdr, 1.0);
+   }
+   m->Finalize();
+
+   Solver       * solver = NULL;
+
+#ifndef MFEM_USE_SUITESPARSE
+   // 6. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //    solve the system A X = B with PCG.
+   cout << "Building CGSolver" << endl;
+   GSSmoother M(m->SpMat());
+   CGSolver * cg_solver = new CGSolver;
+   cg_solver->SetPreconditioner(M);
+   cg_solver->SetRelTol(1.0e-12);
+   solver = cg_solver;
+#else
+   // 7. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   cout << "Building UMFPackSolver" << endl;
+   UMFPackSolver * umf_solver = new UMFPackSolver;
+   umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   solver = umf_solver;
+#endif
+   solver->SetOperator(m->SpMat());
+
+   // 7. Define and configure the ARPACK eigensolver
+   SymGenEigensolver * eig_solver = NULL;
+
+   if (arp_solver)
+   {
+      // ArPackSymGen * arpack = new ArPackSymGen();
+      ArPackSAUPD * arpack = new ArPackSAUPD();
+      arpack->SetMode(2);
+      arpack->SetNumModes(nev);
+      arpack->SetMaxIter(400);
+      arpack->SetTol(1e-8);
+      arpack->SetPrintLevel(2);
+      arpack->SetSolver(*solver);
+
+      eig_solver = arpack;
+   }
+
+   eig_solver->SetOperators(*a, *m);
+
+   // 8. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //    parallel grid function to represent each of the eigenmodes returned by
+   //    the solver.
+   Array<double> eigenvalues;
+   eig_solver->Solve();
+   eig_solver->GetEigenvalues(eigenvalues);
+
+   cout << endl;
+   std::ios::fmtflags old_fmt = cout.flags();
+   cout.setf(std::ios::scientific);
+   std::streamsize old_prec = cout.precision(14);
+   for (int i=0; i<nev; i++)
+   {
+      cout << "Eigenvalue lambda   " << eigenvalues[i] << endl;
+   }
+   cout.precision(old_prec);
+   cout.flags(old_fmt);
+   cout << endl;
+
+   GridFunction x(fespace);
+
+   // 9. Save the refined mesh and the modes in parallel. This output can be
+   //    viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
+   {
+      ostringstream mesh_name, mode_name;
+      mesh_name << "ex11.mesh";
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      mesh->Print(mesh_ofs);
+
+      for (int i=0; i<nev; i++)
+      {
+         // convert eigenvector from Vector to GridFunction
+         x = eig_solver->GetEigenvector(i);
+
+         mode_name << "mode_" << setfill('0') << setw(2) << i;
+
+         ofstream mode_ofs(mode_name.str().c_str());
+         mode_ofs.precision(8);
+         x.Save(mode_ofs);
+         mode_name.str("");
+      }
+   }
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mode_sock(vishost, visport);
+      mode_sock.precision(8);
+
+      for (int i=0; i<nev; i++)
+      {
+         cout << "Eigenmode " << i+1 << '/' << nev
+              << ", Lambda = " << eigenvalues[i] << endl;
+
+         // convert eigenvector from Vector to GridFunction
+         x = eig_solver->GetEigenvector(i);
+
+         mode_sock << "solution\n" << *mesh << x << flush
+                   << "window_title 'Eigenmode " << i+1 << '/' << nev
+                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
+
+         char c;
+         cout << "press (q)uit or (c)ontinue --> " << flush;
+         cin >> c;
+
+         if (c != 'c')
+         {
+            break;
+         }
+      }
+      mode_sock.close();
+   }
+
+   // 11. Free the used memory.
+   delete eig_solver;
+   delete solver;
+   delete m;
+   delete a;
+
+   delete fespace;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete mesh;
+
+   return 0;
+}
+
+#endif // MFEM_USE_ARPACK

examples/ex11p.cpp

@@ -72,6 +72,8 @@ int main(int argc, char *argv[])
    int seed = 75;
    bool slu_solver  = false;
    bool sp_solver = false;
+   bool lob_solver = true;
+   bool arp_solver = false;
    bool cpardiso_solver = false;
    bool visualization = 1;
 
@@ -97,6 +99,10 @@ int main(int argc, char *argv[])
    args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
                   "--no-strumpack", "Use the STRUMPACK Solver.");
 #endif
+#ifdef MFEM_USE_ARPACK
+   args.AddOption(&arp_solver, "-arp", "--arpack", "-no-arp",
+                  "--no-arpack", "Use the Parallel ARPACK Solver.");
+#endif
 #ifdef MFEM_USE_MKL_CPARDISO
    args.AddOption(&cpardiso_solver, "-cpardiso", "--cpardiso", "-no-cpardiso",
                   "--no-cpardiso", "Use the MKL CPardiso Solver.");
@@ -113,6 +119,11 @@ int main(int argc, char *argv[])
               << "         Defaulting to SuperLU." << endl;
       sp_solver = false;
    }
+   if (arp_solver)
+   {
+      lob_solver = false;
+   }
+
    // The command line options are also passed to the STRUMPACK
    // solver. So do not exit if some options are not recognized.
    if (!sp_solver)
@@ -243,70 +254,119 @@ int main(int argc, char *argv[])
    // 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
    //    preconditioner for A to be used within the solver. Set the matrices
    //    which define the generalized eigenproblem A x = lambda M x.
+   Solver * solver = NULL;
    Solver * precond = NULL;
    if (!slu_solver && !sp_solver && !cpardiso_solver)
    {
       HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
       amg->SetPrintLevel(0);
       precond = amg;
-   }
-   else
-   {
-#ifdef MFEM_USE_SUPERLU
-      if (slu_solver)
+
+      if (arp_solver)
+      {
+         HyprePCG * pcg = new HyprePCG(*A);
+         pcg->SetTol(1e-12);
+         pcg->SetPreconditioner(*amg);
+         solver = pcg;
+      }
+   }
+#ifdef MFEM_USE_SUPERLU
+   else if (slu_solver)
+   {
+      SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
+      superlu->SetPrintStatistics(false);
+      superlu->SetSymmetricPattern(true);
+      superlu->SetColumnPermutation(superlu::PARMETIS);
+      superlu->SetOperator(*Arow);
+
+      if (arp_solver)
+      {
+         solver = superlu;
+      }
+      else
       {
-         SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
-         superlu->SetPrintStatistics(false);
-         superlu->SetSymmetricPattern(true);
-         superlu->SetColumnPermutation(superlu::PARMETIS);
-         superlu->SetOperator(*Arow);
          precond = superlu;
       }
+   }
 #endif
 #ifdef MFEM_USE_STRUMPACK
-      if (sp_solver)
+   else if (sp_solver)
+   {
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv,
+                                                        MPI_COMM_WORLD);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      strumpack->SetMatching(strumpack::MatchingJob::NONE);
+      strumpack->SetCompression(strumpack::CompressionType::NONE);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      if (arp_solver)
+      {
+         solver = strumpack;
+      }
+      else
       {
-         STRUMPACKSolver * strumpack = new STRUMPACKSolver(MPI_COMM_WORLD, argc, argv);
-         strumpack->SetPrintFactorStatistics(true);
-         strumpack->SetPrintSolveStatistics(false);
-         strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-         strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-         strumpack->SetMatching(strumpack::MatchingJob::NONE);
-         strumpack->SetCompression(strumpack::CompressionType::NONE);
-         strumpack->SetOperator(*Arow);
-         strumpack->SetFromCommandLine();
          precond = strumpack;
       }
+   }
 #endif
 #ifdef MFEM_USE_MKL_CPARDISO
-      if (cpardiso_solver)
+   else if (cpardiso_solver)
+   {
+      auto cpardiso = new CPardisoSolver(A->GetComm());
+      cpardiso->SetMatrixType(CPardisoSolver::MatType::REAL_STRUCTURE_SYMMETRIC);
+      cpardiso->SetPrintLevel(1);
+      cpardiso->SetOperator(*A);
+      if (arp_solver)
+      {
+         solver = cpardiso;
+      }
+      else
       {
-         auto cpardiso = new CPardisoSolver(A->GetComm());
-         cpardiso->SetMatrixType(CPardisoSolver::MatType::REAL_STRUCTURE_SYMMETRIC);
-         cpardiso->SetPrintLevel(1);
-         cpardiso->SetOperator(*A);
          precond = cpardiso;
       }
-#endif
    }
+#endif
 
-   HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
-   lobpcg->SetNumModes(nev);
-   lobpcg->SetRandomSeed(seed);
-   lobpcg->SetPreconditioner(*precond);
-   lobpcg->SetMaxIter(200);
-   lobpcg->SetTol(1e-8);
-   lobpcg->SetPrecondUsageMode(1);
-   lobpcg->SetPrintLevel(1);
-   lobpcg->SetMassMatrix(*M);
-   lobpcg->SetOperator(*A);
+   SymGenEigensolver * eig_solver = NULL;
+
+   if (lob_solver)
+   {
+      HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
+      lobpcg->SetNumModes(nev);
+      lobpcg->SetRandomSeed(seed);
+      lobpcg->SetPreconditioner(*precond);
+      lobpcg->SetMaxIter(200);
+      lobpcg->SetTol(1e-8);
+      lobpcg->SetPrecondUsageMode(1);
+      lobpcg->SetPrintLevel(1);
+
+      eig_solver = lobpcg;
+   }
+#ifdef MFEM_USE_ARPACK
+   else if (arp_solver)
+   {
+      ArPackPSAUPD * arpack = new ArPackPSAUPD(MPI_COMM_WORLD);
+      arpack->SetNumModes(nev);
+      arpack->SetMaxIter(400);
+      arpack->SetTol(1e-8);
+      arpack->SetMode(3);
+      arpack->SetPrintLevel(2);
+      arpack->SetSolver(*solver);
+
+      eig_solver = arpack;
+   }
+#endif
+   eig_solver->SetOperators(*A, *M);
 
    // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
    //    parallel grid function to represent each of the eigenmodes returned by
    //    the solver.
    Array<real_t> eigenvalues;
-   lobpcg->Solve();
-   lobpcg->GetEigenvalues(eigenvalues);
+   eig_solver->Solve();
+   eig_solver->GetEigenvalues(eigenvalues);
    ParGridFunction x(fespace);
 
    // 10. Save the refined mesh and the modes in parallel. This output can be
@@ -321,8 +381,8 @@ int main(int argc, char *argv[])
 
       for (int i=0; i<nev; i++)
       {
-         // convert eigenvector from HypreParVector to ParGridFunction
-         x = lobpcg->GetEigenvector(i);
+         // convert eigenvector from Vector to ParGridFunction
+         x.Distribute(eig_solver->GetEigenvector(i));
 
          mode_name << "mode_" << setfill('0') << setw(2) << i << "."
                    << setfill('0') << setw(6) << myid;
@@ -350,8 +410,8 @@ int main(int argc, char *argv[])
                  << ", Lambda = " << eigenvalues[i] << endl;
          }
 
-         // convert eigenvector from HypreParVector to ParGridFunction
-         x = lobpcg->GetEigenvector(i);
+         // convert eigenvector from Vector to ParGridFunction
+         x.Distribute(eig_solver->GetEigenvector(i));
 
          mode_sock << "parallel " << num_procs << " " << myid << "\n"
                    << "solution\n" << *pmesh << x << flush
@@ -375,7 +435,8 @@ int main(int argc, char *argv[])
    }
 
    // 12. Free the used memory.
-   delete lobpcg;
+   delete eig_solver;
+   delete solver;
    delete precond;
    delete M;
    delete A;

examples/ex11p_arpack.cpp

@@ -0,0 +1,381 @@
+//                       MFEM Example 11 - Parallel Version
+//
+// Compile with: make ex11p
+//
+// Sample runs:  mpirun -np 4 ex11p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex11p -m ../data/star.mesh
+//               mpirun -np 4 ex11p -m ../data/escher.mesh
+//               mpirun -np 4 ex11p -m ../data/fichera.mesh
+//               mpirun -np 4 ex11p -m ../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex11p -m ../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex11p -m ../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex11p -m ../data/disc-nurbs.mesh -o -1 -n 20
+//               mpirun -np 4 ex11p -m ../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex11p -m ../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex11p -m ../data/star-surf.mesh
+//               mpirun -np 4 ex11p -m ../data/square-disc-surf.mesh
+//               mpirun -np 4 ex11p -m ../data/inline-segment.mesh
+//               mpirun -np 4 ex11p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex11p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex11p -m ../data/mobius-strip.mesh -n 8
+//               mpirun -np 4 ex11p -m ../data/klein-bottle.mesh -n 10
+//
+// Description:  This example code demonstrates the use of MFEM to solve the
+//               eigenvalue problem -Delta u = lambda u with homogeneous
+//               Dirichlet boundary conditions.
+//
+//               We compute a number of the lowest eigenmodes by discretizing
+//               the Laplacian and Mass operators using a FE space of the
+//               specified order, or an isoparametric/isogeometric space if
+//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of the LOBPCG and ARPACK
+//               eigenvalue solvers together with the BoomerAMG preconditioner
+//               in HYPRE, as well as optionally the SuperLU parallel direct
+//               solver. Reusing a single GLVis visualization window for
+//               multiple eigenfunctions is also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int ser_ref_levels = 2;
+   int par_ref_levels = 1;
+   int order = 1;
+   int nev = 5;
+   bool slu_solver = false;
+   bool use_arpack = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nev, "-n", "--num-eigs",
+                  "Number of desired eigenmodes.");
+#ifdef MFEM_USE_SUPERLU
+   args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
+                  "--no-superlu", "Use the SuperLU Solver.");
+#endif
+#ifdef MFEM_USE_ARPACK
+   args.AddOption(&use_arpack, "-arpack", "--use-arpack", "-no-arpack",
+                  "--no-arpack",
+                  "Enable or disable the use of ARPACK.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      if (myid == 0)
+      {
+         cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      }
+      MPI_Finalize();
+      return 2;
+   }
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+   int dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
+   //    specified on the command line with -rs).
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution (1 time by
+   //    default, or specified on the command line with -rp). Once the parallel
+   //    mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int lev = 0; lev < par_ref_levels; lev++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of unknowns: " << size << endl;
+   }
+
+   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   //    element space. The first corresponds to the Laplacian operator -Delta,
+   //    while the second is a simple mass matrix needed on the right hand side
+   //    of the generalized eigenvalue problem below. The boundary conditions
+   //    are implemented by elimination with special values on the diagonal to
+   //    shift the Dirichlet eigenvalues out of the computational range. After
+   //    serial and parallel assembly we extract the corresponding parallel
+   //    matrices A and M.
+   ConstantCoefficient one(1.0);
+   Array<int> ess_bdr;
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   if (pmesh->bdr_attributes.Size() == 0)
+   {
+      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
+      // closed surface.
+      a->AddDomainIntegrator(new MassIntegrator(one));
+   }
+   a->Assemble();
+   a->EliminateEssentialBCDiag(ess_bdr, 1.0);
+   a->Finalize();
+
+   ParBilinearForm *m = new ParBilinearForm(fespace);
+   m->AddDomainIntegrator(new MassIntegrator(one));
+   m->Assemble();
+   // shift the eigenvalue corresponding to eliminated dofs to a large value
+   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m->Finalize();
+
+   HypreParMatrix *A = a->ParallelAssemble();
+   HypreParMatrix *M = m->ParallelAssemble();
+
+#ifdef MFEM_USE_SUPERLU
+   Operator * Arow = NULL;
+   if (slu_solver)
+   {
+      Arow = new SuperLURowLocMatrix(*A);
+   }
+#endif
+
+   delete a;
+   delete m;
+
+   // 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
+   //    preconditioner for A to be used within the solver. Set the matrices
+   //    which define the generalized eigenproblem A x = lambda M x.
+   Eigensolver * esolver = NULL;
+   Solver      *  solver = NULL;
+   Solver      * precond = NULL;
+
+   if (!slu_solver)
+   {
+      HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
+      amg->SetPrintLevel(0);
+      precond = amg;
+
+#ifdef MFEM_USE_ARPACK
+      if ( use_arpack )
+      {
+         HyprePCG * pcg = new HyprePCG(*A);
+         pcg->SetTol(1e-12);
+         pcg->SetMaxIter(200);
+         pcg->SetPreconditioner(*amg);
+         pcg->SetPrintLevel(0);
+
+         solver = pcg;
+      }
+#endif
+   }
+#ifdef MFEM_USE_SUPERLU
+   else
+   {
+      SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
+      superlu->SetPrintStatistics(false);
+      superlu->SetSymmetricPattern(true);
+      superlu->SetColumnPermutation(superlu::PARMETIS);
+      superlu->SetOperator(*Arow);
+
+      solver  = use_arpack?superlu:NULL;
+      precond = use_arpack?NULL:superlu;
+   }
+#endif
+
+   if ( use_arpack )
+   {
+      ParArPackSym * arpack = new ParArPackSym(MPI_COMM_WORLD);
+      arpack->SetMode(3);
+      arpack->SetPrintLevel(2);
+      arpack->SetSolver(*solver);
+
+      esolver = arpack;
+   }
+   else
+   {
+      HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
+      lobpcg->SetPreconditioner(*precond);
+      lobpcg->SetPrecondUsageMode(1);
+      lobpcg->SetPrintLevel(1);
+
+      esolver = lobpcg;
+   }
+
+   esolver->SetNumModes(nev);
+   esolver->SetMaxIter(100);
+   esolver->SetTol(1e-8);
+
+   esolver->SetMassMatrix(*M);
+   esolver->SetOperator(*A);
+
+   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //    parallel grid function to represent each of the eigenmodes returned by
+   //    the solver.
+   Array<double> eigenvalues;
+   esolver->Solve();
+   esolver->GetEigenvalues(eigenvalues);
+
+   if ( myid == 0 && use_arpack )
+   {
+      cout << endl;
+      for (int i=0; i<eigenvalues.Size(); i++)
+      {
+         cout << "Eigenvalue lambda " << eigenvalues[i] << endl;
+      }
+      cout << endl;
+   }
+
+   ParGridFunction x(fespace);
+
+   // 10. Save the refined mesh and the modes in parallel. This output can be
+   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
+   {
+      ostringstream mesh_name, mode_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      for (int i=0; i<nev; i++)
+      {
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x.Distribute(esolver->GetEigenvector(i));
+
+         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
+                   << setfill('0') << setw(6) << myid;
+
+         ofstream mode_ofs(mode_name.str().c_str());
+         mode_ofs.precision(8);
+         x.Save(mode_ofs);
+         mode_name.str("");
+      }
+   }
+
+   // 11. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mode_sock(vishost, visport);
+      mode_sock.precision(8);
+
+      for (int i=0; i<nev; i++)
+      {
+         if ( myid == 0 )
+         {
+            cout << "Eigenmode " << i+1 << '/' << nev
+                 << ", Lambda = " << eigenvalues[i] << endl;
+         }
+
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x.Distribute(esolver->GetEigenvector(i));
+
+         mode_sock << "parallel " << num_procs << " " << myid << "\n"
+                   << "solution\n" << *pmesh << x << flush
+                   << "window_title 'Eigenmode " << i+1 << '/' << nev
+                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
+
+         char c;
+         if (myid == 0)
+         {
+            cout << "press (q)uit or (c)ontinue --> " << flush;
+            cin >> c;
+         }
+         MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
+
+         if (c != 'c')
+         {
+            break;
+         }
+      }
+      mode_sock.close();
+   }
+
+   // 12. Free the used memory.
+   delete esolver;
+   delete solver;
+   delete precond;
+   delete M;
+   delete A;
+
+   delete fespace;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}

examples/ex12p.cpp

@@ -276,8 +276,8 @@ int main(int argc, char *argv[])
 
       for (int i=0; i<nev; i++)
       {
-         // convert eigenvector from HypreParVector to ParGridFunction
-         x = lobpcg->GetEigenvector(i);
+         // convert eigenvector from Vector to ParGridFunction
+         x.Distribute(lobpcg->GetEigenvector(i));
 
          mode_name << "mode_" << setfill('0') << setw(2) << i << "."
                    << setfill('0') << setw(6) << myid;
@@ -303,7 +303,7 @@ int main(int argc, char *argv[])
       pmesh->Print(adios2output);
       for (int i=0; i<nev; i++)
       {
-         x = lobpcg->GetEigenvector(i);
+         x.Distribute(lobpcg->GetEigenvector(i));
          // x is a temporary that must be saved immediately
          x.Save(adios2output, "mode_" + std::to_string(i));
       }
@@ -326,8 +326,8 @@ int main(int argc, char *argv[])
                  << ", Lambda = " << eigenvalues[i] << endl;
          }
 
-         // convert eigenvector from HypreParVector to ParGridFunction
-         x = lobpcg->GetEigenvector(i);
+         // convert eigenvector from Vector to ParGridFunction
+         x.Distribute(lobpcg->GetEigenvector(i));
 
          mode_sock << "parallel " << num_procs << " " << myid << "\n"
                    << "solution\n" << *pmesh << x << flush

examples/ex13.cpp

@@ -0,0 +1,282 @@
+//                       MFEM Example 13
+//
+// Compile with: make ex3p
+//
+// Sample runs:  ex13 -m ../data/star.mesh -s 5
+//               ex13 -m ../data/square-disc.mesh -o 2 -n 4 // minres fails to conv.
+//               ex13 -m ../data/beam-hex.mesh
+//               ex13 -m ../data/square-disc.mesh -rs 1 -s 26
+//               ex13 -m ../data/square-disc-nurbs.mesh -rs 3 -s 26
+//               ex13 -m ../data/amr-quad.mesh -o 2 // minres fails to conv.
+//               ex13 -m ../data/mobius-strip.mesh -n 8
+//
+// Description:  This example code solves a simple 3D electromagnetic
+//               eigenmode problem corresponding to the second order
+//               Maxwell equation curl curl E = lambda E with boundary
+//               condition E x n = 0. We discretize with Nedelec finite
+//               elements.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the use of the ARPACK eigenmode
+//               solver for symmetric matrices using the shift-invert mode.
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+#ifdef MFEM_USE_ARPACK
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/beam-tet.mesh";
+   int order = 1;
+   int nev = 5;
+   int sr = 2;
+   double sigma = 11.0;
+   bool visualization = 1;
+   bool arp_solver = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&nev, "-n", "--num-eigs",
+                  "Number of desired eigenmodes.");
+   args.AddOption(&sr, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&sigma, "-s", "--shift",
+                  "Average of the desired eigenvalue range.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the given mesh file.  We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
+   //    with the same code.
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+   int dim = mesh->Dimension();
+
+   // 3. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement.
+   {
+      int ref_levels = sr;
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 4. Define a finite element space on the mesh. Here we use the lowest
+   //    order Nedelec finite elements, but we can easily switch
+   //    to higher-order spaces by changing the value of p.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   int size = fespace->GetVSize();
+
+   cout << "Number of unknowns: " << size << endl;
+   cout << "Number of boundary attributes: " << mesh->bdr_attributes.Max()
+        << endl;
+
+   // 5. Set up the parallel bilinear form corresponding to the EM diffusion
+   //    operator curl muinv curl - sigma I, by adding the curl-curl and the
+   //    mass domain integrators and finally imposing homogeneous Dirichlet
+   //    boundary conditions. The boundary conditions are implemented by
+   //    marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet). After serial and parallel assembly we extract the
+   //    parallel matrices A and M.
+   Coefficient *muinv    = new ConstantCoefficient(1.0);
+   Coefficient *negSigma = new ConstantCoefficient(-sigma);
+
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*negSigma));
+   a->Assemble();
+   Array<int> ess_bdr(mesh->bdr_attributes.Max());
+   ess_bdr = 1;
+   a->EliminateEssentialBC(ess_bdr);
+   a->Finalize();
+
+   BilinearForm *m = new BilinearForm(fespace);
+   m->AddDomainIntegrator(new VectorFEMassIntegrator());
+   m->Assemble();
+   m->EliminateEssentialBCDiag(ess_bdr, sqrt(numeric_limits<double>::min()));
+   m->Finalize();
+
+   // 6. Define a parallel grid function to approximate each of the
+   // eigenmodes returned by the solver.  Use this as a template to
+   // create a special multi-vector object needed by the eigensolver
+   // which is then initialized with random values.
+   GridFunction x(fespace);
+   x = 0.0;
+
+   // 7. Define and configure the GMRES
+   //    solver to be used within the eigensolver.
+   Solver    * solver = NULL;
+   if ( false )
+   {
+      GMRESSolver *  gmres = new GMRESSolver();
+
+      gmres->SetOperator(*a);
+      gmres->SetRelTol(1e-8);
+      gmres->SetMaxIter(1000);
+      gmres->SetPrintLevel(0);
+      solver = gmres;
+   }
+   else
+   {
+#ifndef MFEM_USE_SUITESPARSE
+      cout << "Building MINRESSolver" << endl;
+      MINRESSolver * minres = new MINRESSolver();
+
+      minres->SetRelTol(1e-12);
+      minres->SetMaxIter(1000);
+      minres->SetPrintLevel(0);
+      solver = minres;
+#else
+      // 7. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+      cout << "Building UMFPackSolver" << endl;
+      UMFPackSolver * umf_solver = new UMFPackSolver;
+      umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      solver = umf_solver;
+#endif
+   }
+   solver->SetOperator(a->SpMat());
+
+   // 7. Define and configure the ARPACK eigensolver
+   SymGenEigensolver * eig_solver = NULL;
+   if (arp_solver)
+   {
+      ArPackSAUPD * arpack = new ArPackSAUPD();
+
+      arpack->SetNumModes(nev);
+      arpack->SetMaxIter(400);
+      arpack->SetTol(1e-8);
+      arpack->SetShift(sigma);
+      arpack->SetMode(3);
+      arpack->SetPrintLevel(2);
+      arpack->SetSolver(*solver);
+
+      eig_solver = arpack;
+   }
+
+   eig_solver->SetOperators(*a, *m);
+
+   // Obtain the eigenvalues and eigenvectors
+   Array<double> eigenvalues(nev);
+   eigenvalues = -1.0;
+
+   // arpack->Solve(eigenvalues, *eigenvectors);
+   eig_solver->Solve();
+
+   eig_solver->GetEigenvalues(eigenvalues);
+
+   cout << endl;
+   std::ios::fmtflags old_fmt = cout.flags();
+   cout.setf(std::ios::scientific);
+   std::streamsize old_prec = cout.precision(14);
+   for (int i=0; i<min(nev,eigenvalues.Size()); i++)
+   {
+      cout << "Eigenvalue lambda   " << eigenvalues[i] << endl;
+   }
+   cout.precision(old_prec);
+   cout.flags(old_fmt);
+   cout << endl;
+
+   VisItDataCollection visit_dc("Example13", mesh);
+   GridFunction ** mode = new GridFunction*[min(nev,eigenvalues.Size())];
+   for (int i=0; i<min(nev,eigenvalues.Size()); i++)
+   {
+      mode[i] = new GridFunction(fespace);
+      *mode[i] = eig_solver->GetEigenvector(i);
+
+      ostringstream modeName;
+      modeName << "mode_" << setfill('0') << setw(2) << i;
+      visit_dc.RegisterField(modeName.str().c_str(),mode[i]);
+   }
+   visit_dc.Save();
+
+   // 8. Save the refined mesh and the modes. This output can
+   //    be viewed later using GLVis: "glvis -m mesh -g mode".
+   {
+      ofstream mesh_ofs("refined.mesh");
+      mesh_ofs.precision(8);
+      mesh->Print(mesh_ofs);
+
+      for (int i=0; i<min(nev,eigenvalues.Size()); i++)
+      {
+         x = eig_solver->GetEigenvector(i);
+
+         ostringstream modeName;
+         modeName << "mode_" << setfill('0') << setw(2) << i;
+
+         ofstream mode_ofs(modeName.str().c_str());
+         mode_ofs.precision(8);
+         x.Save(mode_ofs);
+         modeName.str("");
+      }
+   }
+
+   // 9. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mode_sock(vishost, visport);
+      mode_sock.precision(8);
+
+      for (int i=0; i<min(nev,eigenvalues.Size()); i++)
+      {
+         x = eig_solver->GetEigenvector(i);
+
+         mode_sock << "solution\n" << *mesh << x << flush;
+
+         char c;
+         cout << "press (q)uit or (c)ontinue --> " << flush;
+         cin >> c;
+
+         if (c != 'c')
+         {
+            break;
+         }
+      }
+      mode_sock.close();
+   }
+
+   // 10. Free the used memory.
+   delete a;
+   delete m;
+   delete negSigma;
+   delete muinv;
+   delete eig_solver;
+   delete solver;
+   // delete X;
+   delete fespace;
+   delete fec;
+   delete mesh;
+
+   return 0;
+}
+
+#endif // MFEM_USE_ARPACK

examples/ex13p.cpp

@@ -215,8 +215,8 @@ int main(int argc, char *argv[])
 
       for (int i=0; i<nev; i++)
       {
-         // convert eigenvector from HypreParVector to ParGridFunction
-         x = ame->GetEigenvector(i);
+         // convert eigenvector from Vector to ParGridFunction
+         x.Distribute(ame->GetEigenvector(i));
 
          mode_name << "mode_" << setfill('0') << setw(2) << i << "."
                    << setfill('0') << setw(6) << myid;
@@ -244,8 +244,8 @@ int main(int argc, char *argv[])
                  << ", Lambda = " << eigenvalues[i] << endl;
          }
 
-         // convert eigenvector from HypreParVector to ParGridFunction
-         x = ame->GetEigenvector(i);
+         // convert eigenvector from Vector to ParGridFunction
+         x.Distribute(ame->GetEigenvector(i));
 
          mode_sock << "parallel " << num_procs << " " << myid << "\n"
                    << "solution\n" << *pmesh << x << flush

examples/ex32p.cpp

@@ -228,7 +228,7 @@ int main(int argc, char *argv[])
       for (int i=0; i<nev; i++)
       {
          // convert eigenvector from HypreParVector to ParGridFunction
-         x = ame->GetEigenvector(i);
+         x.Distribute(ame->GetEigenvector(i));
          curl.Mult(x, dx);
 
          mode_name << "mode_" << setfill('0') << setw(2) << i << "."
@@ -295,7 +295,7 @@ int main(int argc, char *argv[])
             }
 
             // convert eigenvector from HypreParVector to ParGridFunction
-            x = ame->GetEigenvector(i);
+            x.Distribute(ame->GetEigenvector(i));
             curl.Mult(x, dx);
 
             {
@@ -469,7 +469,7 @@ int main(int argc, char *argv[])
             }
 
             // convert eigenvector from HypreParVector to ParGridFunction
-            x = ame->GetEigenvector(i);
+            x.Distribute(ame->GetEigenvector(i));
             curl.Mult(x, dx);
 
             {
@@ -599,7 +599,7 @@ int main(int argc, char *argv[])
             }
 
             // convert eigenvector from HypreParVector to ParGridFunction
-            x = ame->GetEigenvector(i);
+            x.Distribute(ame->GetEigenvector(i));
             curl.Mult(x, dx);
 
             mode_sock << "parallel " << num_procs << " " << myid << "\n"

examples/ex35p.cpp

@@ -658,7 +658,7 @@ void ScalarWaveGuide(int mode, ParGridFunction &x)
    lobpcg.SetOperator(*A);
    lobpcg.Solve();
 
-   x = lobpcg.GetEigenvector(mode);
+   x.Distribute(lobpcg.GetEigenvector(mode));
 
    delete A;
    delete M;
@@ -714,7 +714,7 @@ void VectorWaveGuide(int mode, ParGridFunction &x)
    ame.SetOperator(*A);
    ame.Solve();
 
-   x = ame.GetEigenvector(mode);
+   x.Distribute(ame.GetEigenvector(mode));
 
    delete A;
    delete M;
@@ -780,7 +780,7 @@ void PseudoScalarWaveGuide(int mode, ParGridFunction &x_l2)
    lobpcg.SetOperator(*A);
    lobpcg.Solve();
 
-   x = lobpcg.GetEigenvector(mode);
+   x.Distribute(lobpcg.GetEigenvector(mode));
 
    x_l2.ProjectCoefficient(xCoef);
 

examples/jitplayground.cpp

@@ -1,536 +0,0 @@
-#include <mfem.hpp>
-
-#include "../fem/dfem/util.hpp"
-
-#include <proteus/CppJitModule.h>
-
-#include "jitplayground.hpp"
-
-#include <algorithm>
-#include <array>
-#include <cctype>
-#include <cmath>
-#include <fstream>
-#include <initializer_list>
-#include <iostream>
-#include <memory>
-#include <sstream>
-#include <string>
-#include <string_view>
-#include <type_traits>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
-#include <vector>
-
-namespace util
-{
-constexpr std::string_view Dirname(std::string_view path)
-{
-   const size_t last_sep = path.find_last_of("/\\");
-   if (last_sep == std::string_view::npos) { return {}; }
-   return path.substr(0, last_sep);
-}
-
-constexpr std::string_view thisFileDir = Dirname(__FILE__);
-}
-
-template <typename T>
-static std::string TypeNameString()
-{
-   return std::string(mfem::future::get_type_name<T>());
-}
-
-template <typename Tuple, size_t... Is>
-static auto ParamTypeStringsImpl(std::index_sequence<Is...>)
-{
-   return std::array<std::string, sizeof...(Is)>
-   {
-      TypeNameString<std::remove_reference_t<decltype(mfem::future::get<Is>(std::declval<Tuple&>()))>>()...
-   };
-}
-
-template <typename Tuple>
-static auto ParamTypeStrings()
-{
-   return ParamTypeStringsImpl<Tuple>(
-             std::make_index_sequence<mfem::future::tuple_size<Tuple>::value> {});
-}
-
-static std::string_view Trim(std::string_view s)
-{
-   size_t begin = 0;
-   while (begin < s.size() && std::isspace(static_cast<unsigned char>(s[begin])))
-   {
-      ++begin;
-   }
-   size_t end = s.size();
-   while (end > begin &&
-          std::isspace(static_cast<unsigned char>(s[end - 1])))
-   {
-      --end;
-   }
-   return s.substr(begin, end - begin);
-}
-
-static bool IsValidIdentifier(std::string_view s)
-{
-   if (s.empty()) { return false; }
-   const unsigned char c0 = static_cast<unsigned char>(s[0]);
-   if (!(std::isalpha(c0) || c0 == '_')) { return false; }
-   for (size_t i = 1; i < s.size(); ++i)
-   {
-      const unsigned char c = static_cast<unsigned char>(s[i]);
-      if (!(std::isalnum(c) || c == '_')) { return false; }
-   }
-   return true;
-}
-
-static bool ParseJitDirective(std::string_view line,
-                              std::string &type,
-                              std::string &var,
-                              std::string &kind)
-{
-   const size_t jit_pos = line.find("$JIT");
-   if (jit_pos == std::string_view::npos) { return false; }
-
-   const size_t open = line.find('[', jit_pos);
-   const size_t close = line.find(']', jit_pos);
-   MFEM_VERIFY(open != std::string_view::npos &&
-               close != std::string_view::npos &&
-               close > open,
-               "malformed $JIT directive (expected brackets): " << line);
-
-   const std::string_view payload = line.substr(open + 1, close - open - 1);
-   const size_t comma1 = payload.find(',');
-   const size_t comma2 = (comma1 == std::string_view::npos)
-                         ? std::string_view::npos
-                         : payload.find(',', comma1 + 1);
-   MFEM_VERIFY(comma1 != std::string_view::npos &&
-               comma2 != std::string_view::npos,
-               "malformed $JIT directive (expected 3 comma-separated fields): "
-               << line);
-
-   const std::string_view f0 = Trim(payload.substr(0, comma1));
-   const std::string_view f1 = Trim(payload.substr(comma1 + 1,
-                                                   comma2 - comma1 - 1));
-   const std::string_view f2 = Trim(payload.substr(comma2 + 1));
-   MFEM_VERIFY(!f0.empty() && !f1.empty() && !f2.empty(),
-               "malformed $JIT directive (empty field): " << line);
-
-   type.assign(f0);
-   var.assign(f1);
-   kind.assign(f2);
-   return true;
-}
-
-static std::string ReadFileOrEmpty(const std::string &fn)
-{
-   std::ifstream file(fn);
-   if (!file.is_open())
-   {
-      std::cerr << "could not open file " << fn << "\n";
-      return {};
-   }
-   std::stringstream buffer;
-   buffer << file.rdbuf();
-   return buffer.str();
-}
-
-static std::vector<std::string> ExtractJitVarNames(const std::string
-                                                   &kernel_code)
-{
-   std::stringstream ss(kernel_code);
-   std::string line;
-   std::vector<std::string> var_names;
-   std::unordered_set<std::string> seen_vars;
-
-   while (std::getline(ss, line))
-   {
-      std::string type, var, kind;
-      if (ParseJitDirective(line, type, var, kind))
-      {
-         MFEM_VERIFY(IsValidIdentifier(var),
-                     "$JIT variable must be a valid identifier: " << var);
-         MFEM_VERIFY(seen_vars.insert(var).second,
-                     "duplicate $JIT variable name: " << var);
-         var_names.push_back(var);
-      }
-   }
-   return var_names;
-}
-
-static std::string RewriteKernelForJit(std::string kernel_code,
-                                       const std::vector<std::string> &jit_values)
-{
-   std::stringstream ss(kernel_code);
-   std::string line;
-
-   std::string out;
-   out.reserve(kernel_code.size() + 128);
-
-   bool have_pending = false;
-   size_t pending_index = 0;
-   std::string pending_type;
-   std::string pending_var;
-   std::unordered_set<std::string> seen_vars;
-
-   while (std::getline(ss, line))
-   {
-      line.push_back('\n');
-
-      if (have_pending)
-      {
-         MFEM_VERIFY(pending_index < jit_values.size(),
-                     "not enough JIT values provided");
-         const size_t indent_end = line.find_first_not_of(" \t");
-         const std::string indent =
-            (indent_end == std::string::npos) ? std::string() :
-            line.substr(0, indent_end);
-         out += indent + "const " + pending_type + " " + pending_var + " = " +
-                jit_values[pending_index] + ";\n";
-         have_pending = false;
-         ++pending_index;
-         continue;
-      }
-
-      std::string type, var, kind;
-      if (ParseJitDirective(line, type, var, kind))
-      {
-         MFEM_VERIFY(IsValidIdentifier(var),
-                     "$JIT variable must be a valid identifier: " << var);
-         MFEM_VERIFY(kind == "generic",
-                     "unsupported $JIT kind: " << kind);
-         MFEM_VERIFY(seen_vars.insert(var).second,
-                     "duplicate $JIT variable name: " << var);
-
-         pending_type = std::move(type);
-         pending_var = std::move(var);
-         have_pending = true;
-         continue; // drop directive line
-      }
-
-      out += line;
-   }
-
-   MFEM_VERIFY(!have_pending,
-               "$JIT directive must annotate a following line");
-   MFEM_VERIFY(jit_values.size() == pending_index,
-               "JIT value count must match number of $JIT directives");
-   return out;
-}
-
-static std::string GeneratedOutputPath(std::string_view original_path)
-{
-   const size_t last_sep = original_path.find_last_of("/\\");
-   const size_t dot = original_path.find_last_of('.');
-   const bool dot_in_filename =
-      (dot != std::string_view::npos) &&
-      (last_sep == std::string_view::npos || dot > last_sep);
-
-   const std::string_view base =
-      dot_in_filename ? original_path.substr(0, dot) : original_path;
-   return std::string(base) + "_generated.hpp";
-}
-
-static void WriteFileOrWarn(const std::string &path,
-                            const std::string &contents)
-{
-   std::ofstream out(path);
-   if (!out.is_open())
-   {
-      std::cerr << "could not write generated file " << path << "\n";
-      return;
-   }
-   out << contents;
-}
-
-class JitQFunction
-{
-public:
-   template <typename ImplT, size_t N>
-   JitQFunction(ImplT, const std::string &fn,
-                const std::array<bool, N> &activity_map)
-   {
-      using qf_signature = typename
-                           mfem::future::get_function_signature<
-                           decltype(&ImplT::operator())>::type;
-      using qf_param_ts = typename qf_signature::parameter_ts;
-      constexpr size_t nparams = mfem::future::tuple_size<qf_param_ts>::value;
-      static_assert(N == nparams, "activity_map size must match qfunc arity");
-
-      this->fn = fn;
-      this->nparams = nparams;
-      this->activity_map.reserve(N);
-      for (size_t i = 0; i < N; ++i)
-      {
-         this->activity_map.push_back(activity_map[i]);
-      }
-      {
-         const auto param_types_arr = ParamTypeStrings<qf_param_ts>();
-         this->param_types.assign(param_types_arr.begin(), param_types_arr.end());
-      }
-      this->return_type = TypeNameString<typename qf_signature::return_t>();
-      this->return_is_void = std::is_same_v<typename qf_signature::return_t, void>;
-      this->impl_type_name = TypeNameString<ImplT>();
-      this->jit_var_names = ExtractJitVarNames(ReadFileOrEmpty(fn));
-   }
-
-   template <typename ReturnT, typename... Args>
-   ReturnT run(std::string_view name,
-               std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
-               Args&&... args)
-   {
-      auto ordered_values = MatchJitValues(jit_values);
-      auto &mod = GetOrCreateModule(ordered_values);
-      auto &instance = mod.instantiate(std::string(name), std::string());
-      return instance.template run<ReturnT>(std::forward<Args>(args)...);
-   }
-
-   template <typename ReturnT, typename... Args>
-   ReturnT run_primal(
-      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
-      Args&&... args)
-   {
-      return run<ReturnT>(qfunc_name, jit_values,
-                          std::forward<Args>(args)...);
-   }
-
-   template <typename ReturnT, typename... Args>
-   ReturnT run_derivative(
-      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
-      Args&&... args)
-   {
-      return run<ReturnT>(qfunc_name + "_fwddiff", jit_values,
-                          std::forward<Args>(args)...);
-   }
-
-private:
-   std::vector<std::string_view> MatchJitValues(
-      std::initializer_list<std::pair<std::string_view, std::string_view>>
-      named_values) const
-   {
-      std::unordered_map<std::string_view, std::string_view> value_map;
-      for (const auto &[name, value] : named_values)
-      {
-         value_map[name] = value;
-      }
-
-      std::vector<std::string_view> ordered_values;
-      ordered_values.reserve(jit_var_names.size());
-      for (const auto &var_name : jit_var_names)
-      {
-         auto it = value_map.find(var_name);
-         MFEM_VERIFY(it != value_map.end(),
-                     "missing JIT value for variable: " << var_name);
-         ordered_values.push_back(it->second);
-      }
-
-      MFEM_VERIFY(ordered_values.size() == named_values.size(),
-                  "provided " << named_values.size() << " JIT values but expected "
-                  << jit_var_names.size());
-      return ordered_values;
-   }
-
-
-   std::string BuildModuleCode(const std::vector<std::string> &jit_values) const
-   {
-      std::string module_code =
-         RewriteKernelForJit(ReadFileOrEmpty(fn), jit_values);
-      module_code += "\n\n";
-      module_code += "// --- generated ---\n";
-      module_code +=
-         "template <typename return_type, typename... Args>\n"
-         "return_type __enzyme_fwddiff(Args...);\n"
-         "\n"
-         "extern int enzyme_const;\n"
-         "extern int enzyme_dup;\n"
-         "\n";
-
-      // Generate a primal wrapper with the requested symbol name, so the kernel
-      // header can just define the qfunc as a functor.
-      //
-      // Note: Proteus instantiates entrypoints via `qfunc_wrapper<>(...)` even
-      // when there are no user template args, so keep the wrapper itself a
-      // template (with a default parameter) while still doing literal `$JIT`
-      // replacements in the kernel code.
-      module_code += "template <typename = void>\n";
-      module_code += return_type + " " +
-                     std::string(qfunc_name) + "(";
-      bool first = true;
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (!first) { module_code += ", "; }
-         first = false;
-         module_code += param_types[i] + " Arg" + std::to_string(i);
-      }
-      module_code += ")\n";
-      module_code += "{\n";
-      module_code += "   " + impl_type_name + " qf;\n";
-      if (return_is_void)
-      {
-         module_code += "   ";
-      }
-      else
-      {
-         module_code += "   return ";
-      }
-      module_code += "qf(";
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (i) { module_code += ", "; }
-         module_code += "Arg" + std::to_string(i);
-      }
-      module_code += ");\n";
-      module_code += "}\n\n";
-
-      module_code += "template <typename = void>\n";
-      module_code += return_type + " " +
-                     std::string(qfunc_name) + "_fwddiff(";
-
-      first = true;
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (!first) { module_code += ", "; }
-         first = false;
-         module_code += param_types[i] + " Arg" + std::to_string(i);
-         if (activity_map[i])
-         {
-            module_code += ", " + param_types[i] + " dArg" + std::to_string(i);
-         }
-      }
-      module_code += ")\n";
-      module_code += "{\n";
-      if (return_is_void)
-      {
-         module_code += "   __enzyme_fwddiff<void>(\n";
-      }
-      else
-      {
-         module_code += "   return __enzyme_fwddiff<" +
-                        return_type + ">(\n";
-      }
-      module_code += "      (void*)" + std::string(qfunc_name) + "<>";
-      module_code += ",\n";
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (activity_map[i])
-         {
-            module_code += "      enzyme_dup, Arg" + std::to_string(i) +
-                           ", dArg" + std::to_string(i);
-         }
-         else
-         {
-            module_code += "      enzyme_const, Arg" + std::to_string(i);
-         }
-         module_code += (i + 1 == nparams) ? ");\n" : ",\n";
-      }
-      module_code += "}\n";
-
-      WriteFileOrWarn(GeneratedOutputPath(fn), module_code);
-      return module_code;
-   }
-
-   proteus::CppJitModule &GetOrCreateModule(
-      const std::vector<std::string_view> &jit_values)
-   {
-      std::string key;
-      for (const auto &val : jit_values)
-      {
-         if (!key.empty()) { key += ","; }
-         key += val;
-      }
-
-      auto it = modules.find(key);
-      if (it != modules.end())
-      {
-         return *it->second;
-      }
-
-      std::vector<std::string> values(jit_values.begin(), jit_values.end());
-      std::string code = BuildModuleCode(values);
-      auto mod = std::make_unique<proteus::CppJitModule>("host", code,
-                                                         DefaultExtraArgs());
-      auto [inserted, ok] = modules.emplace(key, std::move(mod));
-      MFEM_VERIFY(ok, "failed to cache JIT module");
-      return *inserted->second;
-   }
-
-   static std::vector<std::string> DefaultExtraArgs()
-   {
-      return {"-fplugin=/Users/andrej1/local/enzyme/lib/ClangEnzyme-20.dylib"};
-   }
-
-   std::string qfunc_name = "qfunc_wrapper";
-   std::string fn;
-   size_t nparams = 0;
-   std::vector<bool> activity_map;
-   std::vector<std::string> param_types;
-   std::string return_type;
-   bool return_is_void = false;
-   std::string impl_type_name;
-   std::vector<std::string> jit_var_names;
-   std::unordered_map<std::string, std::unique_ptr<proteus::CppJitModule>> modules;
-};
-
-int main()
-{
-   const size_t N = 4;
-   const size_t M = 5;
-   const double A = 123.4;
-
-   std::vector<double> X(N);
-   std::vector<double> Y(N);
-   for (size_t i = 0; i < N; ++i)
-   {
-      X[i] = static_cast<double>(i + 1);
-      Y[i] = static_cast<double>(N - i);
-   }
-
-   // // >>> user interface calls
-   // const std::string kernel_path = std::string(util::thisFileDir) +
-   //                                 "/jitplayground.hpp";
-   // JitQFunction qf(daxpy_op{}, kernel_path, std::array{false, true, false});
-   // // <<< user interface calls
-
-   // // this will happen internally in dFEM
-
-   daxpy_op op;
-   printf("\n\nfunction call\n");
-   op(&A, X.data(), Y.data(), &N);
-
-   // reset X for the derivative test
-   for (size_t i = 0; i < N; ++i)
-   {
-      X[i] = static_cast<double>(i + 1);
-      Y[i] = static_cast<double>(N - i);
-   }
-
-   std::vector<double> dX(N, 1.0);
-   printf("\n\nforward diff call\n");
-   daxpy_op_fwddiff(&A, X.data(), dX.data(), Y.data(), &N);
-
-   std::vector<double> dX_manual(N, A);
-
-   printf("\n\nderivative checks\n");
-   std::cout << "dX: ";
-   for (size_t i = 0; i < N; ++i)
-   {
-      std::cout << dX[i] << (i + 1 == N ? '\n' : ' ');
-   }
-
-   std::cout << "dX_manual: ";
-   for (size_t i = 0; i < N; ++i)
-   {
-      std::cout << dX_manual[i] << (i + 1 == N ? '\n' : ' ');
-   }
-
-   double max_abs_err = 0.0;
-   for (size_t i = 0; i < N; ++i)
-   {
-      max_abs_err = std::max(max_abs_err, std::abs(dX[i] - dX_manual[i]));
-   }
-   std::cout << "max |dX - dX_manual| = " << max_abs_err << "\n";
-
-   return 0;
-}

examples/jitplayground.hpp

@@ -1,58 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <vector>
-#include <type_traits>
-
-#include "proteus/JitInterface.h"
-
-struct daxpy_op
-{
-   void operator()(
-      const double *a,
-      double *x,
-      const double *y,
-      const size_t *N) const
-   {
-      const size_t n = *N;
-      auto lam = [=, n = proteus::jit_variable(n)]
-                 () __attribute__((annotate("jit")))
-      {
-         printf("N = %zu\n", n);
-         for (size_t i = 0; i < n; ++i)
-         {
-            printf("x[%zu] = %f, y[%zu] = %f\n", i, x[i], i, y[i]);
-            x[i] = *a * x[i] + y[i];
-            printf("updated x[%zu] = %f\n", i, x[i]);
-         }
-      };
-
-      proteus::register_lambda(lam);
-
-      lam();
-   }
-};
-
-template <typename return_type, typename... Args>
-return_type __enzyme_fwddiff(Args...);
-
-extern int enzyme_const;
-extern int enzyme_dup;
-
-void daxpy_op_wrapper(const double * Arg0, double * Arg1,
-                      const double * Arg2, const size_t *Arg3)
-{
-   daxpy_op qf;
-   qf(Arg0, Arg1, Arg2, Arg3);
-}
-
-void daxpy_op_fwddiff(const double * Arg0, double * Arg1,
-                      double * dArg1, const double * Arg2, const size_t *Arg3)
-{
-   __enzyme_fwddiff<void>(
-      (void*)daxpy_op_wrapper,
-      enzyme_const, Arg0,
-      enzyme_dup, Arg1, dArg1,
-      enzyme_const, Arg2,
-      enzyme_const, Arg3);
-}

examples/makefile

@@ -31,6 +31,9 @@ SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
    ex22p ex24p ex25p ex26p ex34p ex35p
 
+ifeq ($(MFEM_USE_ARPACK),YES)
+   SEQ_EXAMPLES += ex11 ex13
+endif
 ifeq ($(MFEM_USE_LAPACK),YES)
    SEQ_EXAMPLES += ex38
 endif

fem/CMakeLists.txt

@@ -121,11 +121,6 @@ set(SRCS
   qinterp/eval_hdiv.cpp
   qinterp/grad_by_nodes.cpp
   qinterp/grad_by_vdim.cpp
-  qinterp/grad_transpose.cpp
-  qinterp/grad_transpose_by_nodes.cpp
-  qinterp/grad_transpose_by_vdim.cpp
-  qinterp/eval_transpose.cpp
-  qinterp/eval_transpose_by_vdim.cpp
   qspace.cpp
   quadinterpolator.cpp
   quadinterpolator_face.cpp
@@ -138,7 +133,7 @@ set(SRCS
   tmop/assemble/diag2.cpp
   tmop/assemble/grad2_limit.cpp
   tmop/assemble/grad2.cpp
-  tmop/assemble/diag3_limit.cpp
+  tmop/assemble/diag3_limit.cpp  
   tmop/assemble/diag3.cpp
   tmop/assemble/grad3_limit.cpp
   tmop/assemble/grad3.cpp
@@ -283,10 +278,8 @@ set(HDRS
   qfunction.hpp
   qinterp/det.hpp
   qinterp/eval.hpp
-  qinterp/eval_transpose.hpp
   qinterp/eval_hdiv.hpp
   qinterp/grad.hpp
-  qinterp/grad_transpose.hpp
   qspace.hpp
   quadinterpolator.hpp
   quadinterpolator_face.hpp
@@ -320,36 +313,36 @@ set(HDRS
   )
 
 if (MFEM_USE_SIDRE)
-    list(APPEND SRCS sidredatacollection.cpp)
-    list(APPEND HDRS sidredatacollection.hpp)
+  list(APPEND SRCS sidredatacollection.cpp)
+  list(APPEND HDRS sidredatacollection.hpp)
 endif()
 
 if (MFEM_USE_CONDUIT)
-    list(APPEND SRCS conduitdatacollection.cpp)
-    list(APPEND HDRS conduitdatacollection.hpp)
+  list(APPEND SRCS conduitdatacollection.cpp)
+  list(APPEND HDRS conduitdatacollection.hpp)
 endif()
 
 if (MFEM_USE_ADIOS2)
-    list(APPEND SRCS adios2datacollection.cpp)
-    list(APPEND HDRS adios2datacollection.hpp)
+  list(APPEND SRCS adios2datacollection.cpp)
+  list(APPEND HDRS adios2datacollection.hpp)
 endif()
 
 if (MFEM_USE_FMS)
-    list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
-    list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
+  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
+  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
 endif()
 
 if (MFEM_USE_MPI)
-    list(APPEND SRCS
+  list(APPEND SRCS
     pbilinearform.cpp
     pfespace.cpp
     pgridfunc.cpp
     plinearform.cpp
     pnonlinearform.cpp
     prestriction.cpp)
-    # If this list (HDRS -> HEADERS) is used for install, we probably want the
-    # headers added all the time.
-    list(APPEND HDRS
+  # If this list (HDRS -> HEADERS) is used for install, we probably want the
+  # headers added all the time.
+  list(APPEND HDRS
     pbilinearform.hpp
     pfespace.hpp
     pgridfunc.hpp

fem/bilininteg.hpp

@@ -2178,22 +2178,18 @@ class DiffusionIntegrator: public BilinearFormIntegrator
 {
 public:
 
-   using DiffusionApplyKernelType = void(*)(const int, const bool,
-                                            const Array<real_t>&,
-                                            const Array<real_t>&, const Array<real_t>&,
-                                            const Array<real_t>&,
-                                            const Vector&, const Vector&,
-                                            Vector&, const int, const int);
+   using ApplyKernelType = void(*)(const int, const bool, const Array<real_t>&,
+                                   const Array<real_t>&, const Array<real_t>&,
+                                   const Array<real_t>&,
+                                   const Vector&, const Vector&,
+                                   Vector&, const int, const int);
 
-   using DiffusionDiagonalKernelType = void(*)(const int, const bool,
-                                               const Array<real_t>&,
-                                               const Array<real_t>&, const Vector&, Vector&,
-                                               const int, const int);
+   using DiagonalKernelType = void(*)(const int, const bool, const Array<real_t>&,
+                                      const Array<real_t>&, const Vector&, Vector&,
+                                      const int, const int);
 
-   MFEM_REGISTER_KERNELS(DiffusionApplyPAKernel, DiffusionApplyKernelType,
-                         (int, int, int));
-   MFEM_REGISTER_KERNELS(DiffusionDiagonalPAKernel, DiffusionDiagonalKernelType,
-                         (int, int, int));
+   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
+   MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
    struct Kernels { Kernels(); };
 
 protected:
@@ -2213,7 +2209,6 @@ private:
    const FiniteElementSpace *fespace;
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
-public:
    int dim, ne, dofs1D, quad1D;
    Vector pa_data;
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
@@ -2355,8 +2350,8 @@ public:
    template <int DIM, int D1D, int Q1D>
    static void AddSpecialization()
    {
-      DiffusionApplyPAKernel::Specialization<DIM,D1D,Q1D>::Add();
-      DiffusionDiagonalPAKernel::Specialization<DIM,D1D,Q1D>::Add();
+      ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
+      DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
    }
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(

fem/dfem/action.hpp

@@ -1,587 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include <cassert>
-#include <cstddef>
-
-// #include "fem/kernels.hpp"
-#include "fem/kernels3d.hpp"
-namespace ker = mfem::kernels::internal;
-namespace low = mfem::kernels::internal::low;
-#include "fem/kernel_dispatch.hpp"
-
-// #include "linalg/kernels.hpp"
-
-#include "util.hpp"
-
-#undef NVTX_COLOR
-#define NVTX_COLOR ::nvtx::kOrchid
-
-namespace mfem::future
-{
-
-/** @brief Zero-copy view of a contiguous block as a `tensor<T, n1>` */
-template<typename T, int n1>
-MFEM_HOST_DEVICE
-const tensor<T, n1>& as_tensor(const T* ptr)
-{
-   // std::launder makes this defined behavior under strict aliasing rules
-   return *std::launder(reinterpret_cast<const tensor<T, n1>*>(ptr));
-}
-
-// convenience overload if you prefer a mutable view
-template<typename T, int n1>
-MFEM_HOST_DEVICE
-tensor<T, n1>& as_tensor(T* ptr)
-{
-   return *std::launder(reinterpret_cast<tensor<T, n1>*>(ptr));
-}
-
-/** @brief Zero-copy view of a contiguous block as a `tensor<T, n1, n2>` */
-template<typename T, int n1, int n2>
-MFEM_HOST_DEVICE
-const tensor<T, n1, n2>& as_tensor(const T* ptr)
-{
-   // std::launder makes this defined behavior under strict aliasing rules
-   return *std::launder(reinterpret_cast<const tensor<T, n1, n2>*>(ptr));
-}
-
-// convenience overload if you prefer a mutable view
-template<typename T, int n1, int n2>
-MFEM_HOST_DEVICE
-tensor<T, n1, n2>& as_tensor(T* ptr)
-{
-   return *std::launder(reinterpret_cast<tensor<T, n1, n2>*>(ptr));
-}
-
-/** @brief Zero-copy view of a contiguous block as a `tensor<T, n1, n2, n3>` */
-template<typename T, int n1, int n2, int n3>
-MFEM_HOST_DEVICE
-const tensor<T, n1, n2, n3>& as_tensor(const T* ptr)
-{
-   // std::launder makes this defined behavior under strict aliasing rules
-   return *std::launder(reinterpret_cast<const tensor<T, n1, n2, n3>*>(ptr));
-}
-
-// convenience overload if you prefer a mutable view
-template<typename T, int n1, int n2, int n3>
-MFEM_HOST_DEVICE
-tensor<T, n1, n2, n3>& as_tensor(T* ptr)
-{
-   return *std::launder(reinterpret_cast<tensor<T, n1, n2, n3>*>(ptr));
-}
-
-/** @brief Zero-copy view of a contiguous block as a `tensor<T, n1, n2, n3, n4>` */
-template<typename T, int n1, int n2, int n3, int n4>
-MFEM_HOST_DEVICE
-const tensor<T, n1, n2, n3, n4>& as_tensor(const T* ptr)
-{
-   // std::launder makes this defined behavior under strict aliasing rules
-   return *std::launder(reinterpret_cast<const tensor<T, n1, n2, n3, n4>*>(ptr));
-}
-
-// convenience overload if you prefer a mutable view
-template<typename T, int n1, int n2, int n3, int n4>
-MFEM_HOST_DEVICE
-tensor<T, n1, n2, n3, n4>& as_tensor(T* ptr)
-{
-   return *std::launder(reinterpret_cast<tensor<T, n1, n2, n3, n4>*>(ptr));
-}
-
-
-template <std::size_t N>
-MFEM_HOST_DEVICE inline
-std::array<real_t*, N>
-load_field_e_ptr(const std::array<DeviceTensor<2>, N> &fields_e,
-                 const int e)
-{
-   std::array<real_t*, N> f;
-   for_constexpr<N>([&](auto i) { f[i] = &fields_e[i](0, e); });
-   return f;
-}
-
-namespace qf
-{
-
-template <int T_Q1D,
-          size_t num_args,
-          typename reg_t,
-          typename qfunc_t,
-          typename args_ts>
-MFEM_HOST_DEVICE inline
-void apply_kernel(reg_t &res /*output*/,
-                  reg_t &reg,
-                  const real_t *rd,
-                  const int qx, const int qy, const int qz,
-                  const qfunc_t &qfunc, args_ts &args)
-{
-   if constexpr (num_args == 2) // PAApply
-   {
-      // ∇u
-      tensor<real_t, 3> &arg_0 = get<0>(args);
-      arg_0[0] = reg[qz][qy][qx][0];
-      arg_0[1] = reg[qz][qy][qx][1];
-      arg_0[2] = reg[qz][qy][qx][2];
-
-      // D (PA data)
-      tensor<real_t, 3, 3> &arg_1 = get<1>(args);
-
-      if constexpr (T_Q1D > 0)
-      {
-         const auto *D = (const real_t (*)[T_Q1D][T_Q1D][3][3]) rd;
-         for (int k = 0; k < 3; k++)
-         {
-            for (int j = 0; j < 3; j++)
-            {
-               arg_1[k][j] = D[qx][qy][qz][k][j];
-            }
-         }
-      }
-      else
-      {
-         static_assert(false);
-         // const auto D = Reshape(r2, 3, 3, Q1D, Q1D, Q1D);
-         // for (int j = 0; j < 3; j++)
-         // {
-         //    for (int k = 0; k < 3; k++)
-         //    {
-         //       arg_1[k][j] = D(j, k, qz, qy, qx);
-         //    }
-         // }
-      }
-   }
-   else
-   {
-      // MFApply comes here
-      assert(false);
-      // MFEM_ABORT("Only two arguments (∇u and D) are supported in apply_kernel for now");
-   }
-
-   const auto r = get<0>(apply(qfunc, args));
-
-   if constexpr (decltype(r)::ndim == 1)
-   {
-      // process_qf_result_from_reg(r0, qx, qy, qz, r);
-      as_tensor<real_t, 3>(&res[qz][qy][qx][0]) = r;
-   }
-   else
-   {
-      static_assert(false);
-   }
-}
-
-} // namespace qf
-
-#define MFEM_D2Q_MAX_SIZE 4
-static MFEM_CONSTANT real_t Bi[MFEM_D2Q_MAX_SIZE][8*8], Bo[8*8];
-static MFEM_CONSTANT real_t Gi[MFEM_D2Q_MAX_SIZE][8*8], Go[8*8];
-
-template<size_t num_fields,
-         size_t num_inputs,
-         size_t num_outputs,
-         typename restriction_cb_t,
-         typename qfunc_t,
-         typename input_t,
-         typename output_fop_t>
-class NewActionCallback
-{
-   restriction_cb_t &restriction_cb;
-   qfunc_t &qfunc;
-   input_t &inputs;
-   const std::array<size_t, num_inputs> &input_to_field;
-   const std::array<DofToQuadMap, num_inputs> &input_dtq_maps;
-   const std::array<DofToQuadMap, num_outputs> &output_dtq_maps;
-   const int num_entities;
-   const int test_vdim;
-   const int num_test_dof;
-   const int dimension;
-   const ThreadBlocks &thread_blocks;
-   SharedMemoryInfo<num_fields, num_inputs, num_outputs> &shmem_info;
-   const Array<int> &attributes;
-   const output_fop_t &output_fop;
-   const Array<int> *elem_attributes;
-   // refs
-   std::vector<Vector> &fields_e;
-   Vector &residual_e;
-   std::function<void(Vector &, Vector &)> &output_restriction_transpose;
-   // args
-   std::vector<Vector> &solutions_l;
-   const std::vector<Vector> &parameters_l;
-   Vector &residual_l;
-
-public:
-   NewActionCallback() = delete;
-
-   NewActionCallback(const bool use_kernels_specialization,
-                     restriction_cb_t &restriction_cb,
-                     qfunc_t &qfunc,
-                     input_t &inputs,
-                     const std::array<size_t, num_inputs> &input_to_field,
-                     const std::array<DofToQuadMap, num_inputs> &input_dtq_maps,
-                     const std::array<DofToQuadMap, num_outputs> &output_dtq_maps,
-                     const int num_entities,
-                     const int test_vdim,
-                     const int num_test_dof,
-                     const int dimension,
-                     const ThreadBlocks &thread_blocks,
-                     SharedMemoryInfo<num_fields, num_inputs, num_outputs> &shmem_info,
-                     const Array<int> &attributes,
-                     const output_fop_t &output_fop,
-                     const Array<int> *elem_attributes,
-                     // refs
-                     std::vector<Vector> &fields_e,
-                     Vector &residual_e,
-                     std::function<void(Vector &, Vector &)> &output_restriction_transpose,
-                     // args
-                     std::vector<Vector> &solutions_l,
-                     const std::vector<Vector> &parameters_l,
-                     Vector &residual_l):
-      restriction_cb(restriction_cb),
-      qfunc(qfunc),
-      inputs(inputs),
-      input_to_field(input_to_field),
-      input_dtq_maps(input_dtq_maps),
-      output_dtq_maps(output_dtq_maps),
-      num_entities(num_entities),
-      test_vdim(test_vdim),
-      num_test_dof(num_test_dof),
-      dimension(dimension),
-      thread_blocks(thread_blocks),
-      shmem_info(shmem_info),
-      attributes(attributes),
-      output_fop(output_fop),
-      elem_attributes(elem_attributes),
-      fields_e(fields_e),
-      residual_e(residual_e),
-      output_restriction_transpose(output_restriction_transpose),
-      solutions_l(solutions_l),
-      parameters_l(parameters_l),
-      residual_l(residual_l)
-   {
-      if (!use_kernels_specialization) { return; }
-      NewActionCallbackKernels::template Specialization<3>::Add(); // 1
-      NewActionCallbackKernels::template Specialization<4>::Add(); // 2
-      NewActionCallbackKernels::template Specialization<5>::Add(); // 3
-      NewActionCallbackKernels::template Specialization<6>::Add(); // 4
-      NewActionCallbackKernels::template Specialization<7>::Add(); // 5
-      NewActionCallbackKernels::template Specialization<8>::Add(); // 6
-   }
-
-   template<int T_Q1D = 0>
-   static void action_callback_new(const int d1d,
-                                   restriction_cb_t &restriction_cb,
-                                   qfunc_t &qfunc,
-                                   [[maybe_unused]] input_t &inputs,
-                                   [[maybe_unused]] const std::array<size_t, num_inputs> &input_to_field,
-                                   const std::array<DofToQuadMap, num_inputs> &input_dtq_maps,
-                                   const std::array<DofToQuadMap, num_outputs> &output_dtq_maps,
-                                   [[maybe_unused]] const int dimension,
-                                   const int num_entities,
-                                   [[maybe_unused]] const int test_vdim,
-                                   [[maybe_unused]] const int num_test_dof,
-                                   const ThreadBlocks &thread_blocks,
-                                   [[maybe_unused]] SharedMemoryInfo<num_fields, num_inputs, num_outputs>
-                                   &shmem_info,
-                                   [[maybe_unused]] const Array<int> &attributes,
-                                   [[maybe_unused]] const output_fop_t &output_fop,
-                                   [[maybe_unused]] const Array<int> *elem_attributes,
-                                   // refs
-                                   std::vector<Vector> &fields_e,
-                                   Vector &residual_e,
-                                   std::function<void(Vector &, Vector &)> &output_restriction_transpose,
-                                   // args
-                                   std::vector<Vector> &solutions_l,
-                                   const std::vector<Vector> &parameters_l,
-                                   Vector &residual_l,
-                                   // fallback arguments
-                                   const int q1d)
-   {
-      NVTX_MARK_FUNCTION;
-      assert(dimension == 3);
-      static_assert(MFEM_D2Q_MAX_SIZE >= num_inputs, "MFEM_D2Q_MAX_SIZE error");
-
-      constexpr int DIM = 3;
-
-      [[maybe_unused]] static bool ini = (for_constexpr<num_inputs>([&](auto i)
-      {
-         const auto dtq = input_dtq_maps[i];
-         {
-            const auto [q, _, p] = dtq.B.GetShape();
-            const auto B = (const real_t*)input_dtq_maps[i].B;
-            dbg("Loading Bi[{}]: q={} p={}", i.value, q, p);
-            if (B) { Gpu(MemcpyToSymbol)(Bi[i], B, (p*q)*sizeof(real_t)); }
-         }
-         {
-            const auto [q, _, p] = dtq.G.GetShape();
-            const auto G = (const real_t*)input_dtq_maps[i].G;
-            if (G) { Gpu(MemcpyToSymbol)(Gi[i], G, (p*q)*sizeof(real_t)); }
-         }
-         if constexpr (i == 0) // output B
-         {
-            const auto dtq_o = output_dtq_maps[0];
-            const auto [q, _, p] = dtq_o.B.GetShape();
-            const auto B = (const real_t*)dtq_o.B;
-            if (B) { Gpu(MemcpyToSymbol)(Bo, B, (p*q)*sizeof(real_t)); }
-         }
-         if constexpr (i == 0) // output G
-         {
-            const auto dtq_o = output_dtq_maps[0];
-            const auto [q, _, p] = dtq_o.G.GetShape();
-            const auto G = (const real_t*)dtq_o.G;
-            if (G) { Gpu(MemcpyToSymbol)(Go, G, (p*q)*sizeof(real_t)); }
-            dbg("Loaded B and G to constant memory");
-         }
-      }), true);
-
-      // types
-      using qf_signature =
-         typename create_function_signature<decltype(&qfunc_t::operator())>::type;
-      using qf_param_ts = typename qf_signature::parameter_ts;
-
-      restriction_cb(solutions_l, parameters_l, fields_e);
-
-      NVTX_INI("res=0");
-      residual_e = 0.0;
-      NVTX_END("res=0");
-
-      // auto wrapped_fields_e =
-      // wrap_fields(fields_e, shmem_info.field_sizes, num_entities);
-
-      const bool has_attr = attributes.Size() > 0;
-      const auto d_attr = attributes.Read();
-      const auto d_elem_attr = elem_attributes->Read();
-
-      // const int vdim = input.vdim;
-      // const auto fields_e_ptr = load_field_e_ptr(wrapped_fields_e, e);
-      // const real_t *field_e_r = fields_e_ptr[input_to_field[i]];
-      // const auto fields_e_ptr = load_field_e_ptr(wrapped_fields_e, e);
-      const int NE = num_entities;
-      constexpr int VDIM = 1;
-
-      const auto XE = Reshape(fields_e[0].Read(), d1d, d1d, d1d, VDIM, NE);
-      const real_t *dx_ptr = fields_e[1].Read();
-
-      auto YE = Reshape(residual_e.ReadWrite(), d1d, d1d, d1d, VDIM, NE);
-
-      const auto B = (const real_t*)input_dtq_maps[0/*i*/].B;
-      const auto G = (const real_t*)input_dtq_maps[0/*i*/].G;
-
-      NVTX_INI("forall");
-      dfem::forall<T_Q1D*T_Q1D*T_Q1D>([=] MFEM_HOST_DEVICE (int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         constexpr int MQ1 = T_Q1D > 0 ? T_Q1D : 8;
-
-         MFEM_SHARED real_t sm0[MQ1][MQ1][MQ1][3];
-         MFEM_SHARED real_t sm1[MQ1][MQ1][MQ1][3];
-         // real_t (&sm0_ptr)[MQ1][MQ1][MQ1][3] = sm0;
-         // real_t (&sm1_ptr)[MQ1][MQ1][MQ1][3] = sm1;
-
-         low::regs3d_t<DIM, MQ1> reg;
-         const real_t *rd = dx_ptr;
-
-         // const auto fields_e_ptr = load_field_e_ptr(wrapped_fields_e, e);
-
-         MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-         // real_t (&sB_ptr)[MD1][MQ1] = sB;
-         // real_t (&sG_ptr)[MD1][MQ1] = sG;
-
-         // Interpolate
-         // for_constexpr<num_inputs>(
-         //    [ D1D, Q1D, MQ1, e,
-         //           &input_dtq_maps,
-         //           &sm0_ptr, &sm1_ptr,
-         //           &sB = sB_ptr, &sG = sG_ptr,
-         //           &inputs,
-         //           //  &fields_e_ptr,
-         //           &reg, &rd,
-         //           &input_to_field ] (auto i)
-         {
-            // const auto input = get<0/*i*/>(inputs);
-            // using field_operator_t = std::decay_t<decltype(input)>;
-
-            // if constexpr (is_gradient_fop<field_operator_t>::value) // Grad
-            {
-               // const int vdim = input.vdim;
-               // const real_t *field_e_r = fields_e_ptr[input_to_field[i]];
-               // const auto XE = Reshape(field_e_r, D1D, D1D, D1D, vdim);
-               // const auto sB = reinterpret_cast<const real_t (*)[MQ1]>(Bi[i]);
-               // const auto sG = reinterpret_cast<const real_t (*)[MQ1]>(Gi[i]);
-               low::LoadMatrix(d1d, q1d, B, sB);
-               low::LoadMatrix(d1d, q1d, G, sG);
-               // for (int c = 0; c < vdim; c++)
-               // constexpr int c = 0;
-               {
-                  low::LoadDofs3d(e, d1d, XE, sm0);
-                  low::Grad3d(d1d, q1d, sB, sG, sm0, sm1, reg);
-               }
-            }
-            // else if constexpr (is_identity_fop<field_operator_t>::value)   // Identity
-            {
-               // db1("Identity");
-               // rd = fields_e_ptr[input_to_field[i]];
-               // rd = dx_ptr;
-            }
-            // else if constexpr (is_weight_fop<field_operator_t>::value)   // Weight
-            // {
-            //    dbg("Weight");
-            //    rw = fields_e_ptr[input_to_field[i]]; // 🔥
-            // }
-            // else
-            {
-               // MFApply comes here
-               // assert(false);
-               // MFEM_ABORT("Only Grad and Identity field operators are supported");
-            }
-         }//); // for_constexpr<num_inputs>
-
-         MFEM_FOREACH_THREAD_DIRECT(qz,z,q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy,y,q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx,x,q1d)
-               {
-#if 0
-                  auto qf_args = decay_tuple<qf_param_ts> {};
-                  qf::apply_kernel<T_Q1D, num_inputs>
-                  (reg, reg, rd, qx, qy, qz, qfunc, qf_args);
-#elif 0
-                  real_t v[3], u[3] = { reg[qz][qy][qx][0],
-                                        reg[qz][qy][qx][1],
-                                        reg[qz][qy][qx][2]
-                                      };
-                  const auto *D = (real_t (*)[T_Q1D][T_Q1D][3][3]) rd;
-                  kernels::Mult(3, 3, &D[qx][qy][qz][0][0], u, v);
-                  reg[qz][qy][qx][0] = v[0];
-                  reg[qz][qy][qx][1] = v[1];
-                  reg[qz][qy][qx][2] = v[2];
-#elif 0
-                  const auto *D = (real_t (*)[T_Q1D][T_Q1D][3][3]) rd;
-                  const auto args = decay_tuple<qf_param_ts>
-                  {
-                     {{ reg[qz][qy][qx][0], reg[qz][qy][qx][1], reg[qz][qy][qx][2] }},
-                     {{
-                           {{ D[qx][qy][qz][0][0], D[qx][qy][qz][0][1], D[qx][qy][qz][0][2] }},
-                           {{ D[qx][qy][qz][1][0], D[qx][qy][qz][1][1], D[qx][qy][qz][1][2] }},
-                           {{ D[qx][qy][qz][2][0], D[qx][qy][qz][2][1], D[qx][qy][qz][2][2] }}
-                        }
-                     }
-                  };
-                  const auto r = get<0>(apply(qfunc, args));
-                  reg[qz][qy][qx][0] = r[0];
-                  reg[qz][qy][qx][1] = r[1];
-                  reg[qz][qy][qx][2] = r[2];
-#elif 0
-                  auto u = as_tensor<real_t, 3>(&reg[qz][qy][qx][0]);
-                  const auto *d = (real_t (*)[T_Q1D][T_Q1D][3][3]) rd;
-                  auto D = as_tensor<real_t, 3, 3>(&d[qx][qy][qz][0][0]);
-                  auto r = D * u;
-                  reg[qz][qy][qx][0] = r[0];
-                  reg[qz][qy][qx][1] = r[1];
-                  reg[qz][qy][qx][2] = r[2];
-#else
-                  auto args = decay_tuple<qf_param_ts> {};
-                  get<0>(args) = as_tensor<real_t, 3>(&reg[qz][qy][qx][0]);
-                  if constexpr (T_Q1D > 0)
-                  {
-                     get<1>(args) = as_tensor<real_t, 3, 3>(rd + 9*(qx*T_Q1D*T_Q1D + qy*T_Q1D + qz));
-                  }
-                  else
-                  {
-                     get<1>(args) = as_tensor<real_t, 3, 3>(rd + 9*(qx*q1d*q1d + qy*q1d + qz));
-                  }
-                  auto r = get<0>(apply(qfunc, args));
-                  if constexpr (decltype(r)::ndim == 1)
-                  {
-                     as_tensor<real_t, 3>(&reg[qz][qy][qx][0]) = r;
-                  }
-                  else { static_assert(false); }
-#endif
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         // Integrate
-         // if constexpr (is_gradient_fop<std::decay_t<output_fop_t>>::value) // Gradient
-         {
-            // const auto sB = reinterpret_cast<const real_t (*)[MQ1]>(Bo);
-            // const auto sG = reinterpret_cast<const real_t (*)[MQ1]>(Go);
-            low::GradTranspose3d(d1d, q1d, sB, sG, reg, sm1, sm0);
-            low::WriteDofs3d(d1d, 0, e, reg, YE);
-         }
-      },
-      num_entities, thread_blocks, 0, nullptr);
-      NVTX_END("forall");
-
-      NVTX_INI("out^T");
-      output_restriction_transpose(residual_e, residual_l);
-      NVTX_END("out^T");
-   }
-
-   using NewActionKernelType = decltype(&NewActionCallback::action_callback_new<>);
-   MFEM_REGISTER_KERNELS(NewActionCallbackKernels, NewActionKernelType, (int));
-
-   void Apply(const int d1d, const int q1d)
-   {
-      db1();
-      NewActionCallbackKernels::Run(q1d,
-                                    // args
-                                    d1d,
-                                    restriction_cb,
-                                    qfunc,
-                                    inputs,
-                                    input_to_field,
-                                    input_dtq_maps,
-                                    output_dtq_maps,
-                                    dimension,
-                                    num_entities,
-                                    test_vdim,
-                                    num_test_dof,
-                                    thread_blocks,
-                                    shmem_info,
-                                    attributes,
-                                    output_fop,
-                                    elem_attributes,
-                                    fields_e,
-                                    residual_e,
-                                    output_restriction_transpose,
-                                    solutions_l,
-                                    parameters_l,
-                                    residual_l,
-                                    // fallback arguments
-                                    q1d);
-   }
-};
-
-template<size_t num_fields, size_t num_inputs, size_t num_outputs,
-         typename restriction_cb_t, typename qfunc_t, typename input_t, typename output_fop_t>
-template<int T_Q1D>
-typename NewActionCallback<num_fields, num_inputs, num_outputs, restriction_cb_t, qfunc_t, input_t, output_fop_t>::NewActionKernelType
-NewActionCallback<num_fields, num_inputs, num_outputs, restriction_cb_t, qfunc_t, input_t, output_fop_t>::NewActionCallbackKernels::Kernel()
-{
-   return action_callback_new<T_Q1D>;
-}
-
-template<size_t num_fields, size_t num_inputs, size_t num_outputs,
-         typename restriction_cb_t, typename qfunc_t, typename input_t, typename output_fop_t>
-typename NewActionCallback<num_fields, num_inputs, num_outputs, restriction_cb_t, qfunc_t, input_t, output_fop_t>::NewActionKernelType
-NewActionCallback<num_fields, num_inputs, num_outputs, restriction_cb_t, qfunc_t, input_t, output_fop_t>::NewActionCallbackKernels::Fallback
-(int q1d)
-{
-   dbg("\x1b[33mFallback q1d:{}", q1d);
-   // MFEM_ABORT("No kernel for q1d=" << q1d);
-   // return nullptr;
-   return action_callback_new<>;
-}
-
-} // namespace mfem::future

fem/dfem/backends/global_qf/action.hpp

@@ -1,111 +0,0 @@
-#pragma once
-
-#include "../util.hpp"
-#include "../../integrator_ctx.hpp"
-
-#include <utility>
-
-namespace mfem::future
-{
-
-namespace GlobalQFImpl
-{
-
-template<
-   typename qfunc_t,
-   typename inputs_t,
-   typename outputs_t,
-   size_t ninputs = tuple_size<inputs_t>::value,
-   size_t noutputs = tuple_size<outputs_t>::value>
-struct Action
-{
-   Action(
-      IntegratorContext ctx,
-      qfunc_t qfunc,
-      inputs_t inputs,
-      outputs_t outputs) :
-      ctx(ctx),
-      qfunc(std::move(qfunc)),
-      inputs(inputs),
-      outputs(outputs)
-   {
-      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(inputs, input_to_infd, ctx.infds);
-      check_consistency(outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
-      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
-
-      const int nqp = ctx.ir.GetNPoints();
-      gnqp = nqp * ctx.nentities;
-
-      xq_offsets.SetSize(ninputs + 1);
-      xq_offsets[0] = 0;
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         const auto input = get<i>(inputs);
-         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
-      });
-      xq_offsets.PartialSum();
-      xq.Update(xq_offsets);
-
-      yq_offsets.SetSize(noutputs + 1);
-      yq_offsets[0] = 0;
-      constexpr_for<0, noutputs>([&](auto i)
-      {
-         const auto output = get<i>(outputs);
-         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
-      });
-      yq_offsets.PartialSum();
-      yq.Update(yq_offsets);
-   }
-
-   void operator()(
-      const std::vector<Vector *> &xe,
-      std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      // E -> Q
-      interpolate(input_to_infd, input_bases, xe, xq);
-
-      // Q -> Q
-      static_assert(
-         detail::supports_tensor_array_qfunc<qfunc_t, inputs_t, outputs_t>::value,
-         "qfunc signature not supported by default backend Action");
-
-      detail::call_qfunc(
-         qfunc, xq, yq, gnqp, input_qlayouts, output_qlayouts,
-         std::make_index_sequence<ninputs> {},
-         std::make_index_sequence<noutputs> {});
-
-      // Q -> E
-      integrate(output_to_outfd, output_bases, yq, ye);
-   }
-
-   IntegratorContext ctx;
-   qfunc_t qfunc;
-   inputs_t inputs;
-   outputs_t outputs;
-
-   std::array<size_t, ninputs> input_to_infd;
-   std::array<size_t, noutputs> output_to_outfd;
-
-   std::array<FieldBasis, ninputs> input_bases;
-   std::array<FieldBasis, noutputs> output_bases;
-
-   std::array<std::vector<int>, ninputs>  input_qlayouts;
-   std::array<std::vector<int>, noutputs> output_qlayouts;
-
-   int gnqp = 0;
-   Array<int> xq_offsets, yq_offsets;
-   mutable BlockVector xq, yq;
-};
-
-}
-}

fem/dfem/backends/global_qf/derivative_action_enzyme.hpp

@@ -1,131 +0,0 @@
-#pragma once
-
-#include "../fem/quadinterpolator.hpp"
-#include "../../integrator_ctx.hpp"
-#include "../util.hpp"
-#include <utility>
-
-namespace mfem::future
-{
-
-namespace GlobalQFImpl
-{
-
-template<
-   int derivative_id,
-   typename qfunc_t,
-   typename inputs_t,
-   typename outputs_t,
-   size_t ninputs = tuple_size<inputs_t>::value,
-   size_t noutputs = tuple_size<outputs_t>::value>
-struct DerivativeActionEnzyme
-{
-   DerivativeActionEnzyme(
-      IntegratorContext ctx,
-      qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs) :
-      ctx(ctx),
-      qfunc(qfunc),
-      inputs(inputs),
-      outputs(outputs)
-   {
-      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(inputs, input_to_infd, ctx.infds);
-      check_consistency(outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
-      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
-
-      const int nqp = ctx.ir.GetNPoints();
-      gnqp = nqp * ctx.nentities;
-
-      xq_offsets.SetSize(ninputs + 1);
-      xq_offsets[0] = 0;
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         const auto input = get<i>(inputs);
-         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
-      });
-      xq_offsets.PartialSum();
-      xq.Update(xq_offsets);
-
-      yq_offsets.SetSize(noutputs + 1);
-      yq_offsets[0] = 0;
-      constexpr_for<0, noutputs>([&](auto i)
-      {
-         const auto output = get<i>(outputs);
-         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
-      });
-      yq_offsets.PartialSum();
-      yq.Update(yq_offsets);
-
-      // For each dependent input in the dependency map we create a shadow
-      // memory variable at the quadrature point level.
-      const auto activity_map = detail::make_activity_map<derivative_id>(inputs);
-      shadow_xq_offsets.SetSize(ninputs + 1);
-      shadow_xq_offsets = 0;
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         if (activity_map[i])
-         {
-            shadow_xq_offsets[i + 1] =
-               xq_offsets[i + 1] - xq_offsets[i];;
-         }
-      });
-      shadow_xq_offsets.PartialSum();
-      shadow_xq.Update(shadow_xq_offsets);
-   }
-
-   void operator()(
-      const std::vector<Vector *> &xe,
-      const Vector *de,
-      std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-      // E -> Q
-      interpolate(input_to_infd, input_bases, xe, xq);
-
-      const auto activity_map = detail::make_activity_map<derivative_id>(inputs);
-      interpolate(input_to_infd, input_bases, xe, shadow_xq, activity_map);
-
-      // Q -> Q
-      static_assert(
-         detail::supports_tensor_array_qfunc<qfunc_t, inputs_t, outputs_t>::value,
-         "qfunc signature not supported by default backend Action");
-
-      detail::enzyme_fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
-         qfunc, xq, shadow_xq, yq, gnqp, input_qlayouts, output_qlayouts,
-         std::make_index_sequence<ninputs> {},
-         std::make_index_sequence<noutputs> {});
-
-      // Q -> E
-      integrate(output_to_outfd, output_bases, yq, ye);
-   }
-
-   IntegratorContext ctx;
-   qfunc_t &qfunc;
-   inputs_t inputs;
-   outputs_t outputs;
-
-   std::array<size_t, ninputs> input_to_infd;
-   std::array<size_t, noutputs> output_to_outfd;
-
-   std::array<FieldBasis, ninputs> input_bases;
-   std::array<FieldBasis, noutputs> output_bases;
-
-   std::array<std::vector<int>, ninputs>  input_qlayouts;
-   std::array<std::vector<int>, noutputs> output_qlayouts;
-
-   int gnqp = 0;
-   Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
-   mutable BlockVector xq, shadow_xq, yq;
-};
-
-}
-}

fem/dfem/backends/global_qf/prelude.hpp

@@ -1,42 +0,0 @@
-#pragma once
-
-#include "action.hpp"
-#include "derivative_action_enzyme.hpp"
-
-namespace mfem::future
-{
-
-struct GlobalQFBackend
-{
-   template<
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeAction(
-      const IntegratorContext &ctx,
-      qfunc_t qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      return GlobalQFImpl::Action(ctx, qfunc, inputs, outputs);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeAction(
-      const IntegratorContext &ctx,
-      qfunc_t qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      return GlobalQFImpl::DerivativeActionEnzyme<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs);
-   }
-
-};
-
-}

fem/dfem/backends/local_qf/action.hpp

@@ -1,166 +0,0 @@
-#pragma once
-
-#include "../util.hpp"
-#include "../../integrator_ctx.hpp"
-
-#include <utility>
-
-namespace mfem::future
-{
-
-namespace LocalQFImpl
-{
-
-template<
-   typename qfunc_t,
-   typename inputs_t,
-   typename outputs_t,
-   size_t ninputs = tuple_size<inputs_t>::value,
-   size_t noutputs = tuple_size<outputs_t>::value>
-struct Action
-{
-   Action(
-      IntegratorContext ctx,
-      qfunc_t qfunc,
-      inputs_t inputs,
-      outputs_t outputs) :
-      ctx(ctx),
-      qfunc(std::move(qfunc)),
-      inputs(inputs),
-      outputs(outputs)
-   {
-      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(inputs, input_to_infd, ctx.infds);
-      check_consistency(outputs, output_to_outfd, ctx.outfds);
-
-      const int nqp = ctx.ir.GetNPoints();
-
-      // Initialize DofToQuad maps for inputs
-      for_constexpr<ninputs>([&](auto i)
-      {
-         const auto &fd = ctx.infds[input_to_infd[i]];
-         std::visit([&](auto* space_ptr)
-         {
-            using T = std::decay_t<decltype(*space_ptr)>;
-            if constexpr (std::is_same_v<T, FiniteElementSpace> ||
-                          std::is_same_v<T, ParFiniteElementSpace>)
-            {
-               const auto *fe = space_ptr->GetTypicalFE();
-               input_dtq_maps[i] = &fe->GetDofToQuad(ctx.ir, DofToQuad::TENSOR);
-            }
-         }, fd.data);
-      });
-
-      // Initialize DofToQuad maps for outputs
-      for_constexpr<noutputs>([&](auto i)
-      {
-         const auto &fd = ctx.outfds[output_to_outfd[i]];
-         std::visit([&](auto* space_ptr)
-         {
-            using T = std::decay_t<decltype(*space_ptr)>;
-            if constexpr (std::is_same_v<T, FiniteElementSpace> ||
-                          std::is_same_v<T, ParFiniteElementSpace>)
-            {
-               const auto *fe = space_ptr->GetTypicalFE();
-               output_dtq_maps[i] = &fe->GetDofToQuad(ctx.ir, DofToQuad::TENSOR);
-            }
-         }, fd.data);
-      });
-   }
-
-   void operator()(
-      const std::vector<Vector *> &xe,
-      std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      // input_dtq_maps
-
-      // const auto B = (const real_t*)input_dtq_maps[0/*i*/].B;
-      // const auto G = (const real_t*)input_dtq_maps[0/*i*/].G;
-
-      //    dfem::forall<T_Q1D*T_Q1D*T_Q1D>([=] MFEM_HOST_DEVICE (int e, void *)
-      //    {
-      //       if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-      //       constexpr int MQ1 = T_Q1D > 0 ? T_Q1D : 8;
-
-      //       MFEM_SHARED real_t sm0[MQ1][MQ1][MQ1][3];
-      //       MFEM_SHARED real_t sm1[MQ1][MQ1][MQ1][3];
-
-      //       low::regs3d_t<DIM, MQ1> reg;
-      //       const real_t *rd = dx_ptr;
-
-      //       MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-      //       {
-      //          low::LoadMatrix(d1d, q1d, B, sB);
-      //          low::LoadMatrix(d1d, q1d, G, sG);
-      //          {
-      //             low::LoadDofs3d(e, d1d, XE, sm0);
-      //             low::Grad3d(d1d, q1d, sB, sG, sm0, sm1, reg);
-      //          }
-      //       }
-      //       // else if constexpr (is_identity_fop<field_operator_t>::value)   // Identity
-      //       {
-      //          // db1("Identity");
-      //          // rd = fields_e_ptr[input_to_field[i]];
-      //          // rd = dx_ptr;
-      //       }
-      //    }
-
-      //    MFEM_FOREACH_THREAD_DIRECT(qz,z,q1d)
-      //    {
-      //       MFEM_FOREACH_THREAD_DIRECT(qy,y,q1d)
-      //       {
-      //          MFEM_FOREACH_THREAD_DIRECT(qx,x,q1d)
-      //          {
-
-      //             auto args = decay_tuple<qf_param_ts> {};
-      //             get<0>(args) = as_tensor<real_t, 3>(&reg[qz][qy][qx][0]);
-      //             if constexpr (T_Q1D > 0)
-      //             {
-      //                get<1>(args) = as_tensor<real_t, 3, 3>(rd + 9*(qx*T_Q1D*T_Q1D + qy*T_Q1D + qz));
-      //             }
-      //             else
-      //             {
-      //                get<1>(args) = as_tensor<real_t, 3, 3>(rd + 9*(qx*q1d*q1d + qy*q1d + qz));
-      //             }
-      //             auto r = get<0>(apply(qfunc, args));
-      //             if constexpr (decltype(r)::ndim == 1)
-      //             {
-      //                as_tensor<real_t, 3>(&reg[qz][qy][qx][0]) = r;
-      //             }
-      //             else { static_assert(false); }
-      //          }
-      //       }
-      //    }
-      //    MFEM_SYNC_THREAD;
-      //    // Integrate
-      //    // if constexpr (is_gradient_fop<std::decay_t<output_fop_t>>::value) // Gradient
-      //    {
-      //       // const auto sB = reinterpret_cast<const real_t (*)[MQ1]>(Bo);
-      //       // const auto sG = reinterpret_cast<const real_t (*)[MQ1]>(Go);
-      //       low::GradTranspose3d(d1d, q1d, sB, sG, reg, sm1, sm0);
-      //       low::WriteDofs3d(d1d, 0, e, reg, YE);
-      //    }
-      // },
-      // num_entities, thread_blocks, 0, nullptr);
-   }
-
-
-   IntegratorContext ctx;
-   qfunc_t qfunc;
-   inputs_t inputs;
-   outputs_t outputs;
-
-   std::array<size_t, ninputs> input_to_infd;
-   std::array<size_t, noutputs> output_to_outfd;
-
-   std::array<const DofToQuad*, ninputs> input_dtq_maps;
-   std::array<const DofToQuad*, noutputs> output_dtq_maps;
-};
-
-}
-}

fem/dfem/backends/local_qf/prelude.hpp

@@ -1,39 +0,0 @@
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "action.hpp"
-
-namespace mfem::future
-{
-
-struct LocalQFBackend
-{
-   template<
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeAction(
-      const IntegratorContext &ctx,
-      qfunc_t qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      return LocalQFImpl::Action(ctx, qfunc, inputs, outputs);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeAction(
-      const IntegratorContext &ctx,
-      qfunc_t qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      MFEM_ABORT("LocalQFBackend does not support derivative actions.");
-   }
-};
-
-}

fem/dfem/backends/util.hpp

@@ -1,659 +0,0 @@
-#pragma once
-
-#include "../fem/quadinterpolator.hpp"
-#include "../util.hpp"
-#include "general/enzyme.hpp"
-
-namespace mfem::future
-{
-
-template <size_t N, size_t... Is>
-constexpr std::array<bool, N> all_true_impl(std::index_sequence<Is...>)
-{
-   return {{((void)Is, true)...}};
-}
-
-template <size_t N>
-constexpr std::array<bool, N> all_true()
-{
-   return all_true_impl<N>(std::make_index_sequence<N> {});
-}
-
-struct FieldBasis
-{
-   // E-vector -> Q-vector
-   std::function<void(const Vector &, Vector &)> forward;
-
-   // Q-vector -> E-vector
-   std::function<void(const Vector &, Vector &)> transpose;
-};
-
-inline FieldBasis FromQI(const QuadratureInterpolator *qi,
-                         QuadratureInterpolator::EvalFlags mode)
-{
-   return
-   {
-      [qi, mode](const Vector &xe, Vector &xq)
-      {
-         qi->SetOutputLayout(QVectorLayout::byVDIM);
-         if (mode == QuadratureInterpolator::VALUES)
-         {
-            qi->Values(xe, xq);
-         }
-         else
-         {
-            qi->Derivatives(xe, xq);
-         }
-      },
-      [qi, mode](const Vector &yq, Vector &ye)
-      {
-         Vector empty;
-         qi->SetOutputLayout(QVectorLayout::byVDIM);
-         if (mode == QuadratureInterpolator::VALUES)
-         {
-            qi->AddMultTranspose(QuadratureInterpolator::VALUES, yq, empty, ye);
-         }
-         else
-         {
-            qi->AddMultTranspose(QuadratureInterpolator::DERIVATIVES, empty, yq, ye);
-         }
-      }
-   };
-}
-
-// QuadratureFunction identity copy
-inline FieldBasis FromQF()
-{
-   return
-   {
-      [](const Vector &xe, Vector &xq) { xq = xe; },
-      [](const Vector &yq, Vector &ye) { ye = yq; }
-   };
-}
-
-// User-defined parameter space B
-inline FieldBasis FromPS(const Operator *B, const Operator *Bt)
-{
-   return
-   {
-      [B](const Vector &xe, Vector &xq) { B->Mult(xe, xq); },
-      [Bt](const Vector &yq, Vector &ye) { Bt->Mult(yq, ye); }
-   };
-}
-
-inline FieldBasis FieldBasisFromWeight(const IntegrationRule &ir)
-{
-   return
-   {
-      [&ir](const Vector &, Vector &xq)
-      {
-         const int nqp = ir.GetNPoints();
-         MFEM_ASSERT(xq.Size() % nqp == 0, "weight block has unexpected size");
-
-         const int ne = xq.Size() / nqp;
-         const real_t *wref = ir.GetWeights().Read();
-
-         for (int e = 0; e < ne; e++)
-         {
-            std::memcpy(xq.GetData() + e*nqp, wref, nqp*sizeof(real_t));
-         }
-      },
-      [](const Vector &, Vector &) {}
-   };
-}
-
-inline const FieldBasis GetFieldBasis(const FieldDescriptor &f,
-                                      const IntegrationRule &ir,
-                                      QuadratureInterpolator::EvalFlags mode)
-{
-   return std::visit([&ir, &mode](auto && arg) -> FieldBasis
-   {
-      using T = std::decay_t<decltype(arg)>;
-
-      if constexpr (std::is_same_v<T, const FiniteElementSpace *>)
-      {
-         return FromQI(arg->GetQuadratureInterpolator(ir), mode);
-      }
-      else if constexpr (std::is_same_v<T, const ParFiniteElementSpace *>)
-      {
-         return FromQI(arg->GetQuadratureInterpolator(ir), mode);
-      }
-      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
-      {
-         return FromQF();
-      }
-      else if constexpr (std::is_same_v<T, const ParameterSpace *>)
-      {
-         return FromPS(arg->GetB(), arg->GetBt());
-      }
-      else if constexpr (std::is_same_v<T, const IntegrationRule *>)
-      {
-         return FieldBasis{};
-      }
-      else
-      {
-         static_assert(dfem::always_false<T>, "internal error");
-      }
-   }, f.data);
-}
-
-template <typename fops_t, size_t nfops>
-void create_fieldbases(
-   fops_t &fops,
-   const std::array<size_t, nfops> &fop_to_fd,
-   const std::vector<FieldDescriptor> &fds,
-   const IntegrationRule &ir,
-   std::array<FieldBasis, nfops> &bases)
-{
-   constexpr_for<0, nfops>([&](auto i)
-   {
-      const auto fop = get<i>(fops);
-      using fop_t = std::decay_t<decltype(fop)>;
-
-      const auto fd = fds[fop_to_fd[i]];
-
-      constexpr QuadratureInterpolator::EvalFlags dummy_mode =
-         QuadratureInterpolator::VALUES;
-      if constexpr (is_identity_fop<fop_t>::value)
-      {
-         bases[i] = GetFieldBasis(fd, ir, dummy_mode);
-      }
-      else if constexpr (is_weight_fop<fop_t>::value)
-      {
-         bases[i] = FieldBasisFromWeight(ir);
-      }
-      else if constexpr (is_value_fop<fop_t>::value)
-      {
-         bases[i] = GetFieldBasis(fd, ir, QuadratureInterpolator::VALUES);
-      }
-      else if constexpr (is_gradient_fop<fop_t>::value)
-      {
-         bases[i] = GetFieldBasis(fd, ir, QuadratureInterpolator::DERIVATIVES);
-      }
-   });
-}
-
-template <typename fops_t, size_t nfops>
-void check_consistency(
-   fops_t &fops,
-   const std::array<size_t, nfops> &fop_to_fd,
-   const std::vector<FieldDescriptor> &fields)
-{
-   constexpr_for<0, nfops>([&](auto i)
-   {
-      const auto input = get<i>(fops);
-      using input_t = std::decay_t<decltype(input)>;
-
-      const auto fd = fields[fop_to_fd[i]];
-
-      if constexpr (is_identity_fop<input_t>::value)
-      {
-         MFEM_ASSERT(std::holds_alternative<const QuadratureFunction *>(fd.data),
-                     "Identity FieldOperator requested on non "
-                     "QuadratureFunction");
-      }
-      else if constexpr (is_weight_fop<input_t>::value)
-      {
-      }
-      else if constexpr (is_value_fop<input_t>::value)
-      {
-         MFEM_ASSERT(std::holds_alternative<const FiniteElementSpace *>(fd.data) ||
-                     std::holds_alternative<const ParFiniteElementSpace *>(fd.data) ||
-                     std::holds_alternative<const ParameterSpace *>(fd.data),
-                     "Value FieldOperator requested on non "
-                     "QuadratureFunction");
-      }
-      else if constexpr (is_gradient_fop<input_t>::value)
-      {
-         MFEM_ASSERT(std::holds_alternative<const FiniteElementSpace *>(fd.data) ||
-                     std::holds_alternative<const ParFiniteElementSpace *>(fd.data),
-                     "Value FieldOperator requested on non "
-                     "QuadratureFunction");
-      }
-   });
-}
-
-template <size_t ninputs>
-void interpolate(
-   const std::array<size_t, ninputs> &input_to_infd,
-   const std::array<FieldBasis, ninputs> &input_bases,
-   const std::vector<Vector *> &xe,
-   BlockVector &xq,
-   const std::array<bool, ninputs> &conditional = all_true<ninputs>())
-{
-   constexpr_for<0, ninputs>([&](auto i)
-   {
-      if (!conditional.empty() && !conditional[i]) { return; }
-
-      input_bases[i].forward(*xe[input_to_infd[i]], xq.GetBlock(i));
-   });
-}
-
-template <size_t noutputs>
-void integrate(
-   const std::array<size_t, noutputs> &output_to_outfd,
-   const std::array<FieldBasis, noutputs> &output_bases,
-   const BlockVector &yq,
-   std::vector<Vector *> &ye)
-{
-   for (auto v : ye) { *v = 0.0; }
-
-   constexpr_for<0, noutputs>([&](auto i)
-   {
-      output_bases[i].transpose(yq.GetBlock(i), *ye[output_to_outfd[i]]);
-   });
-}
-
-
-namespace detail
-{
-
-template <typename T>
-struct is_tensor_array : std::false_type {};
-
-template <typename scalar_t, int... Dims>
-struct is_tensor_array<tensor_array<scalar_t, Dims...>> : std::true_type {};
-
-template <typename T>
-struct is_tensor_array_mut : std::false_type {};
-
-template <typename scalar_t, int... Dims>
-struct is_tensor_array_mut<tensor_array<scalar_t, Dims...>> :
-                                                            std::bool_constant<!std::is_const_v<scalar_t>> {};
-
-
-template <typename ndarray_t>
-inline void set_layout_default(ndarray_t &a)
-{
-   if constexpr (ndarray_t::tensor_rank() == 0) { return; }
-
-   constexpr std::size_t nd = ndarray_t::rank();
-   constexpr std::size_t td = ndarray_t::tensor_rank();
-   std::array<std::size_t, nd + td> perm{};
-
-   for (std::size_t i = 0; i < td; i++) { perm[i] = nd + i; }
-   for (std::size_t i = 0; i < nd; i++) { perm[td + i] = i; }
-
-   a.set_layout(perm);
-}
-
-template <typename ndarray_t>
-inline void set_layout(ndarray_t& a, const std::vector<int>& layout)
-{
-   if constexpr (ndarray_t::tensor_rank() == 0) { return; }
-
-   constexpr std::size_t nd = ndarray_t::rank();
-   constexpr std::size_t td = ndarray_t::tensor_rank();
-   constexpr std::size_t N  = nd + td;
-
-   // missing means default
-   if (layout.empty()) { set_layout_default(a); return; }
-
-   MFEM_VERIFY(layout.size() == N,
-               "layout size mismatch: expected " << N << " got " << layout.size());
-
-   // TODO: make a version of set_layout that takes `std::vector<int>`
-   std::array<std::size_t, N> perm{};
-   for (std::size_t i = 0; i < N; i++)
-   {
-      MFEM_VERIFY(layout[i] >= 0, "layout index must be >=0");
-      perm[i] = static_cast<std::size_t>(layout[i]);
-   }
-
-   a.set_layout(perm);
-}
-
-/// Primary template: intentionally undefined — gives a clear error for unsupported types.
-template <typename T>
-struct tensor_array_traits;
-
-/// Matches tensor<scalar_t, sizes...>
-template <typename scalar_t, int... sizes>
-struct tensor_array_traits<tensor<scalar_t, sizes...>>
-{
-   using scalar_type = scalar_t;
-   template <std::size_t ndims>
-   using array_type = tensor_ndarray<scalar_t, ndims, sizes...>;
-};
-
-/// Matches tensor_ndarray<scalar_t, ndims, tensor_sizes...>
-template <typename scalar_t, int ndims, int... tensor_sizes>
-struct tensor_array_traits<tensor_ndarray<scalar_t, ndims, tensor_sizes...>>
-{
-   using scalar_type = scalar_t;
-   template <std::size_t N>
-   using array_type = tensor_ndarray<scalar_t, N, tensor_sizes...>;
-};
-
-/// Entry point: explicit tensor type T as template argument.
-template <typename T, typename ptr_scalar_t, typename... dyn_sizes_t>
-decltype(auto) make_tensor_array(ptr_scalar_t *ptr,
-                                 const std::vector<int>* layout,
-                                 dyn_sizes_t... dynamic_sizes)
-{
-   using traits = tensor_array_traits<T>;
-   using array_t = typename traits::template array_type<sizeof...(dynamic_sizes)>;
-   auto a = array_t(ptr, {std::size_t(dynamic_sizes)...});
-   if (layout) { set_layout(a, *layout); }
-   else        { set_layout_default(a); }
-   return a;
-}
-
-template <typename qfunc_t, typename inputs_t, typename outputs_t>
-struct supports_tensor_array_qfunc
-{
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-
-   static constexpr int ninputs = tuple_size<inputs_t>::value;
-   static constexpr int noutputs = tuple_size<outputs_t>::value;
-   static constexpr int nparams = tuple_size<qf_param_ts>::value;
-
-   template <std::size_t... Is>
-   static constexpr bool InputsOk(std::index_sequence<Is...>)
-   {
-      return (is_tensor_array<std::remove_cv_t<std::remove_reference_t<
-              typename tuple_element<Is, qf_param_ts>::type>>>::value && ...);
-   }
-
-   template <std::size_t... Is>
-   static constexpr bool OutputsOk(std::index_sequence<Is...>)
-   {
-      return (is_tensor_array_mut<std::remove_cv_t<std::remove_reference_t<
-              typename tuple_element<ninputs + Is, qf_param_ts>::type>>>::value && ...);
-   }
-
-   static constexpr bool value =
-      (nparams == ninputs + noutputs) &&
-      InputsOk(std::make_index_sequence<ninputs> {}) &&
-      OutputsOk(std::make_index_sequence<noutputs> {});
-};
-
-template <typename qfunc_t, std::size_t... Is, std::size_t... Os>
-inline void call_qfunc(
-   const qfunc_t &qfunc,
-   const BlockVector &xq,
-   BlockVector &yq,
-   int gnqp,
-   const std::array<std::vector<int>, sizeof...(Is)>& in_layouts,
-   const std::array<std::vector<int>, sizeof...(Os)>& out_layouts,
-   std::index_sequence<Is...>,
-   std::index_sequence<Os...>)
-{
-   constexpr std::size_t ninputs = sizeof...(Is);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-
-   auto inputs = std::make_tuple(
-                    make_tensor_array<std::remove_cv_t<std::remove_reference_t<
-                    typename tuple_element<Is, qf_param_ts>::type>>>(
-                       xq.GetBlock(Is).Read(), &in_layouts[Is], gnqp)...);
-
-   auto outputs = std::make_tuple(
-                     make_tensor_array<std::remove_cv_t<std::remove_reference_t<
-                     typename tuple_element<ninputs + Os, qf_param_ts>::type>>>(
-                        yq.GetBlock(Os).ReadWrite(), &out_layouts[Os], gnqp)...);
-
-   std::apply([&](auto&&... args)
-   {
-      qfunc(args...);
-   }, std::tuple_cat(inputs, outputs));
-}
-
-template <typename func_t, typename... arg_ts>
-MFEM_HOST_DEVICE inline
-auto qfunction_wrapper(const func_t &f, arg_ts...args)
-{
-   return f(args...);
-}
-
-template <std::size_t derivative_id, std::size_t I, typename Tuple, std::size_t... Is>
-constexpr std::array<bool, sizeof...(Is)>
-make_activity_array(std::index_sequence<Is...>)
-{
-   return { (std::decay_t<typename tuple_element<Is, Tuple>::type>::GetFieldId() == derivative_id)... };
-}
-
-template <std::size_t derivative_id, typename inputs_t, std::size_t... Is>
-constexpr auto make_activity_map_impl(std::index_sequence<Is...>)
-{
-   constexpr std::size_t N = sizeof...(Is);
-
-   if constexpr (N == 0)
-      return std::array<bool, 0> {};
-
-   return make_activity_array<derivative_id, 0, inputs_t>
-          (std::make_index_sequence<N> {});
-}
-
-template <std::size_t derivative_id, typename inputs_t>
-constexpr auto make_activity_map(inputs_t)
-{
-   return make_activity_map_impl<derivative_id, inputs_t>(
-             std::make_index_sequence<tuple_size<inputs_t>::value> {});
-}
-
-namespace enzyme_detail
-{
-
-template <auto wrapper_fn, typename qf_return_t, typename... AccArgs>
-__attribute__((always_inline)) inline void
-do_enzyme_call(AccArgs... acc)
-{
-   __enzyme_fwddiff<qf_return_t>(wrapper_fn, acc...);
-}
-
-template <auto wrapper_fn, typename qf_return_t,
-          size_t CurO, size_t NO,
-          typename primals_t, typename derivs_t,
-          typename... AccArgs>
-__attribute__((always_inline)) inline void
-process_outputs(primals_t &primals, derivs_t &derivs, AccArgs... acc)
-{
-   if constexpr (CurO == NO)
-   {
-      do_enzyme_call<wrapper_fn, qf_return_t>(acc...);
-   }
-   else
-   {
-      process_outputs<wrapper_fn, qf_return_t, CurO + 1, NO>(
-         primals, derivs,
-         acc...,
-         enzyme_dupnoneed,
-         &std::get<CurO>(primals),
-         &std::get<CurO>(derivs));
-   }
-}
-
-template <auto wrapper_fn, typename qf_return_t,
-          size_t CurI, size_t NI, bool... ActivityMap,
-          typename inputs_t, typename shadows_t,
-          typename primals_t, typename derivs_t,
-          typename... AccArgs>
-__attribute__((always_inline)) inline void
-process_inputs(inputs_t &inputs, shadows_t &shadows,
-               primals_t &primals, derivs_t &derivs,
-               AccArgs... acc)
-{
-   if constexpr (CurI == NI)
-   {
-      constexpr size_t NO = std::tuple_size_v<primals_t>;
-      process_outputs<wrapper_fn, qf_return_t, 0, NO>(
-         primals, derivs, acc...);
-   }
-   else
-   {
-      constexpr bool active =
-         std::array<bool, sizeof...(ActivityMap)> {ActivityMap...} [CurI];
-
-      if constexpr (active)
-      {
-         std::cout << "Input[" << CurI << "]: ACTIVE (enzyme_dup)\n"
-                   << "  primal ptr type:  "
-                   << get_type_name<decltype(&std::get<CurI>(inputs))>() << "\n"
-                   << "  shadow ptr type:  "
-                   << get_type_name<decltype(&std::get<CurI>(shadows))>() << "\n";
-      }
-      else
-      {
-         std::cout << "Input[" << CurI << "]: INACTIVE (enzyme_const)\n"
-                   << "  primal ptr type:  "
-                   << get_type_name<decltype(&std::get<CurI>(inputs))>() << "\n";
-      }
-
-      if constexpr (active)
-      {
-         process_inputs<wrapper_fn, qf_return_t, CurI + 1, NI, ActivityMap...>(
-            inputs, shadows, primals, derivs,
-            acc...,
-            enzyme_dup,
-            &std::get<CurI>(inputs),
-            &std::get<CurI>(shadows));
-      }
-      else
-      {
-         process_inputs<wrapper_fn, qf_return_t, CurI + 1, NI, ActivityMap...>(
-            inputs, shadows, primals, derivs,
-            acc...,
-            enzyme_const,
-            &std::get<CurI>(inputs));
-      }
-   }
-}
-
-} // namespace enzyme_detail
-
-template <size_t derivative_id, typename qfunc_t, typename inputs_t, typename outputs_t,
-          std::size_t... Is, std::size_t... Os>
-inline void enzyme_fwddiff(
-   qfunc_t &qfunc,
-   const BlockVector &xq,
-   const BlockVector &shadow_xq,
-   BlockVector &yq,
-   const int &gnqp,
-   const std::array<std::vector<int>, sizeof...(Is)>& in_layouts,
-   const std::array<std::vector<int>, sizeof...(Os)>& out_layouts,
-   std::index_sequence<Is...>,
-   std::index_sequence<Os...>)
-{
-#ifdef MFEM_USE_ENZYME
-   constexpr std::size_t ninputs  = sizeof...(Is);
-   constexpr std::size_t noutputs = sizeof...(Os);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts  = typename qf_signature::parameter_ts;
-   using qf_return_t  = typename qf_signature::return_t;
-
-   constexpr auto activity_map = make_activity_map<derivative_id>(inputs_t{});
-   static_assert(activity_map.size() == ninputs, "activity map size mismatch");
-
-   std::cout << "activity_map: ";
-   for (const auto &v : activity_map)
-   {
-      std::cout << v << " ";
-   }
-   std::cout << "\n";
-
-   auto inputs = std::make_tuple(
-                    make_tensor_array<std::remove_cv_t<std::remove_reference_t<
-                    typename tuple_element<Is, qf_param_ts>::type>>>(
-                       xq.GetBlock(Is).Read(), &in_layouts[Is], gnqp)...);
-
-   auto shadows = std::make_tuple(
-                     make_tensor_array<std::remove_cv_t<std::remove_reference_t<
-                     typename tuple_element<Is, qf_param_ts>::type>>>(
-                        shadow_xq.GetBlock(Is).Read(), &in_layouts[Is], gnqp)...);
-
-   std::array<Vector, noutputs> primal_storage;
-   ((primal_storage[Os].SetSize(yq.GetBlock(Os).Size())), ...);
-
-   auto primals_out = std::make_tuple(
-                         make_tensor_array<std::remove_cv_t<std::remove_reference_t<
-                         typename tuple_element<ninputs + Os, qf_param_ts>::type>>>(
-                            primal_storage[Os].ReadWrite(), &out_layouts[Os], gnqp)...);
-
-   auto derivs_out = std::make_tuple(
-                        make_tensor_array<std::remove_cv_t<std::remove_reference_t<
-                        typename tuple_element<ninputs + Os, qf_param_ts>::type>>>(
-                           yq.GetBlock(Os).ReadWrite(), &out_layouts[Os], gnqp)...);
-
-   using wrapper_fn_t = qf_return_t (*)(
-                           const qfunc_t &,
-                           std::remove_reference_t<decltype(std::get<Is>(inputs))>...,
-                           std::remove_reference_t<decltype(std::get<Os>(primals_out))>...);
-
-   constexpr wrapper_fn_t wrapper_fn =
-      qfunction_wrapper<qfunc_t,
-      std::remove_reference_t<decltype(std::get<Is>(inputs))>...,
-      std::remove_reference_t<decltype(std::get<Os>(primals_out))>...>;
-
-   // wrapper_fn travels as a non-type template parameter throughout without
-   // being stored.
-   enzyme_detail::process_inputs<
-   wrapper_fn,
-   qf_return_t,
-   0,
-   ninputs,
-   activity_map[Is]...
-   >(inputs, shadows,
-     primals_out, derivs_out,
-     enzyme_const, &qfunc // seed: qfunc is always inactive
-    );
-
-#else
-   MFEM_ABORT("enzyme_fwddiff requires MFEM_USE_ENZYME");
-#endif
-}
-
-} // namespace detail
-
-// Create quadrature function fop to fields map
-template <typename fops_t, size_t N = tuple_size<fops_t>::value, size_t M>
-void create_fop_to_fd(const fops_t &fops,
-                      const std::vector<FieldDescriptor> &fields,
-                      std::array<size_t, M> &fop_to_fd)
-{
-   static_assert(N == M, "sizes must match");
-   constexpr_for<0, N>([&](auto i)
-   {
-      const auto fop = get<i>(fops);
-      fop_to_fd[i] = std::numeric_limits<size_t>::max();
-      for (size_t j = 0; j < fields.size(); j++)
-      {
-         // TODO: output.GetFieldId() should probably store/return size_t
-         if (static_cast<int>(fields[j].id) == fop.GetFieldId())
-         {
-            fop_to_fd[i] = j;
-         }
-      }
-      // Handle Weight type. There is no FieldDescriptor for the weight.
-      // TODO: Create weight descriptor for the weight for internal use?
-      // TODO: this is a hack...
-      if (is_weight_fop<std::remove_cv_t<decltype(fop)>>::value)
-      {
-         fop_to_fd[i] = 0;
-      }
-      else if (fop_to_fd[i] == std::numeric_limits<size_t>::max())
-      {
-         MFEM_ABORT("not found");
-      }
-   });
-}
-
-template <typename fops_t, size_t nfops>
-void create_qlayouts(const fops_t &fops,
-                     const std::unordered_map<std::type_index, std::vector<int>> &a,
-                     std::array<std::vector<int>, nfops> &b)
-{
-   constexpr_for<0, nfops>([&](auto i)
-   {
-      using fop_t =
-         std::remove_cv_t<std::remove_reference_t<decltype(get<i>(fops))>>;
-      auto it = a.find(std::type_index(typeid(fop_t)));
-      if (it != a.end()) { b[i] = it->second; }
-      else               { b[i].clear(); }
-   });
-}
-
-}

fem/dfem/doperator.cpp

@@ -11,119 +11,44 @@
 
 #include "doperator.hpp"
 
-#include <algorithm>
-
 #ifdef MFEM_USE_MPI
 
 using namespace mfem;
 using namespace mfem::future;
 
+void DifferentiableOperator::SetParameters(std::vector<Vector *> p) const
+{
+   MFEM_ASSERT(parameters.size() == p.size(),
+               "number of parameters doesn't match descriptors");
+   for (size_t i = 0; i < parameters.size(); i++)
+   {
+      p[i]->Read();
+      parameters_l[i] = *p[i];
+   }
+}
+
 DifferentiableOperator::DifferentiableOperator(
-   const std::vector<FieldDescriptor> &infds,
-   const std::vector<FieldDescriptor> &outfds,
+   const std::vector<FieldDescriptor> &solutions,
+   const std::vector<FieldDescriptor> &parameters,
    const ParMesh &mesh) :
-   Operator(),
    mesh(mesh),
-   infds(infds),
-   outfds(outfds)
+   solutions(solutions),
+   parameters(parameters)
 {
-   unionfds.clear();
-   unionfds.insert(unionfds.end(), infds.begin(), infds.end());
-   unionfds.insert(unionfds.end(), outfds.begin(), outfds.end());
-   std::sort(unionfds.begin(), unionfds.end());
-   auto last = std::unique(unionfds.begin(), unionfds.end());
-   unionfds.erase(last, unionfds.end());
+   fields.resize(solutions.size() + parameters.size());
+   fields_e.resize(fields.size());
+   solutions_l.resize(solutions.size());
+   parameters_l.resize(parameters.size());
 
-   infields_l.resize(infds.size());
-   for (size_t i = 0; i < infds.size(); i++)
+   for (size_t i = 0; i < solutions.size(); i++)
    {
-      infields_l[i] = new Vector(GetVSize(infds[i]));
+      fields[i] = solutions[i];
    }
 
-   infields_e.resize(infds.size());
-}
-
-void DifferentiableOperator::SetMultLevel(MultLevel level)
-{
-   mult_level = level;
-}
-
-void DifferentiableOperator::Mult(const Vector &x, Vector &y) const
-{
-   MFEM_ASSERT(!action_callbacks.empty(),
-               "no integrators have been set");
-
-   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&x),
-               "x needs to be a BlockVector");
-
-   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&y),
-               "y needs to be a BlockVector");
-
-   const auto &bx = static_cast<const BlockVector &>(x);
-   auto &by = static_cast<BlockVector &>(y);
-
-   Mult(bx, by);
-}
-
-void DifferentiableOperator::DisableTensorProductStructure(bool disable)
-{
-   use_tensor_product_structure = !disable;
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
-   size_t derivative_id, const Vector &x)
-{
-   MFEM_ASSERT(derivative_action_callbacks.find(derivative_id) !=
-               derivative_action_callbacks.end(),
-               "no derivative action has been found for ID " << derivative_id);
-
-   const size_t dfidx = FindIdx(derivative_id, infds);
-
-   // Get transpose callbacks if available, otherwise pass empty vector
-   std::vector<derivative_action_t> transpose_callbacks;
-   auto it = daction_transpose_callbacks.find(derivative_id);
-   if (it != daction_transpose_callbacks.end())
+   for (size_t i = 0; i < parameters.size(); i++)
    {
-      transpose_callbacks = it->second;
+      fields[i + solutions.size()] = parameters[i];
    }
-
-   return std::make_shared<DerivativeOperator>(
-             height,
-             GetTrueVSize(infds[dfidx]),
-             derivative_action_callbacks[derivative_id],
-             transpose_callbacks,
-             infds[dfidx],
-             x,
-             infds,
-             outfds);
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
-   size_t derivative_id, const MultiVector &x)
-{
-   MFEM_ASSERT(derivative_action_callbacks.find(derivative_id) !=
-               derivative_action_callbacks.end(),
-               "no derivative action has been found for ID " << derivative_id);
-
-   const size_t dfidx = FindIdx(derivative_id, infds);
-
-   // Get transpose callbacks if available, otherwise pass empty vector
-   std::vector<derivative_action_t> transpose_callbacks;
-   auto it = daction_transpose_callbacks.find(derivative_id);
-   if (it != daction_transpose_callbacks.end())
-   {
-      transpose_callbacks = it->second;
-   }
-
-   return std::make_shared<DerivativeOperator>(
-             height,
-             GetTrueVSize(infds[dfidx]),
-             derivative_action_callbacks[derivative_id],
-             transpose_callbacks,
-             infds[dfidx],
-             x,
-             infds,
-             outfds);
 }
 
 #endif // MFEM_USE_MPI

fem/dfem/fielddescriptor.hpp

@@ -1,63 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../fespace.hpp"
-#include "parameterspace.hpp"
-
-namespace mfem::future
-{
-
-/// @brief FieldDescriptor struct
-///
-/// This struct is used to store information about a field.
-struct FieldDescriptor
-{
-   using data_variant_t =
-      std::variant<const FiniteElementSpace *,
-      const ParFiniteElementSpace *,
-      const QuadratureFunction *,
-      const ParameterSpace *>;
-
-   /// Field ID
-   std::size_t id;
-
-   /// Field variant
-   data_variant_t data;
-
-   /// Default constructor
-   FieldDescriptor() :
-      id(SIZE_MAX), data(data_variant_t{}) {}
-
-   /// Constructor
-   template <typename T>
-   FieldDescriptor(std::size_t field_id, const T* v) :
-      id(field_id), data(v) {}
-
-   bool operator==(const FieldDescriptor& other) const
-   {
-      return id == other.id;
-   }
-
-   bool operator<(const FieldDescriptor& other) const
-   {
-      return id < other.id;
-   }
-
-   friend void swap(FieldDescriptor& a, FieldDescriptor& b)
-   {
-      using std::swap;
-      swap(a.id, b.id);
-      swap(a.data, b.data);
-   }
-};
-
-}

fem/dfem/integrator_ctx.hpp

@@ -1,22 +0,0 @@
-#pragma once
-
-#include "util.hpp"
-
-namespace mfem::future
-{
-
-struct IntegratorContext
-{
-   const ParMesh &mesh;
-   const Array<int> *elem_attr;
-   Array<int> attr;
-   int nentities;
-   const std::vector<FieldDescriptor> &infds;
-   const std::vector<FieldDescriptor> &outfds;
-   const std::vector<FieldDescriptor> &unionfds;
-   const IntegrationRule &ir;
-   std::unordered_map<std::type_index, std::vector<int>> &in_qlayouts;
-   std::unordered_map<std::type_index, std::vector<int>> &out_qlayouts;
-};
-
-}

fem/dfem/interpolate.hpp

@@ -9,23 +9,8 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 #pragma once
-// #define NVTX_COLOR nvtx::kPeru
 
 #include "util.hpp"
-#include "fem/kernels.hpp"
-
-///////////////////////////////////////////////////////////////////////////////
-template <class T>
-inline std::enable_if_t<!std::numeric_limits<T>::is_integer, bool>
-AlmostEq(T x, T y, T tolerance = 15.0 * std::numeric_limits<T>::epsilon())
-{
-   const T neg = std::abs(x - y);
-   constexpr T min = std::numeric_limits<T>::min();
-   constexpr T eps = std::numeric_limits<T>::epsilon();
-   const T min_abs = std::min(std::abs(x), std::abs(y));
-   if (std::abs(min_abs) == 0.0) { return neg < eps; }
-   return (neg / (1.0 + std::max(min, min_abs))) < tolerance;
-}
 
 namespace mfem::future
 {
@@ -45,7 +30,6 @@ void map_field_to_quadrature_data_tensor_product_3d(
 
    if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
    {
-      dbg("Value");
       auto [q1d, unused, d1d] = B.GetShape();
       const int vdim = input.vdim;
       const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
@@ -110,11 +94,10 @@ void map_field_to_quadrature_data_tensor_product_3d(
    else if constexpr (
       is_gradient_fop<std::decay_t<field_operator_t>>::value)
    {
-      // dbg("Gradient");
-      const auto [q1d, B_dim, d1d] = B.GetShape();
+      const auto [q1d, unused, d1d] = B.GetShape();
       const int vdim = input.vdim;
       const int dim = input.dim;
-      const auto field = Reshape(&std::as_const(field_e[0]), d1d, d1d, d1d, vdim);
+      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
       auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d, q1d);
 
       auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
@@ -123,30 +106,7 @@ void map_field_to_quadrature_data_tensor_product_3d(
       auto s3 = Reshape(&scratch_mem[3](0), d1d, q1d, q1d);
       auto s4 = Reshape(&scratch_mem[4](0), d1d, q1d, q1d);
 
-      // constexpr int MQ1 = T_Q1D > 0 ? T_Q1D : 8;
-      // static constexpr int DIM = 3;
-      // MFEM_VERIFY(q1d <= MQ1, "q1d > MQ1");
-      // MFEM_SHARED real_t smem[MQ1][MQ1];
-
-      // kernels::internal::d_regs3d_t<DIM, MQ1> r0, r1;
-      // real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      /*
-      {
-         assert(B_dim == 1 && "1D B required!");
-         kernels::internal::LoadMatrix(d1d, q1d, B, sB);
-         kernels::internal::LoadMatrix(d1d, q1d, G, sG);
-         for (int qx = 0; qx < q1d; qx++)
-         {
-            for (int dx = 0; dx < d1d; dx++)
-            {
-               assert(AlmostEq(B(qx, 0, dx), sB[dx][qx]));
-               assert(AlmostEq(G(qx, 0, dx), sG[dx][qx]));
-            }
-         }
-      }*/
-
-      for (int c = 0; c < vdim; c++)
+      for (int vd = 0; vd < vdim; vd++)
       {
          MFEM_FOREACH_THREAD(dz, z, d1d)
          {
@@ -157,7 +117,7 @@ void map_field_to_quadrature_data_tensor_product_3d(
                   real_t uv[2] = {0.0, 0.0};
                   for (int dx = 0; dx < d1d; dx++)
                   {
-                     const real_t f = field(dx, dy, dz, c);
+                     const real_t f = field(dx, dy, dz, vd);
                      uv[0] += f * B(qx, 0, dx);
                      uv[1] += f * G(qx, 0, dx);
                   }
@@ -203,59 +163,19 @@ void map_field_to_quadrature_data_tensor_product_3d(
                      uvw[1] += s3(dz, qy, qx) * B(qz, 0, dz);
                      uvw[2] += s4(dz, qy, qx) * G(qz, 0, dz);
                   }
-                  fqp(c, 0, qx, qy, qz) = uvw[0];
-                  fqp(c, 1, qx, qy, qz) = uvw[1];
-                  fqp(c, 2, qx, qy, qz) = uvw[2];
+                  fqp(vd, 0, qx, qy, qz) = uvw[0];
+                  fqp(vd, 1, qx, qy, qz) = uvw[1];
+                  fqp(vd, 2, qx, qy, qz) = uvw[2];
                }
             }
          }
          MFEM_SYNC_THREAD;
       }
-
-      /*
-      {
-         for (int c = 0; c < vdim; c++)
-         {
-            kernels::internal::LoadDofs3d(d1d, c, field, r0);
-            for (int d = 0; d < DIM; d++)
-            {
-               for (int dz = 0; dz < d1d; dz++)
-               {
-                  for (int dy = 0; dy < d1d; dy++)
-                  {
-                     for (int dx = 0; dx < d1d; dx++)
-                     {
-                        const real_t f = field(dx, dy, dz, c);
-                        assert(AlmostEq(f, r0[d][dz][dy][dx]));
-                     }
-                  }
-               }
-            }
-
-            kernels::internal::Grad3d(d1d, q1d, smem, sB, sG, r0, r1, c);
-            for (int qz = 0; qz < q1d; qz++)
-            {
-               for (int qy = 0; qy < q1d; qy++)
-               {
-                  for (int qx = 0; qx < q1d; qx++)
-                  {
-                     if (!AlmostEq(fqp(c, d, qx, qy, qz), r1[d][qz][qy][qx]))
-                     {
-                        dbg("\x1b[31m[{}:d] {} {}", c, fqp(c, d, qx, qy, qz), r1[d][qz][qy][qx]);
-                        dbg("❌❌❌"), std::exit(EXIT_FAILURE);
-                     }
-                  }
-               }
-            }
-         }
-         // dbg("✅✅✅✅✅✅✅✅✅✅✅✅✅✅✅");//, std::exit(EXIT_SUCCESS);
-      }*/
    }
    // TODO: Create separate function for clarity
    else if constexpr (
       std::is_same_v<std::decay_t<field_operator_t>, Weight>)
    {
-      // dbg("None");
       const int num_qp = integration_weights.GetShape()[0];
       // TODO: eeek
       const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
@@ -598,9 +518,6 @@ void map_fields_to_quadrature_data(
    const int &dimension,
    const bool &use_sum_factorization = false)
 {
-   // dbg();
-   assert(use_sum_factorization && "❌ use_sum_factorization required");
-
    // When the input_to_field map returns -1, this means the requested input
    // is the integration weight. Weights don't have a user defined field
    // attached to them and we create a dummy field which is not accessed
@@ -661,7 +578,6 @@ void map_field_to_quadrature_data_conditional(
    const int &dimension,
    const bool &use_sum_factorization = false)
 {
-   assert(false && "❌ condition not implemented");
    if (condition)
    {
       if (use_sum_factorization)
@@ -703,7 +619,6 @@ void map_fields_to_quadrature_data_conditional(
    const std::array<bool, num_inputs> &conditions,
    const bool &use_sum_factorization = false)
 {
-   assert(false && "❌ condition not implemented");
    for_constexpr<num_inputs>([&](auto i)
    {
       map_field_to_quadrature_data_conditional(
@@ -712,7 +627,7 @@ void map_fields_to_quadrature_data_conditional(
    });
 }
 
-template <int T_Q1D, size_t num_inputs, typename field_operator_ts>
+template <size_t num_inputs, typename field_operator_ts>
 MFEM_HOST_DEVICE
 void map_direction_to_quadrature_data_conditional(
    std::array<DeviceTensor<2>, num_inputs> &directions_qp,
@@ -745,7 +660,7 @@ void map_direction_to_quadrature_data_conditional(
             }
             else if (dimension == 3)
             {
-               map_field_to_quadrature_data_tensor_product_3d<T_Q1D>(
+               map_field_to_quadrature_data_tensor_product_3d(
                   directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
                   integration_weights, scratch_mem);
             }

fem/dfem/parameterspace.hpp

@@ -43,7 +43,7 @@ public:
    /// Get spatial dimension
    ///
    /// returns always 1.
-   constexpr int Dimension() const
+   int Dimension() const
    {
       return 1;
    }
@@ -74,14 +74,11 @@ public:
       return elem_restr.get();
    }
 
-   virtual const Operator* GetB() const = 0;
-
-   virtual const Operator* GetBt() const = 0;
-
 protected:
    int vdim;
    DofToQuad dtq;
-   mutable std::unique_ptr<Operator> prolongation, elem_restr, B, Bt;
+   mutable std::unique_ptr<Operator> prolongation;
+   mutable std::unique_ptr<Operator> elem_restr;
 };
 
 /// @brief Uniform parameter space
@@ -125,18 +122,6 @@ public:
       return lsize;
    }
 
-   const Operator* GetB() const override
-   {
-      MFEM_ABORT("UniformParameterSpace does not support GetB");
-      return nullptr;
-   }
-
-   const Operator* GetBt() const override
-   {
-      MFEM_ABORT("UniformParameterSpace does not support GetBt");
-      return nullptr;
-   }
-
 private:
    /// T-vector size
    int tsize;

fem/dfem/qfunction_transform.hpp

@@ -243,8 +243,6 @@ void process_qf_arg(
    }
 }
 
-// const tensor<real_t, DIM>       ∇u
-// const tensor<real_t, DIM, DIM>  D (PA_DATA)
 template <typename arg_type>
 MFEM_HOST_DEVICE inline
 void process_qf_arg(const DeviceTensor<2> &u, arg_type &arg, int qp)

fem/dfem/qp.hpp

@@ -1,76 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "tuple.hpp"
-#include "../linalg/tensor.hpp"
-
-using namespace mfem::future;
-using mfem::future::tensor;
-
-// Helper to add dimension to tensor type
-template<typename T, int qp>
-struct AddQPDimension;
-
-// Specialization for tensor<real_t, dim>
-template<typename real_t, int dim, int qp>
-struct AddQPDimension<tensor<real_t, dim>, qp>
-{
-   using type = tensor<real_t, dim, qp>;
-};
-
-// Specialization for tensor<real_t, dim, dim>
-template<typename real_t, int dim, int qp>
-struct AddQPDimension<tensor<real_t, dim, dim>, qp>
-{
-   using type = tensor<real_t, dim, dim, qp>;
-};
-
-// Specialization for real_t (transforms to tensor<real_t, qp>)
-template<typename real_t, int qp>
-struct AddQPDimension
-{
-   using type = tensor<real_t, qp>;
-};
-
-// Helper to transform tuple
-template<typename Tuple, int qp>
-struct TransformTupleQP {};
-
-// Specialization for mfem::future::tuple
-template<int qp, typename... Types>
-struct TransformTupleQP<mfem::future::tuple<Types...>, qp>
-{
-   using type = mfem::future::tuple<typename AddQPDimension<Types, qp>::type...>;
-};
-
-template<int qp, typename... Types>
-struct TransformTupleQP<std::tuple<Types...>, qp>
-{
-   using type = std::tuple<typename AddQPDimension<Types, qp>::type...>;
-};
-
-// Function to transform tuple type with qp dimension
-template<int qp, typename qf_param_ts>
-struct add_qp_dimension
-{
-   using type = typename TransformTupleQP<qf_param_ts, qp>::type;
-};
-
-// Helper alias template for cleaner usage
-template<int qp, typename qf_param_ts>
-using add_qp_dimension_t = typename add_qp_dimension<qp, qf_param_ts>::type;
-
-// ...AddDomainIntegrator...
-// {
-//    constexpr int Q1D = 4;
-//    using qf_param_augmentd_ts = add_qp_dimension_t<Q1D, decay_tuple<qf_param_ts>>;
-// }

fem/dfem/util.hpp

@@ -21,7 +21,6 @@
 #include <type_traits>
 #include <numeric>
 #include <iomanip>
-#include <typeindex>
 
 #include "../../general/communication.hpp"
 #include "../../general/forall.hpp"
@@ -29,19 +28,13 @@
 #include "../fe/fe_base.hpp"
 #include "../fespace.hpp"
 #include "../pfespace.hpp"
-#include "../qfunction.hpp"
 #include "../../mesh/mesh.hpp"
 #include "../../linalg/dtensor.hpp"
-#include "../quadinterpolator.hpp"
 
-#include "fielddescriptor.hpp"
 #include "fieldoperator.hpp"
 #include "parameterspace.hpp"
 #include "tuple.hpp"
 
-#undef NVTX_COLOR
-#define NVTX_COLOR ::nvtx::kLightBlue
-
 namespace mfem::future
 {
 
@@ -82,7 +75,7 @@ constexpr void for_constexpr(lambda&& f,
 }
 
 template <typename lambda>
-constexpr void for_constexpr(lambda&&, std::integer_sequence<std::size_t>) {}
+constexpr void for_constexpr(lambda&& f, std::integer_sequence<std::size_t>) {}
 
 template <int... n, typename lambda>
 constexpr void for_constexpr(lambda&& f)
@@ -91,7 +84,7 @@ constexpr void for_constexpr(lambda&& f)
 }
 
 template <typename lambda, typename arg_t>
-constexpr void for_constexpr_with_arg(lambda&&, arg_t&&,
+constexpr void for_constexpr_with_arg(lambda&& f, arg_t&& arg,
                                       std::integer_sequence<std::size_t>)
 {
    // Base case - do nothing for empty sequence
@@ -115,16 +108,6 @@ constexpr void for_constexpr_with_arg(lambda&& f, arg_t&& arg)
                           indices{});
 }
 
-template <auto start, auto end, auto inc = 1, typename F>
-constexpr void constexpr_for(F&& f)
-{
-   if constexpr (start < end)
-   {
-      f(std::integral_constant<decltype(start), start>());
-      constexpr_for<start + inc, end, inc>(f);
-   }
-}
-
 template <std::size_t I, typename Tuple, std::size_t... Is>
 std::array<bool, sizeof...(Is)>
 make_dependency_array(const Tuple& inputs, std::index_sequence<Is...>)
@@ -461,21 +444,6 @@ struct create_function_signature<output_t (*)(input_ts...)>
    using type = FunctionSignature<output_t(input_ts...)>;
 };
 
-template <typename...>
-using void_t = void;
-
-template <typename T, typename = void>
-struct get_function_signature
-{
-   using type = typename create_function_signature<T>::type;
-};
-
-template <typename T>
-struct get_function_signature<T, void_t<decltype(&T::operator())>>
-{
-   using type = typename create_function_signature<decltype(&T::operator())>::type;
-};
-
 template <typename T>
 constexpr int GetFieldId()
 {
@@ -570,12 +538,38 @@ auto get_marked_entries(
 /// @param t the tuple to filter fields from.
 /// @returns a tuple containing only the fields with field IDs not equal to -1.
 template <typename... Ts>
-constexpr auto filter_fields(const std::tuple<Ts...>&)
+constexpr auto filter_fields(const std::tuple<Ts...>& t)
 {
    return std::tuple_cat(
              std::conditional_t<Ts::GetFieldId() != -1, std::tuple<Ts>, std::tuple<>> {}...);
 }
 
+/// @brief FieldDescriptor struct
+///
+/// This struct is used to store information about a field.
+struct FieldDescriptor
+{
+   using data_variant_t =
+      std::variant<const FiniteElementSpace *,
+      const ParFiniteElementSpace *,
+      const ParameterSpace *>;
+
+   /// Field ID
+   std::size_t id;
+
+   /// Field variant
+   data_variant_t data;
+
+   /// Default constructor
+   FieldDescriptor() :
+      id(SIZE_MAX), data(data_variant_t{}) {}
+
+   /// Constructor
+   template <typename T>
+   FieldDescriptor(std::size_t field_id, const T* v) :
+      id(field_id), data(v) {}
+};
+
 namespace dfem
 {
 template <class... T> constexpr bool always_false = false;
@@ -605,7 +599,7 @@ struct ThreadBlocks
 
 #if defined(MFEM_USE_CUDA_OR_HIP)
 template <typename func_t>
-__global__ void forall_kernel_extern_shmem(func_t f, int n)
+__global__ void forall_kernel_shmem(func_t f, int n)
 {
    int i = blockIdx.x;
    extern __shared__ real_t shmem[];
@@ -614,48 +608,23 @@ __global__ void forall_kernel_extern_shmem(func_t f, int n)
       f(i, shmem);
    }
 }
-template <typename func_t>
-__global__ void forall_kernel_static_smem(func_t f, int n)
-{
-   int i = blockIdx.x;
-   if (i >= n) { return; }
-   f(i, nullptr);
-}
-template <int MAX_THREADS_PER_BLOCK, typename func_t>
-__global__
-MFEM_LAUNCH_BOUNDS(MAX_THREADS_PER_BLOCK)
-static void forall_kernel_static_smem_launch_bounds(func_t f, int n)
-{
-   for (int k = blockIdx.x; k < n; k += gridDim.x) { f(k, nullptr); }
-}
 #endif
 
-template </*typename kernel_tag,*/ typename func_t>
+template <typename func_t>
 void forall(func_t f,
             const int &N,
-            [[maybe_unused]] const ThreadBlocks &blocks,
-            [[maybe_unused]] int num_shmem = 0,
+            const ThreadBlocks &blocks,
+            int num_shmem = 0,
             real_t *shmem = nullptr)
 {
-   db1();
    if (Device::Allows(Backend::CUDA_MASK) ||
        Device::Allows(Backend::HIP_MASK))
    {
 #if defined(MFEM_USE_CUDA_OR_HIP)
       // int gridsize = (N + Z - 1) / Z;
       int num_bytes = num_shmem * sizeof(decltype(shmem));
-      db1("num_bytes:{}", num_bytes);
-      db1("block: {}x{}x{}", blocks.x, blocks.y, blocks.z);
       dim3 block_size(blocks.x, blocks.y, blocks.z);
-      // ForallKernel<kernel_tag>::run<<<N, block_size, num_bytes>>>(f, N);
-      if (num_bytes > 0)
-      {
-         forall_kernel_extern_shmem<<<N, block_size, num_bytes>>>(f, N);
-      }
-      else
-      {
-         forall_kernel_static_smem<<<N, block_size>>>(f, N);
-      }
+      forall_kernel_shmem<<<N, block_size, num_bytes>>>(f, N);
 #if defined(MFEM_USE_CUDA)
       MFEM_GPU_CHECK(cudaGetLastError());
 #elif defined(MFEM_USE_HIP)
@@ -666,7 +635,6 @@ void forall(func_t f,
    }
    else if (Device::Allows(Backend::CPU_MASK))
    {
-      db1("CPU_MASK");
       MFEM_ASSERT(!((bool)num_shmem != (bool)shmem),
                   "Backend::CPU needs a pre-allocated shared memory block");
       for (int i = 0; i < N; i++)
@@ -680,69 +648,6 @@ void forall(func_t f,
    }
 }
 
-namespace dfem
-{
-
-template <int MAX_THREADS_PER_BLOCK = 0, typename func_t>
-void forall(func_t f,
-            const int &N,
-            [[maybe_unused]] const ThreadBlocks &blocks,
-            [[maybe_unused]] int num_shmem = 0,
-            real_t *shmem = nullptr)
-{
-   db1();
-   if (Device::Allows(Backend::CUDA_MASK) ||
-       Device::Allows(Backend::HIP_MASK))
-   {
-#if defined(MFEM_USE_CUDA_OR_HIP)
-      int num_bytes = num_shmem * sizeof(decltype(shmem));
-      db1("num_bytes:{}", num_bytes);
-      db1("block: {}x{}x{}", blocks.x, blocks.y, blocks.z);
-      db1("MAX_THREADS_PER_BLOCK:{}", MAX_THREADS_PER_BLOCK);
-      dim3 block_size(blocks.x, blocks.y, blocks.z);
-      if constexpr (MAX_THREADS_PER_BLOCK > 0)
-      {
-         assert(num_bytes == 0);
-         forall_kernel_static_smem_launch_bounds
-         <MAX_THREADS_PER_BLOCK><<<N, block_size>>> (f, N);
-      }
-      else
-      {
-         static_assert(MAX_THREADS_PER_BLOCK == 0);
-         if (num_bytes == 0)
-         {
-            forall_kernel_static_smem<<<N, block_size>>>(f, N);
-         }
-         else
-         {
-            forall_kernel_extern_shmem<<<N, block_size, num_bytes>>>(f, N);
-         }
-      }
-#if defined(MFEM_USE_CUDA)
-      MFEM_GPU_CHECK(cudaGetLastError());
-#elif defined(MFEM_USE_HIP)
-      MFEM_GPU_CHECK(hipGetLastError());
-#endif
-      // MFEM_DEVICE_SYNC; // ⚠️
-#endif
-   }
-   else if (Device::Allows(Backend::CPU_MASK))
-   {
-      db1("CPU_MASK");
-      MFEM_ASSERT(!((bool)num_shmem != (bool)shmem),
-                  "Backend::CPU needs a pre-allocated shared memory block");
-      for (int i = 0; i < N; i++)
-      {
-         f(i, shmem);
-      }
-   }
-   else
-   {
-      MFEM_ABORT("no compute backend available");
-   }
-}
-}
-
 /// @todo To be removed.
 class FDJacobian : public Operator
 {
@@ -867,10 +772,6 @@ int GetVSize(const FieldDescriptor &f)
       {
          return arg->GetVSize();
       }
-      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
-      {
-         return arg->Size();
-      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->GetVSize();
@@ -909,10 +810,6 @@ void GetElementVDofs(const FieldDescriptor &f, int el, Array<int> &vdofs)
       {
          arg->GetElementVDofs(el, vdofs);
       }
-      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
-      {
-         MFEM_ABORT("internal error");
-      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          MFEM_ABORT("internal error");
@@ -947,10 +844,6 @@ int GetTrueVSize(const FieldDescriptor &f)
       {
          return arg->GetTrueVSize();
       }
-      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
-      {
-         return arg->Size();
-      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->GetTrueVSize();
@@ -981,10 +874,6 @@ int GetVDim(const FieldDescriptor &f)
       {
          return arg->GetVDim();
       }
-      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
-      {
-         return arg->GetVDim();
-      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->GetVDim();
@@ -1020,10 +909,6 @@ int GetDimension(const FieldDescriptor &f)
             return arg->GetMesh()->Dimension() - 1;
          }
       }
-      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
-      {
-         return arg->GetSpace()->GetMesh()->Dimension();
-      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->Dimension();
@@ -1036,36 +921,6 @@ int GetDimension(const FieldDescriptor &f)
    }, f.data);
 }
 
-inline
-std::variant<const QuadratureInterpolator *, const Operator *>get_qinterp(
-   const FieldDescriptor &f,
-   const IntegrationRule &ir)
-{
-   return std::visit([&ir](auto && arg) -> const QuadratureInterpolator*
-   {
-      using T = std::decay_t<decltype(arg)>;
-      if constexpr (std::is_same_v<T, const FiniteElementSpace *> ||
-                    std::is_same_v<T, const ParFiniteElementSpace *>)
-      {
-         return arg->GetQuadratureInterpolator(ir);
-      }
-      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
-      {
-         // QuadratureFunction doesn't need a QuadratureInterpolator
-         return nullptr;
-      }
-      else if constexpr (std::is_same_v<T, const ParameterSpace *>)
-      {
-         return nullptr;
-      }
-      else
-      {
-         static_assert(dfem::always_false<T>, "internal error");
-      }
-
-      return nullptr; // Unreachable, but avoids compiler warning
-   }, f.data);
-}
 
 /// @brief Get the prolongation operator for a field descriptor.
 ///
@@ -1074,7 +929,6 @@ std::variant<const QuadratureInterpolator *, const Operator *>get_qinterp(
 inline
 const Operator *get_prolongation(const FieldDescriptor &f)
 {
-   NVTX("get P");
    return std::visit([](auto&& arg) -> const Operator*
    {
       using T = std::decay_t<decltype(arg)>;
@@ -1083,10 +937,6 @@ const Operator *get_prolongation(const FieldDescriptor &f)
       {
          return arg->GetProlongationMatrix();
       }
-      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
-      {
-         return nullptr;
-      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->GetProlongationMatrix();
@@ -1109,7 +959,6 @@ inline
 const Operator *get_element_restriction(const FieldDescriptor &f,
                                         ElementDofOrdering o)
 {
-   NVTX("get ER");
    return std::visit([&o](auto&& arg) -> const Operator*
    {
       using T = std::decay_t<decltype(arg)>;
@@ -1118,10 +967,6 @@ const Operator *get_element_restriction(const FieldDescriptor &f,
       {
          return arg->GetElementRestriction(o);
       }
-      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
-      {
-         return nullptr;
-      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return arg->GetElementRestriction(o);
@@ -1149,7 +994,6 @@ const Operator *get_face_restriction(const FieldDescriptor &f,
                                      FaceType ft,
                                      L2FaceValues m)
 {
-   NVTX("get FR");
    return std::visit([&o, &ft, &m](auto&& arg) -> const Operator*
    {
       using T = std::decay_t<decltype(arg)>;
@@ -1158,11 +1002,6 @@ const Operator *get_face_restriction(const FieldDescriptor &f,
       {
          return arg->GetFaceRestriction(o, ft, m);
       }
-      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
-      {
-         // QuadratureFunction does not support face restrictions
-         MFEM_ABORT("internal error");
-      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          // ParameterSpace does not support face restrictions
@@ -1188,7 +1027,6 @@ inline
 const Operator *get_restriction(const FieldDescriptor &f,
                                 const ElementDofOrdering &o)
 {
-   NVTX("get R");
    if constexpr (std::is_same_v<entity_t, Entity::Element>)
    {
       return get_element_restriction(f, o);
@@ -1214,14 +1052,12 @@ inline std::tuple<std::function<void(const Vector&, Vector&)>, int>
 get_restriction_transpose(
    const FieldDescriptor &f,
    const ElementDofOrdering &o,
-   [[maybe_unused]] const fop_t &fop)
+   const fop_t &fop)
 {
-   NVTX("get R^T");
    if constexpr (is_sum_fop<fop_t>::value)
    {
       auto RT = [=](const Vector &v_e, Vector &v_l)
       {
-         NVTX("R^T sum");
          v_l += v_e;
       };
       return std::make_tuple(RT, 1);
@@ -1231,7 +1067,6 @@ get_restriction_transpose(
       const Operator *R = get_restriction<entity_t>(f, o);
       std::function<void(const Vector&, Vector&)> RT = [=](const Vector &x, Vector &y)
       {
-         NVTX("R^T+");
          R->AddMultTranspose(x, y);
       };
       return std::make_tuple(RT, R->Height());
@@ -1251,26 +1086,11 @@ get_restriction_transpose(
 inline
 void prolongation(const FieldDescriptor field, const Vector &x, Vector &field_l)
 {
-   NVTX("P");
    const auto P = get_prolongation(field);
-
-   NVTX_INI("SetSize");
    field_l.SetSize(P->Height());
-   NVTX_END("SetSize");
-
-   NVTX_INI("P->Mult");
    P->Mult(x, field_l);
 }
 
-inline
-void prolongation_transpose(
-   const FieldDescriptor &field, const Vector &field_l, Vector &x)
-{
-   const auto P = get_prolongation(field);
-   x.SetSize(P->Width());
-   P->MultTranspose(field_l, x);
-}
-
 /// @brief Apply the prolongation operator to a vector of fields.
 ///
 /// x is a long vector containing the data for all fields on tdofs and
@@ -1287,7 +1107,6 @@ void prolongation(const std::array<FieldDescriptor, N> fields,
                   const Vector &x,
                   std::array<Vector, M> &fields_l)
 {
-   NVTX("P");
    int data_offset = 0;
    for (int i = 0; i < N; i++)
    {
@@ -1295,14 +1114,9 @@ void prolongation(const std::array<FieldDescriptor, N> fields,
       const int width = P->Width();
       // const Vector x_i(x.GetData() + data_offset, width);
       const Vector x_i(const_cast<Vector&>(x), data_offset, width);
-
-      NVTX_INI("SetSize");
       fields_l[i].SetSize(P->Height());
-      NVTX_END("SetSize");
 
-      NVTX_INI("P->Mult");
       P->Mult(x_i, fields_l[i]);
-      NVTX_END("P->Mult");
       data_offset += width;
    }
 }
@@ -1316,259 +1130,20 @@ void prolongation(const std::array<FieldDescriptor, N> fields,
 /// @param fields the array of field descriptors.
 /// @param x the input vector in tdofs.
 /// @param fields_l the array of output vectors in vdofs.
-// inline
-// void prolongation(const std::vector<FieldDescriptor> fields,
-//                   const Vector &x,
-//                   std::vector<Vector> &fields_l)
-// {
-//    int data_offset = 0;
-//    for (std::size_t i = 0; i < fields.size(); i++)
-//    {
-//       const auto P = get_prolongation(fields[i]);
-//       const int width = P->Width();
-//       const Vector x_i(const_cast<Vector&>(x), data_offset, width);
-//       fields_l[i].SetSize(P->Height());
-//       P->Mult(x_i, fields_l[i]);
-//       data_offset += width;
-//    }
-// }
-
 inline
-void prolongation(
-   const std::vector<FieldDescriptor> fields,
-   const BlockVector &x,
-   std::vector<Vector *> &x_l)
+void prolongation(const std::vector<FieldDescriptor> fields,
+                  const Vector &x,
+                  std::vector<Vector> &fields_l)
 {
-   MFEM_ASSERT(x.NumBlocks() == static_cast<int>(x_l.size()),
-               "error " << x.NumBlocks() << " vs " << x_l.size());
-   for (int i = 0; i < x.NumBlocks(); i++)
+   int data_offset = 0;
+   for (std::size_t i = 0; i < fields.size(); i++)
    {
       const auto P = get_prolongation(fields[i]);
-
-      // If nullptr, assume Identity.
-      if (P == nullptr)
-      {
-         *x_l[i] = x.GetBlock(i);
-      }
-      else
-      {
-         const auto P = get_prolongation(fields[i]);
-         MFEM_ASSERT(P->Width() == x.GetBlock(i).Size(),
-                     "prolongation not applicable to given input data size " <<
-                     P->Width() << " vs " << x.GetBlock(i).Size());
-         MFEM_ASSERT(P->Height() == x_l[i]->Size(),
-                     "prolongation not applicable to given output data size " <<
-                     P->Height() << " vs " << x_l[i]->Size());
-         P->Mult(x.GetBlock(i), *x_l[i]);
-      }
-   }
-}
-
-inline
-void prolongation(
-   const std::vector<FieldDescriptor> fields,
-   const MultiVector &x,
-   std::vector<Vector *> &x_l)
-{
-   MFEM_ASSERT(x.NumBlocks() == static_cast<int>(x_l.size()),
-               "error " << x.NumBlocks() << " vs " << x_l.size());
-   for (int i = 0; i < x.NumBlocks(); i++)
-   {
-      const auto P = get_prolongation(fields[i]);
-
-      // If nullptr, assume Identity.
-      if (P == nullptr)
-      {
-         *x_l[i] = x[i];
-      }
-      else
-      {
-         const auto P = get_prolongation(fields[i]);
-         MFEM_ASSERT(P->Width() == x[i].Size(),
-                     "prolongation not applicable to given input data size " <<
-                     P->Width() << " vs " << x[i].Size());
-         MFEM_ASSERT(P->Height() == x_l[i]->Size(),
-                     "prolongation not applicable to given output data size " <<
-                     P->Height() << " vs " << x_l[i]->Size());
-         P->Mult(x[i], *x_l[i]);
-      }
-   }
-}
-
-inline
-void prolongation_transpose(
-   const std::vector<FieldDescriptor> fields,
-   const std::vector<Vector *> &x_l,
-   BlockVector &x)
-{
-   MFEM_ASSERT(static_cast<int>(x_l.size()) == x.NumBlocks(),
-               "error " << x_l.size() << " vs " << x.NumBlocks());
-   for (size_t i = 0; i < x_l.size(); i++)
-   {
-      const auto P = get_prolongation(fields[i]);
-
-      // If nullptr, assume Identity.
-      if (P == nullptr)
-      {
-         x.GetBlock(i) = *x_l[i];
-      }
-      else
-      {
-         MFEM_ASSERT(P->Height() == x_l[i]->Size(),
-                     "prolongation not applicable to given input data size " <<
-                     P->Height() << " vs " << x_l[i]->Size());
-         MFEM_ASSERT(P->Width() == x.GetBlock(i).Size(),
-                     "prolongation not applicable to given output data size " <<
-                     P->Width() << " vs " << x.GetBlock(i).Size());
-         P->MultTranspose(*x_l[i], x.GetBlock(i));
-      }
-   }
-}
-
-inline
-void prolongation_transpose(
-   const std::vector<FieldDescriptor> fields,
-   const std::vector<Vector *> &x_l,
-   MultiVector &x)
-{
-   MFEM_ASSERT(static_cast<int>(x_l.size()) == x.NumBlocks(),
-               "error " << x_l.size() << " vs " << x.NumBlocks());
-   for (size_t i = 0; i < x_l.size(); i++)
-   {
-      const auto P = get_prolongation(fields[i]);
-
-      // If nullptr, assume Identity.
-      if (P == nullptr)
-      {
-         x[i] = *x_l[i];
-      }
-      else
-      {
-         MFEM_ASSERT(P->Height() == x_l[i]->Size(),
-                     "prolongation not applicable to given input data size " <<
-                     P->Height() << " vs " << x_l[i]->Size());
-         MFEM_ASSERT(P->Width() == x[i].Size(),
-                     "prolongation not applicable to given output data size " <<
-                     P->Width() << " vs " << x[i].Size());
-         P->MultTranspose(*x_l[i], x[i]);
-      }
-   }
-}
-
-template <typename entity_t>
-void restriction(
-   const std::vector<FieldDescriptor> fields,
-   const std::vector<Vector *> &x_l,
-   std::vector<Vector *> &x_e)
-{
-   MFEM_ASSERT(x_l.size() == x_e.size(),
-               "internal error " << x_l.size() << " vs " << x_e.size());
-   for (size_t i = 0; i < fields.size(); i++)
-   {
-      int s = 0;
-      const auto R = get_restriction<entity_t>(
-                        fields[i], ElementDofOrdering::LEXICOGRAPHIC);
-
-      // If nullptr, assume Identity.
-      if (R == nullptr)
-      {
-         s = x_l[i]->Size();
-      }
-      else
-      {
-         s = R->Height();
-      }
-
-      // TODO
-      if (x_e[i] == nullptr)
-      {
-         x_e[i] = new Vector(s);
-      }
-      x_e[i]->SetSize(s);
-
-      if (R == nullptr)
-      {
-         x_e[i] = x_l[i];
-      }
-      else
-      {
-         MFEM_ASSERT(R->Width() == x_l[i]->Size(),
-                     "restriction not applicable to given input data size " <<
-                     R->Width() << " vs " << x_l[i]->Size());
-         R->Mult(*x_l[i], *x_e[i]);
-      }
-   }
-}
-
-template <typename entity_t>
-void prepare_residual(
-   const std::vector<FieldDescriptor> &fields,
-   std::vector<Vector *> &r_e)
-{
-   for (size_t i = 0; i < fields.size(); i++)
-   {
-      int s = 0;
-      if (std::holds_alternative<const QuadratureFunction *>(fields[i].data))
-      {
-         const auto fd = std::get<const QuadratureFunction *>(fields[i].data);
-         s = fd->Size();
-      }
-      else
-      {
-         const auto R = get_restriction<entity_t>(
-                           fields[i], ElementDofOrdering::LEXICOGRAPHIC);
-         s = R->Height();
-      }
-
-      // TODO
-      if (r_e[i] == nullptr)
-      {
-         r_e[i] = new Vector(s);
-      }
-      else
-      {
-         r_e[i]->SetSize(s);
-      }
-   }
-}
-
-template <typename entity_t>
-void restriction_transpose(
-   const std::vector<FieldDescriptor> &fields,
-   const std::vector<Vector *> &x_e,
-   std::vector<Vector *> &x_l)
-{
-   for (size_t i = 0; i < fields.size(); i++)
-   {
-      int s = 0;
-      const auto R = get_restriction<entity_t>(
-                        fields[i], ElementDofOrdering::LEXICOGRAPHIC);
-      // TODO: if nullptr, assume Identity
-      if (R == nullptr)
-      {
-         s = x_e[i]->Size();
-      }
-      else
-      {
-         s = R->Width();
-      }
-
-      // TODO
-      if (x_l[i] == nullptr)
-      {
-         x_l[i] = new Vector(s);
-      }
-      x_l[i]->SetSize(s);
-
-      // TODO: if nullptr, assume Identity
-      if (R == nullptr)
-      {
-         x_l[i] = x_e[i];
-      }
-      else
-      {
-         R->MultTranspose(*x_e[i], *x_l[i]);
-      }
+      const int width = P->Width();
+      const Vector x_i(const_cast<Vector&>(x), data_offset, width);
+      fields_l[i].SetSize(P->Height());
+      P->Mult(x_i, fields_l[i]);
+      data_offset += width;
    }
 }
 
@@ -1577,7 +1152,6 @@ void get_lvectors(const std::vector<FieldDescriptor> fields,
                   const Vector &x,
                   std::vector<Vector> &fields_l)
 {
-   NVTX("get_lvectors");
    int data_offset = 0;
    for (std::size_t i = 0; i < fields.size(); i++)
    {
@@ -1604,15 +1178,13 @@ template <typename fop_t>
 inline
 std::function<void(const Vector&, Vector&)> get_prolongation_transpose(
    const FieldDescriptor &f,
-   [[maybe_unused]] const fop_t &fop,
+   const fop_t &fop,
    MPI_Comm mpi_comm)
 {
-   NVTX("get P^T");
    if constexpr (is_sum_fop<fop_t>::value)
    {
       auto PT = [=](const Vector &r_local, Vector &y)
       {
-         NVTX("P^T sum");
          MFEM_ASSERT(y.Size() == 1, "output size doesn't match kernel description");
          real_t local_sum = r_local.Sum();
          MPI_Allreduce(&local_sum, y.GetData(), 1, MPI_DOUBLE, MPI_SUM, mpi_comm);
@@ -1623,7 +1195,6 @@ std::function<void(const Vector&, Vector&)> get_prolongation_transpose(
    {
       auto PT = [=](const Vector &r_local, Vector &y)
       {
-         NVTX("P^T Identity");
          y = r_local;
       };
       return PT;
@@ -1631,7 +1202,6 @@ std::function<void(const Vector&, Vector&)> get_prolongation_transpose(
    const Operator *P = get_prolongation(f);
    auto PT = [=](const Vector &r_local, Vector &y)
    {
-      NVTX("P^T");
       P->MultTranspose(r_local, y);
    };
    return PT;
@@ -1650,19 +1220,12 @@ void restriction(const FieldDescriptor u,
                  Vector &field_e,
                  ElementDofOrdering ordering)
 {
-   NVTX("R");
    const auto R = get_restriction<entity_t>(u, ordering);
    MFEM_ASSERT(R->Width() == u_l.Size(),
                "restriction not applicable to given data size");
    const int height = R->Height();
-
-   NVTX_INI("SetSize");
    field_e.SetSize(height);
-   NVTX_END("SetSize");
-
-   NVTX_INI("R->Mult");
    R->Mult(u_l, field_e);
-   NVTX_END("R->Mult");
 }
 
 /// @brief Apply the restriction operator to a vector of fields.
@@ -1680,29 +1243,14 @@ void restriction(const std::vector<FieldDescriptor> u,
                  ElementDofOrdering ordering,
                  const int offset = 0)
 {
-   NVTX("R");
    for (std::size_t i = 0; i < u.size(); i++)
    {
       const auto R = get_restriction<entity_t>(u[i], ordering);
       MFEM_ASSERT(R->Width() == u_l[i].Size(),
                   "restriction not applicable to given data size");
       const int height = R->Height();
-
-      // NVTX_INI("SetSize");
       fields_e[i + offset].SetSize(height);
-      // NVTX_END("SetSize");
-
-      // NVTX_INI("R->Mult");
-      if (dynamic_cast<const IdentityOperator*>(R))
-      {
-         NVTX("Identity");
-         fields_e[i + offset].NewMemoryAndSize(u_l[i].GetMemory(), u_l[i].Size(), false);
-      }
-      else
-      {
-         R->Mult(u_l[i], fields_e[i + offset]);
-      }
-      // NVTX_END("R->Mult");
+      R->Mult(u_l[i], fields_e[i + offset]);
    }
 }
 
@@ -1714,21 +1262,14 @@ void element_restriction(const std::array<FieldDescriptor, N> u,
                          ElementDofOrdering ordering,
                          const int offset = 0)
 {
-   NVTX("ER");
    for (int i = 0; i < N; i++)
    {
       const auto R = get_element_restriction(u[i], ordering);
       MFEM_ASSERT(R->Width() == u_l[i].Size(),
                   "element restriction not applicable to given data size");
       const int height = R->Height();
-
-      NVTX_INI("SetSize");
       fields_e[i + offset].SetSize(height);
-      NVTX_END("SetSize");
-
-      NVTX_INI("R->Mult");
       R->Mult(u_l[i], fields_e[i + offset]);
-      NVTX_END("R->Mult");
    }
 }
 
@@ -1785,10 +1326,6 @@ const DofToQuad *GetDofToQuad(const FieldDescriptor &f,
             return &arg->GetTypicalTraceElement()->GetDofToQuad(ir, mode);
          }
       }
-      else if constexpr (std::is_same_v<T, const QuadratureFunction *>)
-      {
-         return nullptr;
-      }
       else if constexpr (std::is_same_v<T, const ParameterSpace *>)
       {
          return &arg->GetDofToQuad();
@@ -1920,7 +1457,7 @@ create_descriptors_to_fields_map(
 
    auto f = [&](auto &fop, auto &map)
    {
-      if constexpr (is_weight_fop<std::decay_t<decltype(fop)>>::value)
+      if constexpr (std::is_same_v<std::decay_t<decltype(fop)>, Weight>)
       {
          // TODO-bug: stealing dimension from the first field
          fop.dim = GetDimension<entity_t>(fields[0]);
@@ -2050,7 +1587,7 @@ get_shmem_info(
    const std::array<DofToQuadMap, num_outputs> &output_dtq_maps,
    const std::vector<FieldDescriptor> &fields,
    const int &num_entities,
-   [[maybe_unused]] const input_t &inputs,
+   const input_t &inputs,
    const int &num_qp,
    const std::vector<int> &input_size_on_qp,
    const int &residual_size_on_qp,
@@ -2805,25 +2342,5 @@ std::array<DofToQuadMap, num_fields> create_dtq_maps(
              std::make_index_sequence<num_fields> {});
 }
 
-struct QLayoutEntry
-{
-   std::type_index type;
-   std::vector<int> layout;
-
-   template <class Fop>
-   QLayoutEntry(Fop, std::initializer_list<int> idx) :
-      type(typeid(Fop)), layout(idx) {}
-};
-
-static void ExtractQLayouts(
-   const std::initializer_list<QLayoutEntry> entries,
-   std::unordered_map<std::type_index, std::vector<int>>& out)
-{
-   for (const auto& e : entries)
-   {
-      out[e.type] = e.layout;
-   }
-}
-
 } // namespace mfem::future
 #endif

fem/fem.hpp

@@ -52,7 +52,7 @@
 #include "bounds.hpp"
 #include "particleset.hpp"
 
-// #include "dfem/doperator.hpp"
+#include "dfem/doperator.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"

fem/integ/bilininteg_diffusion_kernels.hpp

@@ -1064,8 +1064,6 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-
-      // Grad X
       MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
       {
          MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
@@ -1086,8 +1084,6 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-
-      // Grad Y
       MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
       {
          MFEM_FOREACH_THREAD_DIRECT(qy,y,Q1D)
@@ -1109,8 +1105,6 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-
-      // Grad Z + Q-function
       MFEM_FOREACH_THREAD_DIRECT(qz,z,Q1D)
       {
          MFEM_FOREACH_THREAD_DIRECT(qy,y,Q1D)
@@ -1223,23 +1217,20 @@ inline void SmemPADiffusionApply3D(const int NE,
 
 namespace
 {
-using DiffusionApplyKernelType =
-   DiffusionIntegrator::DiffusionApplyKernelType;
-
-using DiffusionDiagonalKernelType =
-   DiffusionIntegrator::DiffusionDiagonalKernelType;
+using ApplyKernelType = DiffusionIntegrator::ApplyKernelType;
+using DiagonalKernelType = DiffusionIntegrator::DiagonalKernelType;
 }
 
 template<int DIM, int T_D1D, int T_Q1D>
-DiffusionApplyKernelType DiffusionIntegrator::DiffusionApplyPAKernel::Kernel()
+ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Kernel()
 {
    if constexpr (DIM == 2) { return internal::SmemPADiffusionApply2D<T_D1D,T_Q1D>; }
    else if constexpr (DIM == 3) { return internal::SmemPADiffusionApply3D<T_D1D, T_Q1D>; }
    MFEM_ABORT("");
 }
 
-inline DiffusionApplyKernelType
-DiffusionIntegrator::DiffusionApplyPAKernel::Fallback(int DIM, int, int)
+inline
+ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
 {
    if (DIM == 2) { return internal::PADiffusionApply2D; }
    else if (DIM == 3) { return internal::PADiffusionApply3D; }
@@ -1247,16 +1238,15 @@ DiffusionIntegrator::DiffusionApplyPAKernel::Fallback(int DIM, int, int)
 }
 
 template<int DIM, int D1D, int Q1D>
-DiffusionDiagonalKernelType
-DiffusionIntegrator::DiffusionDiagonalPAKernel::Kernel()
+DiagonalKernelType DiffusionIntegrator::DiagonalPAKernels::Kernel()
 {
    if constexpr (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D,Q1D>; }
    else if constexpr (DIM == 3) { return internal::SmemPADiffusionDiagonal3D<D1D, Q1D>; }
    MFEM_ABORT("");
 }
 
-inline DiffusionDiagonalKernelType
-DiffusionIntegrator::DiffusionDiagonalPAKernel::Fallback(int DIM, int, int)
+inline DiagonalKernelType
+DiffusionIntegrator::DiagonalPAKernels::Fallback(int DIM, int, int)
 {
    if (DIM == 2) { return internal::PADiffusionDiagonal2D; }
    else if (DIM == 3) { return internal::PADiffusionDiagonal3D; }

fem/integ/bilininteg_diffusion_pa.cpp

@@ -31,8 +31,8 @@ void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
       const Array<real_t> &B = maps->B;
       const Array<real_t> &G = maps->G;
       const Vector &Dv = pa_data;
-      DiffusionDiagonalPAKernel::Run(dim, dofs1D, quad1D, ne, symmetric, B, G, Dv,
-                                     diag, dofs1D, quad1D);
+      DiagonalPAKernels::Run(dim, dofs1D, quad1D, ne, symmetric, B, G, Dv,
+                             diag, dofs1D, quad1D);
    }
 }
 
@@ -68,8 +68,8 @@ void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
       }
 #endif // MFEM_USE_OCCA
 
-      DiffusionApplyPAKernel::Run(dim, dofs1D, quad1D, ne, symmetric, B, G, Bt,
-                                  Gt, Dv, x, y, dofs1D, quad1D);
+      ApplyPAKernels::Run(dim, dofs1D, quad1D, ne, symmetric, B, G, Bt,
+                          Gt, Dv, x, y, dofs1D, quad1D);
    }
 }
 
@@ -174,9 +174,9 @@ void DiffusionIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
    abs_pa_data.Abs();
    auto abs_maps = maps->Abs();
 
-   DiffusionApplyPAKernel::Run(dim, dofs1D, quad1D, ne, symmetric,
-                               abs_maps.B, abs_maps.G, abs_maps.Bt, abs_maps.Gt,
-                               abs_pa_data, x, y, dofs1D, quad1D);
+   ApplyPAKernels::Run(dim, dofs1D, quad1D, ne, symmetric,
+                       abs_maps.B, abs_maps.G, abs_maps.Bt, abs_maps.Gt,
+                       abs_pa_data, x, y, dofs1D, quad1D);
 }
 
 void DiffusionIntegrator::AddAbsMultTransposePA(const Vector &x,

fem/kernels.hpp

@@ -207,28 +207,6 @@ inline MFEM_HOST_DEVICE void WriteDofs2d(const int e, const int d1d,
    MFEM_SYNC_THREAD;
 }
 
-/// Load 3D input DIM vector at element offset into given register tensor
-template <int VDIM, int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void LoadDofs3d(const int d1d, const int c,
-                                        const DeviceTensor<4, const real_t> &X,
-                                        vd_regs3d_t<VDIM, DIM, MQ1> &Y)
-{
-   for (int d = 0; d < DIM; d++)
-   {
-      for (int dz = 0; dz < d1d; ++dz)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d)
-            {
-               Y[c][d][dz][dy][dx] = X(dx, dy, dz, c);
-            }
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
 /// Load 3D input VDIM*DIM vector into given register tensor, specific component
 template <int VDIM, int DIM, int MQ1>
 inline MFEM_HOST_DEVICE void LoadDofs3d(const int e, const int d1d, const int c,
@@ -354,28 +332,6 @@ inline MFEM_HOST_DEVICE void WriteDofs3d(const int e, const int d1d,
    MFEM_SYNC_THREAD;
 }
 
-/// Write 3D DIM vector into given device tensor for specific component
-template <int VDIM, int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void WriteDofs3d(const int d1d, const int c,
-                                         vd_regs3d_t<VDIM, DIM, MQ1> &X,
-                                         DeviceTensor<4, real_t> &Y)
-{
-   for (int dz = 0; dz < d1d; ++dz)
-   {
-      MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d)
-         {
-            for (int d = 0; d < DIM; ++d)
-            {
-               Y(dx, dy, dz, c) += X(c, d, dz, dy, dx);
-            }
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
 /// 2D scalar contraction, X direction
 template <bool Transpose, int MQ1>
 inline MFEM_HOST_DEVICE void ContractX2d(const int d1d, const int q1d,

fem/kernels3d.hpp

@@ -1,332 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../config/config.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/dtensor.hpp"
-
-#include "kernels.hpp" // IWYU pragma: keep
-
-namespace mfem::kernels::internal::low
-{
-
-#if ((defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)) ||       \
-     (defined(MFEM_USE_HIP) && defined(__HIP_DEVICE_COMPILE__)))
-template <int DIM, int N>
-// struct regs3d_device_wrapper: mfem::future::tensor<real_t, DIM, 0, 0, 0> {};
-struct regs3d_device_wrapper: mfem::future::tensor<real_t, 0, 0, 0, DIM> {};
-template <int DIM, int N>
-using regs3d_t = regs3d_device_wrapper<DIM, N>;
-#else
-template <int DIM, int N>
-using regs3d_t = mfem::future::tensor<real_t, N, N, N, DIM>;
-// using regs3d_t = mfem::future::tensor<real_t, DIM, N, N, N>;
-#endif
-
-///////////////////////////////////////////////////////////////////////////////
-/// Load 2D matrix into shared memory
-template <int MQ1>
-inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d,
-                                        const real_t *M, real_t (*N)[MQ1])
-{
-   if (MFEM_THREAD_ID(z) == 0)
-   {
-      MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            N[dy][qx] = M[dy * q1d + qx];
-         }
-      }
-   }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-template <int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void LoadDofs3d(const int e, const int d1d,
-                                        const DeviceTensor<5, const real_t> &XE,
-                                        real_t (&sm0)[MQ1][MQ1][MQ1][DIM])
-{
-   MFEM_FOREACH_THREAD_DIRECT(dy,y,d1d)
-   {
-      MFEM_FOREACH_THREAD_DIRECT(dx,x,d1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dz,z,d1d)
-         {
-            sm0[dz][dy][dx][0] = XE(dx, dy, dz, 0, e);
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// 3D Scalar Gradient, 1/3
-template<int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void GradX(const int d1d, const int q1d,
-                                   const real_t (*B)[MQ1],
-                                   const real_t (*G)[MQ1],
-                                   const real_t (&sm0)[MQ1][MQ1][MQ1][DIM],
-                                   real_t (&sm1)[MQ1][MQ1][MQ1][DIM])
-{
-   MFEM_FOREACH_THREAD_DIRECT(dz,z,d1d)
-   {
-      MFEM_FOREACH_THREAD_DIRECT(dy,y,d1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx,x,q1d)
-         {
-            real_t u = 0.0, v = 0.0;
-            MFEM_UNROLL(MQ1)
-            for (int dx = 0; dx < d1d; ++dx)
-            {
-               const auto x = sm0[dz][dy][dx][0];
-               u = std::fma(B[dx][qx], x, u);
-               v = std::fma(G[dx][qx], x, v);
-            }
-            sm1[dz][dy][qx][0] = u;
-            sm1[dz][dy][qx][1] = v;
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// 3D Scalar Gradient, 2/3
-template<int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void GradY(const int d1d, const int q1d,
-                                   const real_t (*B)[MQ1],
-                                   const real_t (*G)[MQ1],
-                                   const real_t (&sm1)[MQ1][MQ1][MQ1][DIM],
-                                   real_t (&sm0)[MQ1][MQ1][MQ1][DIM])
-{
-   MFEM_FOREACH_THREAD_DIRECT(dz,z,d1d)
-   {
-      MFEM_FOREACH_THREAD_DIRECT(qy,y,q1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx,x,q1d)
-         {
-            real_t u = 0.0, v = 0.0, w = 0.0;
-            MFEM_UNROLL(MQ1)
-            for (int dy = 0; dy < d1d; ++dy)
-            {
-               u = std::fma(sm1[dz][dy][qx][1], B[dy][qy], u);
-               v = std::fma(sm1[dz][dy][qx][0], G[dy][qy], v);
-               w = std::fma(sm1[dz][dy][qx][0], B[dy][qy], w);
-            }
-            sm0[dz][qy][qx][0] = u;
-            sm0[dz][qy][qx][1] = v;
-            sm0[dz][qy][qx][2] = w;
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// 3D Scalar Gradient, 3/3
-template<int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void GradZ(const int d1d, const int q1d,
-                                   const real_t (*B)[MQ1],
-                                   const real_t (*G)[MQ1],
-                                   const real_t (&sm0)[MQ1][MQ1][MQ1][DIM],
-                                   regs3d_t<DIM,MQ1> &reg)
-{
-   MFEM_FOREACH_THREAD_DIRECT(qz,z,q1d)
-   {
-      MFEM_FOREACH_THREAD_DIRECT(qy,y,q1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx,x,q1d)
-         {
-            real_t u[3] = {0.0, 0.0, 0.0};
-            MFEM_UNROLL(MQ1)
-            for (int dz = 0; dz < d1d; ++dz)
-            {
-               u[0] = std::fma(B[dz][qz], sm0[dz][qy][qx][0], u[0]);
-               u[1] = std::fma(B[dz][qz], sm0[dz][qy][qx][1], u[1]);
-               u[2] = std::fma(G[dz][qz], sm0[dz][qy][qx][2], u[2]);
-            }
-            reg[qz][qy][qx][0] = u[0];
-            reg[qz][qy][qx][1] = u[1];
-            reg[qz][qy][qx][2] = u[2];
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// 3D scalar gradient
-template <int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void Grad3d(const int d1d, const int q1d,
-                                    const real_t (*B)[MQ1],
-                                    const real_t (*G)[MQ1],
-                                    real_t (&sm0)[MQ1][MQ1][MQ1][DIM],
-                                    real_t (&sm1)[MQ1][MQ1][MQ1][DIM],
-                                    regs3d_t<DIM,MQ1> &reg)
-{
-   GradX(d1d, q1d, B, G, sm0, sm1); // Grad X
-   GradY(d1d, q1d, B, G, sm1, sm0); // Grad Y
-   GradZ(d1d, q1d, B, G, sm0, reg); // Grad Z
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// 3D Scalar Gradient Transposed, 1/3
-template<int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void GradTranspose3dX(const int d1d, const int q1d,
-                                              const real_t (*B)[MQ1],
-                                              const real_t (*G)[MQ1],
-                                              regs3d_t<DIM,MQ1> &reg,
-                                              real_t (&sm1)[MQ1][MQ1][MQ1][DIM],
-                                              real_t (&sm0)[MQ1][MQ1][MQ1][DIM])
-
-{
-   MFEM_FOREACH_THREAD_DIRECT(qz,z,q1d)
-   {
-      MFEM_FOREACH_THREAD_DIRECT(qy,y,q1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx,x,q1d)
-         {
-            sm1[qz][qy][qx][0] = reg[qz][qy][qx][0];
-            sm1[qz][qy][qx][1] = reg[qz][qy][qx][1];
-            sm1[qz][qy][qx][2] = reg[qz][qy][qx][2];
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-
-   MFEM_FOREACH_THREAD_DIRECT(qz,z,q1d)
-   {
-      MFEM_FOREACH_THREAD_DIRECT(qy,y,q1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dx,x,d1d)
-         {
-            real_t u = 0.0, v = 0.0, w = 0.0;
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < q1d; ++qx)
-            {
-               u = std::fma(sm1[qz][qy][qx][0], G[dx][qx], u);
-               v = std::fma(sm1[qz][qy][qx][1], B[dx][qx], v);
-               w = std::fma(sm1[qz][qy][qx][2], B[dx][qx], w);
-            }
-            sm0[qz][qy][dx][0] = u;
-            sm0[qz][qy][dx][1] = v;
-            sm0[qz][qy][dx][2] = w;
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// 3D Scalar Gradient Transposed, 2/3
-template<int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void GradTranspose3dY(const int d1d, const int q1d,
-                                              const real_t (*B)[MQ1],
-                                              const real_t (*G)[MQ1],
-                                              real_t (&sm0)[MQ1][MQ1][MQ1][DIM],
-                                              real_t (&sm1)[MQ1][MQ1][MQ1][DIM])
-{
-   MFEM_FOREACH_THREAD_DIRECT(qz,z,q1d)
-   {
-      MFEM_FOREACH_THREAD_DIRECT(dy,y,d1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dx,x,d1d)
-         {
-            real_t u = 0.0, v = 0.0, w = 0.0;
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < q1d; ++qy)
-            {
-               u = std::fma(sm0[qz][qy][dx][0], B[dy][qy], u);
-               v = std::fma(sm0[qz][qy][dx][1], G[dy][qy], v);
-               w = std::fma(sm0[qz][qy][dx][2], B[dy][qy], w);
-            }
-            sm1[qz][dy][dx][0] = u;
-            sm1[qz][dy][dx][1] = v;
-            sm1[qz][dy][dx][2] = w;
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// 3D Scalar Gradient Transposed, 3/3
-template<int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void GradTranspose3dZ(const int d1d, const int q1d,
-                                              const real_t (*B)[MQ1],
-                                              const real_t (*G)[MQ1],
-                                              real_t (&sm1)[MQ1][MQ1][MQ1][DIM],
-                                              regs3d_t<DIM,MQ1> &reg)
-{
-   MFEM_FOREACH_THREAD_DIRECT(dz,z,d1d)
-   {
-      MFEM_FOREACH_THREAD_DIRECT(dy,y,d1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dx,x,d1d)
-         {
-            real_t u = 0.0, v = 0.0, w = 0.0;
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < q1d; ++qz)
-            {
-               u = std::fma(sm1[qz][dy][dx][0], B[dz][qz], u);
-               v = std::fma(sm1[qz][dy][dx][1], B[dz][qz], v);
-               w = std::fma(sm1[qz][dy][dx][2], G[dz][qz], w);
-            }
-            reg[dz][dy][dx][0] = u;
-            reg[dz][dy][dx][1] = v;
-            reg[dz][dy][dx][2] = w;
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// 3D scalar gradient transposed
-template <int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void GradTranspose3d(const int d1d, const int q1d,
-                                             const real_t (*B)[MQ1],
-                                             const real_t (*G)[MQ1],
-                                             regs3d_t<DIM,MQ1> &reg,
-                                             real_t (&sm1)[MQ1][MQ1][MQ1][DIM],
-                                             real_t (&sm0)[MQ1][MQ1][MQ1][DIM])
-{
-   GradTranspose3dX(d1d, q1d, B, G, reg, sm1, sm0); // Grad^T X
-   GradTranspose3dY(d1d, q1d, B, G, sm0, sm1); // Grad^T Y
-   GradTranspose3dZ(d1d, q1d, B, G, sm1, reg); // Grad^T Z
-}
-
-///////////////////////////////////////////////////////////////////////////////
-/// 3D Scalar Gradient Transposed, 3/3
-template<int DIM, int MQ1>
-inline MFEM_HOST_DEVICE void WriteDofs3d(const int d1d,
-                                         const int c, const int e,
-                                         regs3d_t<DIM,MQ1> &reg,
-                                         const DeviceTensor<5, real_t> &YE)
-{
-   MFEM_FOREACH_THREAD_DIRECT(dz,z,d1d)
-   {
-      MFEM_FOREACH_THREAD_DIRECT(dy,y,d1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dx,x,d1d)
-         {
-            const real_t u = reg[dz][dy][dx][0];
-            const real_t v = reg[dz][dy][dx][1];
-            const real_t w = reg[dz][dy][dx][2];
-            YE(dx, dy, dz, c, e) += (u + v + w);
-         }
-      }
-   }
-}
-
-} // namespace mfem::kernels::internal

fem/nonlinearform.hpp

@@ -224,9 +224,6 @@ public:
    /** @see GetGradient(const Vector &) */
    Operator &GetGradient(const Vector &x, bool finalize) const;
 
-   /// Suppress a warning about hiding overloaded virtual function.
-   using Operator::GetGradient;
-
    /// Update the NonlinearForm to propagate updates of the associated FE space.
    /** After calling this method, the essential boundary conditions need to be
        set again. */

fem/qinterp/eval_transpose.cpp

@@ -1,44 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../quadinterpolator.hpp"
-#include "eval_transpose.hpp"
-
-namespace mfem
-{
-
-/// @cond Suppress_Doxygen_warnings
-
-QuadratureInterpolator::TensorEvalTransposeKernelType
-QuadratureInterpolator::TensorEvalTransposeKernels::Fallback(
-   int DIM, QVectorLayout Q_LAYOUT, int, int, int)
-{
-   using namespace internal::quadrature_interpolator;
-
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      if (DIM == 1) { return ValuesTranspose1D<QVectorLayout::byNODES>; }
-      else if (DIM == 2) { return ValuesTranspose2D<QVectorLayout::byNODES>; }
-      else if (DIM == 3) { return ValuesTranspose3D<QVectorLayout::byNODES>; }
-   }
-   else
-   {
-      if (DIM == 1) { return ValuesTranspose1D<QVectorLayout::byVDIM>; }
-      else if (DIM == 2) { return ValuesTranspose2D<QVectorLayout::byVDIM>; }
-      else if (DIM == 3) { return ValuesTranspose3D<QVectorLayout::byVDIM>; }
-   }
-   MFEM_ABORT("Invalid dimension");
-   return nullptr;
-}
-
-/// @endcond
-
-} // namespace mfem

fem/qinterp/eval_transpose.hpp

@@ -1,300 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#pragma once
-
-#include "../quadinterpolator.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../linalg/kernels.hpp"
-#include "../kernels.hpp"
-
-namespace mfem
-{
-namespace internal
-{
-namespace quadrature_interpolator
-{
-
-template<QVectorLayout Q_LAYOUT>
-static void ValuesTranspose1D(const int NE,
-                              const real_t *b_,
-                              const real_t *q_,
-                              real_t *e_,
-                              const int vdim,
-                              const int d1d,
-                              const int q1d)
-{
-   const auto b = Reshape(b_, q1d, d1d);
-   const auto qd = Q_LAYOUT == QVectorLayout::byNODES ?
-                   Reshape(q_, q1d, vdim, NE) :
-                   Reshape(q_, vdim, q1d, NE);
-   auto e = Reshape(e_, d1d, vdim, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int el)
-   {
-      for (int c = 0; c < vdim; c++)
-      {
-         for (int d = 0; d < d1d; d++)
-         {
-            real_t u = 0.0;
-            for (int q = 0; q < q1d; q++)
-            {
-               const real_t qval = Q_LAYOUT == QVectorLayout::byVDIM ?
-                                   qd(c, q, el) : qd(q, c, el);
-               u += b(q, d) * qval;
-            }
-            e(d, c, el) += u;
-         }
-      }
-   });
-}
-
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int T_NBZ = 1>
-static void ValuesTranspose2D(const int NE,
-                              const real_t *b_,
-                              const real_t *q_,
-                              real_t *e_,
-                              const int vdim = 0,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
-                  Reshape(q_, Q1D, Q1D, VDIM, NE) :
-                  Reshape(q_, VDIM, Q1D, Q1D, NE);
-   auto e = Reshape(e_, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D_batch(NE, D1D, D1D, NBZ, [=] MFEM_HOST_DEVICE (int el)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      const int tidz = MFEM_THREAD_ID(z);
-
-      MFEM_SHARED real_t sB[MQ1*MD1];
-      MFEM_SHARED real_t sm0[NBZ][MDQ*MDQ];
-      MFEM_SHARED real_t sm1[NBZ][MDQ*MDQ];
-
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
-
-      ConstDeviceMatrix B(sB, D1D, Q1D);
-      DeviceMatrix QQ(sm0[tidz], MQ1, MQ1);
-      DeviceMatrix DQ(sm1[tidz], MD1, MQ1);
-      DeviceMatrix DD(sm0[tidz], MD1, MD1);
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         // Load Q data
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               QQ(qx,qy) = Q_LAYOUT == QVectorLayout::byVDIM ?
-                           q(c,qx,qy,el) : q(qx,qy,c,el);
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Transpose in y: QQ -> DQ (apply B^T in y-direction)
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += B(dy,qy) * QQ(qx,qy);
-               }
-               DQ(dy,qx) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Transpose in x: DQ -> DD (apply B^T in x-direction)
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += B(dx,qx) * DQ(dy,qx);
-               }
-               DD(dx,dy) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Store result
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               e(dx,dy,c,el) += DD(dx,dy);
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
-static void ValuesTranspose3D(const int NE,
-                              const real_t *b_,
-                              const real_t *q_,
-                              real_t *e_,
-                              const int vdim = 0,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
-                  Reshape(q_, Q1D, Q1D, Q1D, VDIM, NE) :
-                  Reshape(q_, VDIM, Q1D, Q1D, Q1D, NE);
-   auto e = Reshape(e_, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int el)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_INTERP_1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_INTERP_1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-
-      MFEM_SHARED real_t sB[MQ1*MD1];
-      MFEM_SHARED real_t sm0[MDQ*MDQ*MDQ];
-      MFEM_SHARED real_t sm1[MDQ*MDQ*MDQ];
-
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
-
-      ConstDeviceMatrix B(sB, D1D, Q1D);
-      DeviceCube QQQ(sm0, MQ1, MQ1, MQ1);
-      DeviceCube DQQ(sm1, MD1, MQ1, MQ1);
-      DeviceCube DDQ(sm0, MD1, MD1, MQ1);
-      DeviceCube DDD(sm1, MD1, MD1, MD1);
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         // Load Q data
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  QQQ(qx,qy,qz) = Q_LAYOUT == QVectorLayout::byVDIM ?
-                                  q(c,qx,qy,qz,el) : q(qx,qy,qz,c,el);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Transpose in z
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t u = 0.0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u += B(dz,qz) * QQQ(qx,qy,qz);
-                  }
-                  DQQ(dz,qx,qy) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Transpose in y
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t u = 0.0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     u += B(dy,qy) * DQQ(dz,qx,qy);
-                  }
-                  DDQ(dz,dy,qx) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Transpose in x
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     u += B(dx,qx) * DDQ(dz,dy,qx);
-                  }
-                  DDD(dx,dy,dz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-} // namespace quadrature_interpolator
-} // namespace internal
-
-template<int DIM, QVectorLayout Q_LAYOUT,
-         int VDIM, int D1D, int Q1D, int NBZ>
-QuadratureInterpolator::TensorEvalTransposeKernelType
-QuadratureInterpolator::TensorEvalTransposeKernels::Kernel()
-{
-   if (DIM == 1) { return internal::quadrature_interpolator::ValuesTranspose1D<Q_LAYOUT>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::ValuesTranspose2D<Q_LAYOUT, VDIM, D1D, Q1D, NBZ>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::ValuesTranspose3D<Q_LAYOUT, VDIM, D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
-}
-
-} // namespace mfem

fem/qinterp/eval_transpose_by_vdim.cpp

@@ -1,61 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../quadinterpolator.hpp"
-#include "eval_transpose.hpp"
-
-namespace mfem
-{
-namespace internal
-{
-namespace quadrature_interpolator
-{
-
-void InitEvalTransposeByVDimKernels()
-{
-   using k = QuadratureInterpolator::TensorEvalTransposeKernels;
-   constexpr auto L = QVectorLayout::byVDIM;
-
-   // 2D
-   k::Specialization<2,L,1,2,4>::Opt<8>::Add();
-   k::Specialization<2,L,1,3,6>::Opt<4>::Add();
-   k::Specialization<2,L,1,4,8>::Opt<2>::Add();
-
-   k::Specialization<2,L,2,2,4>::Opt<8>::Add();
-   k::Specialization<2,L,2,3,4>::Opt<8>::Add();
-   k::Specialization<2,L,2,3,6>::Opt<4>::Add();
-   k::Specialization<2,L,2,4,6>::Opt<2>::Add();
-   k::Specialization<2,L,2,4,8>::Opt<2>::Add();
-
-   // 3D
-   k::Specialization<3,L,1,2,4>::Opt<1>::Add();
-   k::Specialization<3,L,1,3,6>::Opt<1>::Add();
-   k::Specialization<3,L,1,4,8>::Opt<1>::Add();
-   k::Specialization<3,L,3,2,4>::Opt<1>::Add();
-   k::Specialization<3,L,3,3,6>::Opt<1>::Add();
-   k::Specialization<3,L,3,4,8>::Opt<1>::Add();
-
-   k::Specialization<3,L,3,2,2>::Opt<1>::Add();
-   k::Specialization<3,L,3,3,3>::Opt<1>::Add();
-   k::Specialization<3,L,3,4,4>::Opt<1>::Add();
-   k::Specialization<3,L,3,5,5>::Opt<1>::Add();
-   k::Specialization<3,L,3,6,6>::Opt<1>::Add();
-   k::Specialization<3,L,3,7,7>::Opt<1>::Add();
-   k::Specialization<3,L,3,8,8>::Opt<1>::Add();
-   k::Specialization<3,L,3,9,9>::Opt<1>::Add();
-
-   k::Specialization<3,L,3,4,6>::Opt<1>::Add();
-   k::Specialization<3,L,3,3,4>::Opt<1>::Add();
-}
-
-} // namespace quadrature_interpolator
-} // namespace internal
-} // namespace mfem

fem/qinterp/grad_transpose.cpp

@@ -1,62 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../quadinterpolator.hpp"
-#include "grad_transpose.hpp"
-
-namespace mfem
-{
-
-/// @cond Suppress_Doxygen_warnings
-
-QuadratureInterpolator::GradTransposeKernelType
-QuadratureInterpolator::GradTransposeKernels::Fallback(
-   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int, int)
-{
-   using namespace internal::quadrature_interpolator;
-
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      if (GRAD_PHYS)
-      {
-         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byNODES, true>; }
-         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byNODES, true>; }
-         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byNODES, true>; }
-      }
-      else
-      {
-         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byNODES, false>; }
-         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byNODES, false>; }
-         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byNODES, false>; }
-      }
-   }
-   else
-   {
-      if (GRAD_PHYS)
-      {
-         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byVDIM, true>; }
-         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byVDIM, true>; }
-         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byVDIM, true>; }
-      }
-      else
-      {
-         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byVDIM, false>; }
-         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byVDIM, false>; }
-         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byVDIM, false>; }
-      }
-   }
-   MFEM_ABORT("Invalid dimension");
-   return nullptr;
-}
-
-/// @endcond
-
-} // namespace mfem

fem/qinterp/grad_transpose.hpp

@@ -1,737 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#pragma once
-
-#include "../quadinterpolator.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../linalg/kernels.hpp"
-#include "../kernels.hpp"
-
-namespace mfem
-{
-namespace internal
-{
-namespace quadrature_interpolator
-{
-
-// Transpose gradient operation: integrate against shape function derivatives
-// This is the adjoint of the Derivatives operation
-
-template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
-static void DerivativesTranspose1D(const int NE,
-                                   const real_t *b_,
-                                   const real_t *g_,
-                                   const real_t *j_,
-                                   const real_t *q_,
-                                   real_t *e_,
-                                   const int sdim,
-                                   const int vdim,
-                                   const int d1d,
-                                   const int q1d)
-{
-   MFEM_CONTRACT_VAR(b_);
-   const int SDIM = GRAD_PHYS ? sdim : 1;
-   const auto g = Reshape(g_, q1d, d1d);
-   const auto j = Reshape(j_, q1d, SDIM, NE);
-   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
-                  Reshape(q_, q1d, vdim, SDIM, NE):
-                  Reshape(q_, vdim, SDIM, q1d, NE);
-   auto e = Reshape(e_, d1d, vdim, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int el)
-   {
-      for (int c = 0; c < vdim; c++)
-      {
-         for (int d = 0; d < d1d; d++)
-         {
-            real_t u = 0.0;
-            for (int qx = 0; qx < q1d; qx++)
-            {
-               // Load gradient from q-vector
-               real_t dq[3] = {0.0, 0.0, 0.0};
-               for (int s = 0; s < SDIM; ++s)
-               {
-                  if (Q_LAYOUT == QVectorLayout::byVDIM)  { dq[s] = q(c, s, qx, el); }
-                  if (Q_LAYOUT == QVectorLayout::byNODES) { dq[s] = q(qx, c, s, el); }
-               }
-
-               // Apply inverse Jacobian transpose (adjoint of physical gradient)
-               real_t du = dq[0];
-               if (GRAD_PHYS)
-               {
-                  if (SDIM == 1) { du = dq[0] / j(qx, 0, el); }
-                  else if (SDIM == 2)
-                  {
-                     const real_t Jloc[2] = {j(qx,0,el), j(qx,1,el)};
-                     real_t Jinv[3];
-                     kernels::CalcLeftInverse<2,1>(Jloc, Jinv);
-                     du = Jinv[0]*dq[0] + Jinv[1]*dq[1];
-                  }
-                  else // SDIM == 3
-                  {
-                     const real_t Jloc[3] = {j(qx,0,el), j(qx,1,el), j(qx,2,el)};
-                     real_t Jinv[3];
-                     kernels::CalcLeftInverse<3,1>(Jloc, Jinv);
-                     du = Jinv[0]*dq[0] + Jinv[1]*dq[1] + Jinv[2]*dq[2];
-                  }
-               }
-
-               // Accumulate contribution (transpose of G matrix)
-               u += g(qx, d) * du;
-            }
-            e(d, c, el) += u;
-         }
-      }
-   });
-}
-
-template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int T_NBZ = 1>
-static void DerivativesTranspose2D(const int NE,
-                                   const real_t *b_,
-                                   const real_t *g_,
-                                   const real_t *j_,
-                                   const real_t *q_,
-                                   real_t *e_,
-                                   const int sdim = 2,
-                                   const int vdim = 0,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   const int SDIM = GRAD_PHYS ? sdim : 2;
-   static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto g = Reshape(g_, Q1D, D1D);
-   const auto j = Reshape(j_, Q1D, Q1D, SDIM, 2, NE);
-   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
-                  Reshape(q_, Q1D, Q1D, VDIM, SDIM, NE):
-                  Reshape(q_, VDIM, SDIM, Q1D, Q1D, NE);
-   auto e = Reshape(e_, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D_batch(NE, D1D, D1D, NBZ, [=] MFEM_HOST_DEVICE (int el)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      const int tidz = MFEM_THREAD_ID(z);
-
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-      DeviceMatrix B(BG[0], D1D, Q1D);
-      DeviceMatrix G(BG[1], D1D, Q1D);
-
-      MFEM_SHARED real_t sm0[NBZ][MDQ*MDQ];
-      MFEM_SHARED real_t sm1[NBZ][MDQ*MDQ];
-
-      DeviceMatrix QQ(sm0[tidz], MQ1, MQ1);
-      DeviceMatrix DQ0(sm1[tidz], MD1, MQ1);
-      DeviceMatrix DQ1(sm1[tidz], MD1, MQ1);  // Reuse sm1 after DQ0 is done
-      DeviceMatrix DD(sm0[tidz], MD1, MD1);   // Reuse sm0 after QQ is done
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         // Load Q data and apply inverse Jacobian
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               // Load gradient components
-               real_t dq[3] = {0.0, 0.0, 0.0};
-               for (int d = 0; d < SDIM; ++d)
-               {
-                  if (Q_LAYOUT == QVectorLayout::byVDIM) { dq[d] = q(c, d, qx, qy, el); }
-                  else { dq[d] = q(qx, qy, c, d, el); }
-               }
-
-               // Apply inverse Jacobian transpose (adjoint of physical gradient)
-               real_t du[2] = {dq[0], dq[1]};
-               if (GRAD_PHYS)
-               {
-                  if (SDIM == 2)
-                  {
-                     real_t Jloc[4], Jinv[4];
-                     Jloc[0] = j(qx,qy,0,0,el);
-                     Jloc[1] = j(qx,qy,1,0,el);
-                     Jloc[2] = j(qx,qy,0,1,el);
-                     Jloc[3] = j(qx,qy,1,1,el);
-                     kernels::CalcInverse<2>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1];
-                     du[0] = U;
-                     du[1] = V;
-                  }
-                  else // SDIM == 3
-                  {
-                     real_t Jloc[6], Jinv[6];
-                     Jloc[0] = j(qx,qy,0,0,el);
-                     Jloc[1] = j(qx,qy,1,0,el);
-                     Jloc[2] = j(qx,qy,2,0,el);
-                     Jloc[3] = j(qx,qy,0,1,el);
-                     Jloc[4] = j(qx,qy,1,1,el);
-                     Jloc[5] = j(qx,qy,2,1,el);
-                     kernels::CalcLeftInverse<3,2>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1] + Jinv[4]*dq[2];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1] + Jinv[5]*dq[2];
-                     du[0] = U;
-                     du[1] = V;
-                  }
-               }
-               QQ(qx, qy) = du[0];  // Store du/dx component
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in y-direction: QQ -> DQ0
-         // (Transpose of d/dx which uses DQ1(dy,qx)*B(dy,qy))
-         // Must produce DQ0(dy,qx) to match forward's DQ1 indexing
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += B(dy,qy) * QQ(qx,qy);
-               }
-               DQ0(dy,qx) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply G^T in x-direction: DQ0 -> DD
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += G(dx,qx) * DQ0(dy,qx);
-               }
-               DD(dx,dy) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Accumulate to output
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               e(dx,dy,c,el) += DD(dx,dy);
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Now process du/dy component
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               // Load gradient components
-               real_t dq[3] = {0.0, 0.0, 0.0};
-               for (int d = 0; d < SDIM; ++d)
-               {
-                  if (Q_LAYOUT == QVectorLayout::byVDIM) { dq[d] = q(c, d, qx, qy, el); }
-                  else { dq[d] = q(qx, qy, c, d, el); }
-               }
-
-               // Apply inverse Jacobian transpose
-               real_t du[2] = {dq[0], dq[1]};
-               if (GRAD_PHYS)
-               {
-                  if (SDIM == 2)
-                  {
-                     real_t Jloc[4], Jinv[4];
-                     Jloc[0] = j(qx,qy,0,0,el);
-                     Jloc[1] = j(qx,qy,1,0,el);
-                     Jloc[2] = j(qx,qy,0,1,el);
-                     Jloc[3] = j(qx,qy,1,1,el);
-                     kernels::CalcInverse<2>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1];
-                     du[0] = U;
-                     du[1] = V;
-                  }
-                  else // SDIM == 3
-                  {
-                     real_t Jloc[6], Jinv[6];
-                     Jloc[0] = j(qx,qy,0,0,el);
-                     Jloc[1] = j(qx,qy,1,0,el);
-                     Jloc[2] = j(qx,qy,2,0,el);
-                     Jloc[3] = j(qx,qy,0,1,el);
-                     Jloc[4] = j(qx,qy,1,1,el);
-                     Jloc[5] = j(qx,qy,2,1,el);
-                     kernels::CalcLeftInverse<3,2>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1] + Jinv[4]*dq[2];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1] + Jinv[5]*dq[2];
-                     du[0] = U;
-                     du[1] = V;
-                  }
-               }
-               QQ(qx, qy) = du[1];  // Store du/dy component
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply G^T in y-direction: QQ -> DQ1
-         // (Transpose of d/dy which uses DQ0(dy,qx)*G(dy,qy))
-         // Must produce DQ1(dy,qx) to match forward's DQ0 indexing
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += G(dy,qy) * QQ(qx,qy);
-               }
-               DQ1(dy,qx) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in x-direction: DQ1 -> DD
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += B(dx,qx) * DQ1(dy,qx);
-               }
-               DD(dx,dy) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Accumulate to output
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               e(dx,dy,c,el) += DD(dx,dy);
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
-static void DerivativesTranspose3D(const int NE,
-                                   const real_t *b_,
-                                   const real_t *g_,
-                                   const real_t *j_,
-                                   const real_t *q_,
-                                   real_t *e_,
-                                   const int sdim = 3,
-                                   const int vdim = 0,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto g = Reshape(g_, Q1D, D1D);
-   const auto j = Reshape(j_, Q1D, Q1D, Q1D, 3, 3, NE);
-   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
-                  Reshape(q_, Q1D, Q1D, Q1D, VDIM, 3, NE):
-                  Reshape(q_, VDIM, 3, Q1D, Q1D, Q1D, NE);
-   auto e = Reshape(e_, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int el)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_INTERP_1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_INTERP_1D;
-
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-      DeviceMatrix B(BG[0], D1D, Q1D);
-      DeviceMatrix G(BG[1], D1D, Q1D);
-
-      MFEM_SHARED real_t sm0[3][MQ1*MQ1*MQ1];
-      MFEM_SHARED real_t sm1[3][MQ1*MQ1*MQ1];
-      DeviceCube QQQ(sm0[0], MQ1, MQ1, MQ1);
-      DeviceCube DQQ(sm1[0], MD1, MQ1, MQ1);
-      DeviceCube DDQ(sm0[0], MD1, MD1, MQ1);
-      DeviceCube DDD(sm1[0], MD1, MD1, MD1);
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         // Process du/dx component
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t dq[3];
-                  if (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     dq[0] = q(c,0,qx,qy,qz,el);
-                     dq[1] = q(c,1,qx,qy,qz,el);
-                     dq[2] = q(c,2,qx,qy,qz,el);
-                  }
-                  else
-                  {
-                     dq[0] = q(qx,qy,qz,c,0,el);
-                     dq[1] = q(qx,qy,qz,c,1,el);
-                     dq[2] = q(qx,qy,qz,c,2,el);
-                  }
-
-                  real_t du[3] = {dq[0], dq[1], dq[2]};
-                  if (GRAD_PHYS)
-                  {
-                     real_t Jloc[9], Jinv[9];
-                     for (int col = 0; col < 3; col++)
-                     {
-                        for (int row = 0; row < 3; row++)
-                        {
-                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
-                        }
-                     }
-                     kernels::CalcInverse<3>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
-                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
-                     du[0] = U; du[1] = V; du[2] = W;
-                  }
-                  QQQ(qx,qy,qz) = du[0];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply G^T in x: QQQ -> DQQ (transpose of G⊗B⊗B)
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     u += G(dx,qx) * QQQ(qx,qy,qz);
-                  }
-                  DQQ(dx,qy,qz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in y: DQQ -> DDQ
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     u += B(dy,qy) * DQQ(dx,qy,qz);
-                  }
-                  DDQ(dx,dy,qz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in z: DDQ -> DDD
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u += B(dz,qz) * DDQ(dx,dy,qz);
-                  }
-                  DDD(dx,dy,dz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Accumulate result
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Process du/dy component
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t dq[3];
-                  if (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     dq[0] = q(c,0,qx,qy,qz,el);
-                     dq[1] = q(c,1,qx,qy,qz,el);
-                     dq[2] = q(c,2,qx,qy,qz,el);
-                  }
-                  else
-                  {
-                     dq[0] = q(qx,qy,qz,c,0,el);
-                     dq[1] = q(qx,qy,qz,c,1,el);
-                     dq[2] = q(qx,qy,qz,c,2,el);
-                  }
-
-                  real_t du[3] = {dq[0], dq[1], dq[2]};
-                  if (GRAD_PHYS)
-                  {
-                     real_t Jloc[9], Jinv[9];
-                     for (int col = 0; col < 3; col++)
-                     {
-                        for (int row = 0; row < 3; row++)
-                        {
-                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
-                        }
-                     }
-                     kernels::CalcInverse<3>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
-                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
-                     du[0] = U; du[1] = V; du[2] = W;
-                  }
-                  QQQ(qx,qy,qz) = du[1];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in x: QQQ -> DQQ (transpose of B⊗G⊗B)
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     u += B(dx,qx) * QQQ(qx,qy,qz);
-                  }
-                  DQQ(dx,qy,qz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply G^T in y: DQQ -> DDQ
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     u += G(dy,qy) * DQQ(dx,qy,qz);
-                  }
-                  DDQ(dx,dy,qz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in z: DDQ -> DDD
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u += B(dz,qz) * DDQ(dx,dy,qz);
-                  }
-                  DDD(dx,dy,dz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Accumulate result
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Process du/dz component
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t dq[3];
-                  if (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     dq[0] = q(c,0,qx,qy,qz,el);
-                     dq[1] = q(c,1,qx,qy,qz,el);
-                     dq[2] = q(c,2,qx,qy,qz,el);
-                  }
-                  else
-                  {
-                     dq[0] = q(qx,qy,qz,c,0,el);
-                     dq[1] = q(qx,qy,qz,c,1,el);
-                     dq[2] = q(qx,qy,qz,c,2,el);
-                  }
-
-                  real_t du[3] = {dq[0], dq[1], dq[2]};
-                  if (GRAD_PHYS)
-                  {
-                     real_t Jloc[9], Jinv[9];
-                     for (int col = 0; col < 3; col++)
-                     {
-                        for (int row = 0; row < 3; row++)
-                        {
-                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
-                        }
-                     }
-                     kernels::CalcInverse<3>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
-                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
-                     du[0] = U; du[1] = V; du[2] = W;
-                  }
-                  QQQ(qx,qy,qz) = du[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply G^T in z: QQQ -> DQQ (transpose of B⊗B⊗G)
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t u = 0.0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u += G(dz,qz) * QQQ(qx,qy,qz);
-                  }
-                  DQQ(qx,qy,dz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in y: DQQ -> DDQ
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t u = 0.0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     u += B(dy,qy) * DQQ(qx,qy,dz);
-                  }
-                  DDQ(qx,dy,dz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in x: DDQ -> DDD
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     u += B(dx,qx) * DDQ(qx,dy,dz);
-                  }
-                  DDD(dx,dy,dz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Accumulate result
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-} // namespace quadrature_interpolator
-} // namespace internal
-
-template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
-         int Q1D, int NBZ>
-QuadratureInterpolator::GradTransposeKernelType
-QuadratureInterpolator::GradTransposeKernels::Kernel()
-{
-   if (DIM == 1) { return internal::quadrature_interpolator::DerivativesTranspose1D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::DerivativesTranspose2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D, NBZ>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::DerivativesTranspose3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
-}
-
-} // namespace mfem

fem/qinterp/grad_transpose_by_nodes.cpp

@@ -1,50 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../quadinterpolator.hpp"
-#include "grad_transpose.hpp"
-
-namespace mfem
-{
-namespace internal
-{
-namespace quadrature_interpolator
-{
-
-template <bool P>
-void InitGradTransposeByNodesKernels()
-{
-   using k = QuadratureInterpolator::GradTransposeKernels;
-   constexpr auto L = QVectorLayout::byNODES;
-   // 2D
-   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
-
-   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
-   // 3D
-   k::Specialization<3,L,P,1,3,4>::Add();
-   k::Specialization<3,L,P,1,4,6>::Add();
-   k::Specialization<3,L,P,1,5,8>::Add();
-   k::Specialization<3,L,P,3,3,4>::Add();
-   k::Specialization<3,L,P,3,4,6>::Add();
-   k::Specialization<3,L,P,3,5,8>::Add();
-}
-
-template void InitGradTransposeByNodesKernels<false>();
-template void InitGradTransposeByNodesKernels<true>();
-
-} // namespace quadrature_interpolator
-} // namespace internal
-} // namespace mfem

fem/qinterp/grad_transpose_by_vdim.cpp

@@ -1,50 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../quadinterpolator.hpp"
-#include "grad_transpose.hpp"
-
-namespace mfem
-{
-namespace internal
-{
-namespace quadrature_interpolator
-{
-
-template <bool P>
-void InitGradTransposeByVDimKernels()
-{
-   using k = QuadratureInterpolator::GradTransposeKernels;
-   constexpr auto L = QVectorLayout::byVDIM;
-   // 2D
-   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
-
-   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
-   // 3D
-   k::Specialization<3,L,P,1,3,4>::Add();
-   k::Specialization<3,L,P,1,4,6>::Add();
-   k::Specialization<3,L,P,1,5,8>::Add();
-   k::Specialization<3,L,P,3,3,4>::Add();
-   k::Specialization<3,L,P,3,4,6>::Add();
-   k::Specialization<3,L,P,3,5,8>::Add();
-}
-
-template void InitGradTransposeByVDimKernels<false>();
-template void InitGradTransposeByVDimKernels<true>();
-
-} // namespace quadrature_interpolator
-} // namespace internal
-} // namespace mfem

fem/quadinterpolator.cpp

@@ -11,7 +11,6 @@
 
 #include "quadinterpolator.hpp"
 #include "qinterp/grad.hpp"
-#include "qinterp/grad_transpose.hpp"
 #include "qinterp/eval.hpp"
 #include "qspace.hpp"
 #include "../general/forall.hpp"
@@ -31,10 +30,7 @@ void InitEvalKernels();
 void InitDetKernels();
 template <bool P> void InitGradByNodesKernels();
 template <bool P> void InitGradByVDimKernels();
-template <bool P> void InitGradTransposeByNodesKernels();
-template <bool P> void InitGradTransposeByVDimKernels();
 void InitTensorEvalHDivKernels();
-void InitEvalTransposeByVDimKernels();
 struct Kernels
 {
    Kernels()
@@ -49,19 +45,12 @@ struct Kernels
       // Phys grad kernels
       InitGradByNodesKernels<true>();
       InitGradByVDimKernels<true>();
-      // Non-phys grad transpose kernels
-      InitGradTransposeByNodesKernels<false>();
-      InitGradTransposeByVDimKernels<false>();
-      // Phys grad transpose kernels
-      InitGradTransposeByNodesKernels<true>();
-      InitGradTransposeByVDimKernels<true>();
       // Determinants
       InitDetKernels();
       // Non-tensor
       InitEvalKernels();
       // Tensor (quad,hex) H(div)
       InitTensorEvalHDivKernels();
-      InitEvalTransposeByVDimKernels();
    }
 };
 }
@@ -657,59 +646,16 @@ void QuadratureInterpolator::MultHDiv(const Vector &e_vec,
    MFEM_CONTRACT_VAR(q_div);
 }
 
-void QuadratureInterpolator::AddMultTranspose(unsigned eval_flags,
-                                              const Vector &q_val,
-                                              const Vector &q_der,
-                                              Vector &e_vec) const
+void QuadratureInterpolator::MultTranspose(unsigned eval_flags,
+                                           const Vector &q_val,
+                                           const Vector &q_der,
+                                           Vector &e_vec) const
 {
-   const int ne = fespace->GetNE();
-   if (ne == 0) { return; }
-   const FiniteElement *fe = fespace->GetFE(0);
-   const int vdim = fespace->GetVDim();
-   const int sdim = fespace->GetMesh()->SpaceDimension();
-
-   const bool use_tensor_eval =
-      use_tensor_products &&
-      dynamic_cast<const TensorBasisElement*>(fe) != nullptr;
-   const IntegrationRule *ir =
-      IntRule ? IntRule : &qspace->GetElementIntRule(0);
-   const DofToQuad::Mode mode =
-      use_tensor_eval ? DofToQuad::TENSOR : DofToQuad::FULL;
-   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
-   const int dim = maps.FE->GetDim();
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-
-   const GeometricFactors *geom = nullptr;
-   if (eval_flags & PHYSICAL_DERIVATIVES)
-   {
-      const int jacobians = GeometricFactors::JACOBIANS;
-      geom = fespace->GetMesh()->GetGeometricFactors(*ir, jacobians);
-   }
-
-   if (use_tensor_eval)
-   {
-      if (eval_flags & (VALUES | PHYSICAL_VALUES))
-      {
-         TensorEvalTransposeKernels::Run(dim, q_layout, vdim, nd, nq, ne,
-                                         maps.B.Read(), q_val.Read(),
-                                         e_vec.ReadWrite(), vdim, nd, nq);
-      }
-      if (eval_flags & (DERIVATIVES | PHYSICAL_DERIVATIVES))
-      {
-         const bool phys = (eval_flags & PHYSICAL_DERIVATIVES);
-         const real_t *J = phys ? geom->J.Read() : nullptr;
-         const int s_dim = phys ? sdim : dim;
-         GradTransposeKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
-                                   maps.B.Read(), maps.G.Read(), J,
-                                   q_der.Read(), e_vec.ReadWrite(),
-                                   s_dim, vdim, nd, nq);
-      }
-   }
-   else
-   {
-      MFEM_ABORT("Non-tensor MultTranspose not yet implemented");
-   }
+   MFEM_CONTRACT_VAR(eval_flags);
+   MFEM_CONTRACT_VAR(q_val);
+   MFEM_CONTRACT_VAR(q_der);
+   MFEM_CONTRACT_VAR(e_vec);
+   MFEM_ABORT("this method is not implemented yet");
 }
 
 void QuadratureInterpolator::Values(const Vector &e_vec,

fem/quadinterpolator.hpp

@@ -152,8 +152,8 @@ public:
    void Determinants(const Vector &e_vec, Vector &q_det) const;
 
    /// Perform the transpose operation of Mult(). (TODO)
-   void AddMultTranspose(unsigned eval_flags, const Vector &q_val,
-                         const Vector &q_der, Vector &e_vec) const;
+   void MultTranspose(unsigned eval_flags, const Vector &q_val,
+                      const Vector &q_der, Vector &e_vec) const;
 
    /// @brief Returns true if the given finite element space is supported by
    /// QuadratureInterpolator.
@@ -178,13 +178,6 @@ public:
    using TensorEvalHDivKernelType =
       void(*)(const int, const real_t *, const real_t *, const real_t *,
               const real_t *, real_t *, const int, const int);
-   using TensorEvalTransposeKernelType = void(*)(const int, const real_t *,
-                                                 const real_t *,
-                                                 real_t *, const int, const int, const int);
-   using GradTransposeKernelType = void(*)(const int, const real_t *,
-                                           const real_t *, const real_t *,
-                                           const real_t *, real_t *,
-                                           const int, const int, const int, const int);
 
    MFEM_REGISTER_KERNELS(TensorEvalKernels, TensorEvalKernelType,
                          (int, QVectorLayout, int, int, int), (int));
@@ -196,10 +189,6 @@ public:
                          (int, QVectorLayout, bool, int, int), (int));
    MFEM_REGISTER_KERNELS(TensorEvalHDivKernels, TensorEvalHDivKernelType,
                          (int, QVectorLayout, unsigned, int, int));
-   MFEM_REGISTER_KERNELS(TensorEvalTransposeKernels, TensorEvalTransposeKernelType,
-                         (int, QVectorLayout, int, int, int), (int));
-   MFEM_REGISTER_KERNELS(GradTransposeKernels, GradTransposeKernelType,
-                         (int, QVectorLayout, bool, int, int, int), (int));
 };
 
 }

general/backends.hpp

@@ -27,14 +27,6 @@
 #endif
 #include "hip.hpp"
 
-#if defined(MFEM_USE_CUDA)
-#define Gpu(...) Cu##__VA_ARGS__
-#elif defined(MFEM_USE_HIP)
-#define Gpu(...) Hip##__VA_ARGS__
-#else
-#define Gpu(...) __VA_ARGS__
-#endif
-
 #ifdef MFEM_USE_OCCA
 #include "occa.hpp"
 #endif
@@ -56,7 +48,6 @@ constexpr bool mfem_use_gpu = false;
 #define MFEM_DEVICE
 #define MFEM_HOST
 #define MFEM_LAMBDA
-#define MFEM_CONSTANT
 // #define MFEM_HOST_DEVICE // defined in config/config.hpp
 // MFEM_DEVICE_SYNC is made available for debugging purposes
 #define MFEM_DEVICE_SYNC
@@ -74,13 +65,6 @@ constexpr bool mfem_use_gpu = false;
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) MFEM_FOREACH_THREAD(i,k,N)
-
-inline const void* MemcpyToSymbol(const void *d_sym, const void *h_src,
-                                  size_t bytes)
-{
-   memcpy(const_cast<void *>(d_sym), h_src, bytes);
-   return d_sym;
-}
 #endif
 
 // 'double' and 'float' atomicAdd implementation for previous versions of CUDA

general/cuda.cpp

@@ -175,17 +175,6 @@ void* CuMemcpyDtoHAsync(void *dst, const void *src, size_t bytes)
    return dst;
 }
 
-const void* CuMemcpyToSymbol(const void *d_sym, const void *h_src,
-                             size_t bytes)
-{
-#ifdef MFEM_USE_CUDA
-   MFEM_GPU_CHECK(cudaMemcpyToSymbol(d_sym, h_src, bytes));
-   return d_sym;
-#endif
-   MFEM_ABORT("CUDA has no shadow host copy of device symbols");
-   return memcpy(const_cast<void*>(d_sym), h_src, bytes);
-}
-
 void CuCheckLastError()
 {
 #ifdef MFEM_USE_CUDA

general/cuda.hpp

@@ -25,8 +25,6 @@ constexpr bool mfem_use_gpu = true;
 #define MFEM_HOST __host__
 #define MFEM_LAMBDA __host__
 #define MFEM_LAUNCH_BOUNDS __launch_bounds__
-#define MFEM_DEVICE __device__
-#define MFEM_CONSTANT __constant__
 // #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
 #define MFEM_DEVICE_SYNC MFEM_GPU_CHECK(cudaDeviceSynchronize())
 #define MFEM_STREAM_SYNC MFEM_GPU_CHECK(cudaStreamSynchronize(0))
@@ -96,10 +94,6 @@ void* CuMemcpyDtoH(void *h_dst, const void *d_src, size_t bytes);
 /// Copies memory from Device to Host
 void* CuMemcpyDtoHAsync(void *h_dst, const void *d_src, size_t bytes);
 
-/// Copies data to the given symbol on the device.
-const void* CuMemcpyToSymbol(const void *d_sym, const void *h_src,
-                             size_t bytes);
-
 /// Check the error code returned by cudaGetLastError(), aborting on error.
 void CuCheckLastError();
 

general/forall.hpp

@@ -1090,12 +1090,6 @@ inline void forall_2D_batch(int N, int X, int Y, int BZ, lambda &&body)
    ForallWrap<2>(true, N, body, X, Y, BZ);
 }
 
-template<int MAX_THREADS_PER_BLOCK, typename lambda>
-inline void forall_2D_batch(int N, int X, int Y, int BZ, lambda &&body)
-{
-   ForallWrap<2, MAX_THREADS_PER_BLOCK>(true, N, body, X, Y, BZ);
-}
-
 template<typename lambda>
 inline void forall_3D(int N, int X, int Y, int Z, lambda &&body)
 {

general/hip.cpp

@@ -175,15 +175,6 @@ void* HipMemcpyDtoHAsync(void *dst, const void *src, size_t bytes)
    return dst;
 }
 
-const void* HipMemcpyToSymbol(const void *d_sym, const void *h_src,
-                              size_t bytes)
-{
-#ifdef MFEM_USE_HIP
-   MFEM_GPU_CHECK(hipMemcpyToSymbol(d_sym, h_src, bytes));
-#endif
-   return memcpy(const_cast<void*>(d_sym), h_src, bytes);
-}
-
 void HipCheckLastError()
 {
 #ifdef MFEM_USE_HIP

general/hip.hpp

@@ -21,9 +21,8 @@
 #if defined(MFEM_USE_HIP) && defined(__HIP__)
 #define MFEM_USE_CUDA_OR_HIP
 constexpr bool mfem_use_gpu = true;
-#define MFEM_HOST __host__
 #define MFEM_DEVICE __device__
-#define MFEM_CONSTANT __constant__
+#define MFEM_HOST __host__
 #define MFEM_LAMBDA __host__ __device__
 #define MFEM_LAUNCH_BOUNDS __launch_bounds__
 // #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
@@ -97,10 +96,6 @@ void* HipMemcpyDtoH(void *h_dst, const void *d_src, size_t bytes);
 /// Copies memory from Device to Host
 void* HipMemcpyDtoHAsync(void *h_dst, const void *d_src, size_t bytes);
 
-/// Copies data to the given symbol on the device.
-const void* HipMemcpyToSymbol(const void *d_sym, const void *h_src,
-                              size_t bytes);
-
 /// Check the error code returned by hipGetLastError(), aborting on error.
 void HipCheckLastError();
 

general/nvtx.hpp

@@ -1 +0,0 @@
-../../stash/debug/nvtx.hpp

linalg/CMakeLists.txt

@@ -23,11 +23,11 @@ list(APPEND SRCS
   complex_operator.cpp
   constraints.cpp
   densemat.cpp
+  eigensolvers.cpp
   filteredsolver.cpp
   handle.cpp
   matrix.cpp
   mma.cpp
-  multivector.cpp
   ode.cpp
   operator.cpp
   ordering.cpp
@@ -56,6 +56,7 @@ list(APPEND HDRS
   dinvariants.hpp
   dtensor.hpp
   dual.hpp
+  eigensolvers.hpp
   filteredsolver.hpp
   handle.hpp
   invariants.hpp
@@ -64,7 +65,6 @@ list(APPEND HDRS
   linalg.hpp
   matrix.hpp
   mma.hpp
-  multivector.hpp
   ode.hpp
   operator.hpp
   ordering.hpp
@@ -103,6 +103,11 @@ if (MFEM_USE_MPI)
   endif()
 endif()
 
+if (MFEM_USE_ARPACK)
+  list(APPEND SRCS arpack.cpp)
+  list(APPEND HDRS arpack.hpp)
+endif()
+
 if (MFEM_USE_SUNDIALS)
   list(APPEND SRCS sundials.cpp)
   list(APPEND HDRS sundials.hpp)

linalg/arpack.hpp

@@ -0,0 +1,271 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_ARPACK
+#define MFEM_ARPACK
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_ARPACK
+
+#include <string>
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#include "hypre.hpp"
+#endif
+
+#include "operator.hpp"
+
+#define SSAUPD  ssaupd_
+#define SSEUPD  sseupd_
+#define DSAUPD  dsaupd_
+#define DSEUPD  dseupd_
+
+#ifdef MFEM_USE_MPI
+#define PSSAUPD pssaupd_
+#define PSSEUPD psseupd_
+#define PDSAUPD pdsaupd_
+#define PDSEUPD pdseupd_
+#endif
+
+extern "C" void SSAUPD(int *ido, char *bmat, int *n,
+                       char *which, int *nev, float *tol, float *resid,
+                       int *ncv, float *v, int *ldv,
+                       int *iparam, int *ipntr,
+                       float *workd, float *workl, int *lworkl, int *info);
+
+extern "C" void SSEUPD(int *, char *, int *, float *,
+                       float *, int *, float *, char *, int *, char *,
+                       int *, float *, float *, int *, float *,
+                       int *, int  *, int *, float *,
+                       float *, int *, int *);
+
+extern "C" void DSAUPD(int *ido, char *bmat, int *n,
+                       char *which, int *nev, double *tol, double *resid,
+                       int *ncv, double *v, int *ldv,
+                       int *iparam, int *ipntr,
+                       double *workd, double *workl, int *lworkl, int *info);
+
+extern "C" void DSEUPD(int *, char *, int *, double *,
+                       double *, int *, double *, char *, int *, char *,
+                       int *, double *, double *, int *, double *,
+                       int *, int  *, int *, double *,
+                       double *, int *, int *);
+
+#ifdef MFEM_USE_MPI
+
+extern "C" void PSSAUPD(int *comm, int *ido, char *bmat, int *n,
+                        char *which, int *nev, float *tol, float *resid,
+                        int *ncv, float *v, int *ldv,
+                        int *iparam, int *ipntr,
+                        float *workd, float *workl, int *lworkl, int *info);
+
+extern "C" void PSSEUPD(int *comm, int *, char *, int *, float *,
+                        float *, int *, float *, char *, int *, char *,
+                        int *, float *, float *, int *, float *,
+                        int *, int  *, int *, float *,
+                        float *, int *, int *);
+
+extern "C" void PDSAUPD(int *comm, int *ido, char *bmat, int *n,
+                        char *which, int *nev, double *tol, double *resid,
+                        int *ncv, double *v, int *ldv,
+                        int *iparam, int *ipntr,
+                        double *workd, double *workl, int *lworkl, int *info);
+
+extern "C" void PDSEUPD(int *comm, int *, char *, int *, double *,
+                        double *, int *, double *, char *, int *, char *,
+                        int *, double *, double *, int *, double *,
+                        int *, int  *, int *, double *,
+                        double *, int *, int *);
+
+#endif
+
+extern "C" {
+   void arpackgetcommdbg_(int *,int *,int *);
+   void arpacksetcommdbg_(int *,int *,int *);
+   void arpacksymdbg_(int *,int *,int *,int *,int *,int *,int *);
+   void arpacknonsymdbg_(int *,int *,int *,int *,int *,int *,int *);
+   void arpackcmplxdbg_(int *,int *,int *,int *,int *,int *,int *);
+}
+
+namespace mfem
+{
+
+/// Wrapper for the ARPACK routine SSAUPD or DSAUPD
+class ArPackSAUPD : public SymEigensolver, public SymGenEigensolver
+{
+public:
+
+   ArPackSAUPD();
+   virtual ~ArPackSAUPD();
+
+   /** ARPACK modes are described in section 3.5 of the ARPACK manual.
+       Mode 1: regular mode to solve A x = lambda x
+               No solver and no mass matrix are needed.
+       Mode 2: regular inverse mode to solve A x = lambda M x
+               Both A and M are needed and the solver should compute M^{-1}.
+       Mode 3: shift-invert mode to solve either A x = lambda x
+               or A x = lambda M x
+               Mass matrix is optional.  The solver should compute
+               (A-sigma I)^{-1} or (A-sigma M)^{-1}.  The shift parameter,
+               sigma, also needs to be set with SetShift().
+       Mode 4: Buckling mode to solve K x = lambda K_G x
+               K is set using SetMassMatrix(), K_G is set using SetOperator(),
+          and the solver should compute (K-sigma K_G)^{-1}.  The shift
+               parameter, sigma, also needs to be set with SetShift().
+       Mode 5: Cayley mode to solve A x = lambda M x
+               Both A and M are needed and the solver should compute
+               (A - sigma M)^{-1}.  The shift parameter, sigma, also needs
+               to be set with SetShift().
+    */
+   void SetMode(int mode);
+
+   inline void SetTol(real_t tol) override         {      tol_ = tol;      }
+   inline void SetMaxIter(int max_iter) override   { max_iter_ = max_iter; }
+   inline void SetPrintLevel(int logging) override {  logging_ = logging;  }
+   inline void SetShift(real_t sigma)              {    sigma_ = sigma;    }
+   inline void SetNumModes(int num_eigs) override  {      nev_ = num_eigs; }
+
+   virtual void SetSolver(Solver & solver);
+   virtual void SetOperator(const Operator & A) override;
+   virtual void SetMassMatrix(const Operator & M);
+   virtual void SetOperators(const Operator & A, const Operator & B) override
+   { SetOperator(A); SetMassMatrix(B); }
+
+   void Solve() override;
+
+   virtual int GetNumConverged() const override { return iparam_[4]; }
+
+   /// Collect the converged eigenvalues
+   virtual void GetEigenvalues(Array<real_t> & eigenvalues) const override;
+
+   /// Extract a single eigenvector
+   virtual const Vector & GetEigenvector(unsigned int i) const override;
+
+   /// Transfer ownership of the converged eigenvectors
+   Vector ** StealEigenvectors() override;
+
+protected:
+
+   int myid_;       // Index of this processor
+   int max_iter_;
+   int logging_;
+
+   // The following variables are for ARPACK
+   int nloc_;       // number of items stored locally
+   int nev_;        // number of requested eigenvalues
+   int ncv_;        // number of ritz vectors
+   int rvec_;       // boolean to return eigenvectors as well
+   int mode_;       // 1 = standard, 2 = generalized, 3 = shift invert,
+   // 4 = buckling, 5 = Cayley
+   int lworkl_;     // length of lworkl_ work array
+   int iparam_[12]; // arpack parameters
+   int ipntr_[12];  // arpack pointers
+
+   char bmat_;      // I for standard problem, G for generalized
+   char which_[3];  // spectrum portion: LA, SA, LM, SM, BE
+   char hwmny_;     // DSEUPD: A for all eigenvalues, S for some
+
+   real_t tol_;     // relative accuracy bound for Ritz values
+   real_t sigma_;   // eigenvalue shift parameter
+
+   int    * select_;// workspace used during eigenvalue computation
+   real_t * dv_;    // Ritz values
+   real_t * v_;     // ncv Lanczos basis vectors
+   real_t * resid_; // residual vector
+   real_t * workd_; // work array for 3 vectors used in Arnoldi iteration
+   real_t * workl_; // work array
+
+   // Operators and Vectors needed outside of ARPACK
+   Solver   * solver_;
+   const Operator * A_;
+   const Operator * B_;
+
+   Vector * w_;
+   Vector * x_;
+   Vector * y_;
+   Vector * z_;
+
+   mutable Vector ** eigenvectors_;
+
+   std::string solverName_;
+
+   void reverseComm();
+
+   int reverseCommMode1();
+   int reverseCommMode2();
+   int reverseCommMode3();
+   int reverseCommMode4();
+   int reverseCommMode5();
+
+   virtual void prepareEigenvectors() const;
+
+   void printErrors(const int & info, const int iparam[],
+                    const char & bmat, const int & n,
+                    const char which[],
+                    const int & nev, const int & ncv,
+                    const int & lworkl );
+
+private:
+
+   virtual int computeNlocf() { return nloc_; }
+   virtual int computeIter(int & ido);
+   virtual int computeEigs();
+
+};
+
+#ifdef MFEM_USE_MPI
+
+class ArPackPSAUPD : public ArPackSAUPD
+{
+public:
+   ArPackPSAUPD(MPI_Comm comm);
+   virtual ~ArPackPSAUPD() {}
+
+   void SetOperator(const Operator & A);
+   void SetMassMatrix(const Operator & M);
+
+   /// Collect the converged eigenvalues
+   void GetEigenvalues(Array<real_t> & eigenvalues) const;
+
+   /// Extract a single eigenvector
+   const Vector & GetEigenvector(unsigned int i) const;
+
+   /// Transfer ownership of the converged eigenvectors
+   // HypreParVector ** StealEigenvectors();
+   Vector ** StealEigenvectors();
+
+protected:
+
+   void prepareEigenvectors() const;
+
+private:
+
+   MPI_Comm comm_;
+   MPI_Fint commf_; // Fortran style MPI communicator
+   int numProcs_;   // Number of processors
+
+   mutable HYPRE_Int * part_; // parallel partitioning for eigenvectors
+
+   int computeNlocf();
+   int computeIter(int & ido);
+   int computeEigs();
+
+};
+
+#endif // MFEM_USE_MPI
+
+};
+
+#endif // MFEM_USE_ARPACK
+
+#endif // MFEM_ARPACK

linalg/eigensolvers.cpp

@@ -0,0 +1,20 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "linalg.hpp"
+#include "eigensolvers.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+};

linalg/eigensolvers.hpp

@@ -0,0 +1,396 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_EIGENSOLVERS
+#define MFEM_EIGENSOLVERS
+
+#include "vector.hpp"
+
+namespace mfem
+{
+
+/// Abstract Eigenequation
+/// Defines the operator of the linear eigenvalue equation
+///    A x_i = lambda_i x_i
+/// Where A is a real-valued operator, the lambda_i are the eigenvalues,
+/// and x_i are the eigenvectors.
+class Eigenequation
+{
+protected:
+   Eigenequation() = default;
+
+public:
+   virtual ~Eigenequation() = default;
+
+   /// @brief Set the operator A of the eigenvalue equation
+   virtual void SetOperator(const Operator & A) = 0;
+};
+
+/// Abstract Complex-valued Eigenequation
+/// Defines the operator of the linear eigenvalue equation
+///    A x_i = lambda_i x_i
+/// Where A is a complex-valued operator, the lambda_i are the eigenvalues,
+/// and x_i are the eigenvectors.
+class ComplexEigenequation
+{
+protected:
+   ComplexEigenequation() = default;
+
+public:
+   virtual ~ComplexEigenequation() = default;
+
+   /// @brief Set the real and imaginary parts of the operator A
+   virtual void SetOperator(const Operator & Ar, const Operator & Ai) = 0;
+};
+
+/// Abstract Generalized Eigenequation
+/// Defines the operator of the linear eigenvalue equation
+///    A x_i = lambda_i B x_i
+/// Where A and B are real-valued operators, the lambda_i are the eigenvalues,
+/// and x_i are the eigenvectors.
+class GenEigenequation
+{
+protected:
+   GenEigenequation() = default;
+
+public:
+   virtual ~GenEigenequation() = default;
+
+   /// @brief Set the operators A and B of the generalized eigenvalue equation
+   virtual void SetOperators(const Operator & A, const Operator & B) = 0;
+};
+
+/// Abstract Complex-valued Generalized Eigenequation
+/// Defines the operator of the linear eigenvalue equation
+///    A x_i = lambda_i B x_i
+/// Where A and B are complex-valued operators, the lambda_i are the
+/// eigenvalues, and x_i are the eigenvectors.
+class ComplexGenEigenequation
+{
+protected:
+   ComplexGenEigenequation() = default;
+
+public:
+   virtual ~ComplexGenEigenequation() = default;
+
+   /// @brief Set the real and imaginary parts of the operators A and B
+   virtual void SetOperators(const Operator & Ar, const Operator & Ai,
+                             const Operator & Br, const Operator & Bi) = 0;
+};
+
+/// Abstract Eigensolver
+/// Computes eigenvalue/eigenvector pairs for the linear system
+///    A x_i = lambda_i x_i
+/// Where the lambda_i are the eigenvalues and x_i are the eigenvectors.
+class EigensolverBase
+{
+protected:
+   EigensolverBase() = default;
+
+public:
+   virtual ~EigensolverBase() = default;
+
+   /// @brief Stopping criteria based on numerical tolerance
+   ///
+   /// @note This may be defined differently by different solvers.
+   virtual void SetTol(real_t tol) = 0;
+
+   /// @brief Stopping criteria based on number of iterations required to
+   /// reach convergence.
+   ///
+   /// @note This may also be defined differently in different solvers.
+   virtual void SetMaxIter(int max_iter) = 0;
+
+   /// @brief Controls the type and amount of information printed to
+   ///        standard output.
+   virtual void SetPrintLevel(int logging) = 0;
+
+   /// @brief Set the number of desired eigenmodes to compute
+   virtual void SetNumModes(int num_eigs) = 0;
+
+   /// @brief Get the number of converged eigenmodes
+   virtual int GetNumConverged() const = 0;
+
+   /// @brief Perform the eigenvalue solve
+   virtual void Solve() = 0;
+};
+
+/// Symmetric Eigensolver
+/// If A^T = A the linear system must have real-valued eigenvalues
+/// and eigenvectors.
+class SymEigensolver : public EigensolverBase, public Eigenequation
+{
+protected:
+   SymEigensolver() = default;
+
+public:
+   virtual ~SymEigensolver() = default;
+
+   /// @brief Collect the converged eigenvalues
+   ///
+   /// The length of the array should equal the number of converged eigenvalues.
+   virtual void GetEigenvalues(Array<real_t> & eigenvalues) const = 0;
+
+   /// @brief Extract a single eigenvector
+   ///
+   /// The index i should be in the range [0, numConverged). The
+   virtual const Vector & GetEigenvector(unsigned int i) const = 0;
+
+   /// @brief Transfer ownership of the converged eigenvectors
+   ///
+   /// The array should contain numConverged vectors.
+   virtual Vector ** StealEigenvectors() = 0;
+};
+
+/// Symmetric Generalized Eigensolver
+/// If A^T = A and M^T = M the linear system must have real-valued eigenvalues
+/// and eigenvectors.
+class SymGenEigensolver : public EigensolverBase, public GenEigenequation
+{
+protected:
+   SymGenEigensolver() = default;
+
+public:
+   virtual ~SymGenEigensolver() = default;
+
+   /// @brief Collect the converged eigenvalues
+   ///
+   /// The length of the array should equal the number of converged eigenvalues.
+   virtual void GetEigenvalues(Array<real_t> & eigenvalues) const = 0;
+
+   /// @brief Extract a single eigenvector
+   ///
+   /// The index i should be in the range [0, numConverged). The
+   virtual const Vector & GetEigenvector(unsigned int i) const = 0;
+
+   /// @brief Transfer ownership of the converged eigenvectors
+   ///
+   /// The array should contain numConverged vectors.
+   virtual Vector ** StealEigenvectors() = 0;
+};
+
+/// Hermetian Eigensolver
+/// If A^H = A the linear system must have real-valued eigenvalues
+/// but may have complex-valued eigenvectors.
+class HermEigensolver : public EigensolverBase, public ComplexEigenequation
+{
+protected:
+   HermEigensolver() = default;
+
+public:
+   virtual ~HermEigensolver() = default;
+
+   /// @brief Collect the converged eigenvalues
+   ///
+   /// The length of the array should equal the number of converged eigenvalues.
+   virtual void GetEigenvalues(Array<real_t> & eigenvalues) const = 0;
+
+   /// @brief Extract a single eigenvector
+   ///
+   /// The index i should be in the range [0, 2*numConverged). The
+   /// vectors corresponding to even indices are the real parts of the
+   /// converged eigenvectors and the odd indices correspond to the
+   /// imaginary parts.
+   virtual const Vector & GetEigenvector(unsigned int i) const = 0;
+
+   /// @brief Transfer ownership of the converged eigenvectors
+   ///
+   /// The array should contain 2*numConverged vectors with the even
+   /// indices corresponding to the real parts of the converged
+   /// eigenvectors and the odd indices corresponding to the imaginary
+   /// parts.
+   virtual Vector ** StealEigenvectors() = 0;
+};
+
+/// Hermetian Generalized Eigensolver
+/// If A^H = A and M^H = M the linear system must have real-valued eigenvalues
+/// but may have complex-valued eigenvectors.
+class HermGenEigensolver :
+   public EigensolverBase, public ComplexGenEigenequation
+{
+protected:
+   HermGenEigensolver() = default;
+
+public:
+   virtual ~HermGenEigensolver() = default;
+
+   /// @brief Collect the converged eigenvalues
+   ///
+   /// The length of the array should equal the number of converged eigenvalues.
+   virtual void GetEigenvalues(Array<real_t> & eigenvalues) const = 0;
+
+   /// @brief Extract a single eigenvector
+   ///
+   /// The index i should be in the range [0, 2*numConverged). The
+   /// vectors corresponding to even indices are the real parts of the
+   /// converged eigenvectors and the odd indices correspond to the
+   /// imaginary parts.
+   virtual const Vector & GetEigenvector(unsigned int i) const = 0;
+
+   /// @brief Transfer ownership of the converged eigenvectors
+   ///
+   /// The array should contain 2*numConverged vectors with the even
+   /// indices corresponding to the real parts of the converged
+   /// eigenvectors and the odd indices corresponding to the imaginary
+   /// parts.
+   virtual Vector ** StealEigenvectors() = 0;
+};
+
+/// Non-Symmetric Eigensolver
+/// For general real-valued operators A the linear system must have
+/// eigenvalues and eigenvectors which form complex conjugate pairs.
+class NonSymEigensolver : public EigensolverBase, public Eigenequation
+{
+protected:
+   NonSymEigensolver() = default;
+
+public:
+   virtual ~NonSymEigensolver() = default;
+
+   /// @brief Collect the converged eigenvalues
+   ///
+   /// The length of the array should be the number of converged
+   /// eigenvalues. The complex-valued eigenvalues can be constructed
+   /// as: lambda_{2*j} = eig[2*j]+i*eig[2*j+1] and
+   /// lambda_{2*j+1} = eig[2*j]-i*eig[2*j+1]
+   /// With j in the range [0, numConverged/2)
+   virtual void GetEigenvalues(Array<real_t> & eig) const = 0;
+
+   /// @brief Extract a single eigenvector
+   ///
+   /// The index i should be in the range [0, numConverged). The
+   /// vectors corresponding to even indices are the real parts of the
+   /// converged eigenvectors and the odd indices correspond to the
+   /// imaginary parts. If needed, the complex conjugate pairs of
+   /// eigenvectors can be constructed in the same manner described
+   /// for the eigenvalues.
+   virtual const Vector & GetEigenvector(unsigned int i) const = 0;
+
+   /// @brief Transfer ownership of the converged eigenvectors
+   ///
+   /// The array should contain numConverged vectors with the even
+   /// indices corresponding to the real parts of the converged
+   /// eigenvectors and the odd indices corresponding to the imaginary
+   /// parts.
+   virtual Vector ** StealEigenvectors() = 0;
+};
+
+/// Non-Symmetric Eigensolver
+/// For general real-valued operators A and M the linear system must have
+/// eigenvalues and eigenvectors which form complex conjugate pairs.
+class NonSymGenEigensolver : public EigensolverBase, public GenEigenequation
+{
+protected:
+   NonSymGenEigensolver() = default;
+
+public:
+   virtual ~NonSymGenEigensolver() = default;
+
+   /// @brief Collect the converged eigenvalues
+   ///
+   /// The length of the array should be the number of converged
+   /// eigenvalues. The complex-valued eigenvalues can be constructed
+   /// as: lambda_{2*j} = eig[2*j]+i*eig[2*j+1] and
+   /// lambda_{2*j+1} = eig[2*j]-i*eig[2*j+1]
+   /// With j in the range [0, numConverged/2)
+   virtual void GetEigenvalues(Array<real_t> & eig) const = 0;
+
+   /// @brief Extract a single eigenvector
+   ///
+   /// The index i should be in the range [0, numConverged). The
+   /// vectors corresponding to even indices are the real parts of the
+   /// converged eigenvectors and the odd indices correspond to the
+   /// imaginary parts. If needed, the complex conjugate pairs of
+   /// eigenvectors can be constructed in the same manner described
+   /// for the eigenvalues.
+   virtual const Vector & GetEigenvector(unsigned int i) const = 0;
+
+   /// @brief Transfer ownership of the converged eigenvectors
+   ///
+   /// The array should contain numConverged vectors with the even
+   /// indices corresponding to the real parts of the converged
+   /// eigenvectors and the odd indices corresponding to the imaginary
+   /// parts.
+   virtual Vector ** StealEigenvectors() = 0;
+};
+
+/// Complex Eigensolver
+/// Can have arbitrary complex-valued eigenvalues and eigenvectors
+class ComplexEigensolver : public EigensolverBase, public ComplexEigenequation
+{
+protected:
+   ComplexEigensolver() = default;
+
+public:
+   virtual ~ComplexEigensolver() = default;
+
+   /// @brief Collect the converged eigenvalues
+   ///
+   /// The length of the array should be twice the number of converged
+   /// eigenvalues. The complex-valued eigenvalues can be constructed
+   /// as: lambda_j = eig[2*j]+i*eig[2*j+1]
+   virtual void GetEigenvalues(Array<real_t> & eig) const = 0;
+
+   /// @brief Extract a single eigenvector
+   ///
+   /// The index i should be in the range [0, 2*numConverged). The
+   /// vectors corresponding to even indices are the real parts of the
+   /// converged eigenvectors and the odd indices correspond to the
+   /// imaginary parts.
+   virtual const Vector & GetEigenvector(unsigned int i) const = 0;
+
+   /// @brief Transfer ownership of the converged eigenvectors
+   ///
+   /// The array should contain 2*numConverged vectors with the even
+   /// indices corresponding to the real parts of the converged
+   /// eigenvectors and the odd indices corresponding to the imaginary
+   /// parts.
+   virtual Vector ** StealEigenvectors() = 0;
+};
+
+/// Complex Generalized Eigensolver
+/// Can have arbitrary complex-valued eigenvalues and eigenvectors
+class ComplexGenEigensolver :
+   public EigensolverBase, public ComplexGenEigenequation
+{
+protected:
+   ComplexGenEigensolver() = default;
+
+public:
+   virtual ~ComplexGenEigensolver() = default;
+
+   /// @brief Collect the converged eigenvalues
+   ///
+   /// The length of the array should be twice the number of converged
+   /// eigenvalues. The complex-valued eigenvalues can be constructed
+   /// as: lambda_j = eig[2*j]+i*eig[2*j+1]
+   virtual void GetEigenvalues(Array<real_t> & eig) const = 0;
+
+   /// @brief Extract a single eigenvector
+   ///
+   /// The index i should be in the range [0, 2*numConverged). The
+   /// vectors corresponding to even indices are the real parts of the
+   /// converged eigenvectors and the odd indices correspond to the
+   /// imaginary parts.
+   virtual const Vector & GetEigenvector(unsigned int i) const = 0;
+
+   /// @brief Transfer ownership of the converged eigenvectors
+   ///
+   /// The array should contain 2*numConverged vectors with the even
+   /// indices corresponding to the real parts of the converged
+   /// eigenvectors and the odd indices corresponding to the imaginary
+   /// parts.
+   virtual Vector ** StealEigenvectors() = 0;
+};
+
+}
+
+#endif

linalg/hypre.cpp

@@ -6556,7 +6556,7 @@ HypreLOBPCG::SetPreconditioner(Solver & precond)
 }
 
 void
-HypreLOBPCG::SetOperator(Operator & A)
+HypreLOBPCG::SetOperator(const Operator & A)
 {
    HYPRE_BigInt locSize = A.Width();
 
@@ -6603,7 +6603,7 @@ HypreLOBPCG::SetOperator(Operator & A)
 }
 
 void
-HypreLOBPCG::SetMassMatrix(Operator & M)
+HypreLOBPCG::SetMassMatrix(const Operator & M)
 {
    matvec_fn.MatvecCreate  = this->OperatorMatvecCreate;
    matvec_fn.Matvec        = this->OperatorMatvec;
@@ -6624,7 +6624,7 @@ HypreLOBPCG::GetEigenvalues(Array<real_t> & eigs) const
    }
 }
 
-const HypreParVector &
+const Vector &
 HypreLOBPCG::GetEigenvector(unsigned int i) const
 {
    return multi_vec->GetVector(i);
@@ -6866,6 +6866,24 @@ HypreAME::SetPreconditioner(HypreSolver & precond)
    ams_precond = &precond;
 }
 
+void
+HypreAME::SetOperators(const Operator & opA, const Operator & opB)
+{
+   const HypreParMatrix * A = dynamic_cast<const HypreParMatrix *>(&opA);
+   if (A == NULL)
+   {
+      mfem_error("HypreAME::SetOperator : first operator not HypreParMatrix!");
+   }
+   SetOperator(*A);
+
+   const HypreParMatrix * B = dynamic_cast<const HypreParMatrix *>(&opB);
+   if (B == NULL)
+   {
+      mfem_error("HypreAME::SetOperator : second operator not HypreParMatrix!");
+   }
+   SetMassMatrix(*B);
+}
+
 void
 HypreAME::SetOperator(const HypreParMatrix & A)
 {
@@ -6924,7 +6942,7 @@ HypreAME::createDummyVectors() const
    }
 }
 
-const HypreParVector &
+const Vector &
 HypreAME::GetEigenvector(unsigned int i) const
 {
    if ( eigenvectors == NULL )
@@ -6935,7 +6953,7 @@ HypreAME::GetEigenvector(unsigned int i) const
    return *eigenvectors[i];
 }
 
-HypreParVector **
+Vector **
 HypreAME::StealEigenvectors()
 {
    if ( eigenvectors == NULL )
@@ -6944,11 +6962,11 @@ HypreAME::StealEigenvectors()
    }
 
    // Set the local pointers to NULL so that they won't be deleted later
-   HypreParVector ** vecs = eigenvectors;
+   Vector ** vecs = (Vector**)eigenvectors;
    eigenvectors = NULL;
    multi_vec = NULL;
 
-   return vecs;
+   return (Vector**)vecs;
 }
 
 }

linalg/hypre.hpp

@@ -18,7 +18,9 @@
 
 #include "../general/globals.hpp"
 #include "sparsemat.hpp"
+#include "eigensolvers.hpp"
 #include "hypre_parcsr.hpp"
+#include "eigensolvers.hpp"
 #include <mpi.h>
 
 // Enable internal hypre timing routines
@@ -2146,7 +2148,7 @@ public:
     A. Knyazev, M. Argentati, I. Lashuk, and E. Ovtchinnikov, SISC, 29(5),
     2224-2239, 2007.
 */
-class HypreLOBPCG
+class HypreLOBPCG : public SymGenEigensolver
 {
 private:
    MPI_Comm comm;
@@ -2236,38 +2238,43 @@ public:
    HypreLOBPCG(MPI_Comm comm);
    ~HypreLOBPCG();
 
-   void SetTol(real_t tol);
+   void SetTol(real_t tol) override;
    // not implemented in HYPRE
    // real_t GetTol() const;
    void SetRelTol(real_t rel_tol);
    // not implemented in HYPRE
    // real_t GetRelTol() const;
-   void SetMaxIter(int max_iter);
+   void SetMaxIter(int max_iter) override;
    // not implemented in HYPRE
    // int GetMaxIter() const;
-   void SetPrintLevel(int logging);
-   void SetNumModes(int num_eigs) { nev = num_eigs; }
+   void SetPrintLevel(int logging) override;
+   void SetNumModes(int num_eigs) override { nev = num_eigs; }
    void SetPrecondUsageMode(int pcg_mode);
    void SetRandomSeed(int s) { seed = s; }
    void SetInitialVectors(int num_vecs, HypreParVector ** vecs);
 
    // The following four methods support general operators
    void SetPreconditioner(Solver & precond);
-   void SetOperator(Operator & A);
-   void SetMassMatrix(Operator & M);
+   void SetOperators(const Operator & A, const Operator & B) override
+   { SetOperator(A); SetMassMatrix(B); }
+   void SetOperator(const Operator & A);
+   void SetMassMatrix(const Operator & M);
    void SetSubSpaceProjector(Operator & proj) { subSpaceProj = &proj; }
 
    /// Solve the eigenproblem
-   void Solve();
+   void Solve() override;
+
+   int GetNumConverged() const override { return nev; }
 
    /// Collect the converged eigenvalues
-   void GetEigenvalues(Array<real_t> & eigenvalues) const;
+   void GetEigenvalues(Array<real_t> & eigenvalues) const override;
 
    /// Extract a single eigenvector
-   const HypreParVector & GetEigenvector(unsigned int i) const;
+   const Vector & GetEigenvector(unsigned int i) const override;
 
    /// Transfer ownership of the converged eigenvectors
-   HypreParVector ** StealEigenvectors() { return multi_vec->StealVectors(); }
+   Vector ** StealEigenvectors() override
+   { return (Vector**)multi_vec->StealVectors(); }
 };
 
 /** AME eigenvalue solver in hypre
@@ -2292,7 +2299,7 @@ public:
     mass matrix but it seems unlikely that this would be useful so it is not the
     default behavior.
 */
-class HypreAME
+class HypreAME : public SymGenEigensolver
 {
 private:
    int myid;
@@ -2321,28 +2328,31 @@ public:
    HypreAME(MPI_Comm comm);
    ~HypreAME();
 
-   void SetTol(real_t tol);
+   void SetTol(real_t tol) override;
    void SetRelTol(real_t rel_tol);
-   void SetMaxIter(int max_iter);
-   void SetPrintLevel(int logging);
-   void SetNumModes(int num_eigs);
+   void SetMaxIter(int max_iter) override;
+   void SetPrintLevel(int logging) override;
+   void SetNumModes(int num_eigs) override;
 
    // The following four methods support operators of type HypreParMatrix.
    void SetPreconditioner(HypreSolver & precond);
+   void SetOperators(const Operator & opA, const Operator & opB) override;
    void SetOperator(const HypreParMatrix & A);
    void SetMassMatrix(const HypreParMatrix & M);
 
    /// Solve the eigenproblem
-   void Solve();
+   void Solve() override;
+
+   int GetNumConverged() const override { return nev; }
 
    /// Collect the converged eigenvalues
-   void GetEigenvalues(Array<real_t> & eigenvalues) const;
+   void GetEigenvalues(Array<real_t> & eigenvalues) const override;
 
    /// Extract a single eigenvector
-   const HypreParVector & GetEigenvector(unsigned int i) const;
+   const Vector & GetEigenvector(unsigned int i) const override;
 
    /// Transfer ownership of the converged eigenvectors
-   HypreParVector ** StealEigenvectors();
+   Vector ** StealEigenvectors() override;
 };
 
 }

linalg/linalg.hpp

@@ -15,7 +15,6 @@
 // Linear algebra header file
 
 #include "vector.hpp"
-#include "multivector.hpp"
 #include "operator.hpp"
 #include "matrix.hpp"
 #include "sparsemat.hpp"
@@ -29,6 +28,7 @@
 #include "symmat.hpp"
 #include "ode.hpp"
 #include "solvers.hpp"
+#include "eigensolvers.hpp"
 #include "handle.hpp"
 #include "invariants.hpp"
 #include "constraints.hpp"
@@ -38,7 +38,6 @@
 #include "batched/gpu_blas.hpp"
 #include "batched/solver.hpp"
 #include "tensor.hpp"
-#include "tensor_arrays.hpp"
 #include "filteredsolver.hpp"
 #include "ordering.hpp"
 #include "particlevector.hpp"
@@ -59,6 +58,10 @@
 #include "ginkgo.hpp"
 #endif
 
+#ifdef MFEM_USE_ARPACK
+#include "arpack.hpp"
+#endif
+
 #ifdef MFEM_USE_MKL_PARDISO
 #include "pardiso.hpp"
 #endif

linalg/multivector.cpp

@@ -1,60 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "multivector.hpp"
-
-namespace mfem
-{
-
-MultiVector::MultiVector(const Array<int> &vector_sizes)
-{
-   SetSizes(vector_sizes);
-}
-
-MultiVector::MultiVector(const Array<int> &vector_sizes, MemoryType mt)
-{
-   SetSizes(vector_sizes, mt);
-}
-
-MultiVector::MultiVector(Vector &base, const Array<int> &vector_sizes)
-{
-   MakeRef(base, vector_sizes);
-}
-
-void MultiVector::SetSizes(const Array<int> &vector_sizes)
-{
-   blocks.resize(vector_sizes.Size());
-   for (int i = 0; i < vector_sizes.Size(); i++)
-   {
-      operator[](i).SetSize(vector_sizes[i]);
-   }
-}
-
-void MultiVector::SetSizes(const Array<int> &vector_sizes, MemoryType mt)
-{
-   blocks.resize(vector_sizes.Size());
-   for (int i = 0; i < vector_sizes.Size(); i++)
-   {
-      operator[](i).SetSize(vector_sizes[i], mt);
-   }
-}
-
-void MultiVector::MakeRef(Vector &base, const Array<int> &vector_sizes)
-{
-   blocks.resize(vector_sizes.Size());
-   for (int offset = 0, i = 0; i < vector_sizes.Size(); i++)
-   {
-      blocks[i].emplace<0>(base, offset, vector_sizes[i]);
-      offset += vector_sizes[i];
-   }
-}
-
-} // namespace mfem

linalg/multivector.hpp

@@ -1,198 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_MULTIVECTOR_HPP
-#define MFEM_MULTIVECTOR_HPP
-
-#include "../general/array.hpp"
-#include "vector.hpp"
-#include <vector>
-#include <array>
-#include <variant>
-
-namespace mfem
-{
-
-/// Class representing an array of Vectors with generally different sizes.
-/** This class is similar to BlockVector with the following two main
-    differences:
-    - the data for the individual Vector blocks does not need to be part of one
-      big contiguous memory allocation;
-    - this class does not inherit from class Vector (as a consequence of the
-      first bullet).
-
-    Internally, each Vector block is represented as either:
-    - (default) a Vector object constructed and owned by this class; this
-      object, in turn, as any Vector object, can own its Memory allocation or
-      refer to a sub-Memory of another Memory object; or
-    - a pointer to an externally allocated Vector or classes derived from
-      Vector. */
-class MultiVector
-{
-private:
-   std::vector<std::variant<Vector,Vector*>> blocks;
-
-public:
-   /// Create an empty MultiVector with zero blocks.
-   MultiVector() = default;
-
-   /** @brief Create a MultiVector with @a num_blocks blocks. The individual
-       Vector blocks are default initialized, i.e. they all have size zero. */
-   MultiVector(int num_blocks)
-      : blocks(num_blocks) { }
-
-   /** @brief Construct a MultiVector with number of blocks and individual block
-       Vector sizes given by @a vector_sizes.
-
-       @note The memory of the individual Vector blocks is NOT initialized. */
-   MultiVector(const Array<int> &vector_sizes);
-
-   /** @brief Construct a MultiVector with number of blocks and individual block
-       Vector sizes given by @a vector_sizes. All Vector blocks use the
-       MemoryType @a mt.
-
-       @note The memory of the individual Vector blocks is NOT initialized. */
-   MultiVector(const Array<int> &vector_sizes, MemoryType mt);
-
-   /** @brief Construct a MultiVector referencing data within a given monolithic
-       Vector @a base.
-
-       With this constructor, the Memory flags of @a base and of the individual
-       Vector blocks may need to be explicitly synchronized when data is moved
-       between host and device. */
-   MultiVector(Vector &base, const Array<int> &vector_sizes);
-
-   /** @brief Construct a MultiVector referencing multiple Vectors given as
-       arguments.
-
-       The VectorTypes reference arguments are expected to be static_cast-able
-       to (Vector &) which is the case if the types are derived from Vector,
-       e.g. HypreParVector, GridFunction, etc.
-
-       With this constructor, operations on individual Vector blocks are
-       performed directly on the objects @a vs. In particular, there is no need
-       to synchronize the Memory flags of @a vs and the ones of the individual
-       Vector blocks when data is moved between host and device. */
-   template <typename... VectorTypes,
-             std::enable_if_t<
-                std::conjunction_v<
-                   std::is_convertible<VectorTypes&,Vector&>...>, bool> = true>
-   MultiVector(VectorTypes &...vs) { MakeRef(vs...); }
-
-   /// Return the number of Vectors in the MultiVector.
-   int NumBlocks() const { return blocks.size(); }
-
-   /** @brief Set the number of Vectors in the MultiVector. Existing Vector
-       blocks will remain unmodified. New Vector blocks will be default
-       initialized, i.e. they all have size zero. */
-   void SetNumBlocks(int num_blocks) { blocks.resize(num_blocks); }
-
-   /// Read-write access to the i-th Vector.
-   inline Vector &operator[](int i);
-
-   /// Read-only access to the i-th Vector.
-   inline const Vector &operator[](int i) const;
-
-   /** @brief Update the MultiVector according to the given @a vector_sizes.
-
-       This method can be used to add or remove blocks. The individual Vector
-       sizes are updated using the method Vector::SetSize(int). */
-   void SetSizes(const Array<int> &vector_sizes);
-
-   /** @brief Update the MultiVector according to the given @a vector_sizes and
-       MemoryType @a mt.
-
-       This method can be used to add or remove blocks. The individual Vector
-       sizes and MemoryType are updated using the method
-       Vector::SetSize(int, MemoryType). */
-   void SetSizes(const Array<int> &vector_sizes, MemoryType mt);
-
-   /** @brief Update the MultiVector to reference data within a given monolithic
-       Vector @a base.
-
-       After calling this method, the Memory flags of @a base and of the
-       individual Vector blocks may need to be explicitly synchronized when data
-       is moved between host and device.*/
-   void MakeRef(Vector &base, const Array<int> &vector_sizes);
-
-   /** @brief Update the @a i-th MultiVector block to reference data within the
-       given monolithic Vector @a base at the given @a offset and with the given
-       @a size.
-
-       After calling this method, the Memory flags of @a base and of the @a i-th
-       Vector block may need to be explicitly synchronized when data is moved
-       between host and device.*/
-   inline void MakeRef(int i, Vector &base, int offset, int size)
-   {
-      blocks[i].emplace<0>(base, offset, size);
-   }
-
-   /** @brief Update the MultiVector to reference multiple Vectors given as
-       arguments.
-
-       The VectorTypes reference arguments are expected to be static_cast-able
-       to (Vector &) which is the case if the types are derived from Vector,
-       e.g. HypreParVector, GridFunction, etc.
-
-       After calling this method, operations on individual Vector blocks are
-       performed directly on the objects @a vs. In particular, there is no need
-       to synchronize the Memory flags of @a vs and the ones of the individual
-       Vector blocks when data is moved between host and device. */
-   template <typename... VectorTypes,
-             std::enable_if_t<
-                std::conjunction_v<
-                   std::is_convertible<VectorTypes&,Vector&>...>, bool> = true>
-   inline void MakeRef(VectorTypes &...vs);
-
-   /** @brief Update the @a i-th MultiVector block to reference the given
-       Vector @a v.
-
-       After calling this method, operations on the @a i-th Vector block are
-       performed directly on the Vector @a v. In particular, there is no need
-       to synchronize the Memory flags of @a v and the ones of the @a i-th
-       Vector blocks when data is moved between host and device. */
-   inline void MakeRef(int i, Vector &v) { blocks[i] = &v; }
-};
-
-// Inline and template methods
-
-inline Vector &MultiVector::operator[](int i)
-{
-   auto &bi = blocks[i];
-   return (bi.index() == 0) ? std::get<0>(bi) : *std::get<1>(bi);
-}
-
-inline const Vector &MultiVector::operator[](int i) const
-{
-   auto &bi = blocks[i];
-   return (bi.index() == 0) ? std::get<0>(bi) : *std::get<1>(bi);
-}
-
-template <typename... VectorTypes,
-          std::enable_if_t<
-             std::conjunction_v<
-                std::is_convertible<VectorTypes&,Vector&>...>, bool>>
-inline void MultiVector::MakeRef(VectorTypes &...vs)
-{
-   blocks.resize(sizeof...(vs));
-   if constexpr (sizeof...(vs) > 0)
-   {
-      const std::array vs_p{&static_cast<Vector&>(vs)...};
-      for (std::size_t i = 0; i < sizeof...(vs); i++)
-      {
-         blocks[i] = vs_p[i];
-      }
-   }
-}
-
-} // namespace mfem
-
-#endif // MFEM_MULTIVECTOR_HPP

linalg/operator.cpp

@@ -111,16 +111,6 @@ void Operator::ArrayAddMultTranspose(const Array<const Vector *> &X,
    }
 }
 
-void Operator::Mult(const MultiVector &, MultiVector &)
-{
-   MFEM_ABORT("this method is not overriden for this class!");
-}
-
-Operator &Operator::GetGradient(const MultiVector &) const
-{
-   MFEM_ABORT("this method is not overriden for this class!");
-}
-
 void Operator::FormLinearSystem(const Array<int> &ess_tdof_list,
                                 Vector &x, Vector &b,
                                 Operator* &Aout, Vector &X, Vector &B,

linalg/operator.hpp

@@ -13,7 +13,6 @@
 #define MFEM_OPERATOR
 
 #include "vector.hpp"
-#include "multivector.hpp"
 
 namespace mfem
 {
@@ -130,16 +129,6 @@ public:
    virtual void ArrayAddMultTranspose(const Array<const Vector *> &X,
                                       Array<Vector *> &Y, const real_t a = 1.0) const;
 
-   /** @brief Operator application, y = A(x), where the input @a x and the
-       output @a y are MultiVector objects, i.e. they generally use
-       non-contiguous memory representation.
-
-       The typical use case for this method are block operators like
-       DifferentiableOperator.
-
-       The base class implementation for the method is to generate an error. */
-   virtual void Mult(const MultiVector &x, MultiVector &y);
-
    /** @brief Evaluate the gradient operator at the point @a x. The default
        behavior in class Operator is to generate an error. */
    virtual Operator &GetGradient(const Vector &x) const
@@ -148,16 +137,6 @@ public:
       return const_cast<Operator &>(*this);
    }
 
-   /** @brief Evaluate the gradient operator at the point @a x. The input @a x
-       is provided as a MultiVector, i.e. it generally uses non-contiguous
-       memory representation.
-
-       The typical use case for this method are block operators like
-       DifferentiableOperator.
-
-       The base class implementation for the method is to generate an error. */
-   virtual Operator &GetGradient(const MultiVector &x) const;
-
    /** @brief Computes the diagonal entries into @a diag. Typically, this
        operation only makes sense for linear Operator%s. In some cases, only an
        approximation of the diagonal is computed. */
@@ -865,6 +844,22 @@ public:
 };
 
 
+/// Zero Operator N: x -> 0.
+class ZeroOperator : public Operator
+{
+public:
+   /// Create an zero operator of size @a n.
+   explicit ZeroOperator(int n) : Operator(n) { }
+
+   /// Operator application
+   void Mult(const Vector &x, Vector &y) const override
+   { y.SetSize(width); y = 0_r; }
+
+   /// Application of the transpose
+   void MultTranspose(const Vector &x, Vector &y) const override
+   { y.SetSize(width); y = 0_r; }
+};
+
 /// Identity Operator I: x -> x.
 class IdentityOperator : public Operator
 {

linalg/tensor.hpp

@@ -33,62 +33,17 @@ struct tensor;
 /// The implementation can be drastically generalized by using concepts of the
 /// c++17 standard.
 
-template <typename T>
+template < typename T >
 struct tensor<T>
 {
    using type = T;
    static constexpr int ndim      = 1;
    static constexpr int first_dim = 0;
-
-   MFEM_HOST_DEVICE tensor() = default;
-   MFEM_HOST_DEVICE tensor(T v) : values(v) {}
-
-   MFEM_HOST_DEVICE T& operator[](int) { return values; }
-   MFEM_HOST_DEVICE const T& operator[](int) const { return values; }
-   MFEM_HOST_DEVICE T& operator()(int) { return values; }
-   MFEM_HOST_DEVICE const T& operator()(int) const { return values; }
-   MFEM_HOST_DEVICE T& operator()() { return values; }
-   MFEM_HOST_DEVICE const T& operator()() const { return values; }
-
+   MFEM_HOST_DEVICE T& operator[](int /*unused*/) { return values; }
+   MFEM_HOST_DEVICE const T& operator[](int /*unused*/) const { return values; }
+   MFEM_HOST_DEVICE T& operator()(int /*unused*/) { return values; }
+   MFEM_HOST_DEVICE const T& operator()(int /*unused*/) const { return values; }
    MFEM_HOST_DEVICE operator T() const { return values; }
-
-   MFEM_HOST_DEVICE constexpr const T& scalar() const { return values; }
-
-   // A * tensor<T>  ->  A * T
-   template <typename A,
-             std::enable_if_t<!std::is_same_v<std::decay_t<A>, tensor>, int> = 0>
-   MFEM_HOST_DEVICE friend auto operator*(const A& a, const tensor& s)
-   -> decltype(a * s.scalar())
-   {
-      return a * s.scalar();
-   }
-
-   // tensor<T> * A  ->  T * A
-   template <typename A,
-             std::enable_if_t<!std::is_same_v<std::decay_t<A>, tensor>, int> = 0>
-   MFEM_HOST_DEVICE friend auto operator*(const tensor& s, const A& a)
-   -> decltype(s.scalar() * a)
-   {
-      return s.scalar() * a;
-   }
-
-   // A / tensor<T>, tensor<T> / A
-   template <typename A,
-             std::enable_if_t<!std::is_same_v<std::decay_t<A>, tensor>, int> = 0>
-   MFEM_HOST_DEVICE friend auto operator/(const A& a, const tensor& s)
-   -> decltype(a / s.scalar())
-   {
-      return a / s.scalar();
-   }
-
-   template <typename A,
-             std::enable_if_t<!std::is_same_v<std::decay_t<A>, tensor>, int> = 0>
-   MFEM_HOST_DEVICE friend auto operator/(const tensor& s, const A& a)
-   -> decltype(s.scalar() / a)
-   {
-      return s.scalar() / a;
-   }
-
    T values;
 };
 
@@ -165,23 +120,6 @@ struct tensor<T, n0, n1, n2>
    tensor < T, n1, n2 > values[n0];
 };
 
-template < typename T, int n1, int n2 >
-struct tensor<T, 0, n1, n2>
-{
-   using type = T;
-   static constexpr int ndim      = 3;
-   static constexpr int first_dim = 0;
-   MFEM_HOST_DEVICE tensor< T, n1, n2 >& operator[](int /*i*/) { return values; }
-   MFEM_HOST_DEVICE const tensor< T, n1, n2 >& operator[](int /*i*/) const { return values; }
-   MFEM_HOST_DEVICE tensor< T, n1, n2 >& operator()(int /*i*/) { return values; }
-   MFEM_HOST_DEVICE const tensor< T, n1, n2 >& operator()(int /*i*/) const { return values; }
-   MFEM_HOST_DEVICE tensor< T, n2 >& operator()(int /*i*/, int j) { return values[j]; }
-   MFEM_HOST_DEVICE const tensor< T, n2 >& operator()(int i, int j) const { return values[i][j]; }
-   MFEM_HOST_DEVICE T& operator()(int /*i*/, int j, int k) { return values[j][k]; }
-   MFEM_HOST_DEVICE const T& operator()(int /*i*/, int j, int k) const { return values[j][k]; }
-   tensor < T, n1, n2 > values;
-};
-
 template < typename T, int n0, int n1, int n2, int n3 >
 struct tensor<T, n0, n1, n2, n3>
 {
@@ -201,26 +139,6 @@ struct tensor<T, n0, n1, n2, n3>
    tensor < T, n1, n2, n3 > values[n0];
 };
 
-template < typename T, int n1, int n2, int n3 >
-struct tensor<T, 0, n1, n2, n3>
-{
-   using type = T;
-   static constexpr int ndim      = 4;
-   static constexpr int first_dim = 0;
-   MFEM_HOST_DEVICE tensor< T, n1, n2, n3 >& operator[](int /*i*/) { return values; }
-   MFEM_HOST_DEVICE const tensor< T, n1, n2, n3 >& operator[](int /*i*/) const { return values; }
-   MFEM_HOST_DEVICE tensor< T, n1, n2, n3 >& operator()(int /*i*/) { return values; }
-   MFEM_HOST_DEVICE const tensor< T, n1, n2, n3 >& operator()(int /*i*/) const { return values; }
-   MFEM_HOST_DEVICE tensor< T, n2, n3 >& operator()(int /*i*/, int j) { return values[j]; }
-   MFEM_HOST_DEVICE const tensor< T, n2, n3 >& operator()(int /*i*/, int j) const { return values[j]; }
-   MFEM_HOST_DEVICE tensor< T, n3 >& operator()(int /*i*/, int j, int k) { return values[j][k]; }
-   MFEM_HOST_DEVICE const tensor< T, n3 >& operator()(int /*i*/, int j,
-                                                      int k) const { return values[j][k]; }
-   MFEM_HOST_DEVICE T& operator()(int /*i*/, int j, int k, int l) { return values[j][k][l]; }
-   MFEM_HOST_DEVICE const T&  operator()(int /*i*/, int j, int k, int l) const { return values[j][k][l]; }
-   tensor < T, n1, n2, n3 > values;
-};
-
 template < typename T, int n0, int n1, int n2, int n3, int n4 >
 struct tensor<T, n0, n1, n2, n3, n4>
 {
@@ -623,21 +541,13 @@ MFEM_HOST_DEVICE auto operator*(S scale, const tensor<T, n...>& A) ->
 tensor<decltype(S {} * T{}), n...>
 {
    tensor<decltype(S{} * T{}), n...> C{};
-   if constexpr (sizeof...(n) == 0)
+   for (int i = 0; i < tensor<T, n...>::first_dim; i++)
    {
-      C.values = scale * A.values;
-   }
-   else
-   {
-      for (int i = 0; i < tensor<T, n...>::first_dim; i++)
-      {
-         C[i] = scale * A[i];
-      }
+      C[i] = scale * A[i];
    }
    return C;
 }
 
-
 /**
  * @brief multiply a tensor by a scalar value
  * @tparam S the scalar value type. Must be arithmetic (e.g. float, real_t, int) or a dual number
@@ -653,16 +563,9 @@ MFEM_HOST_DEVICE auto operator*(const tensor<T, n...>& A, S scale) ->
 tensor<decltype(T {} * S{}), n...>
 {
    tensor<decltype(T{} * S{}), n...> C{};
-   if constexpr (sizeof...(n) == 0)
+   for (int i = 0; i < tensor<T, n...>::first_dim; i++)
    {
-      C.values = A.values * scale;
-   }
-   else
-   {
-      for (int i = 0; i < tensor<T, n...>::first_dim; i++)
-      {
-         C[i] = A[i] * scale;
-      }
+      C[i] = A[i] * scale;
    }
    return C;
 }

linalg/tensor_arrays.hpp

@@ -1,348 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_TENSOR_ARRAYS_HPP
-#define MFEM_TENSOR_ARRAYS_HPP
-
-#include "tensor.hpp"
-#include <array>       // std::array, std::size_t (indirectly)
-#include <type_traits> /* std::remove_cv_t, std::remove_reference_t,
-                          std::is_const_v */
-#include <utility>     /* std::forward, std::index_sequence,
-                          std::make_index_sequence */
-#include <algorithm>   // std::min
-#include <tuple>       // std::apply, std::tuple_size_v
-#include <numeric>     // std::iota
-
-namespace mfem
-{
-
-namespace future
-{
-
-template <std::size_t... Is, typename Fn>
-constexpr inline void for_unrolled_simple(std::index_sequence<Is...>, Fn &&fn)
-{
-   (fn(Is), ...);
-}
-
-
-template <int... loop_sizes>
-__attribute__((annotate("enzyme_inactive")))
-constexpr inline auto to_multiindex(std::size_t i)
-{
-   constexpr auto dims = sizeof...(loop_sizes);
-   constexpr std::array<std::size_t,dims> sizes{loop_sizes...};
-   std::array<std::size_t,dims> is{};  // value initialization with zeros
-   for (std::size_t d = 0; d < dims; d++)
-   {
-      is[d] = i%sizes[d];
-      i /= sizes[d];
-   }
-   return is;
-}
-
-
-/// lambda_t:
-/// - input: const std::array<std::size_t,sizeof...(loop_sizes)> &
-/// - output: void
-/// Note: 0D loop executes the lambda one time with an array of dim 0.
-template <int... loop_sizes, typename lambda_t>
-constexpr inline void for_multiindex(lambda_t f)
-{
-   constexpr auto dims = sizeof...(loop_sizes);
-   if constexpr (dims == 0)
-   {
-      f(std::array<std::size_t,0> {});
-   }
-   else
-   {
-      if constexpr (std::min({loop_sizes...}) <= 0) { return; }
-      constexpr auto total_loop_size = (loop_sizes * ...);
-      for_unrolled_simple(std::make_index_sequence<total_loop_size> {},
-                          [&f](std::size_t i)
-      {
-         f(to_multiindex<loop_sizes...>(i));
-      });
-   }
-}
-
-
-/// Extend std::apply to work with 0-size arrays.
-template <typename Fn, typename Tuple>
-inline constexpr decltype(auto) apply(Fn&& f, Tuple&& t)
-{
-   if constexpr (std::tuple_size_v<std::remove_reference_t<Tuple>> == 0)
-   { return f(); }
-   return std::apply(std::forward<Fn>(f), std::forward<Tuple>(t));
-}
-
-
-/// Multi-dimensional array of tensors of the same size.
-/** The array sizes are dynamic while the tensor sizes are static, i.e. template
-    parameters.
-
-    This class provides flexible global data layout where the dynamic (array)
-    dimnsions and the tensor dimnsions are stored in memory using a runtime
-    defined strided layout. */
-template <typename scalar_t, int ndims, int... tensor_sizes>
-class tensor_ndarray
-{
-public:
-   typedef scalar_t scalar_type;
-   typedef tensor<std::remove_cv_t<scalar_t>,tensor_sizes...> tensor_type;
-
-   static constexpr std::integer_sequence<size_t, tensor_sizes...> tensor_sizes_;
-   static constexpr auto tensor_dims = sizeof...(tensor_sizes);
-   static constexpr auto total_dims = ndims + tensor_dims;
-   static constexpr std::array<std::size_t,tensor_dims>
-   tensor_sizes_array{tensor_sizes...};
-
-private:
-   scalar_t *data;  /// Not owned
-   std::array<std::size_t,ndims> dyn_sizes;
-   std::array<std::size_t,total_dims> strides;
-
-public:
-   /** @brief Constructor with the default, column-major or left, layout where
-       the dynamic dimensions are first, on the left, and the tensor dimensions
-       are second. */
-   tensor_ndarray(scalar_t *ptr, std::array<std::size_t,ndims> dynamic_sizes)
-      : data(ptr), dyn_sizes(dynamic_sizes)
-   {
-      std::array<std::size_t,total_dims> default_perm;
-      std::iota(default_perm.begin(), default_perm.end(), 0); // 0, 1, 2, ...
-      set_layout(default_perm);
-   }
-
-   /// Number of dynamic array dimensions.
-   static constexpr std::size_t rank() { return ndims; }
-
-   /// Array size in the @a k-th dynamic dimension.
-   std::size_t size(int k = 0) const { return dyn_sizes[k]; }
-
-   /// Returns the product of all sizes of the dynamic dimensions.
-   std::size_t total_size() const
-   {
-      std::size_t t = 1;
-      for (int d = 0; d < ndims; d++)
-      {
-         t *= dyn_sizes[d];
-      }
-      return t;
-   }
-
-   /// Number of tensor (static) dimensions.
-   static constexpr std::size_t tensor_rank()
-   { return sizeof...(tensor_sizes); }
-
-   /// Tensor size in the @a k-th tensor (static) dimension.
-   static constexpr std::size_t tensor_size(int k = 0)
-   { return tensor_sizes_array[k]; }
-
-   /// Returns the product of all sizes of the static (tensor) dimensions.
-   static constexpr std::size_t total_tensor_size()
-   { return (tensor_sizes * ...); }
-
-   /// Set the global data layout based on the given permutation @a perm.
-   /** The entries of @a perm are numbers identifying either a dynamic or a
-       tensor (static) dimension. Values in the range [0,rank()) identify the
-       dynamic dimensions and values in the range [rank(),rank()+tensor_rank())
-       identify the tensor dymensions. The first entry in @a perm determines
-       which dynamic or tensor dimension will have stride 1. The k-th entry of
-       @a perm determines which dimension will use the next stride which is
-       defined as the product of the sizes of all k-1 previous dimensions from
-       @a perm.
-
-       @note The default layout corresponds to the identity permutation:
-       { 0, 1, ..., rank()+tensor_rank()-1 }.
-
-       @note This method does not permute the global 1D data array. */
-   void set_layout(std::array<std::size_t,rank()+tensor_rank()> perm)
-   {
-      std::size_t stride = 1;
-      for (std::size_t d_g = 0; d_g < total_dims; d_g++)
-      {
-         const auto d_l = perm[d_g];
-         strides[d_l] = stride;
-         stride *= (d_l < ndims) ? dyn_sizes[d_l] :
-                   tensor_sizes_array[d_l-ndims];
-      }
-   }
-
-   /** @brief Comute the dynamic offset for a given dynamic multi-index @a is.
-       The total offset in the global data array is the sum of the dynamic and
-       static (tensor) offsets. */
-   std::size_t get_dynamic_offset(
-      const std::array<std::size_t,rank()> &is) const
-   {
-      std::size_t dynamic_offset = 0;
-      for (std::size_t d = 0; d < ndims; d++)
-      {
-         dynamic_offset += is[d]*strides[d];
-      }
-      return dynamic_offset;
-   }
-
-   /** @brief Comute the static (tensor) offset for a given tensor multi-index
-       @a js. The total offset in the global data array is the sum of the
-       dynamic and static (tensor) offsets. */
-   std::size_t get_static_offset(
-      const std::array<std::size_t,tensor_rank()> &js) const
-   {
-      std::size_t static_offset = 0;
-      for (std::size_t d = 0; d < tensor_dims; d++)
-      {
-         static_offset += js[d]*strides[ndims+d];
-      }
-      return static_offset;
-   }
-
-   /** @brief Return a local tensor extracted from the global data array
-       corresponding to the given dynamic multi-index @a is. */
-   /** @note Return a const tensor to prevent attempts to assign to the
-       temporary object which is considered a mistake. */
-   const tensor_type get_tensor(std::array<std::size_t,rank()> is) const
-   {
-      tensor_type result;
-      const std::size_t dynamic_offset = get_dynamic_offset(is);
-      for_multiindex<tensor_sizes...>(
-         [&result, this, dynamic_offset](
-            const std::array<std::size_t,tensor_rank()> &js)
-      {
-         ::mfem::future::apply(result, js) =
-            data[dynamic_offset + get_static_offset(js)];
-      });
-      return result;
-   }
-
-   /** @brief Return a local tensor extracted from the global data array
-       corresponding to the given dynamic indices @a is. */
-   /** @note Return a const tensor to prevent attempts to assign to the
-       temporary object which is considered a mistake. */
-   template <typename... index_types>
-   const tensor_type get_tensor(index_types... is) const
-   {
-      static_assert(sizeof...(is) == rank(), "invalid number of indices!");
-      return get_tensor(std::array<std::size_t,rank()> {std::size_t(is)...});
-   }
-
-   /** @brief Returns one of the following depending on the type scalar_t:
-       - get_tensor(std::array<std::size_t,rank()>) iff scalar_t is const,
-       - get_accessor(std::array<std::size_t,rank()>) iff scalar_t is not
-         const. */
-   decltype(auto) operator()(std::array<std::size_t,rank()> is) const
-   {
-      if constexpr (std::is_const_v<scalar_t>) { return get_tensor(is); }
-      else { return get_accessor(is); }
-   }
-
-   /** @brief Returns one of the following depending on the type scalar_t:
-       - get_tensor(index_types...) iff scalar_t is const,
-       - get_accessor(index_types...) iff scalar_t is not const. */
-   template <typename... index_types>
-   decltype(auto) operator()(index_types... is) const
-   {
-      if constexpr (std::is_const_v<scalar_t>) { return get_tensor(is...); }
-      else { return get_accessor(is...); }
-   }
-
-   /** @brief Helper class facilitating the reading/writing of local tensor
-       objects to the global data array of the tensor_ndarray. */
-   class tensor_accessor
-   {
-   private:
-      const tensor_ndarray &base_array;
-      scalar_t *offset_data;  /// Not owned
-
-   public:
-      /** @brief Construct a tensor_accessor to @a base for the given dynamic
-          multi-index @a is.
-
-          During its life time, this object assumes that the @a base object
-          remains unmodified. */
-      tensor_accessor(const tensor_ndarray &base,
-                      const std::array<std::size_t,rank()> &is)
-         : base_array(base)
-      {
-         offset_data = base_array.data + base_array.get_dynamic_offset(is);
-      }
-
-      /// Read-write access to a particular entry of the referenced tensor.
-      /** The returned reference points to the corresponding entry in the global
-          data array of the base tensor_ndarray. */
-      scalar_t &operator()(const std::array<std::size_t,tensor_rank()> &js)
-      {
-         return offset_data[base_array.get_static_offset(js)];
-      }
-
-      /** @brief Write a tensor to the referenced tensor in the global data
-          array of the base tensor_ndarray. */
-      tensor_accessor &operator=(const tensor_type &rhs)
-      {
-         for_multiindex<tensor_sizes...>(
-            [&](const std::array<std::size_t,tensor_rank()> &js)
-         {
-            operator()(js) = ::mfem::future::apply(rhs, js);
-         });
-         return *this;
-      }
-   };
-
-   /** @brief Get a tensor_accessor object referencing the tensor stored at the
-       dynamic multi-index @a is. This object can be used to write tensor
-       objects into the global data array of the tensor_ndarray. */
-   tensor_accessor get_accessor(std::array<std::size_t,rank()> is) const
-   {
-      return tensor_accessor(*this, is);
-   }
-
-   /** @brief Get a tensor_accessor object referencing the tensor stored at the
-       dynamic indices @a is. This object can be used to write tensor objects
-       into the global data array of the tensor_ndarray. */
-   template <typename... index_types>
-   tensor_accessor get_accessor(index_types... is) const
-   {
-      static_assert(sizeof...(is) == rank(), "invalid number of indices!");
-      return get_accessor(std::array<std::size_t,rank()> {std::size_t(is)...});
-   }
-};
-
-
-/** @brief Construct a tensor_ndarray where only the tensor sizes have to be
-    explicitly given as template parameters, the rest can be deduced from the
-    function call arguments. */
-template <int... tensor_sizes, typename scalar_t, typename... dyn_sizes_t>
-decltype(auto) make_tensor_ndarray(scalar_t *ptr, dyn_sizes_t... dynamic_sizes)
-{
-   return tensor_ndarray<scalar_t,sizeof...(dynamic_sizes),tensor_sizes...>(
-             ptr, {std::size_t(dynamic_sizes)...});
-}
-
-
-/// Alias for make_tensor_ndarray = make_tensor_array.
-template <int... tensor_sizes, typename scalar_t, typename... dyn_sizes_t>
-decltype(auto) make_tensor_array(scalar_t *ptr, dyn_sizes_t... dynamic_sizes)
-{
-   return tensor_ndarray<scalar_t,sizeof...(dynamic_sizes),tensor_sizes...>(
-             ptr, {std::size_t(dynamic_sizes)...});
-}
-
-
-/// Short name for one-dimensional tensor_ndarray: tensor_array.
-template <typename scalar_t, int... tensor_sizes>
-using tensor_array = tensor_ndarray<scalar_t, 1, tensor_sizes...>;
-
-} // namespace mfem::future
-
-} // namespace mfem
-
-#endif // MFEM_TENSOR_ARRAYS_HPP

makefile

@@ -302,7 +302,7 @@ endif
 MFEM_REQ_LIB_DEPS = SUPERLU MUMPS METIS FMS CONDUIT SIDRE LAPACK SUNDIALS\
  SUITESPARSE STRUMPACK GINKGO GNUTLS HDF5 NETCDF SLEPC PETSC MPFR PUMI HIOP\
  GSLIB OCCA CEED RAJA UMPIRE MKL_CPARDISO MKL_PARDISO AMGX MAGMA CALIPER PARELAG\
- TRIBOL BENCHMARK MOONOLITH ALGOIM
+ TRIBOL BENCHMARK MOONOLITH ALGOIM ARPACK
 
 
 PETSC_ERROR_MSG = $(if $(PETSC_FOUND),,. PETSC config not found: $(PETSC_VARS))
@@ -371,7 +371,8 @@ MFEM_DEFINES = MFEM_VERSION MFEM_VERSION_STRING MFEM_GIT_STRING MFEM_USE_MPI\
  MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO MFEM_USE_AMGX\
  MFEM_USE_MAGMA MFEM_USE_MUMPS MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_CALIPER\
  MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_TRIBOL MFEM_USE_ALGOIM MFEM_USE_ENZYME\
- MFEM_SOURCE_DIR MFEM_INSTALL_DIR MFEM_SHARED_BUILD MFEM_USE_DOUBLE MFEM_USE_SINGLE
+ MFEM_SOURCE_DIR MFEM_INSTALL_DIR MFEM_SHARED_BUILD MFEM_USE_DOUBLE MFEM_USE_SINGLE\
+MFEM_USE_ARPACK
 
 # List of makefile variables that will be written to config.mk:
 MFEM_CONFIG_VARS = MFEM_CXX MFEM_HOST_CXX MFEM_CPPFLAGS MFEM_CXXFLAGS\
@@ -733,6 +734,7 @@ status info:
 	$(info MFEM_TIMER_TYPE        = $(MFEM_TIMER_TYPE))
 	$(info MFEM_USE_SUNDIALS      = $(MFEM_USE_SUNDIALS))
 	$(info MFEM_USE_SUITESPARSE   = $(MFEM_USE_SUITESPARSE))
+	$(info MFEM_USE_ARPACK        = $(MFEM_USE_ARPACK))
 	$(info MFEM_USE_SUPERLU       = $(MFEM_USE_SUPERLU))
 	$(info MFEM_USE_SUPERLU5      = $(MFEM_USE_SUPERLU5))
 	$(info MFEM_USE_MUMPS         = $(MFEM_USE_MUMPS))
@@ -807,7 +809,7 @@ FORMAT_EXCLUDE = general/tinyxml2.cpp tests/unit/catch.hpp
 FORMAT_LIST = $(filter-out $(FORMAT_EXCLUDE),$(wildcard $(FORMAT_FILES)))
 
 COUT_CERR_FILES = $(foreach dir,$(DIRS),$(dir)/*.[ch]pp)
-COUT_CERR_EXCLUDE = '^general/error\.cpp' '^general/globals\.[ch]pp' '^general/nvtx\.hpp'
+COUT_CERR_EXCLUDE = '^general/error\.cpp' '^general/globals\.[ch]pp'
 
 DEPRECATION_WARNING := \
 "This feature is planned for removal in the next release."\

miniapps/hdiv-linear-solver/change_basis.hpp

@@ -80,8 +80,6 @@ public:
    // limitations
    void MultRT_2D(const Vector &x, Vector &y, Mode mode) const;
    void MultRT_3D(const Vector &x, Vector &y, Mode mode) const;
-   // suppress warning about hiding overloaded virtual function:
-   using Operator::Mult;
 };
 
 } // namespace mfem

miniapps/nurbs/nurbs_ex11p.cpp

@@ -329,8 +329,8 @@ int main(int argc, char *argv[])
 
       for (int i=0; i<nev; i++)
       {
-         // convert eigenvector from HypreParVector to ParGridFunction
-         x = lobpcg->GetEigenvector(i);
+         // convert eigenvector from Vector to ParGridFunction
+         x.Distribute(lobpcg->GetEigenvector(i));
 
          mode_name << "mode_" << setfill('0') << setw(2) << i << "."
                    << setfill('0') << setw(6) << myid;
@@ -357,8 +357,8 @@ int main(int argc, char *argv[])
                  << ", Lambda = " << eigenvalues[i] << endl;
          }
 
-         // convert eigenvector from HypreParVector to ParGridFunction
-         x = lobpcg->GetEigenvector(i);
+         // convert eigenvector from Vector to ParGridFunction
+         x.Distribute(lobpcg->GetEigenvector(i));
 
          mode_sock << "parallel " << num_procs << " " << myid << "\n"
                    << "solution\n" << *pmesh << x << flush

tests/benchmarks/CMakeLists.txt

@@ -32,11 +32,7 @@ function(add_benchmark name)
     endif(MFEM_USE_CUDA)
 
     add_executable(bench_${name} ${${NAME}_BENCH_SRCS})
-    if (fmt_FOUND)
-        target_link_libraries(bench_${name} mfem pthread fmt::fmt)
-    else()
-        target_link_libraries(bench_${name} mfem pthread)
-    endif()
+    target_link_libraries(bench_${name} mfem pthread)
     add_dependencies(${MFEM_ALL_BENCHMARKS_TARGET_NAME} bench_${name})
 
     add_test(NAME bench_${name}_cpu
@@ -55,7 +51,6 @@ endfunction(add_benchmark)
 #-------------------------------------------------------------------------------
 add_benchmark(assembly_levels)
 add_benchmark(ceed)
-add_benchmark(dfem)
 add_benchmark(dg_amr)
 add_benchmark(elasticity)
 add_benchmark(tmop)

tests/benchmarks/bench_dfem.cpp

@@ -1,845 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "bench.hpp" // IWYU pragma: keep
-
-#ifdef MFEM_USE_BENCHMARK
-
-#include <cstdlib>
-#include <memory>
-
-#include <fem/qinterp/det.cpp>
-#include <fem/qinterp/grad.hpp> // IWYU pragma: keep
-#include "fem/integ/lininteg_domain_kernels.hpp" // IWYU pragma: keep
-
-#include "fem/dfem/doperator.hpp"
-#include <linalg/tensor.hpp>
-
-#include <fem/kernels3d.hpp>
-namespace ker = mfem::kernels::internal;
-namespace low = mfem::kernels::internal::low;
-
-#include "bench_dfem_mma.hpp"
-
-#undef NVTX_COLOR
-#define NVTX_COLOR ::nvtx::kNvidia
-
-using namespace mfem;
-
-using mfem::future::tuple;
-using mfem::future::tensor;
-
-using future::DifferentiableOperator;
-using future::UniformParameterSpace;
-using future::ParameterFunction;
-using future::FieldDescriptor;
-using future::make_tensor;
-using future::Gradient;
-using future::Weight;
-using future::Identity;
-
-/// info //////////////////////////////////////////////////////////////////////
-static void DumpVersionInfo()
-{
-   mfem::out << "\x1b[33m";
-   mfem::out << "version 0: PA std" << std::endl;
-   mfem::out << "version 1: PA reg" << std::endl; // can do high order
-   mfem::out << "version 2: PA low" << std::endl;
-   mfem::out << "version 3: PA mma" << std::endl;
-   // mfem::out << "version 4: PA ∂fem new, not specialized" << std::endl;
-   mfem::out << "version 5: PA ∂fem new, specialized" << std::endl;
-   // mfem::out << "version 6: PA ∂fem std" << std::endl; // ⚠️ max p=3
-   // mfem::out << "version 7: MF ∂fem std" << std::endl;
-   // mfem::out << "version 8: MF ∂fem new" << std::endl; // ⚠️ not supported
-   mfem::out << "\x1b[m" << std::endl;
-}
-
-// Custom benchmark arguments generator ///////////////////////////////////////
-static void CustomArguments(bm::Benchmark *b) noexcept
-{
-   constexpr int MAX_NDOFS = 8 * 1024 * (mfem_use_gpu ? 1024 : 8);
-
-   const auto versions = { 0, 1, 2, 3, /*4,*/ 5, /*6, 7, 8*/ };
-
-   const auto orders = { 6, 5, 4, 3, 2, 1 };
-
-   constexpr auto ndofs = [](int n) constexpr noexcept -> int
-   {
-      return (n + 1) * (n + 1) * (n + 1);
-   };
-
-   constexpr auto inc = [](int n) constexpr noexcept -> int
-   {
-      return n < 160 ?  4 : n < 240 ?  8 : n < 320 ? 16 : 32;
-   };
-
-   for (auto k : versions)
-   {
-      for (auto p : orders)
-      {
-         for (int n = 16; ndofs(n) <= MAX_NDOFS; n += inc(n))
-         {
-            b->Args({k, p, n});
-         }
-      }
-   }
-}
-
-/// Basic Kernels Specializations /////////////////////////////////////////////
-static void AddBasicKernelSpecializations()
-{
-   using Det = QuadratureInterpolator::DetKernels;
-   Det::Specialization<3, 3, 2, 2>::Add();
-   Det::Specialization<3, 3, 2, 3>::Add();
-   Det::Specialization<3, 3, 2, 5>::Add();
-   Det::Specialization<3, 3, 2, 6>::Add();
-   // Others might exceed memory limits
-
-   using Grad = QuadratureInterpolator::GradKernels;
-   Grad::Specialization<3, QVectorLayout::byVDIM,  false, 3, 2, 3>::Add();
-   Grad::Specialization<3, QVectorLayout::byVDIM,  false, 3, 2, 4>::Add();
-   Grad::Specialization<3, QVectorLayout::byVDIM,  false, 3, 2, 5>::Add();
-   Grad::Specialization<3, QVectorLayout::byVDIM,  false, 3, 2, 6>::Add();
-   Grad::Specialization<3, QVectorLayout::byVDIM,  false, 3, 2, 7>::Add();
-   Grad::Specialization<3, QVectorLayout::byVDIM,  false, 3, 2, 8>::Add();
-   Grad::Specialization<3, QVectorLayout::byNODES, false, 3, 2, 7>::Add();
-   Grad::Specialization<3, QVectorLayout::byNODES, false, 3, 2, 8>::Add();
-
-   using LIN = DomainLFIntegrator::AssembleKernels;
-   LIN::Specialization<3, 7, 7>::Add();
-   LIN::Specialization<3, 6, 6>::Add();
-   LIN::Specialization<3, 8, 8>::Add();
-}
-
-/// Globals ///////////////////////////////////////////////////////////////////
-Device *device_ptr = nullptr;
-static int gD1D = 0, gQ1D = 0;
-
-/// StiffnessIntegrator ///////////////////////////////////////////////////////
-struct StiffnessIntegrator : public BilinearFormIntegrator
-{
-   const FiniteElementSpace *fes;
-   const real_t *B, *G, *DX;
-   int ne, d1d, q1d;
-   Vector J0, dx;
-   Vector &qdata;
-
-public:
-   StiffnessIntegrator(Vector &qdata): qdata(qdata)
-   {
-      StiffnessKernels::Specialization<2,3>::Add();  // 1
-      StiffnessKernels::Specialization<3,4>::Add();  // 2
-      StiffnessKernels::Specialization<4,5>::Add();  // 3
-      StiffnessKernels::Specialization<5,6>::Add();  // 4
-      StiffnessKernels::Specialization<6,7>::Add();  // 5
-      StiffnessKernels::Specialization<7,8>::Add();  // 6
-      StiffnessKernels::Specialization<9,10>::Add();  // 8
-   }
-
-   void AssemblePA(const FiniteElementSpace &fespace) override
-   {
-      NVTX();
-      fes = &fespace;
-      auto *mesh = fes->GetMesh();
-      const int DIM = mesh->Dimension();
-      ne = mesh->GetNE();
-      const auto p = fes->GetFE(0)->GetOrder();
-      const auto q = 2 * p + mesh->GetElementTransformation(0)->OrderW();
-      const auto type = mesh->GetElementBaseGeometry(0);
-      const IntegrationRule &ir = IntRules.Get(type, q);
-      const int NQPT = ir.GetNPoints();
-      d1d = p + 1;
-      q1d = IntRules.Get(Geometry::SEGMENT, ir.GetOrder()).GetNPoints();
-      MFEM_VERIFY(d1d == gD1D, "D1D mismatch: " << d1d << " != " << gD1D);
-      MFEM_VERIFY(q1d == gQ1D, "Q1D mismatch: " << q1d << " != " << gQ1D);
-      MFEM_VERIFY(NQPT == q1d * q1d * q1d, "");
-      const DofToQuad *maps =
-         &fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
-      const GridFunction *nodes = (mesh->EnsureNodes(), mesh->GetNodes());
-      const FiniteElementSpace *nfes = nodes->FESpace();
-      const int nVDIM = nfes->GetVDim();
-      dx.SetSize(nVDIM * DIM * NQPT * ne, Device::GetDeviceMemoryType());
-      J0.SetSize(nVDIM * DIM * NQPT * ne, Device::GetDeviceMemoryType());
-      dx.UseDevice(true), J0.UseDevice(true);
-      B = maps->B.Read(), G = maps->G.Read(), DX = dx.Read();
-
-      const Operator *NR =
-         nfes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC);
-      const QuadratureInterpolator *nqi = nfes->GetQuadratureInterpolator(ir);
-      nqi->SetOutputLayout(QVectorLayout::byVDIM);
-      const int nd = nfes->GetFE(0)->GetDof();
-      Vector xe(nVDIM * nd * ne, Device::GetDeviceMemoryType());
-      NR->Mult(*nodes, (xe.UseDevice(true), xe));
-      nqi->Derivatives(xe, J0);
-
-      const int Q1D = q1d;
-      const auto w_r = ir.GetWeights().Read();
-      const auto W = Reshape(w_r, q1d, q1d, q1d);
-      const auto J = Reshape(J0.Read(), 3, 3, q1d, q1d, q1d, ne);
-      auto DX_w = Reshape(dx.Write(), 3, 3, q1d, q1d, q1d, ne);
-
-      mfem::forall_3D(ne, Q1D, Q1D, Q1D,[=] MFEM_HOST_DEVICE(int e)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qz, z, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  const real_t w = W(qx, qy, qz);
-                  const real_t *Jtr = &J(0, 0, qx, qy, qz, e);
-                  const real_t detJ = kernels::Det<3>(Jtr);
-                  const real_t wd = w * detJ;
-                  const real_t D[9] = { wd, 0.0, 0.0,
-                                        0.0, wd, 0.0,
-                                        0.0, 0.0, wd
-                                      };
-                  real_t Jrt[9], A[9];
-                  kernels::CalcInverse<3>(Jtr, Jrt);
-                  kernels::MultABt(3, 3, 3, D, Jrt, A);
-                  kernels::Mult(3, 3, 3, A, Jrt, &DX_w(0, 0, qx, qy, qz, e));
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      });
-      qdata = dx;
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template <int T_D1D = 0, int T_Q1D = 0>
-   static void StiffnessMult(const int NE, const real_t *b, const real_t *g,
-                             const real_t *dx, const real_t *xe, real_t *ye,
-                             const int d1d, const int q1d)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      constexpr int DIM = 3, VDIM = 1;
-      const auto XE = Reshape(xe, D1D, D1D, D1D, VDIM, NE);
-      const auto DX = Reshape(dx, 3, 3, Q1D, Q1D, Q1D, NE);
-      auto YE = Reshape(ye, D1D, D1D, D1D, VDIM, NE);
-
-      mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-      {
-         constexpr int MD1 = T_D1D > 0 ? kernels::internal::SetMaxOf(T_D1D) : 8;
-         constexpr int MQ1 = T_Q1D > 0 ? kernels::internal::SetMaxOf(T_Q1D) : 8;
-
-         MFEM_SHARED real_t smem[MQ1][MQ1];
-         MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-         ker::vd_regs3d_t<VDIM, DIM, MQ1> r0, r1;
-
-         ker::LoadMatrix(D1D, Q1D, b, sB);
-         ker::LoadMatrix(D1D, Q1D, g, sG);
-
-         ker::LoadDofs3d(e, D1D, XE, r0);
-         ker::Grad3d(D1D, Q1D, smem, sB, sG, r0, r1);
-
-         for (int qz = 0; qz < Q1D; qz++)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  real_t v[3], u[3] = { r1[0][0][qz][qy][qx],
-                                        r1[0][1][qz][qy][qx],
-                                        r1[0][2][qz][qy][qx]
-                                      };
-                  const real_t *dx = &DX(0, 0, qx, qy, qz, e);
-                  kernels::Mult(3, 3, dx, u, v);
-                  r0[0][0][qz][qy][qx] = v[0];
-                  r0[0][1][qz][qy][qx] = v[1];
-                  r0[0][2][qz][qy][qx] = v[2];
-               }
-            }
-         }
-         ker::GradTranspose3d(D1D, Q1D, smem, sB, sG, r0, r1);
-         ker::WriteDofs3d(e, D1D, r1, YE);
-      });
-   }
-
-   using StiffnessKernelType = decltype(&StiffnessMult<>);
-   MFEM_REGISTER_KERNELS(StiffnessKernels, StiffnessKernelType, (int, int));
-
-   void AddMultPA(const Vector &x, Vector &y) const override
-   {
-      db1("\x1b[32md1d:{} q1d:{}", d1d, q1d);
-      StiffnessKernels::Run(d1d, q1d,
-                            ne, B, G, DX, x.Read(), y.ReadWrite(),
-                            d1d, q1d);
-   }
-};
-
-template <int D1D, int Q1D>
-StiffnessIntegrator::StiffnessKernelType
-StiffnessIntegrator::StiffnessKernels::Kernel()
-{
-   db1("D1D:{} Q1D:{}", D1D, Q1D);
-   return StiffnessMult<D1D, Q1D>;
-}
-
-StiffnessIntegrator::StiffnessKernelType
-StiffnessIntegrator::StiffnessKernels::Fallback([[maybe_unused]] int d1d,
-                                                [[maybe_unused]] int q1d)
-{
-   dbg("\x1b[33mFallback d1d:{} q1d:{}", d1d, q1d);
-   // MFEM_ABORT("No kernel for d1d=" << d1d << " q1d=" << q1d);
-   // return nullptr;
-   return StiffnessMult<>;
-}
-
-/// PADiffLowIntegrator ///////////////////////////////////////////////////////
-struct PADiffLowIntegrator : public BilinearFormIntegrator
-{
-   const FiniteElementSpace *fes;
-   const real_t *B, *G, *DX;
-   int ne, d1d, q1d;
-   Vector J0, dx;
-
-public: // for nvcc
-   //////////////////////////////////////////////////////////////////
-   template <int T_Q1D = 0>
-   static void PADiffLowMult(const int ne, const int d1d,
-                             const real_t *b, const real_t *g,
-                             const real_t *dx, const real_t *xe,
-                             real_t *ye,
-                             const int q1d)
-   {
-      constexpr int DIM = 3, VDIM = 1;
-
-      const auto XE = Reshape(xe, d1d, d1d, d1d, VDIM, ne);
-      auto YE = Reshape(ye, d1d, d1d, d1d, VDIM, ne);
-
-      mfem::forall_3D<T_Q1D*T_Q1D*T_Q1D>(ne, q1d, q1d, q1d,
-                                         [=] MFEM_HOST_DEVICE(int e)
-      {
-         constexpr int MQ1 = T_Q1D;
-
-         MFEM_SHARED real_t sm0[MQ1][MQ1][MQ1][3];
-         MFEM_SHARED real_t sm1[MQ1][MQ1][MQ1][3];
-         MFEM_SHARED real_t sB[MQ1][MQ1];
-         MFEM_SHARED real_t sG[MQ1][MQ1];
-
-         low::regs3d_t<DIM, MQ1> reg;
-
-         low::LoadMatrix(d1d, q1d, b, sB);
-         low::LoadMatrix(d1d, q1d, g, sG);
-         low::LoadDofs3d(e, d1d, XE, sm0); // Load & sync
-
-         // Grad: sm0 -X-> sm1 -Y-> sm0 -Z-> reg
-         low::Grad3d(d1d, q1d, sB, sG, sm0, sm1, reg); // Grad 3D
-
-         // Q-function
-         MFEM_FOREACH_THREAD_DIRECT(qz,z,q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy,y,q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx,x,q1d)
-               {
-                  // pull
-                  real_t v[3], u[3] = { reg[qz][qy][qx][0],
-                                        reg[qz][qy][qx][1],
-                                        reg[qz][qy][qx][2]
-                                      };
-                  //  Q-function
-                  kernels::Mult(3, 3, dx + 9*(qx*q1d*q1d + qy*q1d + qz), u, v);
-                  // push
-                  reg[qz][qy][qx][0] = v[0];
-                  reg[qz][qy][qx][1] = v[1];
-                  reg[qz][qy][qx][2] = v[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Grad^T: reg -=-> sm1 -X^T-> sm0 -Y^T-> sm1 -Z^T-> reg -> YE
-         low::GradTranspose3d(d1d, q1d, sB, sG, reg, sm1, sm0); // Grad^T 3D
-         low::WriteDofs3d(d1d, 0, e, reg, YE); // Write YE
-      });
-   }
-
-   using PADiffLowKernelType = decltype(&PADiffLowMult<>);
-   MFEM_REGISTER_KERNELS(PADiffLowKernels, PADiffLowKernelType, (int));
-
-public:
-   PADiffLowIntegrator()
-   {
-      PADiffLowKernels::Specialization<3>::Add();  // 1
-      PADiffLowKernels::Specialization<4>::Add();  // 2
-      PADiffLowKernels::Specialization<5>::Add();  // 3
-      PADiffLowKernels::Specialization<6>::Add();  // 4
-      PADiffLowKernels::Specialization<7>::Add();  // 5
-      PADiffLowKernels::Specialization<8>::Add();  // 6
-   }
-
-   void AssemblePA(const FiniteElementSpace &fespace) override
-   {
-      NVTX();
-      fes = &fespace;
-      auto *mesh = fes->GetMesh();
-      const int DIM = mesh->Dimension();
-      ne = mesh->GetNE();
-      const auto p = fes->GetFE(0)->GetOrder();
-      const auto q = 2 * p + mesh->GetElementTransformation(0)->OrderW();
-      const auto type = mesh->GetElementBaseGeometry(0);
-      const IntegrationRule &ir = IntRules.Get(type, q);
-      const int NQPT = ir.GetNPoints();
-      d1d = p + 1;
-      q1d = IntRules.Get(Geometry::SEGMENT, ir.GetOrder()).GetNPoints();
-      MFEM_VERIFY(d1d == gD1D, "D1D mismatch: " << d1d << " != " << gD1D);
-      MFEM_VERIFY(q1d == gQ1D, "Q1D mismatch: " << q1d << " != " << gQ1D);
-      MFEM_VERIFY(NQPT == q1d * q1d * q1d, "");
-      const DofToQuad *maps =
-         &fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
-      const GridFunction *nodes = (mesh->EnsureNodes(), mesh->GetNodes());
-      const FiniteElementSpace *nfes = nodes->FESpace();
-      const int nVDIM = nfes->GetVDim();
-      dx.SetSize(nVDIM * DIM * NQPT * ne, Device::GetDeviceMemoryType());
-      J0.SetSize(nVDIM * DIM * NQPT * ne, Device::GetDeviceMemoryType());
-      dx.UseDevice(true), J0.UseDevice(true);
-      B = maps->B.Read(), G = maps->G.Read(), DX = dx.Read();
-
-      const Operator *NR =
-         nfes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC);
-      const QuadratureInterpolator *nqi = nfes->GetQuadratureInterpolator(ir);
-      nqi->SetOutputLayout(QVectorLayout::byVDIM);
-      const int nd = nfes->GetFE(0)->GetDof();
-      Vector xe(nVDIM * nd * ne, Device::GetDeviceMemoryType());
-      NR->Mult(*nodes, (xe.UseDevice(true), xe));
-      nqi->Derivatives(xe, J0);
-
-      const int Q1D = q1d;
-      const auto w_r = ir.GetWeights().Read();
-      const auto W = Reshape(w_r, q1d, q1d, q1d);
-      const auto J = Reshape(J0.Read(), 3, 3, q1d, q1d, q1d, ne);
-      auto DX_w = Reshape(dx.Write(), 3, 3, q1d, q1d, q1d, ne);
-
-      mfem::forall_3D(ne, Q1D, Q1D, Q1D,[=] MFEM_HOST_DEVICE(int e)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qz, z, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  const real_t w = W(qx, qy, qz);
-                  const real_t *Jtr = &J(0, 0, qx, qy, qz, e);
-                  const real_t detJ = kernels::Det<3>(Jtr);
-                  const real_t wd = w * detJ;
-                  const real_t D[9] = { wd, 0.0, 0.0,
-                                        0.0, wd, 0.0,
-                                        0.0, 0.0, wd
-                                      };
-                  real_t Jrt[9], A[9];
-                  kernels::CalcInverse<3>(Jtr, Jrt);
-                  kernels::MultABt(3, 3, 3, D, Jrt, A);
-                  kernels::Mult(3, 3, 3, A, Jrt, &DX_w(0, 0, qz, qy, qx, e));
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      });
-   }
-
-   void AddMultPA(const Vector &x, Vector &y) const override
-   {
-      db1("\x1b[32md1d:{} q1d:{}", d1d, q1d);
-      PADiffLowKernels::Run(q1d,
-                            ne, d1d, B, G, DX, x.Read(), y.ReadWrite(),
-                            q1d);
-   }
-};
-template <int Q1D>
-PADiffLowIntegrator::PADiffLowKernelType
-PADiffLowIntegrator::PADiffLowKernels::Kernel()
-{
-   db1("Q1D:{}", Q1D);
-   return PADiffLowMult<Q1D>;
-}
-
-PADiffLowIntegrator::PADiffLowKernelType
-PADiffLowIntegrator::PADiffLowKernels::Fallback(int q1d)
-{
-   dbg("\x1b[33mFallback d1d:{} q1d:{}", q1d);
-   MFEM_ABORT("No kernel for q1d=" << q1d);
-   return nullptr;
-   // return StiffnessMult<>;
-}
-
-/// BakeOff ///////////////////////////////////////////////////////////////////
-template <int VDIM, bool GLL>
-struct BakeOff
-{
-   static constexpr int DIM = 3;
-   const int p, c, q, n, nx, ny, nz;
-   const bool check_x, check_y, check_z, checked;
-   Mesh smesh;
-   ParMesh pmesh;
-   H1_FECollection fec;
-   ParFiniteElementSpace pfes;
-   const Geometry::Type geom_type;
-   IntegrationRules irs;
-   const IntegrationRule *ir;
-   ConstantCoefficient one;
-   Vector uvec;
-   VectorConstantCoefficient unit_vec;
-   const int dofs;
-   ParGridFunction *nodes;
-   ParFiniteElementSpace& mfes;
-   ParGridFunction x, y;
-   ParBilinearForm a;
-   std::unique_ptr<DifferentiableOperator> dop;
-   const int elem_size, total_size, d1d, q1d;
-   UniformParameterSpace qd_ps;
-   ParameterFunction qdata;
-
-   double mdofs{};
-
-   BakeOff(int p, int side):
-      p(p), c(side), q(2 * p + (GLL ? -1 : 3)), n((assert(c >= p), c / p)),
-      nx(n + (p * (n + 1) * p * n * p * n < c * c * c ? 1 : 0)),
-      ny(n + (p * (n + 1) * p * (n + 1) * p * n < c * c * c ? 1 : 0)), nz(n),
-      check_x(p * nx * p * ny * p * nz <= c * c * c),
-      check_y(p * (nx + 1) * p * (ny + 1) * p * nz > c * c * c),
-      check_z(p * (nx + 1) * p * (ny + 1) * p * (nz + 1) > c * c * c),
-      checked((assert(check_x &&check_y &&check_z), true)),
-      smesh(Mesh::MakeCartesian3D(nx, ny, nz, Element::HEXAHEDRON)),
-      pmesh(MPI_COMM_WORLD, (smesh.EnsureNodes(), smesh)),
-      fec(p, DIM, BasisType::GaussLobatto),
-      pfes(&pmesh, &fec, VDIM),//, Ordering::byNODES),
-      geom_type(pmesh.GetTypicalElementGeometry()),
-      irs(0, GLL ? Quadrature1D::GaussLobatto : Quadrature1D::GaussLegendre),
-      ir(&irs.Get(geom_type, q)), one(1.0), uvec(DIM),
-      unit_vec((uvec = 1.0, uvec /= uvec.Norml2(), uvec)),
-      dofs(pfes.GetTrueVSize()),
-      nodes(static_cast<ParGridFunction*>(pmesh.GetNodes())),
-      mfes(*nodes->ParFESpace()),
-      x(&pfes),
-      y(&pfes),
-      a(&pfes),
-      elem_size(DIM * DIM * ir->GetNPoints()),
-      total_size(elem_size * pmesh.GetNE()),
-      d1d(p + 1),
-      q1d(IntRules.Get(Geometry::SEGMENT, ir->GetOrder()).GetNPoints()),
-      qd_ps(pmesh, *ir, DIM*DIM),
-      qdata(qd_ps)
-   {
-      NVTX_MARK_FUNCTION;
-      dbg("p:{} q:{}", p, q);
-      smesh.Clear();
-      x = 0.0;
-
-      gD1D = d1d, gQ1D = q1d;
-      dbg("D1D: {}, Q1D: {}", gD1D, gQ1D);
-      qdata.UseDevice(true);
-      qdata = 0.0;
-      MFEM_VERIFY(q1d*q1d*q1d == ir->GetNPoints(), "");
-   }
-
-   virtual void Benchmark() { MFEM_ABORT("Not implemented."); }
-
-   [[nodiscard]] double SumMdofs() const noexcept { return mdofs; }
-
-   [[nodiscard]] double MDofs() const noexcept { return 1e-6 * dofs; }
-};
-
-/// Q-Functions ///////////////////////////////////////////////////////////////
-template<int DIM>
-struct MFApply
-{
-   MFEM_HOST_DEVICE inline
-   auto operator()(const tensor<real_t, DIM>& Gu,
-                   const tensor<real_t, DIM, DIM>& J,
-                   const real_t& w) const
-   {
-      auto invJ = inv(J);
-      return tuple{((Gu * invJ)) * transpose(invJ) * det(J) * w};
-   }
-};
-
-template<int DIM>
-struct PASetup
-{
-   MFEM_HOST_DEVICE inline
-   auto operator()([[maybe_unused]] const real_t &u,
-                   const tensor<real_t, DIM, DIM> &J,
-                   const real_t &w)const
-   {
-      return tuple{inv(J) * transpose(inv(J)) * det(J) * w};
-   }
-};
-
-
-template<int DIM>
-struct PAApply
-{
-   MFEM_HOST_DEVICE inline
-   auto operator()(const tensor<real_t, DIM> &Gu,
-                   const tensor<real_t, DIM, DIM> &q) const
-   {
-      return tuple{q * Gu};
-   };
-};
-
-/// Diffusion /////////////////////////////////////////////////////////////////
-template <int VDIM = 1, bool GLL = false>
-struct Diffusion : public BakeOff<VDIM, GLL>
-{
-   static constexpr int DIM = 3;
-   static constexpr int U = 0, Ξ = 1, Q = 2;
-
-   const real_t rtol = 0.0;
-   const int max_it = 32, print_lvl = -1;
-
-   Array<int> ess_tdof_list, ess_bdr, all_domain_attr;
-   ParLinearForm b;
-   FieldDescriptor u_fd, Ξ_fd, q_fd;
-   std::vector<FieldDescriptor> u_sol, q_param, Ξ_q_params;
-   OperatorPtr A;
-   Operator *A_ptr;
-   Vector B, X;
-   CGSolver cg;
-
-   using BakeOff<VDIM, GLL>::a;
-   using BakeOff<VDIM, GLL>::ir;
-   using BakeOff<VDIM, GLL>::one;
-   using BakeOff<VDIM, GLL>::pmesh;
-   using BakeOff<VDIM, GLL>::pfes;
-   using BakeOff<VDIM, GLL>::mfes;
-   using BakeOff<VDIM, GLL>::x;
-   using BakeOff<VDIM, GLL>::y;
-   using BakeOff<VDIM, GLL>::mdofs;
-   using BakeOff<VDIM, GLL>::dop;
-   using BakeOff<VDIM, GLL>::nodes;
-   using BakeOff<VDIM, GLL>::qdata;
-   using BakeOff<VDIM, GLL>::qd_ps;
-   using BakeOff<VDIM, GLL>::dofs;
-
-   Diffusion(int version, int order, int side):
-      BakeOff<VDIM, GLL>(order, side),
-      ess_bdr(pmesh.bdr_attributes.Max()),
-      all_domain_attr(pmesh.bdr_attributes.Max()),
-      b(&pfes),
-      u_fd{U, &pfes}, Ξ_fd{Ξ, &mfes}, q_fd{Q, &qd_ps},
-      u_sol{u_fd},
-      q_param {q_fd},
-      Ξ_q_params {Ξ_fd, q_fd},
-      cg(MPI_COMM_WORLD)
-   {
-      // dbg("pmesh.bdr_attributes.Max():{}",pmesh.bdr_attributes.Max());
-      static_assert(VDIM == 1 && GLL == false);
-
-      ess_bdr = 1;
-      all_domain_attr = 1;
-      pfes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-      b.AddDomainIntegrator(new DomainLFIntegrator(this->one));
-      b.UseFastAssembly(true);
-      b.Assemble();
-
-      // MF setup ///////////////////////////////////////////////////
-      const auto dMFOperatorSetup = [&] (bool use_new_kernels,
-                                         bool use_kernels_specialization)
-      {
-         dbg("MF ∂fem {} kernels", use_new_kernels ? "NEW" : "STD");
-         std::vector<FieldDescriptor> in_fds = {{U, &pfes}, {Ξ, &mfes}};
-         std::vector<FieldDescriptor> out_fds = {{U, &pfes}};
-         dop = std::make_unique<DifferentiableOperator>(in_fds, out_fds, pmesh);
-         // dop->SetParameters({nodes});
-         if (use_kernels_specialization) { dop->UseKernelsSpecialization(); }
-         if (use_new_kernels) { dop->UseNewKernels(); }
-         // MFApply<DIM> mf_apply;
-         // dop->AddDomainIntegrator(mf_apply,
-         //                          tuple{Gradient<U>{}, Gradient<Ξ>{}, Weight{}}, // local API 🔥
-         //                          tuple{Gradient<U>{}},
-         //                          *ir, ess_bdr);
-         dop->FormLinearSystem(ess_tdof_list, x, b, A_ptr, X, B);
-         A.Reset(A_ptr);
-      };
-
-      // PA setup ///////////////////////////////////////////////////
-      const auto dPAOperatorSetup = [&] (bool use_new_kernels,
-                                         bool use_kernels_specialization)
-      {
-#if 0
-         dbg("[PA ∂fem] Setup");
-         auto Iu = Identity<U> {};
-         auto GΞ = Gradient<Ξ> {};
-         auto W = Weight{};
-         tuple Iu_GΞ_W = {Iu, GΞ, W};
-         PASetup<DIM> pa_setup_qf;
-         DifferentiableOperator dSetup(u_sol, Ξ_q_params, pmesh);
-         if (use_kernels_specialization) { dSetup.UseKernelsSpecialization(); }
-         if (use_new_kernels) { dSetup.UseNewKernels(); }
-         dSetup.AddDomainIntegrator(pa_setup_qf, Iu_GΞ_W, tuple{Iq}, *ir, ess_bdr);
-         dSetup.SetParameters({nodes, &qdata});
-         X.SetSize(pfes.GetTrueVSize());
-         pfes.GetRestrictionMatrix()->Mult(x, X);
-         dSetup.Mult(X, qdata);
-#else
-         dbg("[PA ∂fem] Setup (borrowing PA setup)");
-         {
-            ParBilinearForm bf(&pfes);
-            bf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-            bf.AddDomainIntegrator(new StiffnessIntegrator(qdata));
-            bf.Assemble();
-         }
-#endif
-         dbg("[PA ∂fem] Apply");
-         // auto Iq = Identity<Q> {};
-         // auto Gu = Gradient<U> {};
-         // tuple Gu_Iq = {Gu, Iq};
-         // PAApply<DIM> pa_apply_qf;
-         dop = std::make_unique<DifferentiableOperator>(u_sol, q_param, pmesh);
-         dop->SetMultLevel(DifferentiableOperator::MultLevel::LVECTOR);
-         if (use_kernels_specialization) { dop->UseKernelsSpecialization(); }
-         if (use_new_kernels) { dop->UseNewKernels(); }
-         else { dbg("[PA ∂fem] NOT using kernels specialization"); }
-         // dop->AddDomainIntegrator(pa_apply_qf, Gu_Iq, tuple{Gu}, *ir, ess_bdr); // local API 🔥
-         assert(qdata*qdata > 0.0);
-         // dop->SetParameters({ &qdata });
-         dop->FormLinearSystem(ess_tdof_list, x, b, A_ptr, X, B);
-         A.Reset(A_ptr);
-         dbg("[PA ∂fem] done");
-      };
-
-      if (version <= 3) // std, reg, low & mma
-      {
-         a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-         if (version == 0) { a.AddDomainIntegrator(new DiffusionIntegrator(ir)); }
-         if (version == 1) { a.AddDomainIntegrator(new StiffnessIntegrator(qdata)); }
-         if (version == 2) { a.AddDomainIntegrator(new PADiffLowIntegrator()); }
-         if (version == 3) { a.AddDomainIntegrator(new PADiffMmaIntegrator()); }
-         a.Assemble();
-         a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-         if (version == 0)
-         {
-            BilinearFormIntegrator *bfi = a.GetDBFI()->operator[](0);
-            auto *di = dynamic_cast<DiffusionIntegrator*>(bfi);
-            assert(di);
-            const int d1d = di->dofs1D, q1d = di->quad1D;
-            // dbg("\x1b[33md1d:{} q1d:{}", d1d, q1d);
-            MFEM_VERIFY(d1d == gD1D, "D1D mismatch: " << d1d << " != " << gD1D);
-            MFEM_VERIFY(q1d == gQ1D, "Q1D mismatch: " << q1d << " != " << gQ1D);
-         }
-      }
-      else if (version == 4) // PA ∂fem new kernels, not specialized
-      {
-         dPAOperatorSetup(true, false);
-      }
-      else if (version == 5) // PA ∂fem new kernels, specialized
-      {
-         dPAOperatorSetup(true, true);
-      }
-      else if (version == 6) // PA ∂fem std
-      {
-         dPAOperatorSetup(false, false);
-      }
-      else if (version == 7) // MF ∂fem std
-      {
-         dMFOperatorSetup(false, false);
-      }
-      else if (version == 8) // MF ∂fem new kernels
-      {
-         MFEM_ABORT("MF ∂fem new kernels not implemented");
-         // dMFOperatorSetup(true, true);
-      }
-      else { MFEM_ABORT("Invalid version"); }
-
-      cg.SetOperator(*A);
-      cg.iterative_mode = false;
-      cg.SetAbsTol(0.0);
-      if (dofs < 128 * 1024) // check
-      {
-         cg.SetPrintLevel(3/*-1*/);
-         cg.SetMaxIter(2000);
-         cg.SetRelTol(1e-8);
-         cg.Mult(B, X);
-         MFEM_VERIFY(cg.GetConverged(), "❌ CG solver did not converge.");
-         // mfem::out << (cg.GetConverged() ? "✅" : "❌") << std::endl;
-         // mfem::out << "✅" << std::endl;
-      }
-      cg.SetPrintLevel(print_lvl);
-      cg.SetMaxIter(max_it);
-      cg.SetRelTol(rtol);
-      Benchmark();
-      mdofs = 0.0;
-   }
-
-   void Benchmark() override
-   {
-      NVTX_MARK_FUNCTION;
-      cg.Mult(B, X);
-      MFEM_DEVICE_SYNC;
-      mdofs += this->MDofs() * cg.GetNumIterations();
-   }
-};
-
-///////////////////////////////////////////////////////////////////////////////
-#define BakeOff_Problem(i, Problem)                                  \
-   static void BP##i(bm::State &state)                               \
-   {                                                                 \
-      const auto version = static_cast<int>(state.range(0));         \
-      const auto order = static_cast<int>(state.range(1));           \
-      const auto side = static_cast<int>(state.range(2));            \
-      Problem ker(version, order, side);                             \
-      while (state.KeepRunning()) { ker.Benchmark(); }               \
-      bm::Counter::Flags flags = bm::Counter::kIsRate;               \
-      state.counters["MDof/s"] = bm::Counter(ker.SumMdofs(), flags); \
-      state.counters["Dofs"] = bm::Counter(ker.dofs);                \
-      state.counters["p"] = bm::Counter(order);                      \
-      state.counters["version"] = bm::Counter(version);              \
-   }                                                                 \
-   BENCHMARK(BP##i)                                                  \
-      ->Apply(CustomArguments)                                       \
-      ->Unit(bm::kMillisecond)
-
-BakeOff_Problem(3, Diffusion);
-
-/// main //////////////////////////////////////////////////////////////////////
-int main(int argc, char *argv[])
-{
-   dbg();
-   DumpVersionInfo();
-   AddBasicKernelSpecializations();
-   static mfem::MPI_Session mpi(argc, argv);
-
-   bm::ConsoleReporter CR;
-   bm::Initialize(&argc, argv);
-
-   // Device setup, cpu by default
-   std::string device_context = "cpu",
-               kernels_context = "std",
-               kernels_specialization = "yes";
-   const auto global_context = bmi::GetGlobalContext();
-   if (global_context != nullptr)
-   {
-      const auto device = global_context->find("device");
-      if (device != global_context->end())
-      {
-         mfem::out << device->first << " : "
-                   << device->second << std::endl;
-         device_context = device->second;
-      }
-   }
-   dbg("device_config: {}", device_context);
-   Device device(device_context.c_str());
-   device_ptr = &device;
-   device.Print();
-
-   if (bm::ReportUnrecognizedArguments(argc, argv)) { return EXIT_FAILURE; }
-
-   bm::RunSpecifiedBenchmarks(&CR);
-
-   return EXIT_SUCCESS;
-}
-
-#endif // MFEM_USE_BENCHMARK

tests/benchmarks/bench_dfem_mma.hpp

@@ -1,841 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "fem/bilininteg.hpp"
-#include <fem/quadinterpolator.hpp>
-#include "general/forall.hpp"
-#include "linalg/dtensor.hpp"
-#include "linalg/kernels.hpp"
-
-using namespace mfem;
-
-/// MMA ///////////////////////////////////////////////////////////////////////
-namespace mma
-{
-
-MFEM_HOST_DEVICE inline int getThreadIdx()
-{
-#ifdef __CUDA_ARCH__
-   return threadIdx.x + blockDim.x * (threadIdx.y + blockDim.y * threadIdx.z);
-#else
-   return 0;
-#endif
-}
-
-MFEM_HOST_DEVICE inline int getWarpId(int thread)
-{
-   return thread / 32;
-}
-
-MFEM_HOST_DEVICE inline int getLaneId(int thread)
-{
-   return thread % 32;
-}
-
-MFEM_HOST_DEVICE inline int getGroupId(int laneId)
-{
-   return laneId / 4;
-}
-
-MFEM_HOST_DEVICE inline int getThreadIdInGroup(int laneId)
-{
-   return laneId % 4;
-}
-
-/// Load B1d & G1d matrices into shared memory
-template<int MD1, int MQ1>
-MFEM_HOST_DEVICE inline void LoadBG(const int D1D, const int Q1D,
-                                    const ConstDeviceMatrix &b,
-                                    const ConstDeviceMatrix &g,
-                                    real_t (&sBG)[2][MQ1*MD1])
-{
-   DeviceMatrix B(sBG[0], D1D, Q1D);
-   DeviceMatrix G(sBG[1], D1D, Q1D);
-   int tid = getThreadIdx();
-   if (tid < D1D * Q1D)
-   {
-      int q = tid / D1D;
-      int d = tid % D1D;
-      B(d,q) = b(q,d);
-      G(d,q) = g(q,d);
-   }
-}
-
-/// Load Bt1d & Gt1d matrices into shared memory
-template<int MD1, int MQ1>
-MFEM_HOST_DEVICE inline void LoadBtGt(const int D1D, const int Q1D,
-                                      //   const ConstDeviceMatrix &bt,
-                                      //   const ConstDeviceMatrix &gt,
-                                      const ConstDeviceMatrix &b,
-                                      const ConstDeviceMatrix &g,
-                                      real_t (&sBG)[2][MQ1*MD1])
-{
-   DeviceMatrix Bt(sBG[0], Q1D, D1D);
-   DeviceMatrix Gt(sBG[1], Q1D, D1D);
-
-   int thread = getThreadIdx();
-   if (thread < D1D * Q1D)
-   {
-      int q = thread % Q1D;
-      int d = thread / Q1D;
-      //   Bt(q,d) = bt(d,q);
-      //   Gt(q,d) = gt(d,q);
-      Bt(q,d) = b(q,d);
-      Gt(q,d) = g(q,d);
-   }
-}
-
-/// Load 3D input vector into shared memory
-template<int MQ1>
-MFEM_HOST_DEVICE inline void LoadX(const int e, const int D1D,
-                                   const DeviceTensor<4, const real_t> &x,
-                                   real_t (&sm)[3][MQ1*MQ1*MQ1])
-{
-   const int DDD = D1D * D1D * D1D;
-   DeviceCube X(sm[0], D1D,D1D,D1D);
-   int tid = getThreadIdx();
-   if (tid < DDD)
-   {
-      int dx = tid % D1D;
-      int div = tid / D1D;
-      int dy = div % D1D;
-      int dz = div / D1D;
-      X(dx,dy,dz) = x(dx,dy,dz,e);
-   }
-}
-
-// using the m8n8k4 DMMA instriction
-constexpr int mmaM = 8;
-[[maybe_unused]] constexpr int mmaN = 8;
-constexpr int mmaK = 4;
-
-MFEM_HOST_DEVICE inline void dmmaSync([[maybe_unused]] double aReg[1],
-                                      [[maybe_unused]] double bReg[1],
-                                      [[maybe_unused]] double cReg[2])
-{
-#ifdef __CUDA_ARCH__
-   asm volatile("mma.sync.aligned.m8n8k4.row.col.f64.f64.f64.f64 {%0,%1}, {%2}, {%3}, {%0,%1};"
-                : "+d"(cReg[0]), "+d"(cReg[1]) : "d"(aReg[0]), "d"(bReg[0]));
-#endif
-}
-
-template<int MD1, int MQ1, int MDQ = (MQ1 > MD1 ? MQ1 : MD1)>
-MFEM_HOST_DEVICE inline void dmma_GradX(const int m, const int n, const int k,
-                                        const real_t (&BG)[2][MQ1*MD1],
-                                        const real_t (*A)[MDQ*MDQ*MDQ],
-                                        real_t (*C)[MDQ*MDQ*MDQ])
-{
-   ConstDeviceMatrix B(BG[0], k, n);
-   ConstDeviceMatrix G(BG[1], k, n);
-
-   int thread = getThreadIdx();
-   int warpId = getWarpId(thread);
-   int laneId = getLaneId(thread);
-   int groupId = getGroupId(laneId);
-   int threadIdInGroup = getThreadIdInGroup(laneId);
-
-   // using the m8n8k4 DMMA instriction
-
-   int mPass = (m + mmaM - 1) / mmaM;
-   if (warpId < mPass)   // Spread the warps.
-   {
-
-      int aRowInWarp = groupId;
-      int aColumnInWarp = threadIdInGroup;
-      int bRowInWarp = threadIdInGroup;
-      int bColumnInWarp = groupId;
-
-      constexpr int magicNumber =
-         0b100011111010110001101000; // jump table [0,5,1,6,2,7,3,4]
-      int mM = warpId;
-      double cReg[4] = {};
-      for (int mK = 0; mK < (k + mmaK - 1) / mmaK; mK++)
-      {
-         double bReg[1];
-         double gReg[1];
-         int bRow = bRowInWarp + mK * mmaK;
-         int bColumn = (magicNumber >> (3 * bColumnInWarp)) & 0b111;
-         if (bColumn < n && bRow < k)
-         {
-            bReg[0] = B(bRow, bColumn);
-            gReg[0] = G(bRow, bColumn);
-         }
-         else
-         {
-            bReg[0] = 0;
-            gReg[0] = 0;
-         }
-         double aReg[1];
-         int aRow = aRowInWarp * mPass +  mM;
-         int aColumn = aColumnInWarp + mK * mmaK;
-         if (aRow < m && aColumn < k)
-         {
-            ConstDeviceMatrix aA(A[0], k, m);
-            aReg[0] = aA(aColumn, aRow);
-         }
-         else
-         {
-            aReg[0] = 0;
-         }
-         dmmaSync(aReg, gReg, &cReg[0]);
-         dmmaSync(aReg, bReg, &cReg[2]);
-      }
-      for (int d = 0; d < 2; d++)
-      {
-#pragma unroll
-         for (int i = 0; i < 2; i++)
-         {
-            int cRow = groupId * mPass + mM;
-            int cColumn = (magicNumber >> (3 * (threadIdInGroup * 2 + i))) & 0b111;
-            if (cRow < m && cColumn < n)
-            {
-               DeviceMatrix cC(C[d], m, n);
-               cC(cRow, cColumn) = cReg[d * 2 + i];
-            }
-         }
-      }
-   }
-}
-
-/// 3D Gradient, 1/3
-template<int MD1, int MQ1, int MDQ = (MQ1 > MD1 ? MQ1 : MD1)>
-MFEM_HOST_DEVICE inline void GradX(const int D1D, const int Q1D,
-                                   const real_t (&sBG)[2][MQ1*MD1],
-                                   const real_t (*sDDD)[MDQ*MDQ*MDQ],
-                                   real_t (*sDDQ)[MDQ*MDQ*MDQ])
-{
-   dmma_GradX<MD1, MQ1>(D1D * D1D, Q1D, D1D, sBG, sDDD, sDDQ);
-}
-
-template<int MD1, int MQ1, int MDQ = (MQ1 > MD1 ? MQ1 : MD1)>
-MFEM_HOST_DEVICE inline void dmma_GradY(const int m, const int n,
-                                        const int k,
-                                        const real_t (&BG)[2][MQ1*MD1],
-                                        const real_t (*A)[MDQ*MDQ*MDQ],
-                                        real_t (*C)[MDQ*MDQ*MDQ])
-{
-   ConstDeviceMatrix B(BG[0], k, n);
-   ConstDeviceMatrix G(BG[1], k, n);
-
-   int thread = getThreadIdx();
-   int warpId = getWarpId(thread);
-   int laneId = getLaneId(thread);
-   int groupId = getGroupId(laneId);
-   int threadIdInGroup = getThreadIdInGroup(laneId);
-
-   // using the m8n8k4 DMMA instriction
-
-   int mPass = (m + mmaM - 1) / mmaM;
-   if (warpId < mPass)   // Spread the warps.
-   {
-
-      int aRowInWarp = groupId;
-      int aColumnInWarp = threadIdInGroup;
-      int bRowInWarp = threadIdInGroup;
-      int bColumnInWarp = groupId;
-
-      constexpr int magicNumber =
-         0b100011111010110001101000; // jump table [0,5,1,6,2,7,3,4]
-      int mM = warpId;
-      double cReg[6] = {};
-      for (int mK = 0; mK < (k + mmaK - 1) / mmaK; mK++)
-      {
-         double bReg[1];
-         double gReg[1];
-         int bRow = bRowInWarp + mK * mmaK;
-         int bColumn = (magicNumber >> (3 * bColumnInWarp)) & 0b111;
-         if (bColumn < n && bRow < k)
-         {
-            bReg[0] = B(bRow, bColumn);
-            gReg[0] = G(bRow, bColumn);
-         }
-         else
-         {
-            bReg[0] = 0;
-            gReg[0] = 0;
-         }
-         double agReg[1];
-         double abReg[1];
-         int aRow = aRowInWarp * mPass +  mM;
-         int aColumn = aColumnInWarp + mK * mmaK;
-         if (aRow < m && aColumn < k)
-         {
-            ConstDeviceMatrix gA(A[0], k, m);
-            ConstDeviceMatrix bA(A[1], k, m);
-            agReg[0] = gA(aColumn, aRow);
-            abReg[0] = bA(aColumn, aRow);
-         }
-         else
-         {
-            agReg[0] = 0;
-            abReg[0] = 0;
-         }
-         dmmaSync(agReg, bReg, &cReg[0]);
-         dmmaSync(abReg, gReg, &cReg[2]);
-         dmmaSync(abReg, bReg, &cReg[4]);
-      }
-      for (int d = 0; d < 3; d++)
-      {
-#pragma unroll
-         for (int i = 0; i < 2; i++)
-         {
-            int cRow = groupId * mPass + mM;
-            int cColumn = (magicNumber >> (3 * (threadIdInGroup * 2 + i))) & 0b111;
-            if (cRow < m && cColumn < n)
-            {
-               DeviceMatrix cC(C[d], m, n);
-               cC(cRow, cColumn) = cReg[d * 2 + i];
-            }
-         }
-      }
-   }
-}
-
-/// 3D Gradient, 2/3
-template<int MD1, int MQ1, int MDQ = (MQ1 > MD1 ? MQ1 : MD1)>
-MFEM_HOST_DEVICE inline void GradY(const int D1D, const int Q1D,
-                                   const real_t (&sBG)[2][MQ1*MD1],
-                                   const real_t (*sDDQ)[MDQ*MDQ*MDQ],
-                                   real_t (*sDQQ)[MDQ*MDQ*MDQ])
-{
-   dmma_GradY<MD1, MQ1>(D1D * Q1D, Q1D, D1D, sBG, sDDQ, sDQQ);
-}
-
-template<int MD1, int MQ1, int MDQ = (MQ1 > MD1 ? MQ1 : MD1)>
-MFEM_HOST_DEVICE inline void dmma_GradZ(const int m, const int n,
-                                        const int k,
-                                        const real_t (&BG)[2][MQ1*MD1],
-                                        const real_t (*A)[MDQ*MDQ*MDQ],
-                                        real_t (*C)[MDQ*MDQ*MDQ],
-                                        int gIdx)
-{
-   ConstDeviceMatrix B(BG[0], k, n);
-   ConstDeviceMatrix G(BG[1], k, n);
-
-   int thread = getThreadIdx();
-   int warpId = getWarpId(thread);
-   int laneId = getLaneId(thread);
-   int groupId = getGroupId(laneId);
-   int threadIdInGroup = getThreadIdInGroup(laneId);
-
-   // using the m8n8k4 DMMA instriction
-
-   int mPass = (m + mmaM - 1) / mmaM;
-   if (warpId < mPass)   // Spread the warps.
-   {
-
-      int aRowInWarp = groupId;
-      int aColumnInWarp = threadIdInGroup;
-      int bRowInWarp = threadIdInGroup;
-      int bColumnInWarp = groupId;
-
-      constexpr int magicNumber =
-         0b100011111010110001101000; // jump table [0,5,1,6,2,7,3,4]
-      int mM = warpId;
-      double cReg[6] = {};
-      for (int mK = 0; mK < (k + mmaK - 1) / mmaK; mK++)
-      {
-         double bReg[1];
-         double gReg[1];
-         int bRow = bRowInWarp + mK * mmaK;
-         int bColumn = (magicNumber >> (3 * bColumnInWarp)) & 0b111;
-         if (bColumn < n && bRow < k)
-         {
-            bReg[0] = B(bRow, bColumn);
-            gReg[0] = G(bRow, bColumn);
-         }
-         else
-         {
-            bReg[0] = 0;
-            gReg[0] = 0;
-         }
-         for (int d = 0; d < 3; d++)
-         {
-            double aReg[1];
-            int aRow = aRowInWarp * mPass +  mM;
-            int aColumn = aColumnInWarp + mK * mmaK;
-            if (aRow < m && aColumn < k)
-            {
-               ConstDeviceMatrix aA(A[d], k, m);
-               aReg[0] = aA(aColumn, aRow);
-            }
-            else
-            {
-               aReg[0] = 0;
-            }
-            dmmaSync(aReg, d == gIdx ? gReg : bReg, &cReg[d * 2]);
-         }
-      }
-      for (int d = 0; d < 3; d++)
-      {
-#pragma unroll
-         for (int i = 0; i < 2; i++)
-         {
-            int cRow = groupId * mPass + mM;
-            int cColumn = (magicNumber >> (3 * (threadIdInGroup * 2 + i))) & 0b111;
-            if (cRow < m && cColumn < n)
-            {
-               DeviceMatrix cC(C[d], m, n);
-               cC(cRow, cColumn) = cReg[d * 2 + i];
-            }
-         }
-      }
-   }
-}
-
-/// 3D Gradient, 3/3
-template<int MD1, int MQ1, int MDQ = (MQ1 > MD1 ? MQ1 : MD1)>
-MFEM_HOST_DEVICE inline void GradZ(const int D1D, const int Q1D,
-                                   const real_t (&sBG)[2][MQ1*MD1],
-                                   const real_t (*sDQQ)[MDQ*MDQ*MDQ],
-                                   real_t (*sQQQ)[MDQ*MDQ*MDQ])
-{
-   dmma_GradZ<MD1, MQ1>(Q1D * Q1D, Q1D, D1D, sBG, sDQQ, sQQQ, 2);
-}
-
-/// 3D Transposed Gradient, 1/3
-template<int MD1, int MQ1, int MDQ = (MQ1 > MD1 ? MQ1 : MD1)>
-MFEM_HOST_DEVICE inline void GradZt(const int D1D, const int Q1D,
-                                    const real_t (&sBG)[2][MQ1*MD1],
-                                    const real_t (*sQQQ)[MDQ*MDQ*MDQ],
-                                    real_t (*sDQQ)[MDQ*MDQ*MDQ])
-{
-   ConstDeviceMatrix Bt(sBG[0], Q1D, D1D);
-   ConstDeviceMatrix Gt(sBG[1], Q1D, D1D);
-   int thread = getThreadIdx();
-   int warpId = getWarpId(thread);
-   int laneId = getLaneId(thread);
-   int groupId = getGroupId(laneId);
-   int threadIdInGroup = getThreadIdInGroup(laneId);
-
-   // using the m8n8k4 DMMA instriction
-   // qy (Q1D), qz (Q1D) === M, dx (D1D) === N, qx (Q1D) === K
-
-   int mPass = (Q1D * Q1D + mmaM - 1) / mmaM;
-   if (warpId < mPass)   // Spread the warps to calculate the 3 directions.
-   {
-
-      int aRowInWarp = groupId;
-      int aColumnInWarp = threadIdInGroup;
-      int bRowInWarp = threadIdInGroup;
-      int bColumnInWarp = groupId;
-
-      constexpr int magicNumber =
-         0b100011111010110001101000; // jump table [0,5,1,6,2,7,3,4]
-      int mM = warpId;
-      double cReg[6] = {};
-      for (int mK = 0; mK < (Q1D + mmaK - 1) / mmaK; mK++)
-      {
-         double BtReg[1];
-         double GtReg[1];
-         int bRow = bRowInWarp + mK * mmaK;
-         int bColumn = (magicNumber >> (3 * bColumnInWarp)) & 0b111;
-         if (bColumn < D1D && bRow < Q1D)
-         {
-            BtReg[0] = Bt(bRow, bColumn);
-            GtReg[0] = Gt(bRow, bColumn);
-         }
-         else
-         {
-            BtReg[0] = 0;
-            GtReg[0] = 0;
-         }
-         for (int d = 0; d < 3; d++)
-         {
-            double aReg[1];
-            int aRow = aRowInWarp * mPass +  mM;
-            int aColumn = aColumnInWarp + mK * mmaK;
-            if (aRow < Q1D * Q1D && aColumn < Q1D)
-            {
-               ConstDeviceMatrix XxBBG(sQQQ[d], Q1D, Q1D * Q1D);
-               aReg[0] = XxBBG(aColumn, aRow);
-            }
-            else
-            {
-               aReg[0] = 0;
-            }
-
-            dmmaSync(aReg, d == 0 ? GtReg : BtReg, &cReg[d * 2]);
-         }
-      }
-      for (int d = 0; d < 3; d++)
-      {
-#pragma unroll
-         for (int i = 0; i < 2; i++)
-         {
-            int cRow = groupId * mPass + mM;
-            int cColumn = (magicNumber >> (3 * (threadIdInGroup * 2 + i))) & 0b111;
-            if (cRow < Q1D * Q1D && cColumn < D1D)
-            {
-               DeviceMatrix Xx(sDQQ[d], Q1D * Q1D, D1D); // qy, qz, dx
-               Xx(cRow, cColumn) = cReg[d * 2 + i];
-            }
-         }
-      }
-   }
-}
-
-/// 3D Transposed Gradient, 2/3
-template<int MD1, int MQ1, int MDQ = (MQ1 > MD1 ? MQ1 : MD1)>
-MFEM_HOST_DEVICE inline void GradYt(const int D1D, const int Q1D,
-                                    const real_t (&sBG)[2][MQ1*MD1],
-                                    const real_t (*sDQQ)[MDQ*MDQ*MDQ],
-                                    real_t (*sDDQ)[MDQ*MDQ*MDQ])
-{
-   ConstDeviceMatrix Bt(sBG[0], Q1D, D1D);
-   ConstDeviceMatrix Gt(sBG[1], Q1D, D1D);
-   int thread = getThreadIdx();
-   int warpId = getWarpId(thread);
-   int laneId = getLaneId(thread);
-   int groupId = getGroupId(laneId);
-   int threadIdInGroup = getThreadIdInGroup(laneId);
-
-   // using the m8n8k4 DMMA instriction
-   // dx (D1D), qz (Q1D) === M, dy (D1D) === N, qy (Q1D) === K
-
-   int mPass = (D1D * Q1D + mmaM - 1) / mmaM;
-   if (warpId < mPass)   // Spread the warps.
-   {
-
-      int aRowInWarp = groupId;
-      int aColumnInWarp = threadIdInGroup;
-      int bRowInWarp = threadIdInGroup;
-      int bColumnInWarp = groupId;
-
-      constexpr int magicNumber =
-         0b100011111010110001101000; // jump table [0,5,1,6,2,7,3,4]
-      int mM = warpId;
-      double cReg[6] = {}; // initialized to zero
-      for (int mK = 0; mK < (Q1D + mmaK - 1) / mmaK; mK++)
-      {
-         double BtReg[1];
-         double GtReg[1];
-         int bRow = bRowInWarp + mK * mmaK;
-         int bColumn = (magicNumber >> (3 * bColumnInWarp)) & 0b111;
-         if (bColumn < D1D && bRow < Q1D)
-         {
-            BtReg[0] = Bt(bRow, bColumn);
-            GtReg[0] = Gt(bRow, bColumn);
-         }
-         else
-         {
-            BtReg[0] = 0;
-            GtReg[0] = 0;
-         }
-         for (int d = 0; d < 3; d++)
-         {
-            double aReg[1];
-
-            int aRow = aRowInWarp * mPass + mM;
-            int aColumn = aColumnInWarp + mK * mmaK;
-            if (aRow < D1D * Q1D && aColumn < Q1D)
-            {
-               ConstDeviceMatrix XxBB(sDQQ[d], Q1D, D1D * Q1D); // qy, qz, dx
-               aReg[0] = XxBB(aColumn, aRow);
-            }
-            else
-            {
-               aReg[0] = 0;
-            }
-
-            dmmaSync(aReg, d == 1 ? GtReg : BtReg, &cReg[d * 2]);
-         }
-      }
-      for (int d = 0; d < 3; d++)
-      {
-#pragma unroll
-         for (int i = 0; i < 2; i++)
-         {
-            int cRow = groupId * mPass + mM;
-            int cColumn = (magicNumber >> (3 * (threadIdInGroup * 2 + i))) & 0b111;
-            if (cRow < D1D * Q1D && cColumn < D1D)
-            {
-               DeviceMatrix Xx(sDDQ[d], D1D * Q1D, D1D); // qz, dx, dy
-               Xx(cRow, cColumn) = cReg[d * 2 + i];
-            }
-         }
-      }
-   }
-}
-
-/// 3D Transposed Gradient, 3/3
-template<int MD1, int MQ1, int MDQ = (MQ1 > MD1 ? MQ1 : MD1)>
-MFEM_HOST_DEVICE inline void GradXt(const int D1D, const int Q1D,
-                                    const real_t (&sBG)[2][MQ1*MD1],
-                                    const real_t (&sDDQ)[3][MDQ*MDQ*MDQ],
-                                    const DeviceTensor<4> &Y, // output
-                                    const int e)
-{
-   ConstDeviceMatrix Bt(sBG[0], Q1D, D1D);
-   ConstDeviceMatrix Gt(sBG[1], Q1D, D1D);
-   int thread = getThreadIdx();
-   int warpId = getWarpId(thread);
-   int laneId = getLaneId(thread);
-   int groupId = getGroupId(laneId);
-   int threadIdInGroup = getThreadIdInGroup(laneId);
-
-   // using the m8n8k4 DMMA instriction
-   // dx (D1D), dy (D1D) === M, dz (D1D) === N, qz (Q1D) === K
-
-   int mPass = (D1D * D1D + mmaM - 1) / mmaM;
-   if (warpId < mPass)   // Spread the warps to calculate the 3 directions.
-   {
-
-      int aRowInWarp = groupId;
-      int aColumnInWarp = threadIdInGroup;
-      int bRowInWarp = threadIdInGroup;
-      int bColumnInWarp = groupId;
-
-      constexpr int magicNumber =
-         0b100011111010110001101000; // jump table [0,5,1,6,2,7,3,4]
-      int mM = warpId;
-      {
-         double BtReg[1];
-         double GtReg[1];
-         double cReg[2] = {}; // initialized to zero
-
-         for (int mK = 0; mK < (Q1D + mmaK - 1) / mmaK; mK++)
-         {
-            int bRow = bRowInWarp + mK * mmaK;
-            int bColumn = (magicNumber >> (3 * bColumnInWarp)) & 0b111;
-            if (bColumn < D1D && bRow < Q1D)
-            {
-               BtReg[0] = Bt(bRow, bColumn);
-               GtReg[0] = Gt(bRow, bColumn);
-            }
-            else
-            {
-               BtReg[0] = 0;
-               GtReg[0] = 0;
-            }
-            for (int d = 0; d < 3; d++)
-            {
-               double aReg[1];
-               int aRow = aRowInWarp * mPass + mM;
-               int aColumn = aColumnInWarp + mK * mmaK;
-               if (aRow < D1D * D1D && aColumn < Q1D)
-               {
-                  ConstDeviceMatrix Xx(sDDQ[d], Q1D, D1D * D1D); // qz, dx, dy
-                  aReg[0] = Xx(aColumn, aRow);
-               }
-               else
-               {
-                  aReg[0] = 0;
-               }
-
-               dmmaSync(aReg, d == 2 ? GtReg : BtReg, cReg);
-            }
-         }
-#pragma unroll
-         for (int i = 0; i < 2; i++)
-         {
-            int cRow = groupId * mPass + mM;
-            int cColumn = (magicNumber >> (3 * (threadIdInGroup * 2 + i))) & 0b111;
-            if (cRow < D1D * D1D && cColumn < D1D)
-            {
-               int dx = cRow % D1D;
-               int dy = cRow / D1D;
-               int dz = cColumn;
-               Y(dx,dy,dz,e) += cReg[i];
-            }
-         }
-      }
-   }
-}
-
-} // namespace mma
-
-/// PADiffMmaIntegrator ///////////////////////////////////////////////////////
-struct PADiffMmaIntegrator : public BilinearFormIntegrator
-{
-   const FiniteElementSpace *fes;
-   const real_t *B, *G, *DX;
-   int ne, d1d, q1d;
-   Vector J0, dx;
-
-public: // for nvcc
-   //////////////////////////////////////////////////////////////////
-   template <int T_D1D = 0, int T_Q1D = 0>
-   static void PADiffMmaMult(const int ne,
-                             const real_t *b, const real_t *g,
-                             const real_t *dx, const real_t *xe,
-                             real_t *ye,
-                             const int, const int)
-   {
-      constexpr int Q1D = T_Q1D, D1D = T_D1D;
-
-      const auto B = Reshape(b, Q1D, D1D);
-      const auto G = Reshape(g, Q1D, D1D);
-
-      const auto XE = Reshape(xe, D1D, D1D, D1D, ne);
-      const auto DX = Reshape(dx, 3, 3, Q1D, Q1D, Q1D, ne);
-      auto YE = Reshape(ye, D1D, D1D, D1D, ne);
-
-      mfem::forall_3D(ne, ((Q1D * Q1D * Q1D + 31) / 32) * 32, 1, 1,
-                      [=] MFEM_HOST_DEVICE(int e)
-      {
-         constexpr int MQ1 = T_Q1D, MD1 = T_D1D;
-
-         MFEM_SHARED real_t sm0[3][MQ1*MQ1*MQ1];
-         MFEM_SHARED real_t sm1[3][MQ1*MQ1*MQ1];
-         MFEM_SHARED real_t BG[2][MD1*MQ1];
-
-         mma::LoadBG<MD1, MQ1>(D1D, Q1D, B, G, BG);
-         mma::LoadX<MQ1>(e, D1D, XE, sm0);
-         MFEM_SYNC_THREAD;
-
-         mma::GradX<MD1, MQ1>(D1D, Q1D, BG, sm0, sm1);
-         MFEM_SYNC_THREAD;
-         mma::GradY<MD1, MQ1>(D1D, Q1D, BG, sm1, sm0);
-         MFEM_SYNC_THREAD;
-         mma::GradZ<MD1, MQ1>(D1D, Q1D, BG, sm0, sm1);
-         MFEM_SYNC_THREAD;
-
-         int thread = mma::getThreadIdx();
-         if (thread < Q1D * Q1D * Q1D)
-         {
-            int qx = thread % Q1D;
-            int div = thread / Q1D;
-            int qy = div % Q1D;
-            int qz = div / Q1D;
-
-            {
-               // pull
-               real_t v[3], u[3] = { sm1[0][qz + qy*Q1D + qx*Q1D*Q1D],
-                                     sm1[1][qz + qy*Q1D + qx*Q1D*Q1D],
-                                     sm1[2][qz + qy*Q1D + qx*Q1D*Q1D]
-                                   };
-               //  Q-function
-               const real_t *dx = &DX(0, 0, qx, qy, qz, e);
-               kernels::Mult(3, 3, dx, u, v);
-               // push
-               sm0[0][qz + qy*Q1D + qx*Q1D*Q1D] = v[0];
-               sm0[1][qz + qy*Q1D + qx*Q1D*Q1D] = v[1];
-               sm0[2][qz + qy*Q1D + qx*Q1D*Q1D] = v[2];
-            }
-         }
-
-         mma::LoadBtGt<MD1,MQ1>(D1D, Q1D, B, G, BG);
-         MFEM_SYNC_THREAD;
-         mma::GradZt<MD1, MQ1>(D1D, Q1D, BG, sm0, sm1);
-         MFEM_SYNC_THREAD;
-         mma::GradYt<MD1, MQ1>(D1D, Q1D, BG, sm1, sm0);
-         MFEM_SYNC_THREAD;
-         mma::GradXt<MD1,MQ1>(D1D, Q1D, BG, sm0, YE, e);
-      });
-   }
-
-   using PADiffMmaKernelType = decltype(&PADiffMmaMult<>);
-   MFEM_REGISTER_KERNELS(PADiffMmaKernels, PADiffMmaKernelType, (int, int));
-
-public:
-   PADiffMmaIntegrator()
-   {
-      // PADiffMmaKernels::Specialization<2,3>::Add();  // 1 ❌
-      PADiffMmaKernels::Specialization<3,4>::Add();  // 2
-      PADiffMmaKernels::Specialization<4,5>::Add();  // 3
-      PADiffMmaKernels::Specialization<5,6>::Add();  // 4
-      PADiffMmaKernels::Specialization<6,7>::Add();  // 5
-      PADiffMmaKernels::Specialization<7,8>::Add();  // 6
-   }
-
-   void AssemblePA(const FiniteElementSpace &fespace) override
-   {
-      NVTX();
-      fes = &fespace;
-      auto *mesh = fes->GetMesh();
-      const int DIM = mesh->Dimension();
-      ne = mesh->GetNE();
-      const auto p = fes->GetFE(0)->GetOrder();
-      const auto q = 2 * p + mesh->GetElementTransformation(0)->OrderW();
-      const auto type = mesh->GetElementBaseGeometry(0);
-      const IntegrationRule &ir = IntRules.Get(type, q);
-      const int NQPT = ir.GetNPoints();
-      d1d = p + 1;
-      q1d = IntRules.Get(Geometry::SEGMENT, ir.GetOrder()).GetNPoints();
-      MFEM_VERIFY(NQPT == q1d * q1d * q1d, "");
-      const DofToQuad *maps =
-         &fes->GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
-      const GridFunction *nodes = (mesh->EnsureNodes(), mesh->GetNodes());
-      const FiniteElementSpace *nfes = nodes->FESpace();
-      const int nVDIM = nfes->GetVDim();
-      dx.SetSize(nVDIM * DIM * NQPT * ne, Device::GetDeviceMemoryType());
-      J0.SetSize(nVDIM * DIM * NQPT * ne, Device::GetDeviceMemoryType());
-      dx.UseDevice(true), J0.UseDevice(true);
-      B = maps->B.Read(), G = maps->G.Read(), DX = dx.Read();
-
-      const Operator *NR =
-         nfes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC);
-      const QuadratureInterpolator *nqi = nfes->GetQuadratureInterpolator(ir);
-      nqi->SetOutputLayout(QVectorLayout::byVDIM);
-      const int nd = nfes->GetFE(0)->GetDof();
-      Vector xe(nVDIM * nd * ne, Device::GetDeviceMemoryType());
-      NR->Mult(*nodes, (xe.UseDevice(true), xe));
-      nqi->Derivatives(xe, J0);
-
-      const int Q1D = q1d;
-      const auto w_r = ir.GetWeights().Read();
-      const auto W = Reshape(w_r, q1d, q1d, q1d);
-      const auto J = Reshape(J0.Read(), 3, 3, q1d, q1d, q1d, ne);
-      auto DX_w = Reshape(dx.Write(), 3, 3, q1d, q1d, q1d, ne);
-
-      mfem::forall_3D(ne, Q1D, Q1D, Q1D,[=] MFEM_HOST_DEVICE(int e)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qz, z, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  const real_t w = W(qx, qy, qz);
-                  const real_t *Jtr = &J(0, 0, qx, qy, qz, e);
-                  const real_t detJ = kernels::Det<3>(Jtr);
-                  const real_t wd = w * detJ;
-                  const real_t D[9] = { wd, 0.0, 0.0,
-                                        0.0, wd, 0.0,
-                                        0.0, 0.0, wd
-                                      };
-                  real_t Jrt[9], A[9];
-                  kernels::CalcInverse<3>(Jtr, Jrt);
-                  kernels::MultABt(3, 3, 3, D, Jrt, A);
-                  kernels::Mult(3, 3, 3, A, Jrt, &DX_w(0, 0, qz, qy, qx, e));
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      });
-   }
-
-   void AddMultPA(const Vector &x, Vector &y) const override
-   {
-      db1("\x1b[32md1d:{} q1d:{}", d1d, q1d);
-      PADiffMmaKernels::Run(d1d, q1d,
-                            ne, B, G, DX, x.Read(), y.ReadWrite(),
-                            d1d, q1d);
-   }
-};
-template <int D1D, int Q1D>
-PADiffMmaIntegrator::PADiffMmaKernelType
-PADiffMmaIntegrator::PADiffMmaKernels::Kernel()
-{
-   db1("D1D:{} Q1D:{}", D1D, Q1D);
-   return PADiffMmaMult<D1D, Q1D>;
-}
-
-PADiffMmaIntegrator::PADiffMmaKernelType
-PADiffMmaIntegrator::PADiffMmaKernels::Fallback(int d1d, int q1d)
-{
-   dbg("\x1b[33mFallback d1d:{} q1d:{}", d1d, q1d);
-   MFEM_ABORT("No kernel for q1d=" << q1d);
-   return nullptr;
-   //    return PADiffMmaMult;
-}

tests/benchmarks/makefile

@@ -20,8 +20,8 @@ CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
-SEQ_TESTS = bench_assembly_levels bench_ceed bench_dfem bench_dg_amr \
-			bench_elasticity bench_tmop bench_vector bench_virtuals
+SEQ_TESTS = bench_assembly_levels bench_ceed bench_dg_amr bench_elasticity \
+            bench_tmop bench_vector bench_virtuals
 PAR_TESTS = 
 ifeq ($(MFEM_USE_MPI),NO)
    TESTS = $(SEQ_TESTS)

tests/unit/CMakeLists.txt

@@ -25,8 +25,8 @@ add_custom_target(copy_test_data
 # Add 'copy_test_data' as a prerequisite for test executables, if the source and the
 # build directories are not the same.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
-    add_dependencies(${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME} copy_test_data)
-    add_dependencies(${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME} copy_data)
+  add_dependencies(${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME} copy_test_data)
+  add_dependencies(${MFEM_TEST_EXEC_PREREQUISITES_TARGET_NAME} copy_data)
 endif()
 
 # Include the source directory for the unit tests - catch.hpp is there.
@@ -35,12 +35,10 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR})
 # The following list can be updated using (in bash):
 #    for d in dfem general linalg mesh fem enzyme; do ls -1 $d/*.cpp; done
 set(UNIT_TESTS_SRCS
-  # dfem/test_diffusion.cpp
-  # dfem/test_diffusion_q1d.cpp
-  # dfem/test_divergence.cpp
-  # dfem/test_lvector_interface.cpp
-  # dfem/test_mass.cpp
-  dfem/test_multiple_outputs.cpp
+  dfem/test_diffusion.cpp
+  dfem/test_divergence.cpp
+  dfem/test_lvector_interface.cpp
+  dfem/test_mass.cpp
   general/test_array.cpp
   general/test_scan.cpp
   general/test_arrays_by_name.cpp
@@ -258,7 +256,7 @@ function(add_serial_miniapp_test name test_uvm)
         set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
     endif(MFEM_USE_CUDA)
     if (MFEM_USE_HIP)
-        set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE)
+       set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE)
     endif(MFEM_USE_HIP)
 
     mfem_add_executable(${name}_tests_cpu ${${NAME}_TESTS_SRCS})

tests/unit/dfem/test_diffusion.cpp

@@ -302,7 +302,7 @@ void diffusion(const char *filename, int p)
 
 TEST_CASE("dFEM Diffusion", "[Parallel][dFEM][GPU]")
 {
-   // const bool all_tests = launch_all_non_regression_tests;
+   const bool all_tests = launch_all_non_regression_tests;
 
    const auto p = !all_tests ? 2 : GENERATE(1, 2, 3);
 

tests/unit/dfem/test_diffusion_q1d.cpp

@@ -1,338 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#define NVTX_COLOR nvtx::kGold
-
-#include "unit_tests.hpp"
-#include "mfem.hpp"
-// #include <type_traits>
-#include "fem/dfem/doperator.hpp"
-#include "fem/dfem/util.hpp"
-
-#include <fem/integ/bilininteg_diffusion_kernels.hpp>
-
-#ifdef MFEM_USE_MPI
-
-using namespace mfem;
-using namespace mfem::future;
-using mfem::future::tensor;
-using mfem::future::dual;
-
-using DOperator = future::DifferentiableOperator;
-
-enum class MQ1Settings : int { kRuntime,
-                               kCompileTime,
-                               kDefault
-                             };
-
-namespace dfem_pa_kernels
-{
-
-///////////////////////////////////////////////////////////////////////////////
-template <typename T, int DIM, int T_MQ1 = 0> struct Diffusion
-{
-   using dvecd_t = tensor<T, DIM>;
-   using matd_t = tensor<real_t, DIM, DIM>;
-
-   struct MFApply
-   {
-      static constexpr int MQ1 = T_MQ1;
-      MFEM_HOST_DEVICE inline auto operator()(const dvecd_t &dudxi,
-                                              const real_t &rho,
-                                              const matd_t &J,
-                                              const real_t &w) const
-      {
-         const auto invJ = inv(J), TinJ = transpose(invJ);
-         return mfem::future::tuple{ (dudxi * invJ) * TinJ * det(J) * w * rho };
-      }
-   };
-
-   struct PASetup
-   {
-      MFEM_HOST_DEVICE inline auto operator()(const real_t u,
-                                              const real_t &rho,
-                                              const matd_t &J,
-                                              const real_t &w) const
-      {
-         return mfem::future::tuple{ inv(J) * transpose(inv(J)) * det(J) * w * rho };
-      }
-   };
-
-   struct PAApply
-   {
-      MFEM_HOST_DEVICE inline auto operator()(const dvecd_t &dudxi,
-                                              const matd_t &q) const
-      {
-         return mfem::future::tuple{ q * dudxi };
-      };
-   };
-};
-
-///////////////////////////////////////////////////////////////////////////////
-template <typename T, int DIM, std::size_t... MQ1s>
-struct MFDiffusionFactory
-{
-   static auto All()
-   {
-      // could also use a map instead of a tuple
-      return mfem::future::make_tuple(typename Diffusion<T, DIM, MQ1s>::MFApply{}...);
-   }
-};
-
-template <typename T, int DIM>
-using MFDiffusionFactory_1_4 = MFDiffusionFactory<T, DIM, 1, 2, 3, 4>;
-
-template <typename T, int DIM>
-class MFDiffusionQFs
-{
-   using MFApplyTuple = decltype(MFDiffusionFactory_1_4<T, DIM>::All());
-   MFApplyTuple mf_qfs;
-
-public:
-   MFDiffusionQFs(): mf_qfs(MFDiffusionFactory_1_4<T, DIM>::All()) {}
-
-   template <typename F>
-   void run(int i, F&& f)
-   {
-      MFEM_VERIFY(i >= 1, "Index must be >= 1");
-      const auto I = static_cast<size_t>(i - 1);
-      runtime_get_impl(I, std::forward<F>(f),
-                       std::make_index_sequence<mfem::future::tuple_size<MFApplyTuple>::value>());
-   }
-
-private:
-   template <typename F, size_t... I>
-   void runtime_get_impl(size_t index, F&& f, std::index_sequence<I...>)
-   {
-      using fun_ptr = std::function<void(F&&)>;
-      fun_ptr table[] = { [&](F&& f) { f(mfem::future::get<I>(mf_qfs)); } ... };
-      if (index < mfem::future::tuple_size<MFApplyTuple>::value)
-      {
-         table[index](std::forward<F>(f));
-      }
-      else
-      {
-         throw std::out_of_range("Index out of bounds");
-      }
-   }
-};
-
-///////////////////////////////////////////////////////////////////////////////
-template <int DIM>
-void DFemDiffusion(const char *filename, int p, const int r,
-                   const MQ1Settings mq1_setting)
-{
-   dbg("DIM:{}", DIM);
-   CAPTURE(filename, DIM, p, r);
-
-   Mesh smesh(filename);
-   ParMesh pmesh(MPI_COMM_WORLD, smesh);
-   MFEM_VERIFY(pmesh.Dimension() == DIM, "Mesh dimension mismatch");
-
-   pmesh.EnsureNodes();
-   auto *nodes = static_cast<ParGridFunction *>(pmesh.GetNodes());
-   p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder());
-   smesh.Clear();
-
-   Array<int> all_domain_attr;
-   if (pmesh.attributes.Size() > 0)
-   {
-      all_domain_attr.SetSize(pmesh.attributes.Max());
-      all_domain_attr = 1;
-   }
-
-   H1_FECollection fec(p, DIM);
-   ParFiniteElementSpace pfes(&pmesh, &fec);
-   ParFiniteElementSpace *mfes = nodes->ParFESpace();
-
-   const int NE = pfes.GetNE(), d1d(p + 1), q = 2 * p + r;
-   const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), q);
-   const int q1d(IntRules.Get(Geometry::SEGMENT, ir->GetOrder()).GetNPoints());
-   MFEM_VERIFY(d1d <= q1d, "q1d should be >= d1d");
-
-   ParGridFunction x(&pfes), y(&pfes), z(&pfes);
-   Vector X(pfes.GetTrueVSize()), Y(pfes.GetTrueVSize()), Z(pfes.GetTrueVSize());
-
-   X.Randomize(1);
-   x.SetFromTrueDofs(X);
-
-   auto rho = [](const Vector &xyz)
-   {
-      const real_t x = xyz(0), y = xyz(1), z = DIM == 3 ? xyz(2) : 0.0;
-      real_t r = M_PI * pow(x, 2);
-      if (DIM >= 2) { r += pow(y, 3); }
-      if (DIM >= 3) { r += pow(z, 4); }
-      return r;
-   };
-   FunctionCoefficient rho_coeff(rho);
-
-   ParBilinearForm blf_fa(&pfes);
-   blf_fa.AddDomainIntegrator(new DiffusionIntegrator(rho_coeff, ir));
-   blf_fa.Assemble();
-   blf_fa.Finalize();
-
-   QuadratureSpace qs(pmesh, *ir);
-   CoefficientVector rho_coeff_cv(rho_coeff, qs);
-   MFEM_VERIFY(rho_coeff_cv.GetVDim() == 1, "Coefficient should be scalar");
-   MFEM_VERIFY(rho_coeff_cv.Size() == q1d * q1d * (DIM == 3 ? q1d : 1) * NE, "");
-
-   UniformParameterSpace rho_ps(pmesh, *ir, 1);
-
-   static constexpr int U = 0, Coords = 1, Rho = 3;
-   const auto sol = std::vector{ FieldDescriptor{ U, &pfes } };
-
-   SECTION("DFEM Matrix free")
-   {
-      // fields = {solutions, parameters}
-      dbg("fields = {{solutions, parameters}} = {{{{U}}, {{Rho, Coords}}}}");
-      DOperator dop_mf(sol, {{Rho, &rho_ps}, {Coords, mfes}}, pmesh);
-
-      dbg("AddDomainIntegrator: {{∇U, Rho, ∇Coords, Weight}} -> {{∇U}}");
-      if (mq1_setting == MQ1Settings::kRuntime)
-      {
-         dbg("MQ1Settings::kRuntime");
-         MFEM_VERIFY(q1d == (int)floor(std::pow(ir->GetNPoints(), 1.0/DIM) + 0.5),
-                     "q1d and ir->GetNPoints() have to match");
-         auto add_domain_integrator = [&](auto &qf)
-         {
-            dbg("q1d:{} MQ1:{}", q1d, qf.MQ1);
-            MFEM_VERIFY(q1d == qf.MQ1, "q1d and qf.MQ1 have to match");
-            dop_mf.AddDomainIntegrator(qf,
-                                       tuple{ Gradient<U>{}, Identity<Rho>{},
-                                              Gradient<Coords>{}, Weight{} },
-                                       tuple{ Gradient<U>{} }, *ir,
-                                       all_domain_attr);
-         };
-         // select the right qf from the factory
-         MFDiffusionQFs<real_t, DIM> {}.run(q1d, add_domain_integrator);
-      }
-      else if (mq1_setting == MQ1Settings::kCompileTime) // hardcoded, MQ1 = 2,3,4,5
-      {
-         dbg("MQ1Settings::kCompileTime");
-         dbg("q1d:{}", q1d);
-         if (q1d == 2)
-         {
-            typename Diffusion<real_t, DIM, 2>::MFApply mf_apply_qf;
-            MFEM_VERIFY(q1d == 2, "q1d and 2 have to match");
-            dop_mf.AddDomainIntegrator(mf_apply_qf,
-                                       tuple{ Gradient<U>{}, Identity<Rho>{},
-                                              Gradient<Coords>{}, Weight{} },
-                                       tuple{ Gradient<U>{} }, *ir,
-                                       all_domain_attr);
-         }
-         else if (q1d == 3)
-         {
-            typename Diffusion<real_t, DIM, 3>::MFApply mf_apply_qf;
-            MFEM_VERIFY(q1d == 3, "q1d and 3 have to match");
-            dop_mf.AddDomainIntegrator(mf_apply_qf,
-                                       tuple{ Gradient<U>{}, Identity<Rho>{},
-                                              Gradient<Coords>{}, Weight{} },
-                                       tuple{ Gradient<U>{} }, *ir,
-                                       all_domain_attr);
-         }
-         else if (q1d == 4)
-         {
-            typename Diffusion<real_t, DIM, 4>::MFApply mf_apply_qf;
-            MFEM_VERIFY(q1d == 4, "q1d and 4 have to match");
-            dop_mf.AddDomainIntegrator(mf_apply_qf,
-                                       tuple{ Gradient<U>{}, Identity<Rho>{},
-                                              Gradient<Coords>{}, Weight{} },
-                                       tuple{ Gradient<U>{} }, *ir,
-                                       all_domain_attr);
-         }
-         else if (q1d == 5)
-         {
-            typename Diffusion<real_t, DIM, 5>::MFApply mf_apply_qf;
-            MFEM_VERIFY(q1d == 5, "q1d and 5 have to match");
-            dop_mf.AddDomainIntegrator(mf_apply_qf,
-                                       tuple{ Gradient<U>{}, Identity<Rho>{},
-                                              Gradient<Coords>{}, Weight{} },
-                                       tuple{ Gradient<U>{} }, *ir,
-                                       all_domain_attr);
-         }
-         else { MFEM_ABORT("Not supported q1d:" << q1d); }
-      }
-      else // MQ1Settings::kDefault, MQ1 = 0
-      {
-         dbg("MQ1Settings::kDefault");
-         typename Diffusion<real_t, DIM>::MFApply mf_apply_qf;
-         dop_mf.AddDomainIntegrator(mf_apply_qf,
-                                    tuple{ Gradient<U>{}, Identity<Rho>{},
-                                           Gradient<Coords>{}, Weight{} },
-                                    tuple{ Gradient<U>{} }, *ir,
-                                    all_domain_attr);
-      }
-      dop_mf.SetParameters({ &rho_coeff_cv, nodes });
-
-      pfes.GetRestrictionMatrix()->Mult(x, X);
-      dop_mf.Mult(X, Z);
-
-      blf_fa.Mult(x, y);
-      pfes.GetProlongationMatrix()->MultTranspose(y, Y);
-      Y -= Z;
-
-      real_t norm_global = 0.0;
-      real_t norm_local = Y.Normlinf();
-      MPI_Allreduce(&norm_local, &norm_global, 1, MPI_DOUBLE, MPI_MAX,
-                    pmesh.GetComm());
-
-      REQUIRE(norm_global == MFEM_Approx(0.0));
-      MPI_Barrier(MPI_COMM_WORLD);
-   }
-
-}
-
-TEST_CASE("DFEM Diffusion Q1D", "[Parallel][DFEM][MQ1]")
-{
-   // const bool all_tests = launch_all_non_regression_tests;
-
-   // const auto p = !all_tests ? 2 : GENERATE(1, 2, 3);
-   // const auto r = !all_tests ? 1 : GENERATE(0, 1, 2, 3);
-   const int p = 2, r = 1;
-   dbg("p:{} r:{}", p, r);
-
-   const auto mq1_setting = MQ1Settings::kCompileTime;
-   /*const auto mq1_setting = GENERATE(MQ1Settings::kRuntime,
-                                     MQ1Settings::kCompileTime,
-                                     MQ1Settings::kDefault);*/
-
-   DiffusionIntegrator::AddSpecialization<3,3,3>();
-
-   /*SECTION("2D p=" + std::to_string(p) + " r=" + std::to_string(r))
-   {
-      const auto filename =
-         GENERATE("../../data/star.mesh",
-                  "../../data/star-q3.mesh",
-                  "../../data/rt-2d-q3.mesh",
-                  "../../data/inline-quad.mesh",
-                  "../../data/periodic-square.mesh");
-      DFemDiffusion<2>(filename, p, r);
-   }*/
-
-   // SECTION("3D p=" + std::to_string(p) + " r=" + std::to_string(r))
-   {
-#if 0
-      const auto filename =
-         GENERATE("../../data/fichera.mesh",
-                  "../../data/fichera-q3.mesh",
-                  "../../data/inline-hex.mesh",
-                  "../../data/toroid-hex.mesh",
-                  "../../data/periodic-cube.mesh");
-#else
-      const auto filename = "../../data/fichera.mesh";
-#endif
-      dbg("DFemDiffusion");
-      DFemDiffusion<3>(filename, p, r, mq1_setting);
-   }
-}
-
-} // namespace dfem_pa_kernels
-
-#endif

tests/unit/dfem/test_divergence.cpp

@@ -11,7 +11,6 @@
 
 #include "../unit_tests.hpp"
 #include "mfem.hpp"
-#include <fem/dfem/doperator.hpp>
 
 #ifdef MFEM_USE_MPI
 

tests/unit/dfem/test_mass.cpp

@@ -12,7 +12,6 @@
 #include "../unit_tests.hpp"
 #include "../linalg/test_same_matrices.hpp"
 #include "mfem.hpp"
-#include <fem/dfem/doperator.hpp>
 
 #ifdef MFEM_USE_MPI
 

tests/unit/dfem/test_multiple_outputs.cpp

@@ -1,383 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../unit_tests.hpp"
-#include "mfem.hpp"
-#include "../fem/dfem/doperator.hpp"
-#include "../fem/dfem/backends/local_qf/prelude.hpp"
-#include "linalg/tensor_arrays.hpp"
-
-#include <proteus/JitInterface.h>
-
-#ifdef MFEM_USE_MPI
-
-using namespace mfem;
-using namespace mfem::future;
-using mfem::future::tensor;
-
-#ifdef MFEM_USE_ENZYME
-using dscalar_t = real_t;
-#else
-using mfem::future::dual;
-using dscalar_t = dual<real_t, real_t>;
-#endif
-
-constexpr int DIM = 2;
-
-class DummyParameterSpace : public ParameterSpace
-{
-public:
-   class Bimpl : public Operator
-   {
-      virtual void Mult(const Vector &x, Vector &y) const
-      {
-         for (int i = 0; i < y.Size(); i++)
-         {
-            y(i) = x(0);
-         }
-      }
-   };
-
-   class Btimpl : public Operator
-   {
-      virtual void Mult(const Vector &x, Vector &y) const
-      {
-         y(0) = x(0);
-      }
-   };
-
-   DummyParameterSpace() : ParameterSpace(1) {}
-
-   virtual int GetTrueVSize() const override
-   {
-      return 1;
-   }
-
-   virtual int GetVSize() const override
-   {
-      return 1;
-   }
-
-   virtual const Operator* GetB() const override
-   {
-      if (!B)
-      {
-         B.reset(new Bimpl());
-      }
-      return B.get();
-   }
-
-   virtual const Operator* GetBt() const override
-   {
-      if (!Bt)
-      {
-         Bt.reset(new Btimpl());
-      }
-      return Bt.get();
-   }
-};
-
-struct massqf
-{
-   inline MFEM_HOST_DEVICE
-   void operator()(
-      tensor_array<const real_t> &u,
-      tensor_array<const real_t, DIM, DIM> &J,
-      tensor_array<const real_t> &w,
-      tensor_array<real_t> &out1,
-      tensor_array<real_t> &out2) const
-   {
-      for (size_t q = 0; q < u.size(); q++)
-      {
-         const auto v = u(q) * det(J(q)) * w(q);
-         out1(q) = v;
-         out2(q) = v;
-      }
-   }
-};
-
-struct mass_diffusion_qdata_qf
-{
-   inline MFEM_HOST_DEVICE
-   void operator()(
-      tensor_array<const real_t> &u,
-      tensor_array<const real_t, DIM> &dudxi,
-      tensor_array<const real_t, DIM, DIM> &J,
-      tensor_array<const real_t, DIM, DIM> &qdata,
-      tensor_array<const real_t> &w,
-      tensor_array<const real_t> &dummy_parameter,
-      tensor_array<real_t> &out1,
-      tensor_array<real_t, DIM> &out2,
-      tensor_array<real_t, DIM, DIM> &out3) const
-   {
-      for (size_t q = 0; q < u.size(); q++)
-      {
-         const auto invJq = inv(J(q));
-         const auto detJq = det(J(q));
-
-         out1(q) = u(q) * detJq * w(q);
-         // out2(q) = (dudxi(q) * invJq) * transpose(invJq) * (detJq * w(q));
-         out3(q) = J(q);
-      }
-
-      jit_bounds(dudxi, J, w, out2, u.size());
-   }
-
-   // XXX: Attribute instrumentation does not work due to ABI differences that
-   // change the argument number.
-   //__attribute__((annotate("jit", 5)))
-   void jit_bounds(
-      tensor_array<const real_t, DIM> &dudxi,
-      tensor_array<const real_t, DIM, DIM> &J,
-      tensor_array<const real_t> &w,
-      tensor_array<real_t, DIM> &out,
-      size_t NQ) const
-   {
-      for (size_t q = 0; q < NQ; q++)
-      {
-         const auto invJq = inv(J(q));
-         const auto detJq = det(J(q));
-         out(q) = (dudxi(q) * invJq) * transpose(invJq) * (detJq * w(q));
-      }
-   }
-};
-
-TEST_CASE("dFEM Multiple Outputs", "[Parallel][dFEM]")
-{
-   const bool all_tests = launch_all_non_regression_tests;
-
-   const auto p = !all_tests ? 2 : GENERATE(1, 2, 3);
-   const char *filename = "../../data/inline-quad.mesh";
-   CAPTURE(filename, DIM, p);
-
-   Mesh smesh(filename);
-   MFEM_ASSERT(smesh.Dimension() == DIM, "DIM and mesh dimension have to match");
-
-   ParMesh pmesh(MPI_COMM_WORLD, smesh);
-   pmesh.EnsureNodes();
-   auto* nodes = static_cast<ParGridFunction*>(pmesh.GetNodes());
-   smesh.Clear();
-
-   H1_FECollection fec(p, DIM);
-   ParFiniteElementSpace fes(&pmesh, &fec);
-
-   const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p);
-
-   ParGridFunction x(&fes), y(&fes), z(&fes);
-
-   ConstantCoefficient one(1.0);
-
-   Array<int> all_domain_attr;
-   if (pmesh.attributes.Size() > 0)
-   {
-      all_domain_attr.SetSize(pmesh.attributes.Max());
-      all_domain_attr = 1;
-   }
-
-   // {
-   //    Array<int> inoffsets(3);
-   // inoffsets[0] = 0;
-   // inoffsets[1] = fes.GetTrueVSize();
-   // inoffsets[2] = nodes->ParFESpace()->GetTrueVSize();
-   // inoffsets.PartialSum();
-
-   //    BlockVector X(inoffsets);
-   //    X.GetBlock(0).Randomize(1);
-   //    X.GetBlock(1) = *nodes;
-   //    x.SetFromTrueDofs(X.GetBlock(0));
-
-   //    Array<int> outoffsets(2);
-   //    outoffsets[0] = 0;
-   //    outoffsets[1] = fes.GetTrueVSize();
-   //    outoffsets.PartialSum();
-   //    BlockVector Z(outoffsets);
-
-   //    ParBilinearForm blf(&fes);
-   //    blf.AddDomainIntegrator(new MassIntegrator(one, ir));
-   //    blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   //    blf.Assemble();
-   //    blf.Mult(x, y);
-   //    Vector Y(fes.GetTrueVSize());
-   //    fes.GetProlongationMatrix()->MultTranspose(y, Y);
-
-   //    static constexpr int U = 0, COORDINATES = 1, V = 2;
-   //    const std::vector<FieldDescriptor> in
-   //    {
-   //       {U, &fes},
-   //       {COORDINATES, nodes->ParFESpace()}
-   //    };
-
-   //    const std::vector<FieldDescriptor> out // test spaces?
-   //    {
-   //       {V, &fes},
-   //    };
-   //    DifferentiableOperator dop(in, out, pmesh);
-
-   //    auto derivatives = std::integer_sequence<size_t, U> {};
-   //    auto mass_qfunc = massqf{};
-   //    dop.AddDomainIntegrator(mass_qfunc,
-   //                            tuple{ Value<U>{}, Gradient<COORDINATES>{}, Weight{} },
-   //                            tuple{ Value<V>{}, Value<V>{} },
-   //                            *ir, all_domain_attr, derivatives);
-
-   //    fes.GetRestrictionMatrix()->Mult(x, X.GetBlock(0));
-   //    dop.Mult(X, Z);
-
-   //    Vector Y0(Y);
-   //    Y0 *= 2.0;
-   //    Y0 -= Z.GetBlock(0);
-
-   //    real_t norm_g, norm_l = Y0.Normlinf();
-   //    MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
-   //    REQUIRE(norm_g == MFEM_Approx(0.0));
-   //    MPI_Barrier(MPI_COMM_WORLD);
-
-   //    auto ddop = dop.GetDerivative(U, X);
-
-   //    ddop->Mult(X.GetBlock(0), Z);
-   //    Y0 = Y;
-   //    Y0 *= 2.0;
-   //    Y0 -= Z.GetBlock(0);
-
-   //    norm_l = Y0.Normlinf();
-   //    MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
-   //    REQUIRE(norm_g == MFEM_Approx(0.0));
-   //    MPI_Barrier(MPI_COMM_WORLD);
-   // }
-
-   {
-      QuadratureSpace qs(pmesh, *ir);
-      QuadratureFunction qdata(qs, DIM*DIM);
-
-      DummyParameterSpace dps;
-      ParameterFunction dpf(dps);
-      dpf = 9.12345;
-
-      auto coef_func = [](const Vector &coords)
-      {
-         return coords[0] * coords[1] * (DIM == 3 ? coords[2] : 1.0);
-      };
-      FunctionCoefficient coef(coef_func);
-      x.ProjectCoefficient(coef);
-
-      Vector xtvec, ytvec, ytvecmfem;
-      x.GetTrueDofs(xtvec);
-      ytvec.SetSize(xtvec.Size());
-      ytvecmfem.SetSize(xtvec.Size());
-
-      Vector nodestvec;
-      nodes->GetTrueDofs(nodestvec);
-
-      qdata = 123.0;
-      Vector yqdata(qdata.Size());
-
-      MultiVector X{xtvec, nodestvec, qdata, dpf};
-      MultiVector Z{ytvec, yqdata};
-
-      ParBilinearForm blf(&fes);
-      blf.AddDomainIntegrator(new MassIntegrator(ir));
-      blf.AddDomainIntegrator(new DiffusionIntegrator(ir));
-      blf.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      blf.Assemble();
-      blf.Mult(x, y);
-      fes.GetProlongationMatrix()->MultTranspose(y, ytvecmfem);
-
-      std::cout << "mfem: ";
-      pretty_print(ytvecmfem);
-
-      static constexpr int U = 0, COORDINATES = 1, V = 2, S = 3, L = 4;
-      const std::vector<FieldDescriptor> in
-      {
-         {U, &fes},
-         {COORDINATES, nodes->ParFESpace()},
-         {S, &qdata},
-         {L, &dps}
-      };
-
-      const std::vector<FieldDescriptor> out
-      {
-         {V, &fes},
-         {S, &qdata}
-      };
-
-      {
-         DifferentiableOperator dop(in, out, pmesh);
-
-         dop.SetQLayouts({{Value<U>{}, {1, 0}}}, {});
-
-         auto derivatives = std::integer_sequence<size_t, U> {};
-         auto mass_diffusion_qfunc = mass_diffusion_qdata_qf{};
-         dop.AddDomainIntegrator(mass_diffusion_qfunc,
-                                 tuple{Value<U>{}, Gradient<U>{}, Gradient<COORDINATES>{}, Identity<S>{}, Weight{}, Value<L>{}},
-                                 tuple{Value<V>{}, Gradient<V>{}, Identity<S>{}},
-                                 *ir, all_domain_attr, derivatives);
-
-         fes.GetRestrictionMatrix()->Mult(x, xtvec);
-         dop.Mult(X, Z);
-
-         std::cout << "dfem: ";
-         pretty_print(Z[0]);
-
-         Vector Y0(ytvecmfem);
-         Y0 -= Z[0];
-
-         real_t norm_l = Y0.Normlinf();
-         real_t norm_g = norm_l;
-         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
-         REQUIRE(norm_g == MFEM_Approx(0.0));
-         MPI_Barrier(MPI_COMM_WORLD);
-
-         auto ddop = dop.GetDerivative(U, X);
-
-         ddop->Mult(X[0], Z);
-         Y0 = ytvecmfem;
-         Y0 -= Z[0];
-
-         std::cout << "∂dfem: ";
-         pretty_print(Z[0]);
-
-         norm_l = Y0.Normlinf();
-         norm_g = norm_l;
-         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
-         REQUIRE(norm_g == MFEM_Approx(0.0));
-         MPI_Barrier(MPI_COMM_WORLD);
-      }
-      {
-         DifferentiableOperator dop(in, out, pmesh);
-
-         dop.SetQLayouts({{Value<U>{}, {1, 0}}}, {});
-
-         auto mass_diffusion_qfunc = mass_diffusion_qdata_qf{};
-         dop.AddDomainIntegrator<LocalQFBackend>(
-            mass_diffusion_qfunc,
-            tuple{Value<U>{}, Gradient<U>{}, Gradient<COORDINATES>{}, Identity<S>{}, Weight{}, Value<L>{}},
-            tuple{Value<V>{}, Gradient<V>{}, Identity<S>{}},
-            *ir, all_domain_attr);
-
-         fes.GetRestrictionMatrix()->Mult(x, xtvec);
-         dop.Mult(X, Z);
-
-         std::cout << "dfem: ";
-         pretty_print(Z[0]);
-
-         Vector Y0(ytvecmfem);
-         Y0 -= Z[0];
-
-         real_t norm_l = Y0.Normlinf();
-         real_t norm_g = norm_l;
-         MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm());
-         REQUIRE(norm_g == MFEM_Approx(0.0));
-         MPI_Barrier(MPI_COMM_WORLD);
-      }
-   }
-}
-
-#endif // MFEM_USE_MPI

tests/unit/fem/test_pa_kernels.cpp

@@ -962,9 +962,9 @@ TEST_CASE("Dispatch Map Specializations")
 
    DiffusionIntegrator{};
    REQUIRE_FALSE(
-      DiffusionIntegrator::DiffusionApplyPAKernel::GetDispatchTable().empty());
+      DiffusionIntegrator::ApplyPAKernels::GetDispatchTable().empty());
    REQUIRE_FALSE(
-      DiffusionIntegrator::DiffusionDiagonalPAKernel::GetDispatchTable().empty());
+      DiffusionIntegrator::DiagonalPAKernels::GetDispatchTable().empty());
 
    Mesh mesh = Mesh::MakeCartesian2D(2, 2, Element::QUADRILATERAL);
    H1_FECollection fec(1, mesh.Dimension());