Remove unnecessary SetCurvature

.gitignore

@@ -214,7 +214,6 @@ miniapps/adjoint/adjoint_advection_diffusion
 
 miniapps/dfem/dfem-minimal-surface
 miniapps/dfem/dfem-minimal-surface-output
-miniapps/dfem/dfem-hyperelasticity_energy
 
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
@@ -261,7 +260,6 @@ miniapps/meshing/polar-nc
 miniapps/meshing/mesh-quality
 miniapps/meshing/hpref
 miniapps/meshing/phpref
-miniapps/meshing/pref321
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh

.gitlab/dane-baseline.yml

@@ -102,14 +102,12 @@ report_baseline:
         mkdir -p ${MACHINE_NAME}
         rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
         rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        status=0
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir} || { status=1; }
+        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
         printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
           >> ${rundir}/pipeline.txt
         # We create an autotest-email.html file, because that's how we signal
         # that there was an error / diff (temporary).
-        if [[ $status -ne 0 ]] || \
-           [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
+        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
            [[ -f ${rundir}/${BASELINE_TEST}-${MACHINE_NAME}.diff ]]; then
           cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
         fi

CHANGELOG

@@ -44,24 +44,10 @@ Discretization improvements
 - Added methods to estimate function extremum using piecewise linear bounds plus
   recursive subdivision.
 
-- Added class VectorQuadratureSpace that represents a vector (multi-component)
-  version of the scalar (single-component) class QuadratureSpaceBase.
-
 - Extend FindPointsGSLIB to support surface meshes.
 
-- Added support for complex-valued mixed bilinear forms via the new classes
-  MixedSesquilinearForm and ParMixedSesquilinearForm, mirroring the existing
-  SesquilinearForm classes. Rectangular complex operators are now also
-  handled correctly by ComplexSparseMatrix::GetSystemMatrix and
-  ComplexHypreParMatrix::GetSystemMatrix, which previously assumed equal
-  trial and test spaces.
-
 Meshing improvements
 --------------------
-- Added support for nonuniform anisotropic mesh refinement on parallel quad/hex
-  meshes with arbitrary spacing in each direction. This enables in particular
-  3:1 refinement in parallel, as demonstrated in the new meshing miniapp pref321.
-
 - Added option to guarantee mesh validity during TMOP-based r-adaptivity, using
   bounds on the determinant of the mesh transformation Jacobian.
 
@@ -84,15 +70,6 @@ Linear and nonlinear solvers
 
 GPU computing
 -------------
-- Improved partial assembly for VectorDivergenceIntegrator with shared-memory
-  kernels, kernel registration, and transpose support.
-
-- Improved partial-assembly diagonal kernels for VectorMassIntegrator (shared-
-  memory specializations) and ElasticityIntegrator (no scratch Q-vector).
-
-- Added PA gradient and diagonal support for VectorConvectionNLFIntegrator
- (AssembleGradPA, AddMultGradPA, AssembleGradDiagonalPA).
-
 - Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
 
 - Added NVIDIA cuDSS library interface. Implementation examples have been
@@ -101,12 +78,6 @@ GPU computing
 
 - Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
 
-- Changed VectorFEMassIntegrator to use kernel specialization dispatch for
-  partial assembly.
-
-- Added support for FiniteElement::MapType::INTEGRAL spaces to
-  QuadratureInterpolator.
-
 New and updated examples and miniapps
 -------------------------------------
 - The Lorentz miniapp (in miniapps/electromagnetics) has been updated to

CMakeLists.txt

@@ -184,45 +184,6 @@ if (MFEM_USE_CUDA)
    set(CMAKE_CUDA_EXTENSIONS OFF CACHE BOOL "Enable CUDA standard extensions.")
    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
    find_package(CUDAToolkit REQUIRED)
-   if(CMAKE_CUDA_COMPILER_ID STREQUAL "Clang")
-     set(_real_fatbinary "${CMAKE_CUDA_FATBINARY}")
-     set(_fatbinary_wrapper
-       "${CMAKE_BINARY_DIR}/cmake-fatbinary-cuda13")
-
-     file(WRITE "${_fatbinary_wrapper}"
-"#!/usr/bin/env bash
-real_fatbinary=\"${_real_fatbinary}\"
-"
-[=[
-translated=()
-
-for argument in "$@"; do
-  case "$argument" in
-    -im=profile=sm_*,file=*)
-      value=${argument#-im=profile=sm_}
-      architecture=${value%%,*}
-      filename=${value#*,file=}
-      translated+=(
-        "--image3=kind=elf,sm=${architecture},file=${filename}"
-      )
-      ;;
-    *)
-      translated+=("$argument")
-      ;;
-  esac
-done
-
-exec "$real_fatbinary" "${translated[@]}"
-]=])
-
-    file(CHMOD "${_fatbinary_wrapper}"
-    PERMISSIONS
-      OWNER_READ OWNER_WRITE OWNER_EXECUTE
-      GROUP_READ GROUP_EXECUTE
-      WORLD_READ WORLD_EXECUTE)
-
-    set(CMAKE_CUDA_FATBINARY "${_fatbinary_wrapper}")
-endif()
    set(CUSPARSE_FOUND TRUE)
    set(CUBLAS_FOUND TRUE)
    # Initialize CUSPARSE_LIBRARIES and CUBLAS_LIBRARIES:
@@ -278,13 +239,6 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
-# Shadow warnings for clang only; GCC's -Wshadow flags more.
-if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall -Wshadow")
-elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall")
-endif()
-
 # Shared build on Windows
 if (WIN32 AND BUILD_SHARED_LIBS)
   # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?
@@ -647,13 +601,6 @@ if (MFEM_USE_ENZYME)
     set(ENZYME_INCLUDE_DIRS ${ENZYME_DIR}/include)
 endif()
 
-if (MFEM_USE_PROTEUS)
-    enable_language(C)
-    find_package(proteus REQUIRED PATHS "${PROTEUS_DIR}")
-    message(STATUS "${PROTEUS_DIR}/include")
-    include_directories("${PROTEUS_DIR}/include")
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -790,16 +737,6 @@ mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
-
-if (MFEM_USE_PROTEUS)
-    add_library(ClangProteusFlags INTERFACE IMPORTED)
-    set_target_properties(ClangProteusFlags PROPERTIES
-        INTERFACE_COMPILE_OPTIONS "-fpass-plugin=$<TARGET_FILE:ProteusPass>"
-    )
-    target_link_libraries(mfem PUBLIC ClangProteusFlags)
-    target_link_libraries(mfem PUBLIC proteus)
-endif()
-
 if (TPL_TARGETS)
   add_dependencies(mfem ${TPL_TARGETS})
 endif()
@@ -807,7 +744,7 @@ if (MINGW)
   target_link_libraries(mfem PRIVATE ws2_32)
 endif()
 if (MSVC)
-  target_compile_options(mfem PUBLIC "/wd4819" "/bigobj")
+  target_compile_options(mfem PUBLIC "/wd4819")
 endif()
 message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
 target_include_directories(mfem
@@ -836,7 +773,7 @@ set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
 # If building out-of-source, define MFEM_CONFIG_FILE to point to the config file
 # inside the build directory.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
-  target_compile_definitions(mfem PUBLIC
+  target_compile_definitions(mfem PRIVATE
     "MFEM_CONFIG_FILE=\"${PROJECT_BINARY_DIR}/config/_config.hpp\"")
 endif()
 
@@ -896,15 +833,16 @@ if (MFEM_ENABLE_TESTING)
   add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
   add_subdirectory(tests EXCLUDE_FROM_ALL)
 
-  # Create a target for all benchmarks and, optionally, enable it. Some simple
-  # timer-based benchmarks in tests/benchmarks do not require Google Benchmark.
-  set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
-  add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
-                  ${MFEM_ENABLE_BENCHMARKS})
-  if (MFEM_ENABLE_BENCHMARKS)
-    add_subdirectory(tests/benchmarks) #install benchmarks if enabled
-  else()
-    add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
+  if (MFEM_USE_BENCHMARK)
+    # Create a target for all benchmarks and, optionally, enable it.
+    set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
+    add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
+                    ${MFEM_ENABLE_BENCHMARKS})
+    if (MFEM_ENABLE_BENCHMARKS)
+      add_subdirectory(tests/benchmarks) #install benchmarks if enabled
+    else()
+      add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
+    endif()
   endif()
 endif()
 
@@ -1084,7 +1022,6 @@ install(EXPORT ${PROJECT_NAME_UC}Targets
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
   DESTINATION ${MFEM_INSTALL_DIR} OPTIONAL)
 
-
 #-------------------------------------------------------------------------------
 # Create 'config.mk' from 'config.mk.in' for the build and install locations and
 # define install rules for 'config.mk' and 'test.mk'

config/config.hpp

@@ -18,9 +18,9 @@
 #define MFEM_CONFIG_HPP
 
 #ifdef MFEM_CONFIG_FILE
-#include MFEM_CONFIG_FILE // IWYU pragma: export
+#include MFEM_CONFIG_FILE
 #else
-#include "_config.hpp" // IWYU pragma: export
+#include "_config.hpp"
 #endif
 
 #include <cstdint>

config/defaults.mk

@@ -27,13 +27,7 @@ MPICXX = mpicxx
 
 BASE_FLAGS  = -std=c++17
 OPTIM_FLAGS = -O3 $(BASE_FLAGS)
-
-# Shadow warnings for clang only; GCC's -Wshadow flags more.
-SHADOW_WARNING_FLAG = $(if $(findstring clang,\
- $(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
-WARNING_FLAGS = -pedantic -Wall $(SHADOW_WARNING_FLAG)
-
-DEBUG_FLAGS = $(strip -g $(addprefix $(XCOMPILER),$(WARNING_FLAGS)) $(BASE_FLAGS))
+DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS)
 
 # Prefixes for passing flags to the compiler and linker when using CXX or MPICXX
 CXX_XCOMPILER =

data/tinyzoo-2d.mesh

@@ -1,38 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-
-dimension
-2
-
-elements
-2
-1 3 0 1 4 3
-1 2 1 2 4
-
-boundary
-5
-1 1 0 1
-1 1 1 2
-1 1 2 4
-1 1 4 3
-1 1 3 0
-
-vertices
-5
-2
-0 0
-1 0
-2 0
-0 1
-1 1

doc/CodeDocumentation.dox

@@ -119,6 +119,8 @@ namespace mfem {
  * - <a class="el" href="ex40p_8cpp_source.html">Example 40p</a>: parallel eikonal equation
  * - <a class="el" href="ex41_8cpp_source.html">Example 41</a>: DG/CG IMEX time-dependent advection-diffusion
  * - <a class="el" href="ex41p_8cpp_source.html">Example 41p</a>: parallel DG/CG IMEX time-dependent advection-diffusion
+ * - <a class="el" href="ex42_8cpp_source.html">Example 42</a>: clamped biharmonic equation
+ * - <a class="el" href="ex42p_8cpp_source.html">Example 42p</a>: parallel clamped biharmonic equation
  *
  * <H4>AmgX Examples</H4>
  * - Variants of Examples
@@ -201,7 +203,6 @@ namespace mfem {
  * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
  * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
  * - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
- * - <a class="el" href="nurbs__surface_8cpp_source.html">NURBS Surface</a>: interpolate a 3D Surface in a NURBS Patch
  *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -246,9 +247,6 @@ namespace mfem {
  * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
  * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
  * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
- * - <a class="el" href="reflector_8cpp_source.html">Reflector Miniapp</a>: reflect a mesh about a plane
- * - <a class="el" href="ref321_8cpp_source.html">3:1 Refinement Miniapp</a>: perform 3:1 anisotropic mesh refinements
- * - <a class="el" href="pref321_8cpp_source.html">3:1 Refinement Miniapp</a>: parallel 3:1 anisotropic mesh refinements
  *
  * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
  */

examples/CMakeLists.txt

@@ -47,7 +47,7 @@ list(APPEND ALL_EXE_SRCS
   ex39.cpp
   ex40.cpp
   ex41.cpp
-  # jitplayground.cpp
+  ex42.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -92,6 +92,7 @@ if (MFEM_USE_MPI)
     ex39p.cpp
     ex40p.cpp
     ex41p.cpp
+    ex42p.cpp
   )
 endif()
 

examples/ex42.cpp

@@ -0,0 +1,311 @@
+//                                MFEM Example 42
+//
+// Compile with: make ex42p
+//
+// Sample runs: ex42 -r 3
+//              ex42 -m ../data/hexagon.mesh -r 3 -o 3
+//              ex42 -m ../data/square-mixed.mesh -r 2 -eta 5
+//              ex42 -m ../data/l-shape.mesh -r 3
+//
+// Description: This example solves the clamped biharmonic equation,
+//
+//                      ∇⁴u = f in Ω,  u = 0 and ∇u⋅n = 0 on ∂Ω,
+//
+//              in 2D using just H¹-conforming finite elements by employing the interior penalty
+//              method outlined in [1]. This example demonstrates an approach to solving higher-order
+//              PDEs in MFEM and implementation of custom domain and face integrators to solve the
+//              weak form
+//
+//                      (H(u), H(v))_D - <{{n^T⋅H(u)⋅n}}, [[∇v⋅n]]>_F
+//                                     - <{{n^T⋅H(v)⋅n}}, [[∇u⋅n]]>_F
+//                                     + (η/h_e)<[[∇u⋅n]], [[∇v⋅n]]>_F = (f,v)_D ,
+//
+//               where (⋅,⋅)_D is domain integration, <⋅,⋅>_F is face
+//               integration, and H(⋅) is the Hessian.
+//
+//              [1] Brenner, Susanne & Sung, Li-yeng. (2005). C0 Interior Penalty Methods
+//                  for Fourth Order Elliptic Boundary Value Problems on Polygonal Domains.
+//                  Journal of Scientific Computing. 22-23. 83-118. 10.1007/s10915-004-4135-7.
+
+#include <mfem.hpp>
+
+using namespace mfem;
+using namespace std;
+
+class BiharmonicIntegrator : public BilinearFormIntegrator
+{
+private:
+   Coefficient &D;
+
+   inline static const Vector factors_2D{1.0, 2.0, 1.0};
+   mutable DenseMatrix hessian;
+   mutable Vector factors;
+public:
+   BiharmonicIntegrator(Coefficient &D_) : D(D_) {}
+
+   void AssembleElementMatrix(const FiniteElement &el,
+                              ElementTransformation &Trans, DenseMatrix &elmat) override;
+};
+
+class C0InteriorPenaltyIntegrator : public BilinearFormIntegrator
+{
+private:
+   const double eta;
+
+   mutable Vector normal[2], dnshape[2], nv[2], nd2nshape[2];
+   mutable DenseMatrix dshape[2], hessian[2], blockJ[2][2], blockC[2][2], elmatJ_p,
+           elmatC_p;
+public:
+   C0InteriorPenaltyIntegrator(double eta_) : eta(eta_) {};
+
+   void AssembleFaceMatrix(const FiniteElement &el1, const FiniteElement &el2,
+                           FaceElementTransformations &Trans, DenseMatrix &elmat) override;
+};
+
+int main(int argc, char *argv[])
+{
+   // Parse command line args
+   const char *mesh_file = "../data/star.mesh";
+   int order = 2;
+   int ref_levels = 0;
+   real_t eta = 10;
+   int max_it = 10000;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&eta, "-eta", "--penalty-coeff",
+                  "Penalty coefficient.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // Read the mesh file
+   Mesh mesh(mesh_file);
+   int dim = mesh.Dimension();
+   MFEM_ASSERT(dim == 2, "This example only supports 2D meshes.");
+
+   // Refine the mesh
+   for (int i = 0; i < ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // Initialize the FE collection and FiniteElementSpace
+   H1_FECollection fe_coll(order, dim);
+   FiniteElementSpace fespace(&mesh, &fe_coll, 1);
+
+   // Get the degrees-of-freedom (DOFs) associated with the sides of the panel
+   Array<int> all_bdr_marker(mesh.bdr_attributes.Size());
+   all_bdr_marker = 1; // Mark all sides
+   Array<int> ess_tdof_list;
+   fespace.GetEssentialTrueDofs(all_bdr_marker, ess_tdof_list);
+
+   ConstantCoefficient one(1.0);
+
+   // Initialize the bilinear form
+   BilinearForm a(&fespace);
+   a.AddDomainIntegrator(new BiharmonicIntegrator(one));
+   a.AddInteriorFaceIntegrator(new C0InteriorPenaltyIntegrator(eta));
+   a.AddBdrFaceIntegrator(new C0InteriorPenaltyIntegrator(eta));
+   a.Assemble();
+
+   // Initialize the linear form f=1.0
+   LinearForm b(&fespace);
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.Assemble();
+
+   // Form the linear system
+   GridFunction x(&fespace);
+   x = 0.0; // initial guess
+   SparseMatrix A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   // Solve the system using CG with symmetric Gauss-Seidel preconditioner
+   GSSmoother M(A);
+   PCG(A, M, B, X, 1, max_it, 1e-12, 0.0);
+
+   // Recover solution and visualize
+   a.RecoverFEMSolution(X, B, x);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int visport    = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << mesh << x << flush;
+   }
+
+   return 0;
+}
+
+
+void BiharmonicIntegrator::AssembleElementMatrix(const FiniteElement &el,
+                                                 ElementTransformation &Trans, DenseMatrix &elmat)
+{
+   int ndof = el.GetDof();
+   int dim = el.GetDim();
+
+   MFEM_ASSERT(dim == 2, "Dimension must be 2.");
+
+   hessian.SetSize(ndof, dim * (dim + 1) / 2);
+   elmat.SetSize(ndof);
+   factors.SetSize(dim * (dim + 1) / 2);
+
+   elmat = 0.0;
+
+   const IntegrationRule *ir = GetIntegrationRule(el, Trans);
+   if (ir == NULL)
+   {
+      int order = 2*el.GetOrder();
+      ir = &IntRules.Get(el.GetGeomType(), order);
+   }
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const mfem::IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint(&ip);
+
+      el.CalcPhysHessian(Trans, hessian);
+
+      factors = factors_2D;
+      factors *= D.Eval(Trans, ip) * ip.weight * Trans.Weight();
+
+      AddMultADAt(hessian, factors, elmat);
+   }
+}
+
+void C0InteriorPenaltyIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
+                                                     const FiniteElement &el2, FaceElementTransformations &Trans, DenseMatrix &elmat)
+{
+   int dim = el1.GetDim();
+   MFEM_ASSERT(dim == 2, "Dimension must be 2.");
+
+   int ndof[2] = {el1.GetDof(), 0};
+   int num_elems = 1;
+   if (Trans.Elem2No >= 0)
+   {
+      ndof[1] = el2.GetDof();
+      num_elems++;
+   }
+
+   for (int i = 0; i < num_elems; i++)
+   {
+      normal[i].SetSize(dim);
+      dshape[i].SetSize(ndof[i], dim);
+      hessian[i].SetSize(ndof[i], dim * (dim + 1) / 2);
+      nv[i].SetSize(dim * (dim + 1) / 2);
+      dnshape[i].SetSize(ndof[i]);
+      nd2nshape[i].SetSize(ndof[i]);
+   }
+
+   for (int i = 0; i < num_elems; i++)
+   {
+      for (int j = 0; j < num_elems; j++)
+      {
+         blockJ[i][j].SetSize(ndof[i], ndof[j]);
+         blockC[i][j].SetSize(ndof[i], ndof[j]);
+      }
+   }
+
+   elmatJ_p.SetSize(ndof[0] + ndof[1]);
+   elmatC_p.SetSize(ndof[0] + ndof[1]);
+   elmat.SetSize(ndof[0] + ndof[1]);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * max(el1.GetOrder(), ndof[1] ? el2.GetOrder() : 0);
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   // Compute edge length
+   double h_e  = 0.0;
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      Trans.SetAllIntPoints(&ip);
+      h_e += ip.weight * Trans.Weight();
+   }
+
+   const FiniteElement *els[2] = {&el1, &el2};
+   ElementTransformation *el_trans[2] = {Trans.Elem1, Trans.Elem2};
+
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      elmatJ_p = 0.0;
+      elmatC_p = 0.0;
+
+      const IntegrationPoint &ip = ir->IntPoint(p);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Compute normal gradients + Hessians
+      for (int i = 0; i < num_elems; i++)
+      {
+         if (i == 0)
+         {
+            CalcOrtho(Trans.Jacobian(), normal[i]);
+            normal[i] /= normal[i].Norml2();
+         }
+         else
+         {
+            normal[i] = normal[0];
+            normal[i] *= -1;
+         }
+         els[i]->CalcPhysDShape(*el_trans[i], dshape[i]);
+         els[i]->CalcPhysHessian(*el_trans[i], hessian[i]);
+         dshape[i].Mult(normal[i], dnshape[i]);
+         nv[i][0] = normal[i][0]*normal[i][0];
+         nv[i][1] = 2*normal[i][0]*normal[i][1];
+         nv[i][2] = normal[i][1]*normal[i][1];
+         hessian[i].Mult(nv[i], nd2nshape[i]);
+      }
+
+      // Compute blocks
+      for (int i = 0; i < num_elems; i++)
+      {
+         for (int j = 0; j < num_elems; j++)
+         {
+            blockJ[i][j] = 0.0;
+            blockC[i][j] = 0.0;
+            AddMult_a_VWt(-1.0, dnshape[i], nd2nshape[j], blockJ[i][j]);
+            elmatJ_p.SetSubMatrix(i*ndof[0], j*ndof[0], blockJ[i][j]);
+
+            AddMult_a_VWt(eta/h_e, dnshape[i], dnshape[j], blockC[i][j]);
+            elmatC_p.SetSubMatrix(i*ndof[0], j*ndof[0], blockC[i][j]);
+         }
+      }
+
+      // Symmetrize the jump term
+      elmatJ_p.Symmetrize();
+      if (!ndof[1])
+      {
+         elmatJ_p *= 2;
+      }
+
+      // Add penalty term
+      elmatJ_p += elmatC_p;
+      elmatJ_p *= ip.weight * Trans.Weight();
+      elmat += elmatJ_p;
+   }
+}

examples/ex42p.cpp

@@ -0,0 +1,331 @@
+//                       MFEM Example 42 - Parallel Version
+//
+// Compile with: make ex42p
+//
+// Sample runs: mpirun -np 4 ex42p -r 3
+//              mpirun -np 4 ex42p -m ../data/hexagon.mesh -r 3 -o 3
+//              mpirun -np 4 ex42p -m ../data/square-mixed.mesh -r 2 -eta 5
+//              mpirun -np 4 ex42p -m ../data/l-shape.mesh -r 3
+//
+// Description: This example solves the clamped biharmonic equation,
+//
+//                      ∇⁴u = f in Ω,  u = 0 and ∇u⋅n = 0 on ∂Ω,
+//
+//              in 2D using just H¹-conforming finite elements by employing the interior penalty
+//              method outlined in [1]. This example demonstrates an approach to solving higher-order
+//              PDEs in MFEM and implementation of custom domain and face integrators to solve the
+//              weak form
+//
+//                      (H(u), H(v))_D - <{{n^T⋅H(u)⋅n}}, [[∇v⋅n]]>_F
+//                                     - <{{n^T⋅H(v)⋅n}}, [[∇u⋅n]]>_F
+//                                     + (η/h_e)<[[∇u⋅n]], [[∇v⋅n]]>_F = (f,v)_D ,
+//
+//               where (⋅,⋅)_D is domain integration, <⋅,⋅>_F is face
+//               integration, and H(⋅) is the Hessian.
+//
+//              [1] Brenner, Susanne & Sung, Li-yeng. (2005). C0 Interior Penalty Methods
+//                  for Fourth Order Elliptic Boundary Value Problems on Polygonal Domains.
+//                  Journal of Scientific Computing. 22-23. 83-118. 10.1007/s10915-004-4135-7.
+
+#include <mfem.hpp>
+
+using namespace mfem;
+using namespace std;
+
+class BiharmonicIntegrator : public BilinearFormIntegrator
+{
+private:
+   Coefficient &D;
+
+   inline static const Vector factors_2D{1.0, 2.0, 1.0};
+   mutable DenseMatrix hessian;
+   mutable Vector factors;
+public:
+   BiharmonicIntegrator(Coefficient &D_) : D(D_) {}
+
+   void AssembleElementMatrix(const FiniteElement &el,
+                              ElementTransformation &Trans, DenseMatrix &elmat) override;
+};
+
+class C0InteriorPenaltyIntegrator : public BilinearFormIntegrator
+{
+private:
+   const double eta;
+
+   mutable Vector normal[2], dnshape[2], nv[2], nd2nshape[2];
+   mutable DenseMatrix dshape[2], hessian[2], blockJ[2][2], blockC[2][2], elmatJ_p,
+           elmatC_p;
+public:
+   C0InteriorPenaltyIntegrator(double eta_) : eta(eta_) {};
+
+   void AssembleFaceMatrix(const FiniteElement &el1, const FiniteElement &el2,
+                           FaceElementTransformations &Trans, DenseMatrix &elmat) override;
+};
+
+int main(int argc, char *argv[])
+{
+   // Initialize MPI and HYPRE.
+   Mpi::Init(argc, argv);
+   int size = Mpi::WorldSize();
+   int rank = Mpi::WorldRank();
+   Hypre::Init();
+
+   // Parse command line args
+   const char *mesh_file = "../data/star.mesh";
+   int order = 2;
+   int ref_levels = 0;
+   real_t eta = 10;
+   int max_it = 10000;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&eta, "-eta", "--penalty-coeff",
+                  "Penalty coefficient.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (rank == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // Read the mesh file
+   Mesh mesh(mesh_file);
+   int dim = mesh.Dimension();
+   MFEM_ASSERT(dim == 2, "This example only supports 2D meshes.");
+
+   // Refine the mesh
+   for (int i = 0; i < ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // Partition the mesh
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Initialize the FE collection and FiniteElementSpace
+   H1_FECollection fe_coll(order, dim);
+   ParFiniteElementSpace fespace(&pmesh, &fe_coll, 1);
+
+   // Get the degrees-of-freedom (DOFs) associated with the sides of the panel
+   Array<int> all_bdr_marker(pmesh.bdr_attributes.Size());
+   all_bdr_marker = 1; // Mark all sides
+   Array<int> ess_tdof_list;
+   fespace.GetEssentialTrueDofs(all_bdr_marker, ess_tdof_list);
+
+   ConstantCoefficient one(1.0);
+
+   // Initialize the bilinear form
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new BiharmonicIntegrator(one));
+   a.AddInteriorFaceIntegrator(new C0InteriorPenaltyIntegrator(eta));
+   a.AddBdrFaceIntegrator(new C0InteriorPenaltyIntegrator(eta));
+   a.Assemble();
+
+   // Initialize the linear form f=1.0
+   ParLinearForm b(&fespace);
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.Assemble();
+
+   // Form the linear system
+   ParGridFunction x(&fespace);
+   x = 0.0; // initial guess
+   HypreParMatrix A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   // Solve the system using CG with hypre's BoomerAMG preconditioner
+   HypreBoomerAMG amg(A);
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(max_it);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(amg);
+   cg.SetOperator(A);
+   cg.Mult(B, X);
+
+   // Recover solution and visualize
+   a.RecoverFEMSolution(X, B, x);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int visport    = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << size << " " << rank << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << x << flush;
+   }
+
+   return 0;
+}
+
+
+void BiharmonicIntegrator::AssembleElementMatrix(const FiniteElement &el,
+                                                 ElementTransformation &Trans, DenseMatrix &elmat)
+{
+   int ndof = el.GetDof();
+   int dim = el.GetDim();
+
+   MFEM_ASSERT(dim == 2, "Dimension must be 2.");
+
+   hessian.SetSize(ndof, dim * (dim + 1) / 2);
+   elmat.SetSize(ndof);
+   factors.SetSize(dim * (dim + 1) / 2);
+
+   elmat = 0.0;
+
+   const IntegrationRule *ir = GetIntegrationRule(el, Trans);
+   if (ir == NULL)
+   {
+      int order = 2*el.GetOrder();
+      ir = &IntRules.Get(el.GetGeomType(), order);
+   }
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const mfem::IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint(&ip);
+
+      el.CalcPhysHessian(Trans, hessian);
+
+      factors = factors_2D;
+      factors *= D.Eval(Trans, ip) * ip.weight * Trans.Weight();
+
+      AddMultADAt(hessian, factors, elmat);
+   }
+}
+
+void C0InteriorPenaltyIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
+                                                     const FiniteElement &el2, FaceElementTransformations &Trans, DenseMatrix &elmat)
+{
+   int dim = el1.GetDim();
+   MFEM_ASSERT(dim == 2, "Dimension must be 2.");
+
+   int ndof[2] = {el1.GetDof(), 0};
+   int num_elems = 1;
+   if (Trans.Elem2No >= 0)
+   {
+      ndof[1] = el2.GetDof();
+      num_elems++;
+   }
+
+   for (int i = 0; i < num_elems; i++)
+   {
+      normal[i].SetSize(dim);
+      dshape[i].SetSize(ndof[i], dim);
+      hessian[i].SetSize(ndof[i], dim * (dim + 1) / 2);
+      nv[i].SetSize(dim * (dim + 1) / 2);
+      dnshape[i].SetSize(ndof[i]);
+      nd2nshape[i].SetSize(ndof[i]);
+   }
+
+   for (int i = 0; i < num_elems; i++)
+   {
+      for (int j = 0; j < num_elems; j++)
+      {
+         blockJ[i][j].SetSize(ndof[i], ndof[j]);
+         blockC[i][j].SetSize(ndof[i], ndof[j]);
+      }
+   }
+
+   elmatJ_p.SetSize(ndof[0] + ndof[1]);
+   elmatC_p.SetSize(ndof[0] + ndof[1]);
+   elmat.SetSize(ndof[0] + ndof[1]);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * max(el1.GetOrder(), ndof[1] ? el2.GetOrder() : 0);
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   // Compute edge length
+   double h_e  = 0.0;
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      Trans.SetAllIntPoints(&ip);
+      h_e += ip.weight * Trans.Weight();
+   }
+
+   const FiniteElement *els[2] = {&el1, &el2};
+   ElementTransformation *el_trans[2] = {Trans.Elem1, Trans.Elem2};
+
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      elmatJ_p = 0.0;
+      elmatC_p = 0.0;
+
+      const IntegrationPoint &ip = ir->IntPoint(p);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Compute normal gradients + Hessians
+      for (int i = 0; i < num_elems; i++)
+      {
+         if (i == 0)
+         {
+            CalcOrtho(Trans.Jacobian(), normal[i]);
+            normal[i] /= normal[i].Norml2();
+         }
+         else
+         {
+            normal[i] = normal[0];
+            normal[i] *= -1;
+         }
+         els[i]->CalcPhysDShape(*el_trans[i], dshape[i]);
+         els[i]->CalcPhysHessian(*el_trans[i], hessian[i]);
+         dshape[i].Mult(normal[i], dnshape[i]);
+         nv[i][0] = normal[i][0]*normal[i][0];
+         nv[i][1] = 2*normal[i][0]*normal[i][1];
+         nv[i][2] = normal[i][1]*normal[i][1];
+         hessian[i].Mult(nv[i], nd2nshape[i]);
+      }
+
+      // Compute blocks
+      for (int i = 0; i < num_elems; i++)
+      {
+         for (int j = 0; j < num_elems; j++)
+         {
+            blockJ[i][j] = 0.0;
+            blockC[i][j] = 0.0;
+            AddMult_a_VWt(-1.0, dnshape[i], nd2nshape[j], blockJ[i][j]);
+            elmatJ_p.SetSubMatrix(i*ndof[0], j*ndof[0], blockJ[i][j]);
+
+            AddMult_a_VWt(eta/h_e, dnshape[i], dnshape[j], blockC[i][j]);
+            elmatC_p.SetSubMatrix(i*ndof[0], j*ndof[0], blockC[i][j]);
+         }
+      }
+
+      // Symmetrize the jump term
+      elmatJ_p.Symmetrize();
+      if (!ndof[1])
+      {
+         elmatJ_p *= 2;
+      }
+
+      // Add penalty term
+      elmatJ_p += elmatC_p;
+      elmatJ_p *= ip.weight * Trans.Weight();
+      elmat += elmatJ_p;
+   }
+}

examples/jitplayground.cpp

@@ -1,548 +0,0 @@
-#include <mfem.hpp>
-
-#ifdef MFEM_USE_PROTEUS
-
-#include "../fem/dfem/util.hpp"
-
-#include <proteus/CppJitModule.h>
-
-#include "jitplayground.hpp"
-
-#include <algorithm>
-#include <array>
-#include <cctype>
-#include <cmath>
-#include <fstream>
-#include <initializer_list>
-#include <iostream>
-#include <memory>
-#include <sstream>
-#include <string>
-#include <string_view>
-#include <type_traits>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
-#include <vector>
-
-namespace util
-{
-constexpr std::string_view Dirname(std::string_view path)
-{
-   const size_t last_sep = path.find_last_of("/\\");
-   if (last_sep == std::string_view::npos) { return {}; }
-   return path.substr(0, last_sep);
-}
-
-constexpr std::string_view thisFileDir = Dirname(__FILE__);
-}
-
-template <typename T>
-static std::string TypeNameString()
-{
-   return std::string(mfem::future::get_type_name<T>());
-}
-
-template <typename Tuple, size_t... Is>
-static auto ParamTypeStringsImpl(std::index_sequence<Is...>)
-{
-   return std::array<std::string, sizeof...(Is)>
-   {
-      TypeNameString<std::remove_reference_t<decltype(mfem::future::get<Is>(std::declval<Tuple&>()))>>()...
-   };
-}
-
-template <typename Tuple>
-static auto ParamTypeStrings()
-{
-   return ParamTypeStringsImpl<Tuple>(
-             std::make_index_sequence<mfem::future::tuple_size<Tuple>::value> {});
-}
-
-static std::string_view Trim(std::string_view s)
-{
-   size_t begin = 0;
-   while (begin < s.size() && std::isspace(static_cast<unsigned char>(s[begin])))
-   {
-      ++begin;
-   }
-   size_t end = s.size();
-   while (end > begin &&
-          std::isspace(static_cast<unsigned char>(s[end - 1])))
-   {
-      --end;
-   }
-   return s.substr(begin, end - begin);
-}
-
-static bool IsValidIdentifier(std::string_view s)
-{
-   if (s.empty()) { return false; }
-   const unsigned char c0 = static_cast<unsigned char>(s[0]);
-   if (!(std::isalpha(c0) || c0 == '_')) { return false; }
-   for (size_t i = 1; i < s.size(); ++i)
-   {
-      const unsigned char c = static_cast<unsigned char>(s[i]);
-      if (!(std::isalnum(c) || c == '_')) { return false; }
-   }
-   return true;
-}
-
-static bool ParseJitDirective(std::string_view line,
-                              std::string &type,
-                              std::string &var,
-                              std::string &kind)
-{
-   const size_t jit_pos = line.find("$JIT");
-   if (jit_pos == std::string_view::npos) { return false; }
-
-   const size_t open = line.find('[', jit_pos);
-   const size_t close = line.find(']', jit_pos);
-   MFEM_VERIFY(open != std::string_view::npos &&
-               close != std::string_view::npos &&
-               close > open,
-               "malformed $JIT directive (expected brackets): " << line);
-
-   const std::string_view payload = line.substr(open + 1, close - open - 1);
-   const size_t comma1 = payload.find(',');
-   const size_t comma2 = (comma1 == std::string_view::npos)
-                         ? std::string_view::npos
-                         : payload.find(',', comma1 + 1);
-   MFEM_VERIFY(comma1 != std::string_view::npos &&
-               comma2 != std::string_view::npos,
-               "malformed $JIT directive (expected 3 comma-separated fields): "
-               << line);
-
-   const std::string_view f0 = Trim(payload.substr(0, comma1));
-   const std::string_view f1 = Trim(payload.substr(comma1 + 1,
-                                                   comma2 - comma1 - 1));
-   const std::string_view f2 = Trim(payload.substr(comma2 + 1));
-   MFEM_VERIFY(!f0.empty() && !f1.empty() && !f2.empty(),
-               "malformed $JIT directive (empty field): " << line);
-
-   type.assign(f0);
-   var.assign(f1);
-   kind.assign(f2);
-   return true;
-}
-
-static std::string ReadFileOrEmpty(const std::string &fn)
-{
-   std::ifstream file(fn);
-   if (!file.is_open())
-   {
-      std::cerr << "could not open file " << fn << "\n";
-      return {};
-   }
-   std::stringstream buffer;
-   buffer << file.rdbuf();
-   return buffer.str();
-}
-
-static std::vector<std::string> ExtractJitVarNames(const std::string
-                                                   &kernel_code)
-{
-   std::stringstream ss(kernel_code);
-   std::string line;
-   std::vector<std::string> var_names;
-   std::unordered_set<std::string> seen_vars;
-
-   while (std::getline(ss, line))
-   {
-      std::string type, var, kind;
-      if (ParseJitDirective(line, type, var, kind))
-      {
-         MFEM_VERIFY(IsValidIdentifier(var),
-                     "$JIT variable must be a valid identifier: " << var);
-         MFEM_VERIFY(seen_vars.insert(var).second,
-                     "duplicate $JIT variable name: " << var);
-         var_names.push_back(var);
-      }
-   }
-   return var_names;
-}
-
-static std::string RewriteKernelForJit(std::string kernel_code,
-                                       const std::vector<std::string> &jit_values)
-{
-   std::stringstream ss(kernel_code);
-   std::string line;
-
-   std::string out;
-   out.reserve(kernel_code.size() + 128);
-
-   bool have_pending = false;
-   size_t pending_index = 0;
-   std::string pending_type;
-   std::string pending_var;
-   std::unordered_set<std::string> seen_vars;
-
-   while (std::getline(ss, line))
-   {
-      line.push_back('\n');
-
-      if (have_pending)
-      {
-         MFEM_VERIFY(pending_index < jit_values.size(),
-                     "not enough JIT values provided");
-         const size_t indent_end = line.find_first_not_of(" \t");
-         const std::string indent =
-            (indent_end == std::string::npos) ? std::string() :
-            line.substr(0, indent_end);
-         out += indent + "const " + pending_type + " " + pending_var + " = " +
-                jit_values[pending_index] + ";\n";
-         have_pending = false;
-         ++pending_index;
-         continue;
-      }
-
-      std::string type, var, kind;
-      if (ParseJitDirective(line, type, var, kind))
-      {
-         MFEM_VERIFY(IsValidIdentifier(var),
-                     "$JIT variable must be a valid identifier: " << var);
-         MFEM_VERIFY(kind == "generic",
-                     "unsupported $JIT kind: " << kind);
-         MFEM_VERIFY(seen_vars.insert(var).second,
-                     "duplicate $JIT variable name: " << var);
-
-         pending_type = std::move(type);
-         pending_var = std::move(var);
-         have_pending = true;
-         continue; // drop directive line
-      }
-
-      out += line;
-   }
-
-   MFEM_VERIFY(!have_pending,
-               "$JIT directive must annotate a following line");
-   MFEM_VERIFY(jit_values.size() == pending_index,
-               "JIT value count must match number of $JIT directives");
-   return out;
-}
-
-static std::string GeneratedOutputPath(std::string_view original_path)
-{
-   const size_t last_sep = original_path.find_last_of("/\\");
-   const size_t dot = original_path.find_last_of('.');
-   const bool dot_in_filename =
-      (dot != std::string_view::npos) &&
-      (last_sep == std::string_view::npos || dot > last_sep);
-
-   const std::string_view base =
-      dot_in_filename ? original_path.substr(0, dot) : original_path;
-   return std::string(base) + "_generated.hpp";
-}
-
-static void WriteFileOrWarn(const std::string &path,
-                            const std::string &contents)
-{
-   std::ofstream out(path);
-   if (!out.is_open())
-   {
-      std::cerr << "could not write generated file " << path << "\n";
-      return;
-   }
-   out << contents;
-}
-
-class JitQFunction
-{
-public:
-   template <typename ImplT, size_t N>
-   JitQFunction(ImplT, const std::string &fn,
-                const std::array<bool, N> &activity_map)
-   {
-      using qf_signature = typename
-                           mfem::future::get_function_signature<
-                           decltype(&ImplT::operator())>::type;
-      using qf_param_ts = typename qf_signature::parameter_ts;
-      constexpr size_t nparams = mfem::future::tuple_size<qf_param_ts>::value;
-      static_assert(N == nparams, "activity_map size must match qfunc arity");
-
-      this->fn = fn;
-      this->nparams = nparams;
-      this->activity_map.reserve(N);
-      for (size_t i = 0; i < N; ++i)
-      {
-         this->activity_map.push_back(activity_map[i]);
-      }
-      {
-         const auto param_types_arr = ParamTypeStrings<qf_param_ts>();
-         this->param_types.assign(param_types_arr.begin(), param_types_arr.end());
-      }
-      this->return_type = TypeNameString<typename qf_signature::return_t>();
-      this->return_is_void = std::is_same_v<typename qf_signature::return_t, void>;
-      this->impl_type_name = TypeNameString<ImplT>();
-      this->jit_var_names = ExtractJitVarNames(ReadFileOrEmpty(fn));
-   }
-
-   template <typename ReturnT, typename... Args>
-   ReturnT run(std::string_view name,
-               std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
-               Args&&... args)
-   {
-      auto ordered_values = MatchJitValues(jit_values);
-      auto &mod = GetOrCreateModule(ordered_values);
-      auto &instance = mod.instantiate(std::string(name), std::string());
-      return instance.template run<ReturnT>(std::forward<Args>(args)...);
-   }
-
-   template <typename ReturnT, typename... Args>
-   ReturnT run_primal(
-      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
-      Args&&... args)
-   {
-      return run<ReturnT>(qfunc_name, jit_values,
-                          std::forward<Args>(args)...);
-   }
-
-   template <typename ReturnT, typename... Args>
-   ReturnT run_derivative(
-      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
-      Args&&... args)
-   {
-      return run<ReturnT>(qfunc_name + "_fwddiff", jit_values,
-                          std::forward<Args>(args)...);
-   }
-
-private:
-   std::vector<std::string_view> MatchJitValues(
-      std::initializer_list<std::pair<std::string_view, std::string_view>>
-      named_values) const
-   {
-      std::unordered_map<std::string_view, std::string_view> value_map;
-      for (const auto &[name, value] : named_values)
-      {
-         value_map[name] = value;
-      }
-
-      std::vector<std::string_view> ordered_values;
-      ordered_values.reserve(jit_var_names.size());
-      for (const auto &var_name : jit_var_names)
-      {
-         auto it = value_map.find(var_name);
-         MFEM_VERIFY(it != value_map.end(),
-                     "missing JIT value for variable: " << var_name);
-         ordered_values.push_back(it->second);
-      }
-
-      MFEM_VERIFY(ordered_values.size() == named_values.size(),
-                  "provided " << named_values.size() << " JIT values but expected "
-                  << jit_var_names.size());
-      return ordered_values;
-   }
-
-
-   std::string BuildModuleCode(const std::vector<std::string> &jit_values) const
-   {
-      std::string module_code =
-         RewriteKernelForJit(ReadFileOrEmpty(fn), jit_values);
-      module_code += "\n\n";
-      module_code += "// --- generated ---\n";
-      module_code +=
-         "template <typename return_type, typename... Args>\n"
-         "return_type __enzyme_fwddiff(Args...);\n"
-         "\n"
-         "extern int enzyme_const;\n"
-         "extern int enzyme_dup;\n"
-         "\n";
-
-      // Generate a primal wrapper with the requested symbol name, so the kernel
-      // header can just define the qfunc as a functor.
-      //
-      // Note: Proteus instantiates entrypoints via `qfunc_wrapper<>(...)` even
-      // when there are no user template args, so keep the wrapper itself a
-      // template (with a default parameter) while still doing literal `$JIT`
-      // replacements in the kernel code.
-      module_code += "template <typename = void>\n";
-      module_code += return_type + " " +
-                     std::string(qfunc_name) + "(";
-      bool first = true;
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (!first) { module_code += ", "; }
-         first = false;
-         module_code += param_types[i] + " Arg" + std::to_string(i);
-      }
-      module_code += ")\n";
-      module_code += "{\n";
-      module_code += "   " + impl_type_name + " qf;\n";
-      if (return_is_void)
-      {
-         module_code += "   ";
-      }
-      else
-      {
-         module_code += "   return ";
-      }
-      module_code += "qf(";
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (i) { module_code += ", "; }
-         module_code += "Arg" + std::to_string(i);
-      }
-      module_code += ");\n";
-      module_code += "}\n\n";
-
-      module_code += "template <typename = void>\n";
-      module_code += return_type + " " +
-                     std::string(qfunc_name) + "_fwddiff(";
-
-      first = true;
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (!first) { module_code += ", "; }
-         first = false;
-         module_code += param_types[i] + " Arg" + std::to_string(i);
-         if (activity_map[i])
-         {
-            module_code += ", " + param_types[i] + " dArg" + std::to_string(i);
-         }
-      }
-      module_code += ")\n";
-      module_code += "{\n";
-      if (return_is_void)
-      {
-         module_code += "   __enzyme_fwddiff<void>(\n";
-      }
-      else
-      {
-         module_code += "   return __enzyme_fwddiff<" +
-                        return_type + ">(\n";
-      }
-      module_code += "      (void*)" + std::string(qfunc_name) + "<>";
-      module_code += ",\n";
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (activity_map[i])
-         {
-            module_code += "      enzyme_dup, Arg" + std::to_string(i) +
-                           ", dArg" + std::to_string(i);
-         }
-         else
-         {
-            module_code += "      enzyme_const, Arg" + std::to_string(i);
-         }
-         module_code += (i + 1 == nparams) ? ");\n" : ",\n";
-      }
-      module_code += "}\n";
-
-      WriteFileOrWarn(GeneratedOutputPath(fn), module_code);
-      return module_code;
-   }
-
-   proteus::CppJitModule &GetOrCreateModule(
-      const std::vector<std::string_view> &jit_values)
-   {
-      std::string key;
-      for (const auto &val : jit_values)
-      {
-         if (!key.empty()) { key += ","; }
-         key += val;
-      }
-
-      auto it = modules.find(key);
-      if (it != modules.end())
-      {
-         return *it->second;
-      }
-
-      std::vector<std::string> values(jit_values.begin(), jit_values.end());
-      std::string code = BuildModuleCode(values);
-      auto mod = std::make_unique<proteus::CppJitModule>("host", code,
-                                                         DefaultExtraArgs());
-      auto [inserted, ok] = modules.emplace(key, std::move(mod));
-      MFEM_VERIFY(ok, "failed to cache JIT module");
-      return *inserted->second;
-   }
-
-   static std::vector<std::string> DefaultExtraArgs()
-   {
-      return {"-fplugin=/Users/andrej1/local/enzyme/lib/ClangEnzyme-20.dylib"};
-   }
-
-   std::string qfunc_name = "qfunc_wrapper";
-   std::string fn;
-   size_t nparams = 0;
-   std::vector<bool> activity_map;
-   std::vector<std::string> param_types;
-   std::string return_type;
-   bool return_is_void = false;
-   std::string impl_type_name;
-   std::vector<std::string> jit_var_names;
-   std::unordered_map<std::string, std::unique_ptr<proteus::CppJitModule>> modules;
-};
-
-int main()
-{
-   const size_t N = 4;
-   const size_t M = 5;
-   const double A = 123.4;
-
-   std::vector<double> X(N);
-   std::vector<double> Y(N);
-   for (size_t i = 0; i < N; ++i)
-   {
-      X[i] = static_cast<double>(i + 1);
-      Y[i] = static_cast<double>(N - i);
-   }
-
-   // // >>> user interface calls
-   // const std::string kernel_path = std::string(util::thisFileDir) +
-   //                                 "/jitplayground.hpp";
-   // JitQFunction qf(daxpy_op{}, kernel_path, std::array{false, true, false});
-   // // <<< user interface calls
-
-   // // this will happen internally in dFEM
-
-   daxpy_op op;
-   printf("\n\nfunction call\n");
-   op(&A, X.data(), Y.data(), &N);
-
-   // reset X for the derivative test
-   for (size_t i = 0; i < N; ++i)
-   {
-      X[i] = static_cast<double>(i + 1);
-      Y[i] = static_cast<double>(N - i);
-   }
-
-   std::vector<double> dX(N, 1.0);
-   printf("\n\nforward diff call\n");
-   daxpy_op_fwddiff(&A, X.data(), dX.data(), Y.data(), &N);
-
-   std::vector<double> dX_manual(N, A);
-
-   printf("\n\nderivative checks\n");
-   std::cout << "dX: ";
-   for (size_t i = 0; i < N; ++i)
-   {
-      std::cout << dX[i] << (i + 1 == N ? '\n' : ' ');
-   }
-
-   std::cout << "dX_manual: ";
-   for (size_t i = 0; i < N; ++i)
-   {
-      std::cout << dX_manual[i] << (i + 1 == N ? '\n' : ' ');
-   }
-
-   double max_abs_err = 0.0;
-   for (size_t i = 0; i < N; ++i)
-   {
-      max_abs_err = std::max(max_abs_err, std::abs(dX[i] - dX_manual[i]));
-   }
-   std::cout << "max |dX - dX_manual| = " << max_abs_err << "\n";
-
-   return 0;
-}
-
-#else
-
-int main()
-{
-   std::cerr << "Proteus is required to run this example.\n";
-   return EXIT_FAILURE;
-}
-
-#endif // MFEM_USE_PROTEUS

examples/jitplayground.hpp

@@ -1,58 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <vector>
-#include <type_traits>
-
-#include "proteus/JitInterface.h"
-
-struct daxpy_op
-{
-   void operator()(
-      const double *a,
-      double *x,
-      const double *y,
-      const size_t *N) const
-   {
-      const size_t n = *N;
-      auto lam = [=, n = proteus::jit_variable(n)]
-                 () __attribute__((annotate("jit")))
-      {
-         printf("N = %zu\n", n);
-         for (size_t i = 0; i < n; ++i)
-         {
-            printf("x[%zu] = %f, y[%zu] = %f\n", i, x[i], i, y[i]);
-            x[i] = *a * x[i] + y[i];
-            printf("updated x[%zu] = %f\n", i, x[i]);
-         }
-      };
-
-      proteus::register_lambda(lam);
-
-      lam();
-   }
-};
-
-template <typename return_type, typename... Args>
-return_type __enzyme_fwddiff(Args...);
-
-extern int enzyme_const;
-extern int enzyme_dup;
-
-void daxpy_op_wrapper(const double * Arg0, double * Arg1,
-                      const double * Arg2, const size_t *Arg3)
-{
-   daxpy_op qf;
-   qf(Arg0, Arg1, Arg2, Arg3);
-}
-
-void daxpy_op_fwddiff(const double * Arg0, double * Arg1,
-                      double * dArg1, const double * Arg2, const size_t *Arg3)
-{
-   __enzyme_fwddiff<void>(
-      (void*)daxpy_op_wrapper,
-      enzyme_const, Arg0,
-      enzyme_dup, Arg1, dArg1,
-      enzyme_const, Arg2,
-      enzyme_const, Arg3);
-}

fem/CMakeLists.txt

@@ -57,8 +57,6 @@ set(SRCS
   integ/lininteg_domain_grad.cpp
   integ/lininteg_domain_vectorfe.cpp
   integ/nonlininteg_vecconvection_pa.cpp
-  integ/nonlininteg_vecconvection_pa_diag.cpp
-  integ/nonlininteg_vecconvection_pa_grad.cpp
   integ/nonlininteg_vecconvection_mf.cpp
   coefficient.cpp
   complex_fem.cpp
@@ -67,7 +65,6 @@ set(SRCS
   dgmassinv.cpp
   doftrans.cpp
   dfem/doperator.cpp
-  dfem/backends/local_qf/kernels.cpp
   eltrans.cpp
   batchitrans.cpp
   estimators.cpp
@@ -124,11 +121,6 @@ set(SRCS
   qinterp/eval_hdiv.cpp
   qinterp/grad_by_nodes.cpp
   qinterp/grad_by_vdim.cpp
-  qinterp/grad_transpose.cpp
-  qinterp/grad_transpose_by_nodes.cpp
-  qinterp/grad_transpose_by_vdim.cpp
-  qinterp/eval_transpose.cpp
-  qinterp/eval_transpose_by_vdim.cpp
   qspace.cpp
   quadinterpolator.cpp
   quadinterpolator_face.cpp
@@ -212,11 +204,7 @@ set(HDRS
   integ/bilininteg_mass_kernels.hpp
   integ/bilininteg_mass_pa_simplices.hpp
   integ/bilininteg_vecdiffusion_pa.hpp
-  integ/bilininteg_vecdiv_pa.hpp
   integ/bilininteg_vecmass_pa.hpp
-  integ/nonlininteg_vecconvection_pa.hpp
-  integ/nonlininteg_vecconvection_pa_diag.hpp
-  integ/nonlininteg_vecconvection_pa_grad.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp
@@ -225,32 +213,13 @@ set(HDRS
   dgmassinv_kernels.hpp
   doftrans.hpp
   dfem/doperator.hpp
-  dfem/fielddescriptor.hpp
   dfem/fieldoperator.hpp
-  dfem/integrator_ctx.hpp
+  dfem/integrate.hpp
   dfem/parameterspace.hpp
-  dfem/tensor_functions.hpp
+  dfem/qfunction_apply.hpp
+  dfem/qfunction_transform.hpp
   dfem/tuple.hpp
   dfem/util.hpp
-  dfem/backends/util.hpp
-  dfem/backends/global_qf/action.hpp
-  dfem/backends/global_qf/derivative_action.hpp
-  dfem/backends/global_qf/derivative_apply.hpp
-  dfem/backends/global_qf/derivative_apply_transpose.hpp
-  dfem/backends/global_qf/derivative_setup.hpp
-  dfem/backends/global_qf/prelude.hpp
-  dfem/backends/local_qf/action.hpp
-  dfem/backends/local_qf/derivative_action.hpp
-  dfem/backends/local_qf/derivative_apply.hpp
-  dfem/backends/local_qf/derivative_apply_transpose.hpp
-  dfem/backends/local_qf/derivative_assemble.hpp
-  dfem/backends/local_qf/derivative_assemble_diagonal.hpp
-  dfem/backends/local_qf/derivative_setup.hpp
-  dfem/backends/local_qf/kernels.hpp
-  dfem/backends/local_qf/kernels_ho.hpp
-  dfem/backends/local_qf/kernels_lo.hpp
-  dfem/backends/local_qf/prelude.hpp
-  dfem/backends/local_qf/util.hpp
   eltrans.hpp
   estimators.hpp
   fe.hpp
@@ -315,10 +284,8 @@ set(HDRS
   qfunction.hpp
   qinterp/det.hpp
   qinterp/eval.hpp
-  qinterp/eval_transpose.hpp
   qinterp/eval_hdiv.hpp
   qinterp/grad.hpp
-  qinterp/grad_transpose.hpp
   qspace.hpp
   quadinterpolator.hpp
   quadinterpolator_face.hpp
@@ -353,36 +320,36 @@ set(HDRS
   )
 
 if (MFEM_USE_SIDRE)
-    list(APPEND SRCS sidredatacollection.cpp)
-    list(APPEND HDRS sidredatacollection.hpp)
+  list(APPEND SRCS sidredatacollection.cpp)
+  list(APPEND HDRS sidredatacollection.hpp)
 endif()
 
 if (MFEM_USE_CONDUIT)
-    list(APPEND SRCS conduitdatacollection.cpp)
-    list(APPEND HDRS conduitdatacollection.hpp)
+  list(APPEND SRCS conduitdatacollection.cpp)
+  list(APPEND HDRS conduitdatacollection.hpp)
 endif()
 
 if (MFEM_USE_ADIOS2)
-    list(APPEND SRCS adios2datacollection.cpp)
-    list(APPEND HDRS adios2datacollection.hpp)
+  list(APPEND SRCS adios2datacollection.cpp)
+  list(APPEND HDRS adios2datacollection.hpp)
 endif()
 
 if (MFEM_USE_FMS)
-    list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
-    list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
+  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
+  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
 endif()
 
 if (MFEM_USE_MPI)
-    list(APPEND SRCS
+  list(APPEND SRCS
     pbilinearform.cpp
     pfespace.cpp
     pgridfunc.cpp
     plinearform.cpp
     pnonlinearform.cpp
     prestriction.cpp)
-    # If this list (HDRS -> HEADERS) is used for install, we probably want the
-    # headers added all the time.
-    list(APPEND HDRS
+  # If this list (HDRS -> HEADERS) is used for install, we probably want the
+  # headers added all the time.
+  list(APPEND HDRS
     pbilinearform.hpp
     pfespace.hpp
     pgridfunc.hpp

fem/bilininteg.hpp

@@ -2689,22 +2689,14 @@ public:
    void AddMultMF(const Vector &x, Vector &y) const override;
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
 
-   // PA AddMultPA kernels
    using VectorMassAddMultPAType =
       void(*)(const int,  const int,
               const Array<real_t>&, const Vector&,
               const Vector&, Vector&, const int, const int);
+
    MFEM_REGISTER_KERNELS(VectorMassAddMultPA,
                          VectorMassAddMultPAType,
                          (int, int, int));
-
-   // PA DiagonalPA kernels
-   using VectorMassAssembleDiagonalPAType =
-      void(*)(const int, const int, const int,
-              const real_t*, const real_t*, real_t*);
-   MFEM_REGISTER_KERNELS(VectorMassAssembleDiagonalPA,
-                         VectorMassAssembleDiagonalPAType,
-                         (int /*dim*/, int /*q1d*/));
 };
 
 
@@ -3003,10 +2995,11 @@ public:
     vector (diagonal matrix), or matrix), trial function $u$ is in $H(curl$ or
     $H(div)$, and test function $v$ is in $H(curl$, $H(div)$, or $v=(v_1,\dots,v_n)$, where
     $v_i$ are in $H^1$. */
-class VectorFEMassIntegrator : public BilinearFormIntegrator
+class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
-   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq);
+   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq)
+   { Q = q; DQ = dq; MQ = mq; }
 
 #ifndef MFEM_THREAD_SAFE
    Vector shape;
@@ -3029,8 +3022,7 @@ protected:
    const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
    const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
    const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D;
-   FiniteElement::DerivType trial_fetype, test_fetype;
+   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D, trial_fetype, test_fetype;
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
@@ -3061,29 +3053,6 @@ public:
                    const bool add) override;
 
    const Coefficient *GetCoefficient() const { return Q; }
-
-   using ApplyKernelType =
-      void (*)(const int NE, bool symmetric, const bool scalar_coeff,
-               const Array<real_t> &trialBO, const Array<real_t> &trialBC,
-               const Array<real_t> &testBOt, const Array<real_t> &testBCt,
-               const Vector &pa_data, const Vector &x, Vector &y,
-               const int triald1d, const int testd1d, const int q1d);
-
-   /// parameters: trial_fetype, test_fetype, ndims, trial_d1d, test_d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType,
-                         (FiniteElement::DerivType, FiniteElement::DerivType,
-                          int, int, int, int));
-
-   struct Kernels { Kernels(); };
-
-   template <FiniteElement::DerivType TrialType,
-             FiniteElement::DerivType TestType, int DIM, int TRIAL_D1D,
-             int TEST_D1D, int Q1D>
-   static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<TrialType, TestType, DIM, TRIAL_D1D,
-                     TEST_D1D, Q1D>::Add();
-   }
 };
 
 /** Integrator for $(Q \nabla \cdot u, v)$ where $u=(u_1,\cdots,u_n)$ and all $u_i$ are in the same
@@ -3129,24 +3098,6 @@ public:
    void AddMultPA(const Vector &x, Vector &y) const override;
    void AddMultTransposePA(const Vector &x, Vector &y) const override;
 
-   using VectorDivergenceAddMultPAType =
-      void (*)(const int ne,
-               const Array<real_t> &b, const Array<real_t> &g, const Array<real_t> &bt,
-               const Vector &op, const Vector &x, Vector &y,
-               const int tr_d1d, const int te_d1d, const int q1d);
-   MFEM_REGISTER_KERNELS(VectorDivergenceAddMultPA,
-                         VectorDivergenceAddMultPAType,
-                         (int, int, int, int));
-
-   using VectorDivergenceAddMultTransposePAType =
-      void (*)(const int ne,
-               const Array<real_t> &bt, const Array<real_t> &gt, const Array<real_t> &b,
-               const Vector &q, const Vector &x, Vector &y,
-               const int tr_d1d, const int te_d1d, const int q1d);
-   MFEM_REGISTER_KERNELS(VectorDivergenceAddMultTransposePA,
-                         VectorDivergenceAddMultTransposePAType,
-                         (int, int, int, int));
-
    static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                          const FiniteElement &test_fe,
                                          const ElementTransformation &Trans);

fem/complex_fem.cpp

@@ -718,8 +718,8 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
+   if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
+        A_i.Type() == Operator::MFEM_SPARSEMAT )
    {
       ComplexSparseMatrix * A_sp =
          new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
@@ -779,8 +779,8 @@ SesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
+   if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
+        A_i.Type() == Operator::MFEM_SPARSEMAT )
    {
       ComplexSparseMatrix * A_sp =
          new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
@@ -843,426 +843,6 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
    if ( blfi ) { blfi->Update(nfes); }
 }
 
-bool
-MixedSesquilinearForm::RealInteg()
-{
-   int nint = mblfr->GetDBFI()->Size() + mblfr->GetBBFI()->Size() +
-              mblfr->GetFBFI()->Size() + mblfr->GetBFBFI()->Size() +
-              mblfr->GetTFBFI()->Size() + mblfr->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-bool
-MixedSesquilinearForm::ImagInteg()
-{
-   int nint = mblfi->GetDBFI()->Size() + mblfi->GetBBFI()->Size() +
-              mblfi->GetFBFI()->Size() + mblfi->GetBFBFI()->Size() +
-              mblfi->GetTFBFI()->Size() + mblfi->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-MixedSesquilinearForm::MixedSesquilinearForm(FiniteElementSpace * trial_fes,
-                                             FiniteElementSpace * test_fes,
-                                             ComplexOperator::Convention convention)
-   : conv(convention),
-     mblfr(new mfem::MixedBilinearForm(trial_fes, test_fes)),
-     mblfi(new mfem::MixedBilinearForm(trial_fes, test_fes))
-{
-}
-
-MixedSesquilinearForm::MixedSesquilinearForm(FiniteElementSpace * trial_fes,
-                                             FiniteElementSpace * test_fes,
-                                             MixedBilinearForm * bfr,
-                                             MixedBilinearForm * bfi,
-                                             ComplexOperator::Convention convention)
-   : conv(convention),
-     mblfr(new MixedBilinearForm(trial_fes, test_fes, bfr)),
-     mblfi(new MixedBilinearForm(trial_fes, test_fes, bfi))
-{
-}
-
-MixedSesquilinearForm::~MixedSesquilinearForm()
-{
-   delete mblfr;
-   delete mblfi;
-}
-
-void
-MixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                           BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddDomainIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddDomainIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                           BilinearFormIntegrator * bfi_imag,
-                                           Array<int> & elem_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddDomainIntegrator(bfi_real, elem_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddDomainIntegrator(bfi_imag, elem_marker);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                                             BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBoundaryIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBoundaryIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                                             BilinearFormIntegrator * bfi_imag,
-                                             Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBoundaryIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBoundaryIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-MixedSesquilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator *
-                                                 bfi_real,
-                                                 BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddInteriorFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddInteriorFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                            BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                            BilinearFormIntegrator * bfi_imag,
-                                            Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void MixedSesquilinearForm::AddTraceFaceIntegrator(BilinearFormIntegrator *
-                                                   bfi_real,
-                                                   BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void MixedSesquilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator
-                                                      *bfi_real,
-                                                      BilinearFormIntegrator *bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void MixedSesquilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator
-                                                      *bfi_real,
-                                                      BilinearFormIntegrator *bfi_imag,
-                                                      Array<int> &bdr_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrTraceFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrTraceFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-MixedSesquilinearForm::Assemble(int skip_zeros)
-{
-   mblfr->Assemble(skip_zeros);
-   mblfi->Assemble(skip_zeros);
-}
-
-void
-MixedSesquilinearForm::Finalize(int skip_zeros)
-{
-   mblfr->Finalize(skip_zeros);
-   mblfi->Finalize(skip_zeros);
-}
-
-ComplexSparseMatrix *
-MixedSesquilinearForm::AssembleComplexSparseMatrix()
-{
-   return new mfem::ComplexSparseMatrix(
-             &mblfr->SpMat(), &mblfi->SpMat(), false, false, conv);
-}
-
-void
-MixedSesquilinearForm::FormRectangularLinearSystem(const Array<int> &
-                                                   ess_trial_tdof_list,
-                                                   const Array<int> & ess_test_tdof_list,
-                                                   Vector & x,
-                                                   Vector & b,
-                                                   OperatorHandle & A,
-                                                   Vector & X,
-                                                   Vector & B)
-{
-   FiniteElementSpace * fes_trial = mblfr->TrialFESpace();
-   FiniteElementSpace * fes_test = mblfr->TestFESpace();
-   const int vsize_trial = fes_trial->GetVSize();
-   const int vsize_test = fes_test->GetVSize();
-
-   // Allocate temporary Vector
-   Vector b_0;
-   b_0.UseDevice(true);
-   b_0.SetSize(vsize_test);
-   b_0 = 0.0;
-
-   // Extract the real and imaginary parts of the input Vectors
-   MFEM_ASSERT(x.Size() == 2 * vsize_trial,
-               "Input GridFunction of incorrect size!");
-   x.Read();
-   Vector x_r;
-   x_r.MakeRef(x, 0, vsize_trial);
-   Vector x_i;
-   x_i.MakeRef(x, vsize_trial, vsize_trial);
-
-   MFEM_ASSERT(b.Size() == 2 * vsize_test, "Input LinearForm of incorrect size!");
-   b.Read();
-   Vector b_r;
-   b_r.MakeRef(b, 0, vsize_test);
-   Vector b_i;
-   b_i.MakeRef(b, vsize_test, vsize_test);
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      b_i *= -1.0;
-   }
-
-   const int tvsize_trial = fes_trial->GetTrueVSize();
-   const int tvsize_test = fes_test->GetTrueVSize();
-   OperatorHandle A_r, A_i;
-
-   X.UseDevice(true);
-   X.SetSize(2 * tvsize_trial);
-   X = 0.0;
-
-   B.UseDevice(true);
-   B.SetSize(2 * tvsize_test);
-   B = 0.0;
-
-   Vector X_r;
-   X_r.MakeRef(X, 0, tvsize_trial);
-   Vector X_i;
-   X_i.MakeRef(X, tvsize_trial, tvsize_trial);
-   Vector B_r;
-   B_r.MakeRef(B, 0, tvsize_test);
-   Vector B_i;
-   B_i.MakeRef(B, tvsize_test, tvsize_test);
-
-   Vector X_0, B_0;
-
-   if (RealInteg())
-   {
-      b_0 = b_r;
-      mblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_r, X_0, B_0);
-      X_r = X_0;
-      B_r = B_0;
-
-      b_0 = b_i;
-      mblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_r, X_0, B_0);
-      X_i = X_0;
-      B_i = B_0;
-
-      if (ImagInteg())
-      {
-         b_0 = 0.0;
-         mblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-         B_r -= B_0;
-
-         b_0 = 0.0;
-         mblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-         B_i += B_0;
-      }
-   }
-   else if (ImagInteg())
-   {
-      b_0 = b_i;
-      mblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-      X_r = X_0;
-      B_i = B_0;
-
-      b_0 = b_r;
-      b_0 *= -1.0;
-      mblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-      X_i = X_0;
-      B_r = B_0;
-      B_r *= -1.0;
-   }
-   else
-   {
-      MFEM_ABORT("Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      B_i *= -1.0;
-      b_i *= -1.0;
-   }
-
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
-   b_r.SyncAliasMemory(b);
-   b_i.SyncAliasMemory(b);
-
-   X_r.SyncAliasMemory(X);
-   X_i.SyncAliasMemory(X);
-   B_r.SyncAliasMemory(B);
-   B_i.SyncAliasMemory(B);
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
-   {
-      ComplexSparseMatrix * A_hyp =
-         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
-                                 A_i.As<SparseMatrix>(),
-                                 A_r.OwnsOperator(),
-                                 A_i.OwnsOperator(),
-                                 conv);
-      A.Reset<ComplexSparseMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-MixedSesquilinearForm::FormRectangularSystemMatrix(const mfem::Array<int> &
-                                                   ess_trial_tdof_list,
-                                                   const mfem::Array<int> & ess_test_tdof_list,
-                                                   mfem::OperatorHandle & A)
-{
-   OperatorHandle A_r, A_i;
-   if (RealInteg())
-   {
-      mblfr->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                         A_r);
-   }
-   if (ImagInteg())
-   {
-      mblfi->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                         A_i);
-   }
-   if (!RealInteg() && !ImagInteg())
-   {
-      MFEM_ABORT("Both Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
-   {
-      ComplexSparseMatrix * A_hyp =
-         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
-                                 A_i.As<SparseMatrix>(),
-                                 A_r.OwnsOperator(),
-                                 A_i.OwnsOperator(),
-                                 conv);
-      A.Reset<ComplexSparseMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-MixedSesquilinearForm::Update()
-{
-   mblfr->Update();
-   mblfi->Update();
-}
-
 
 #ifdef MFEM_USE_MPI
 
@@ -2034,8 +1614,8 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
+   if ( A_r.Type() == Operator::Hypre_ParCSR ||
+        A_i.Type() == Operator::Hypre_ParCSR )
    {
       ComplexHypreParMatrix * A_hyp =
          new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
@@ -2102,8 +1682,8 @@ ParSesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
+   if ( A_r.Type() == Operator::Hypre_ParCSR ||
+        A_i.Type() == Operator::Hypre_ParCSR )
    {
       ComplexHypreParMatrix * A_hyp =
          new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
@@ -2161,434 +1741,6 @@ ParSesquilinearForm::Update(FiniteElementSpace *nfes)
    if ( pblfi ) { pblfi->Update(nfes); }
 }
 
-bool
-ParMixedSesquilinearForm::RealInteg()
-{
-   int nint = pmblfr->GetDBFI()->Size() + pmblfr->GetBBFI()->Size() +
-              pmblfr->GetFBFI()->Size() + pmblfr->GetBFBFI()->Size() +
-              pmblfr->GetTFBFI()->Size() + pmblfr->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-bool
-ParMixedSesquilinearForm::ImagInteg()
-{
-   int nint = pmblfi->GetDBFI()->Size() + pmblfi->GetBBFI()->Size() +
-              pmblfi->GetFBFI()->Size() + pmblfi->GetBFBFI()->Size() +
-              pmblfi->GetTFBFI()->Size() + pmblfi->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-ParMixedSesquilinearForm::ParMixedSesquilinearForm(ParFiniteElementSpace *
-                                                   trial_fes,
-                                                   ParFiniteElementSpace * test_fes,
-                                                   ComplexOperator::Convention convention)
-   : conv(convention),
-     pmblfr(new ParMixedBilinearForm(trial_fes, test_fes)),
-     pmblfi(new ParMixedBilinearForm(trial_fes, test_fes))
-{
-}
-
-ParMixedSesquilinearForm::ParMixedSesquilinearForm(ParFiniteElementSpace *
-                                                   trial_fes,
-                                                   ParFiniteElementSpace * test_fes,
-                                                   ParMixedBilinearForm * pbfr,
-                                                   ParMixedBilinearForm * pbfi,
-                                                   ComplexOperator::Convention convention)
-   : conv(convention),
-     pmblfr(new ParMixedBilinearForm(trial_fes, test_fes, pbfr)),
-     pmblfi(new ParMixedBilinearForm(trial_fes, test_fes, pbfi))
-{
-}
-
-ParMixedSesquilinearForm::~ParMixedSesquilinearForm()
-{
-   delete pmblfr;
-   delete pmblfi;
-}
-
-void
-ParMixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                              BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddDomainIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddDomainIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                              BilinearFormIntegrator * bfi_imag,
-                                              Array<int> & elem_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddDomainIntegrator(bfi_real, elem_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddDomainIntegrator(bfi_imag, elem_marker);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *
-                                                bfi_real,
-                                                BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBoundaryIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBoundaryIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *
-                                                bfi_real,
-                                                BilinearFormIntegrator * bfi_imag,
-                                                Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBoundaryIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBoundaryIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator *
-                                                    bfi_real,
-                                                    BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddInteriorFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddInteriorFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *
-                                               bfi_real,
-                                               BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *
-                                               bfi_real,
-                                               BilinearFormIntegrator * bfi_imag,
-                                               Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void ParMixedSesquilinearForm::AddTraceFaceIntegrator(BilinearFormIntegrator *
-                                                      bfi_real,
-                                                      BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void ParMixedSesquilinearForm::AddBdrTraceFaceIntegrator(
-   BilinearFormIntegrator *bfi_real,
-   BilinearFormIntegrator *bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void ParMixedSesquilinearForm::AddBdrTraceFaceIntegrator(
-   BilinearFormIntegrator *bfi_real,
-   BilinearFormIntegrator *bfi_imag,
-   Array<int> &bdr_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrTraceFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrTraceFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-ParMixedSesquilinearForm::Assemble(int skip_zeros)
-{
-   pmblfr->Assemble(skip_zeros);
-   pmblfi->Assemble(skip_zeros);
-}
-
-void
-ParMixedSesquilinearForm::Finalize(int skip_zeros)
-{
-   pmblfr->Finalize(skip_zeros);
-   pmblfi->Finalize(skip_zeros);
-}
-
-ComplexHypreParMatrix *
-ParMixedSesquilinearForm::ParallelAssemble()
-{
-   return new ComplexHypreParMatrix(
-             pmblfr->ParallelAssemble(), pmblfi->ParallelAssemble(), true, true, conv);
-}
-
-void
-ParMixedSesquilinearForm::FormRectangularLinearSystem(const Array<int> &
-                                                      ess_trial_tdof_list,
-                                                      const Array<int> & ess_test_tdof_list,
-                                                      Vector & x,
-                                                      Vector & b,
-                                                      OperatorHandle & A,
-                                                      Vector & X,
-                                                      Vector & B)
-{
-   FiniteElementSpace * pfes_trial = pmblfr->TrialFESpace();
-   FiniteElementSpace * pfes_test = pmblfr->TestFESpace();
-   const int vsize_trial = pfes_trial->GetVSize();
-   const int vsize_test = pfes_test->GetVSize();
-
-   // Allocate temporary Vector
-   Vector b_0;
-   b_0.UseDevice(true);
-   b_0.SetSize(vsize_test);
-   b_0 = 0.0;
-
-   // Extract the real and imaginary parts of the input Vectors
-   MFEM_ASSERT(x.Size() == 2 * vsize_trial,
-               "Input GridFunction of incorrect size!");
-   x.Read();
-   Vector x_r;
-   x_r.MakeRef(x, 0, vsize_trial);
-   Vector x_i;
-   x_i.MakeRef(x, vsize_trial, vsize_trial);
-
-   MFEM_ASSERT(b.Size() == 2 * vsize_test, "Input LinearForm of incorrect size!");
-   b.Read();
-   Vector b_r;
-   b_r.MakeRef(b, 0, vsize_test);
-   Vector b_i;
-   b_i.MakeRef(b, vsize_test, vsize_test);
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      b_i *= -1.0;
-   }
-
-   const int tvsize_trial = pfes_trial->GetTrueVSize();
-   const int tvsize_test = pfes_test->GetTrueVSize();
-   OperatorHandle A_r, A_i;
-
-   X.UseDevice(true);
-   X.SetSize(2 * tvsize_trial);
-   X = 0.0;
-
-   B.UseDevice(true);
-   B.SetSize(2 * tvsize_test);
-   B = 0.0;
-
-   Vector X_r;
-   X_r.MakeRef(X, 0, tvsize_trial);
-   Vector X_i;
-   X_i.MakeRef(X, tvsize_trial, tvsize_trial);
-   Vector B_r;
-   B_r.MakeRef(B, 0, tvsize_test);
-   Vector B_i;
-   B_i.MakeRef(B, tvsize_test, tvsize_test);
-
-   Vector X_0, B_0;
-
-   if (RealInteg())
-   {
-      b_0 = b_r;
-      pmblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_r, X_0, B_0);
-      X_r = X_0;
-      B_r = B_0;
-
-      b_0 = b_i;
-      pmblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_r, X_0, B_0);
-      X_i = X_0;
-      B_i = B_0;
-
-      if (ImagInteg())
-      {
-         b_0 = 0.0;
-         pmblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-         B_r -= B_0;
-
-         b_0 = 0.0;
-         pmblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-         B_i += B_0;
-      }
-   }
-   else if (ImagInteg())
-   {
-      b_0 = b_i;
-      pmblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-      X_r = X_0;
-      B_i = B_0;
-
-      b_0 = b_r;
-      b_0 *= -1.0;
-      pmblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-      X_i = X_0;
-      B_r = B_0;
-      B_r *= -1.0;
-   }
-   else
-   {
-      MFEM_ABORT("Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      B_i *= -1.0;
-      b_i *= -1.0;
-   }
-
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
-   b_r.SyncAliasMemory(b);
-   b_i.SyncAliasMemory(b);
-
-   X_r.SyncAliasMemory(X);
-   X_i.SyncAliasMemory(X);
-   B_r.SyncAliasMemory(B);
-   B_i.SyncAliasMemory(B);
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
-   {
-      ComplexHypreParMatrix * A_hyp =
-         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
-                                   A_i.As<HypreParMatrix>(),
-                                   A_r.OwnsOperator(),
-                                   A_i.OwnsOperator(),
-                                   conv);
-      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-ParMixedSesquilinearForm::FormRectangularSystemMatrix(const Array<int> &
-                                                      ess_trial_tdof_list,
-                                                      const Array<int> & ess_test_tdof_list,
-                                                      OperatorHandle & A)
-{
-   OperatorHandle A_r, A_i;
-   if (RealInteg())
-   {
-      pmblfr->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                          A_r);
-   }
-   if (ImagInteg())
-   {
-      pmblfi->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                          A_i);
-   }
-   if (!RealInteg() && !ImagInteg())
-   {
-      MFEM_ABORT("Both Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
-   {
-      ComplexHypreParMatrix * A_hyp =
-         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
-                                   A_i.As<HypreParMatrix>(),
-                                   A_r.OwnsOperator(),
-                                   A_i.OwnsOperator(),
-                                   conv);
-      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-ParMixedSesquilinearForm::Update()
-{
-   pmblfr->Update();
-   pmblfi->Update();
-}
-
-
-
 #endif // MFEM_USE_MPI
 
 }

fem/complex_fem.hpp

@@ -505,186 +505,6 @@ public:
    virtual ~SesquilinearForm();
 };
 
-/** Class for a mixed sesquilinear form
-
-    A mixed sesquilinear form is a generalization of a mixed bilinear form to
-    complex-valued fields. Mixed sesquilinear forms are linear in the second
-    argument but the first argument involves a complex conjugate in the sense
-    that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the MixedSesquilinearForm either the real or
-    imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class MixedSesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   MixedBilinearForm * mblfr;
-   MixedBilinearForm * mblfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are not
-      empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   MixedSesquilinearForm(
-      FiniteElementSpace * trial_fes,
-      FiniteElementSpace * test_fes,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a MixedSesquilinearForm on the given trial and test
-       FiniteElementSpaces, using the same integrators as the
-       MixedBilinearForms @a bfr and @a bfi.
-
-       The FiniteElementSpace pointers are not owned by the newly constructed
-       object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed MixedSesquilinearForm. */
-   MixedSesquilinearForm(
-      FiniteElementSpace * trial_fes,
-      FiniteElementSpace * test_fes,
-      MixedBilinearForm * bfr,
-      MixedBilinearForm * bfi,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
-
-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::LEGACY (default)
-       - AssemblyLevel::FULL
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      mblfr->SetAssemblyLevel(assembly_level);
-      mblfi->SetAssemblyLevel(assembly_level);
-   }
-
-   MixedBilinearForm & real() { return *mblfr; }
-   MixedBilinearForm & imag() { return *mblfi; }
-   const MixedBilinearForm & real() const { return *mblfr; }
-   const MixedBilinearForm & imag() const { return *mblfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag,
-                            Array<int> & elem_marker);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag,
-                              Array<int> & bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag,
-                             Array<int> & bdr_marker);
-
-   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
-
-       This type of integrator assembles terms over all faces of the mesh using
-       the face FE from the trial space and the two adjacent volume FEs from
-       the test space. */
-   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                               BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag,
-                                  Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Updates the internal mixed forms with the new finite element space.
-   virtual void Update();
-
-   /** @brief Return a ComplexSparseMatrix wrapping the local (L-dof) real
-       and imaginary matrices of the form.
-
-       The returned wrapper has to be deleted by the caller, but it does not
-       own the wrapped real and imaginary matrices, which remain owned by
-       this form. */
-   ComplexSparseMatrix *AssembleComplexSparseMatrix();
-
-   /// Return the trial FE space associated with the MixedSesquilinearForm.
-   FiniteElementSpace *TrialFESpace() { return mblfr->TrialFESpace(); }
-
-   /// Read-only access to the associated trial FiniteElementSpace.
-   const FiniteElementSpace *TrialFESpace() const { return mblfr->TrialFESpace(); }
-
-   /// Return the test FE space associated with the MixedSesquilinearForm.
-   FiniteElementSpace *TestFESpace() { return mblfr->TestFESpace(); }
-
-   /// Read-only access to the associated test FiniteElementSpace.
-   const FiniteElementSpace *TestFESpace() const { return mblfr->TestFESpace(); }
-
-
-   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    Vector & x,
-                                    Vector & b,
-                                    OperatorHandle & A,
-                                    Vector & X,
-                                    Vector & B);
-
-   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    OperatorHandle & A);
-
-   virtual ~MixedSesquilinearForm();
-};
-
 #ifdef MFEM_USE_MPI
 
 /// Class for parallel complex-valued grid function - real + imaginary part
@@ -1101,169 +921,6 @@ public:
    virtual ~ParSesquilinearForm();
 };
 
-/** Class for a parallel mixed sesquilinear form
-
-    A mixed sesquilinear form is a generalization of a mixed bilinear form to
-    complex-valued fields. Mixed sesquilinear forms are linear in the second
-    argument but the first argument involves a complex conjugate in the sense
-    that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the ParMixedSesquilinearForm either the real
-    or imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class ParMixedSesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   ParMixedBilinearForm * pmblfr;
-   ParMixedBilinearForm * pmblfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are
-    not empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   ParMixedSesquilinearForm(
-      ParFiniteElementSpace * trial_fes,
-      ParFiniteElementSpace * test_fes,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a ParMixedSesquilinearForm on the given trial and test
-       ParFiniteElementSpaces, using the same integrators as the
-       ParMixedBilinearForms @a pbfr and @a pbfi.
-
-       The ParFiniteElementSpace pointers are not owned by the newly
-       constructed object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed ParMixedSesquilinearForm. */
-   ParMixedSesquilinearForm(
-      ParFiniteElementSpace * trial_fes,
-      ParFiniteElementSpace * test_fes,
-      ParMixedBilinearForm * pbfr,
-      ParMixedBilinearForm * pbfi,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
-
-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::LEGACY (default)
-       - AssemblyLevel::FULL
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      pmblfr->SetAssemblyLevel(assembly_level);
-      pmblfi->SetAssemblyLevel(assembly_level);
-   }
-
-   ParMixedBilinearForm & real() { return *pmblfr; }
-   ParMixedBilinearForm & imag() { return *pmblfi; }
-   const ParMixedBilinearForm & real() const { return *pmblfr; }
-   const ParMixedBilinearForm & imag() const { return *pmblfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag,
-                            Array<int> & elem_marker);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag,
-                              Array<int> & bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag,
-                             Array<int> & bdr_marker);
-
-   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
-
-      This type of integrator assembles terms over all faces of the mesh using
-      the face FE from the trial space and the two adjacent volume FEs from
-      the test space. */
-   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                               BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag,
-                                  Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Updates the internal mixed forms with the new finite element space.
-   virtual void Update();
-
-   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
-   /** The returned matrix has to be deleted by the caller. */
-   ComplexHypreParMatrix * ParallelAssemble();
-
-   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    Vector & x,
-                                    Vector & b,
-                                    OperatorHandle & A,
-                                    Vector & X,
-                                    Vector & B);
-
-   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    OperatorHandle & A);
-
-   virtual ~ParMixedSesquilinearForm();
-};
-
 #endif // MFEM_USE_MPI
 
 }

fem/dfem/assemble.hpp

@@ -0,0 +1,403 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "util.hpp"
+
+namespace mfem::future
+{
+
+/// @brief Assemble element matrix for three dimensional data.
+///
+/// Note: In the below layouts, total_trial_op_dim is > 1 if
+/// there are more than one inputs dependent on the derivative variable.
+///
+/// @param A Memory for one element matrix with layout
+/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
+/// @param fhat Memory to hold the residual computation with layout
+/// [test_vdim, test_op_dim, nqp].
+/// @param qpdc The quadrature point data cache with data layout
+/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
+/// @param itod Input Trial Operator Dimension array. If the trial
+/// operator is not dependent, the dimension is 0 to indicate that.
+/// @param inputs The input field operator types.
+/// @param output The output field operator types.
+/// @param input_dtqmaps The input DofToQuad maps.
+/// @param output_dtqmap The output DofToQuad maps.
+/// @param scratch_shmem Scratch shared memory for computations.
+/// @param q1d The number of quadrature points in one dimension.
+/// @param td1d The number of trial dofs in one dimension.
+template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
+MFEM_HOST_DEVICE void assemble_element_mat_t3d(
+   const DeviceTensor<4, real_t>& A,
+   const DeviceTensor<3, real_t>& fhat,
+   const DeviceTensor<5, const real_t>& qpdc,
+   const DeviceTensor<1, const real_t>& itod,
+   const input_fop_ts& inputs,
+   const output_fop_t& output,
+   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
+   const DofToQuadMap& output_dtqmap,
+   std::array<DeviceTensor<1>, 6>& scratch_shmem,
+   const int& q1d,
+   const int& td1d)
+{
+   constexpr int dimension = 3;
+
+   // [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
+   const int test_vdim = qpdc.GetShape()[0];
+   const int test_op_dim = qpdc.GetShape()[1];
+   const int trial_vdim = qpdc.GetShape()[2];
+
+   // [num_test_dof, ...]
+   const auto num_test_dof = A.GetShape()[0];
+
+   for (int Jx = 0; Jx < td1d; Jx++)
+   {
+      for (int Jy = 0; Jy < td1d; Jy++)
+      {
+         for (int Jz = 0; Jz < td1d; Jz++)
+         {
+            const int J = Jx + td1d * (Jy + td1d * Jz);
+
+            for (int j = 0; j < trial_vdim; j++)
+            {
+               for (int tv = 0; tv < test_vdim; tv++)
+               {
+                  for (int tod = 0; tod < test_op_dim; tod++)
+                  {
+                     MFEM_FOREACH_THREAD(qx, x, q1d)
+                     {
+                        MFEM_FOREACH_THREAD(qy, y, q1d)
+                        {
+                           MFEM_FOREACH_THREAD(qz, z, q1d)
+                           {
+                              const int q = qx + q1d * (qy + q1d * qz);
+                              fhat(tv, tod, q) = 0.0;
+                           }
+                        }
+                     }
+                  }
+               }
+
+               // MSVC lambda capture workaround
+               [[maybe_unused]] const auto& inputs_ref = inputs;
+
+               int m_offset = 0;
+               for_constexpr<num_inputs>([&](auto s)
+               {
+                  using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
+
+                  const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
+                  if (trial_op_dim == 0)
+                  {
+                     // This is inside a lambda so we have to return
+                     // instead of idiomatic 'continue'.
+                     return;
+                  }
+
+                  auto& B = input_dtqmaps[s].B;
+                  auto& G = input_dtqmaps[s].G;
+
+                  if constexpr (is_value_fop<fop_t>::value)
+                  {
+                     MFEM_FOREACH_THREAD(qx, x, q1d)
+                     {
+                        MFEM_FOREACH_THREAD(qy, y, q1d)
+                        {
+                           MFEM_FOREACH_THREAD(qz, z, q1d)
+                           {
+                              const int q = qx + q1d * (qy + q1d * qz);
+
+                              for (int m = 0; m < trial_op_dim; m++)
+                              {
+                                 for (int i = 0; i < test_vdim; i++)
+                                 {
+                                    for (int k = 0; k < test_op_dim; k++)
+                                    {
+                                       const real_t f = qpdc(i, k, j, m + m_offset, q);
+                                       fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
+                                    }
+                                 }
+                              }
+                           }
+                        }
+                     }
+                  }
+                  else if constexpr (is_gradient_fop<fop_t>::value)
+                  {
+                     MFEM_FOREACH_THREAD(qx, x, q1d)
+                     {
+                        MFEM_FOREACH_THREAD(qy, y, q1d)
+                        {
+                           MFEM_FOREACH_THREAD(qz, z, q1d)
+                           {
+                              const int q = qx + q1d * (qy + q1d * qz);
+                              for (int m = 0; m < trial_op_dim; m++)
+                              {
+                                 for (int i = 0; i < test_vdim; i++)
+                                 {
+                                    for (int k = 0; k < test_op_dim; k++)
+                                    {
+                                       const real_t f = qpdc(i, k, j, m + m_offset, q);
+                                       if (m == 0)
+                                       {
+                                          fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
+                                       }
+                                       else if (m == 1)
+                                       {
+                                          fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy) * B(qz, 0, Jz);
+                                       }
+                                       else if (m == 2)
+                                       {
+                                          fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * G(qz, 0, Jz);
+                                       }
+                                    }
+                                 }
+                              }
+                           }
+                        }
+                     }
+                  }
+                  else
+                  {
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+                     MFEM_ABORT("sum factorized sparse matrix assemble routine "
+                                "not implemented for field operator");
+#endif
+                  }
+                  MFEM_SYNC_THREAD;
+                  m_offset += trial_op_dim;
+               });
+
+               auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
+               map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
+                                             scratch_shmem, dimension, true);
+            }
+         }
+      }
+   }
+}
+
+/// @brief Assemble element matrix for two dimensional data.
+///
+/// Note: In the below layouts, total_trial_op_dim is > 1 if
+/// there are more than one inputs dependent on the derivative variable.
+///
+/// @param A Memory for one element matrix with layout
+/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
+/// @param fhat Memory to hold the residual computation with layout
+/// [test_vdim, test_op_dim, nqp].
+/// @param qpdc The quadrature point data cache with data layout
+/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
+/// @param itod Input Trial Operator Dimension array. If the trial
+/// operator is not dependent, the dimension is 0 to indicate that.
+/// @param inputs The input field operator types.
+/// @param output The output field operator types.
+/// @param input_dtqmaps The input DofToQuad maps.
+/// @param output_dtqmap The output DofToQuad maps.
+/// @param scratch_shmem Scratch shared memory for computations.
+/// @param q1d The number of quadrature points in one dimension.
+/// @param td1d The number of trial dofs in one dimension.
+template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
+MFEM_HOST_DEVICE void assemble_element_mat_t2d(
+   const DeviceTensor<4, real_t>& A,
+   const DeviceTensor<3, real_t>& fhat,
+   const DeviceTensor<5, const real_t>& qpdc,
+   const DeviceTensor<1, const real_t>& itod,
+   const input_fop_ts& inputs,
+   const output_fop_t& output,
+   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
+   const DofToQuadMap& output_dtqmap,
+   std::array<DeviceTensor<1>, 6>& scratch_shmem,
+   const int& q1d,
+   const int& td1d)
+{
+   constexpr int dimension = 2;
+
+   // [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
+   const int test_vdim = qpdc.GetShape()[0];
+   const int test_op_dim = qpdc.GetShape()[1];
+   const int trial_vdim = qpdc.GetShape()[2];
+
+   // [num_test_dof, ...]
+   const auto num_test_dof = A.GetShape()[0];
+
+   for (int Jx = 0; Jx < td1d; Jx++)
+   {
+      for (int Jy = 0; Jy < td1d; Jy++)
+      {
+         const int J = Jy + Jx * td1d;
+
+         for (int j = 0; j < trial_vdim; j++)
+         {
+            for (int tv = 0; tv < test_vdim; tv++)
+            {
+               for (int tod = 0; tod < test_op_dim; tod++)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, q1d)
+                  {
+                     MFEM_FOREACH_THREAD(qy, y, q1d)
+                     {
+                        const int q = qy + qx * q1d;
+                        fhat(tv, tod, q) = 0.0;
+                     }
+                  }
+               }
+            }
+
+            // MSVC lambda capture workaround
+            [[maybe_unused]] const auto& inputs_ref = inputs;
+
+            int m_offset = 0;
+            for_constexpr<num_inputs>([&](auto s)
+            {
+               using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
+
+               const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
+               if (trial_op_dim == 0)
+               {
+                  // This is inside a lambda so we have to return
+                  // instead of idiomatic 'continue'.
+                  return;
+               }
+
+               auto& B = input_dtqmaps[s].B;
+               auto& G = input_dtqmaps[s].G;
+
+               if constexpr (is_value_fop<fop_t>::value)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, q1d)
+                  {
+                     MFEM_FOREACH_THREAD(qy, y, q1d)
+                     {
+                        const int q = qy + qx * q1d;
+                        for (int m = 0; m < trial_op_dim; m++)
+                        {
+                           for (int i = 0; i < test_vdim; i++)
+                           {
+                              for (int k = 0; k < test_op_dim; k++)
+                              {
+                                 const real_t f = qpdc(i, k, j, m + m_offset, q);
+                                 fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy);
+                              }
+                           }
+                        }
+                     }
+                  }
+               }
+               else if constexpr (is_gradient_fop<fop_t>::value)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, q1d)
+                  {
+                     MFEM_FOREACH_THREAD(qy, y, q1d)
+                     {
+                        const int q = qy + qx * q1d;
+                        for (int m = 0; m < trial_op_dim; m++)
+                        {
+                           for (int i = 0; i < test_vdim; i++)
+                           {
+                              for (int k = 0; k < test_op_dim; k++)
+                              {
+                                 const real_t f = qpdc(i, k, j, m + m_offset, q);
+                                 if (m == 0)
+                                 {
+                                    fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy);
+                                 }
+                                 else
+                                 {
+                                    fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy);
+                                 }
+                              }
+                           }
+                        }
+                     }
+                  }
+               }
+               else
+               {
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+                  MFEM_ABORT("sum factorized sparse matrix assemble routine "
+                             "not implemented for field operator");
+#endif
+               }
+               MFEM_SYNC_THREAD;
+               m_offset += trial_op_dim;
+            });
+
+            auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
+            map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
+                                          scratch_shmem, dimension, true);
+         }
+      }
+   }
+}
+
+/// @brief Assemble element matrix for two or three dimensional data.
+///
+/// Note: In the below layouts, total_trial_op_dim is > 1 if
+/// there are more than one inputs dependent on the derivative variable.
+///
+/// @param A Memory for one element matrix with layout
+/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
+/// @param fhat Memory to hold the residual computation with layout
+/// [test_vdim, test_op_dim, nqp].
+/// @param qpdc The quadrature point data cache with data layout
+/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
+/// @param itod Input Trial Operator Dimension array. If the trial
+/// operator is not dependent, the dimension is 0 to indicate that.
+/// @param inputs The input field operator types.
+/// @param output The output field operator types.
+/// @param input_dtqmaps The input DofToQuad maps.
+/// @param output_dtqmap The output DofToQuad maps.
+/// @param scratch_shmem Scratch shared memory for computations.
+/// @param dimension The spatial dimension.
+/// @param q1d The number of quadrature points in one dimension.
+/// @param td1d The number of trial dofs in one dimension.
+/// @param use_sum_factorization Indicator if sum factorization is used.
+template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
+MFEM_HOST_DEVICE void assemble_element_mat_naive(
+   const DeviceTensor<4, real_t>& A,
+   const DeviceTensor<3, real_t>& fhat,
+   const DeviceTensor<5, const real_t>& qpdc,
+   const DeviceTensor<1, const real_t>& itod,
+   const input_fop_ts& inputs,
+   const output_fop_t& output,
+   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
+   const DofToQuadMap& output_dtqmap,
+   std::array<DeviceTensor<1>, 6>& scratch_shmem,
+   const int& dimension,
+   const int& q1d,
+   const int& td1d,
+   const bool& use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 2)
+      {
+         assemble_element_mat_t2d(A, fhat, qpdc, itod, inputs, output,
+                                  input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
+      }
+      else if (dimension == 3)
+      {
+         assemble_element_mat_t3d(A, fhat, qpdc, itod, inputs, output,
+                                  input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
+      }
+   }
+   else
+   {
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+      MFEM_ABORT("element matrix assemble not implemented for non tensor "
+                 "product basis");
+#endif
+   }
+}
+
+} // namespace mfem::future

fem/dfem/backends/global_qf/action.hpp

@@ -1,122 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../util.hpp"
-#include "../../integrator_ctx.hpp"
-
-#include <utility>
-
-namespace mfem::future
-{
-
-namespace GlobalQFImpl
-{
-
-template<
-   typename qfunc_t,
-   typename inputs_t,
-   typename outputs_t,
-   size_t ninputs = tuple_size<inputs_t>::value,
-   size_t noutputs = tuple_size<outputs_t>::value>
-struct Action
-{
-   Action(
-      IntegratorContext ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs) :
-      ctx(ctx),
-      qfunc(qfunc),
-      inputs(inputs),
-      outputs(outputs)
-   {
-      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(inputs, input_to_infd, ctx.infds);
-      check_consistency(outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
-      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
-
-      const int nqp = ctx.ir.GetNPoints();
-      gnqp = nqp * ctx.nentities;
-
-      // prepare xq and yq BlockVectors
-      xq_offsets.SetSize(ninputs + 1);
-      xq_offsets[0] = 0;
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         const auto input = get<i>(inputs);
-         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
-      });
-      xq_offsets.PartialSum();
-      InitBlockVector(xq, xq_offsets);
-
-      yq_offsets.SetSize(noutputs + 1);
-      yq_offsets[0] = 0;
-      constexpr_for<0, noutputs>([&](auto i)
-      {
-         const auto output = get<i>(outputs);
-         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
-      });
-      yq_offsets.PartialSum();
-      InitBlockVector(yq, yq_offsets);
-   }
-
-   void operator()(
-      const std::vector<Vector *> &xe,
-      std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      // E -> Q
-      interpolate(input_to_infd, input_bases, xe, xq);
-
-      // Q -> Q
-      static_assert(
-         detail::supports_tensor_array_qfunc<qfunc_t, inputs_t, outputs_t>::value,
-         "qfunc signature not supported by default backend Action");
-
-      detail::call_qfunc(
-         qfunc, xq, yq, gnqp, input_qlayouts, output_qlayouts,
-         std::make_index_sequence<ninputs> {},
-         std::make_index_sequence<noutputs> {});
-
-      // Q -> E
-      integrate(output_to_outfd, output_bases, yq, ye);
-   }
-
-   IntegratorContext ctx;
-   qfunc_t qfunc;
-   inputs_t inputs;
-   outputs_t outputs;
-
-   std::array<size_t, ninputs> input_to_infd;
-   std::array<size_t, noutputs> output_to_outfd;
-
-   std::array<FieldBasis, ninputs> input_bases;
-   std::array<FieldBasis, noutputs> output_bases;
-
-   std::array<std::vector<int>, ninputs>  input_qlayouts;
-   std::array<std::vector<int>, noutputs> output_qlayouts;
-
-   int gnqp = 0;
-   Array<int> xq_offsets, yq_offsets;
-   mutable BlockVector xq, yq;
-};
-
-}
-}

fem/dfem/backends/global_qf/derivative_action.hpp

@@ -1,180 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "../util.hpp"
-#include <utility>
-
-namespace mfem::future::GlobalQFImpl
-{
-
-template<
-   int derivative_id,
-   typename qfunc_t,
-   typename inputs_t,
-   typename outputs_t,
-   size_t ninputs = tuple_size<inputs_t>::value,
-   size_t noutputs = tuple_size<outputs_t>::value>
-struct DerivativeAction
-{
-   using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
-
-   DerivativeAction(
-      IntegratorContext ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs):
-      ctx(ctx), qfunc(qfunc),
-      qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
-      inputs(inputs), outputs(outputs)
-   {
-      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(inputs, input_to_infd, ctx.infds);
-      check_consistency(outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(
-         outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
-      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
-
-      const int nqp = ctx.ir.GetNPoints();
-      gnqp = nqp * ctx.nentities;
-
-      xq_offsets.SetSize(ninputs + 1);
-      xq_offsets[0] = 0;
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         const auto input = get<i>(inputs);
-         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
-      });
-      xq_offsets.PartialSum();
-      InitBlockVector(xq, xq_offsets);
-
-      yq_offsets.SetSize(noutputs + 1);
-      yq_offsets[0] = 0;
-      constexpr_for<0, noutputs>([&](auto i)
-      {
-         const auto output = get<i>(outputs);
-         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
-      });
-      yq_offsets.PartialSum();
-      InitBlockVector(yq, yq_offsets);
-
-      // Shadow blocks use the same offsets as xq so tensor_array views
-      shadow_xq_offsets.SetSize(xq_offsets.Size());
-      shadow_xq_offsets = xq_offsets;
-      InitBlockVector(shadow_xq, shadow_xq_offsets);
-
-      dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
-
-      const size_t direction_fd_idx = FindIdx(derivative_id, ctx.infds);
-      MFEM_ASSERT(direction_fd_idx != SIZE_MAX,
-                  "derivative direction field not found in infds");
-      direction_fd = ctx.infds[direction_fd_idx];
-   }
-
-   void operator()(
-      const std::vector<Vector *> &xe,
-      const Vector *de,
-      std::vector<Vector *> &ye)
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      // E -> Q
-      interpolate(input_to_infd, input_bases, xe, xq);
-
-      constexpr auto input_active =
-         detail::make_activity_map<derivative_id>(inputs_t{});
-
-      MFEM_ASSERT(de != nullptr, "derivative direction vector is null");
-      restriction(direction_fd, direction_rcache, *de, direction_e,
-                  dof_ordering);
-
-      shadow_xq = 0.0;
-      shadow_xq.SyncToBlocks();
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         if (!input_active[i]) { return; }
-         input_bases[i].forward(direction_e, shadow_xq.GetBlock(i));
-      });
-
-      static_assert(detail::supports_tensor_array_qfunc<qfunc_t,
-                    inputs_t,
-                    outputs_t>::value,
-                    "qfunc signature not supported by default backend Action");
-
-      // Q -> Q
-      yq = 0.0;
-      yq.SyncToBlocks();
-      if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
-      {
-         detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t, inputs_t,
-                outputs_t>(
-                   qfunc,
-                   qfunc_shadow,
-                   xq,
-                   shadow_xq,
-                   yq,
-                   gnqp,
-                   input_qlayouts,
-                   output_qlayouts,
-                   std::make_index_sequence<ninputs> {},
-                   std::make_index_sequence<noutputs> {});
-      }
-      else
-      {
-         detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
-            qfunc,
-            xq,
-            shadow_xq,
-            yq,
-            gnqp,
-            input_qlayouts,
-            output_qlayouts,
-            std::make_index_sequence<ninputs> {},
-            std::make_index_sequence<noutputs> {});
-      }
-
-      // Q -> E
-      integrate(output_to_outfd, output_bases, yq, ye);
-   }
-
-   IntegratorContext ctx;
-   qfunc_t qfunc;
-   qfunc_shadow_t qfunc_shadow;
-   inputs_t inputs;
-   outputs_t outputs;
-
-   std::array<size_t, ninputs> input_to_infd;
-   std::array<size_t, noutputs> output_to_outfd;
-
-   std::array<FieldBasis, ninputs> input_bases;
-   std::array<FieldBasis, noutputs> output_bases;
-
-   std::array<std::vector<int>, ninputs> input_qlayouts;
-   std::array<std::vector<int>, noutputs> output_qlayouts;
-
-   int gnqp = 0;
-   Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
-   mutable BlockVector xq, shadow_xq, yq;
-
-   FieldDescriptor direction_fd;
-   ElementDofOrdering dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   mutable Vector direction_e;
-   mutable RestrictionCache<Entity::Element> direction_rcache;
-};
-
-} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_apply.hpp

@@ -1,242 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "../util.hpp"
-
-#include <array>
-#include <utility>
-#include <vector>
-
-namespace mfem::future::GlobalQFImpl
-{
-
-// Q-function-shape-agnostic cached forward apply (J·v)
-template<
-   int derivative_id,
-   typename inputs_t,
-   typename outputs_t>
-struct DerivativeApply
-{
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-
-   DerivativeApply(
-      IntegratorContext ctx,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache):
-      ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
-      qp_cache(qp_cache)
-   {
-      create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(this->inputs, input_to_infd, ctx.infds);
-      check_consistency(this->outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(
-         this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(
-         this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      const int nqp = ctx.ir.GetNPoints();
-      const int ne = ctx.nentities;
-      num_qp = nqp;
-      gnqp = nqp * ne;
-
-      // Precompute Q-space BlockVector layouts
-      dir_q_offsets.SetSize(n_inputs + 1);
-      dir_q_offsets[0] = 0;
-      constexpr_for<0, n_inputs>([&](auto i)
-      {
-         dir_q_offsets[i + 1] =
-            dir_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
-      });
-      InitBlockVector(dir_q_local, dir_q_offsets);
-
-      result_q_offsets.SetSize(n_outputs + 1);
-      result_q_offsets[0] = 0;
-      constexpr_for<0, n_outputs>([&](auto i)
-      {
-         result_q_offsets[i + 1] =
-            result_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
-      });
-      InitBlockVector(result_q_local, result_q_offsets);
-
-      // Cache layout metadata (must match DerivativeSetup)
-      residual_size_on_qp = 0;
-      trial_vdim = 0;
-      total_trial_op_dim = 0;
-
-      constexpr auto activity =
-         detail::make_activity_map<derivative_id>(inputs_t{});
-
-      constexpr_for<0, n_inputs>([&](auto i)
-      {
-         if (!activity[i]) { return; }
-         const auto &fop = get<i>(this->inputs);
-         trial_vdim = fop.vdim;
-         total_trial_op_dim += fop.size_on_qp / fop.vdim;
-      });
-
-      constexpr_for<0, n_outputs>([&](auto i)
-      { residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
-      residual_size_on_qp *= trial_vdim * total_trial_op_dim;
-   }
-
-   void operator()(
-      const std::vector<Vector *> & /*xe*/,
-      const Vector *direction_l,
-      std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      MFEM_ASSERT(direction_l != nullptr,
-                  "Global DerivativeApply: direction vector is null");
-
-      // Re-zero pre-allocated Q temporaries
-      dir_q_local = 0.0;
-      dir_q_local.SyncToBlocks();
-      result_q_local = 0.0;
-      result_q_local.SyncToBlocks();
-
-      // Restrict trial direction from the derivative field
-      size_t in_fd = SIZE_MAX;
-      constexpr_for<0, n_inputs>([&](auto i)
-      {
-         if (get<i>(inputs).GetFieldId() == derivative_id)
-         {
-            in_fd = input_to_infd[i.value];
-         }
-      });
-      MFEM_ASSERT(in_fd != SIZE_MAX,
-                  "DerivativeApply: derivative field not found among inputs");
-
-      const auto &fd = ctx.infds[in_fd];
-
-      Vector dir_e;
-      restriction(fd, direction_rcache, *direction_l, dir_e,
-                  ElementDofOrdering::LEXICOGRAPHIC);
-
-      // Forward the trial direction into active input Q block
-      constexpr_for<0, n_inputs>([&](auto s)
-      {
-         if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
-         input_bases[s.value].forward(dir_e, dir_q_local.GetBlock(s.value));
-      });
-
-      const real_t *cache_ptr = qp_cache.Read();
-      const int res_sz = residual_size_on_qp;
-      const int gnqp_local = gnqp;
-      const int num_qp_local = num_qp;
-      const int trial_vdim_local = trial_vdim;
-      const int total_trial_op_dim_local = total_trial_op_dim;
-
-      constexpr_for<0, n_outputs>([&](auto o)
-      {
-         const int tv_o = get<o>(outputs).vdim;
-         const int to_o = get<o>(outputs).size_on_qp / tv_o;
-         const int out_base = [&]
-         {
-            int off = 0;
-            constexpr_for<0, o.value>([&](auto prev)
-            { off += get<prev>(outputs).size_on_qp; });
-            return off;
-         }();
-
-         real_t *res_o = result_q_local.GetBlock(o.value).ReadWrite();
-
-         int m_offset = 0;
-         constexpr_for<0, n_inputs>([&](auto s)
-         {
-            if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
-
-            const int tv = get<s>(inputs).vdim;
-            const int to = get<s>(inputs).size_on_qp / tv;
-            const real_t *dir_s = dir_q_local.GetBlock(s.value).Read();
-
-            mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
-            {
-               // Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
-               // read adjacent addresses for a fixed cache_idx.
-               const int cache_base =
-                  (gq % num_qp_local) +
-                  num_qp_local * res_sz * (gq / num_qp_local);
-
-               for (int j = 0; j < tv; ++j)
-               {
-                  for (int m = 0; m < to; ++m)
-                  {
-                     const real_t v = dir_s[(j * to + m) + (tv * to) * gq];
-                     const int m_global = m + m_offset;
-
-                     for (int i = 0; i < tv_o; ++i)
-                     {
-                        for (int k = 0; k < to_o; ++k)
-                        {
-                           const int out_comp = out_base + i * to_o + k;
-
-                           const int cache_idx =
-                              out_comp * trial_vdim_local * total_trial_op_dim_local +
-                              j * total_trial_op_dim_local + m_global;
-
-                           const real_t c =
-                              cache_ptr[cache_base + num_qp_local * cache_idx];
-                           res_o[(i * to_o + k) + (tv_o * to_o) * gq] += c * v;
-                        }
-                     }
-                  }
-               }
-            });
-            m_offset += to;
-         });
-      });
-
-      result_q_local.SyncToBlocks();
-
-      // Map result Q back to output fields
-      constexpr_for<0, n_outputs>([&](auto o)
-      {
-         const size_t out_fd = output_to_outfd[o.value];
-         output_bases[o.value].transpose(result_q_local.GetBlock(o.value),
-                                         *ye[out_fd]);
-      });
-   }
-
-private:
-   IntegratorContext ctx;
-   inputs_t inputs;
-   outputs_t outputs;
-   const Vector &qp_cache;
-
-   std::array<size_t, n_inputs> input_to_infd;
-   std::array<size_t, n_outputs> output_to_outfd;
-
-   std::array<FieldBasis, n_inputs> input_bases;
-   std::array<FieldBasis, n_outputs> output_bases;
-
-   int gnqp = 0;
-   int num_qp = 0;
-
-   Array<int> dir_q_offsets;
-   Array<int> result_q_offsets;
-   mutable BlockVector dir_q_local;
-   mutable BlockVector result_q_local;
-   mutable RestrictionCache<Entity::Element> direction_rcache;
-
-   int residual_size_on_qp = 0;
-   int trial_vdim = 0;
-   int total_trial_op_dim = 0;
-};
-
-} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_apply_transpose.hpp

@@ -1,260 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "../util.hpp"
-
-#include <array>
-#include <utility>
-#include <vector>
-
-namespace mfem::future::GlobalQFImpl
-{
-
-// Q-function-shape-agnostic cached transpose apply (Jᵀ·w)
-template<
-   int derivative_id,
-   typename qfunc_t,
-   typename inputs_t,
-   typename outputs_t>
-struct DerivativeApplyTranspose
-{
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-
-   DerivativeApplyTranspose(
-      IntegratorContext ctx,
-      qfunc_t /*qfunc*/,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache):
-      ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
-      qp_cache(qp_cache), dir_out_l(n_outputs), dir_out_e(n_outputs)
-   {
-      create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(this->inputs, input_to_infd, ctx.infds);
-      check_consistency(this->outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(
-         this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(
-         this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      const int nqp = ctx.ir.GetNPoints();
-      const int ne = ctx.nentities;
-      num_qp = nqp;
-      gnqp = nqp * ne;
-
-      // Precompute Q-space BlockVector layouts
-      dir_q_offsets.SetSize(n_outputs + 1);
-      dir_q_offsets[0] = 0;
-      constexpr_for<0, n_outputs>([&](auto i)
-      {
-         dir_q_offsets[i + 1] =
-            dir_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
-      });
-      InitBlockVector(dir_q_local, dir_q_offsets);
-
-      result_q_offsets.SetSize(n_inputs + 1);
-      result_q_offsets[0] = 0;
-      constexpr_for<0, n_inputs>([&](auto i)
-      {
-         result_q_offsets[i + 1] =
-            result_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
-      });
-      InitBlockVector(result_q_local, result_q_offsets);
-
-      // Cache layout metadata
-      residual_size_on_qp = 0;
-      trial_vdim = 0;
-      total_trial_op_dim = 0;
-
-      constexpr auto activity =
-         detail::make_activity_map<derivative_id>(inputs_t{});
-
-      constexpr_for<0, n_inputs>([&](auto i)
-      {
-         if (!activity[i]) { return; }
-         const auto &fop = get<i>(this->inputs);
-         trial_vdim = fop.vdim;
-         total_trial_op_dim += fop.size_on_qp / fop.vdim;
-      });
-
-      constexpr_for<0, n_outputs>([&](auto i)
-      { residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
-      residual_size_on_qp *= trial_vdim * total_trial_op_dim;
-   }
-
-   void operator()(
-      const std::vector<Vector *> & /*xe*/,
-      const Vector *direction_l,
-      std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      MFEM_ASSERT(direction_l != nullptr,
-                  "Global DerivativeApplyTranspose: direction vector is null");
-
-      // Re-zero the pre-allocated Q temporaries
-      dir_q_local = 0.0;
-      result_q_local = 0.0;
-      dir_q_local.SyncToBlocks();
-      result_q_local.SyncToBlocks();
-
-      // Bring test cotangent to quadrature points
-      pull_output_cotangents_to_q(direction_l, dir_q_local);
-
-      // Contract qp_cache with test directions at quadrature points
-      const real_t *cache_ptr = qp_cache.Read();
-      const int res_sz = residual_size_on_qp;
-      const int gnqp_local = gnqp;
-      const int num_qp_local = num_qp;
-      const int trial_vdim_local = trial_vdim;
-      const int total_trial_op_dim_local = total_trial_op_dim;
-
-      constexpr_for<0, n_outputs>([&](auto o)
-      {
-         const int tv_o = get<o>(outputs).vdim;
-         const int to_o = get<o>(outputs).size_on_qp / tv_o;
-         const int out_base = [&]
-         {
-            int off = 0;
-            constexpr_for<0, o.value>([&](auto prev)
-            { off += get<prev>(outputs).size_on_qp; });
-            return off;
-         }();
-
-         const int size_o = get<o>(outputs).size_on_qp;
-         const real_t *dir_o = dir_q_local.GetBlock(o.value).Read();
-
-         int m_offset = 0;
-         constexpr_for<0, n_inputs>([&](auto s)
-         {
-            if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
-
-            const int size_s = get<s>(inputs).size_on_qp;
-            const int to_s = size_s / trial_vdim_local;
-            real_t *res_s = result_q_local.GetBlock(s.value).ReadWrite();
-
-            mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
-            {
-               // Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
-               // read adjacent addresses for a fixed cache_idx.
-               const int cache_base =
-                  (gq % num_qp_local) +
-                  num_qp_local * res_sz * (gq / num_qp_local);
-
-               for (int i = 0; i < tv_o; ++i)
-               {
-                  for (int k = 0; k < to_o; ++k)
-                  {
-                     const int out_comp = out_base + i * to_o + k;
-                     const real_t w = dir_o[(i * to_o + k) + size_o * gq];
-
-                     for (int j = 0; j < trial_vdim_local; ++j)
-                     {
-                        for (int m = 0; m < to_s; ++m)
-                        {
-                           const int m_global = m + m_offset;
-                           const int cache_idx =
-                              out_comp * trial_vdim_local * total_trial_op_dim_local +
-                              j * total_trial_op_dim_local + m_global;
-
-                           const real_t c =
-                              cache_ptr[cache_base + num_qp_local * cache_idx];
-                           res_s[(j * to_s + m) + size_s * gq] += c * w;
-                        }
-                     }
-                  }
-               }
-            });
-            m_offset += to_s;
-         });
-      });
-
-      // Map result Q back to the trial (input) fields
-      constexpr_for<0, n_inputs>([&](auto s)
-      {
-         if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
-
-         const size_t in_fd = input_to_infd[s.value];
-         input_bases[s.value].transpose(
-            result_q_local.GetBlock(s.value), *ye[in_fd]);
-      });
-   }
-
-private:
-   IntegratorContext ctx;
-   inputs_t inputs;
-   outputs_t outputs;
-   const Vector &qp_cache;
-
-   std::array<size_t, n_inputs> input_to_infd;
-   std::array<size_t, n_outputs> output_to_outfd;
-
-   std::array<FieldBasis, n_inputs> input_bases;
-   std::array<FieldBasis, n_outputs> output_bases;
-
-   int gnqp = 0;
-   int num_qp = 0;
-
-   // Pre-allocated Q-space temporaries
-   Array<int> dir_q_offsets;
-   Array<int> result_q_offsets;
-   mutable BlockVector dir_q_local;
-   mutable BlockVector result_q_local;
-
-   // Pre-allocated owning storage for output cotangent temporaries
-   mutable std::array<Vector, n_outputs> dir_out_l_owned;
-   mutable std::array<Vector, n_outputs> dir_out_e_owned;
-   mutable std::vector<Vector *> dir_out_l;
-   mutable std::vector<Vector *> dir_out_e;
-   mutable RestrictionCache<Entity::Element> out_rcache;
-
-   int residual_size_on_qp = 0;
-   int trial_vdim = 0;
-   int total_trial_op_dim = 0;
-
-   /// Pull output cotangents from L-space into the pre-allocated Q BlockVector
-   void pull_output_cotangents_to_q(const Vector *direction_l,
-                                    BlockVector &dir_q) const
-   {
-      int l_offset = 0;
-      constexpr_for<0, n_outputs>([&](auto i)
-      {
-         const size_t outfd = output_to_outfd[i];
-         const auto &fd = ctx.outfds[outfd];
-         const int l_size = GetVSize(fd);
-
-         dir_out_l_owned[i] =
-            Vector(*const_cast<Vector *>(direction_l), l_offset, l_size);
-         dir_out_e_owned[i].SetSize(0);
-         dir_out_e_owned[i].UseDevice(true);
-
-         dir_out_l[i] = &dir_out_l_owned[i];
-         dir_out_e[i] = &dir_out_e_owned[i];
-         l_offset += l_size;
-      });
-
-      restriction(ctx.outfds, out_rcache, dir_out_l, dir_out_e);
-
-      constexpr_for<0, n_outputs>([&](auto i)
-      {
-         output_bases[i.value].forward(*dir_out_e[i], dir_q.GetBlock(i.value));
-      });
-      dir_q.SyncToBlocks();
-   }
-};
-
-} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_setup.hpp

@@ -1,244 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "../util.hpp"
-#include <utility>
-
-namespace mfem::future::GlobalQFImpl
-{
-
-template<
-   int derivative_id,
-   typename qfunc_t,
-   typename inputs_t,
-   typename outputs_t,
-   size_t ninputs = tuple_size<inputs_t>::value,
-   size_t noutputs = tuple_size<outputs_t>::value>
-struct DerivativeSetup
-{
-   using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
-
-   DerivativeSetup(
-      IntegratorContext ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      Vector &qp_cache) :
-      ctx(ctx),
-      qfunc(qfunc),
-      qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
-      inputs(inputs),
-      outputs(outputs),
-      qp_cache(qp_cache)
-   {
-      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(inputs, input_to_infd, ctx.infds);
-      check_consistency(outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
-      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
-
-      const int nqp = ctx.ir.GetNPoints();
-      num_qp = nqp;
-      nentities = ctx.nentities;
-      gnqp = nqp * nentities;
-
-      xq_offsets.SetSize(ninputs + 1);
-      xq_offsets[0] = 0;
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         xq_offsets[i + 1] = nqp * get<i>(inputs).size_on_qp * nentities;
-      });
-      xq_offsets.PartialSum();
-      InitBlockVector(xq, xq_offsets);
-
-      shadow_xq_offsets.SetSize(xq_offsets.Size());
-      shadow_xq_offsets = xq_offsets;
-      InitBlockVector(shadow_xq, shadow_xq_offsets);
-
-      yq_offsets.SetSize(noutputs + 1);
-      yq_offsets[0] = 0;
-      constexpr_for<0, noutputs>([&](auto o)
-      {
-         yq_offsets[o + 1] = nqp * get<o>(outputs).size_on_qp * nentities;
-      });
-      yq_offsets.PartialSum();
-      InitBlockVector(yq, yq_offsets);
-
-      total_out_size_on_qp = 0;
-      constexpr_for<0, noutputs>([&](auto o)
-      {
-         total_out_size_on_qp += get<o>(outputs).size_on_qp;
-         out_vdim[o] = get<o>(outputs).vdim;
-         out_op_dim[o] = get<o>(outputs).size_on_qp / get<o>(outputs).vdim;
-      });
-
-      activity_map = detail::make_activity_map<derivative_id>(inputs_t {});
-
-      trial_vdim = 0;
-      total_trial_op_dim = 0;
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         if (!activity_map[i]) { return; }
-         const auto inp = get<i>(inputs);
-         trial_vdim = inp.vdim;
-         total_trial_op_dim += inp.size_on_qp / inp.vdim;
-      });
-
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         input_size_on_qp_arr[i] = get<i>(inputs).size_on_qp;
-      });
-
-      residual_size_on_qp = total_out_size_on_qp * trial_vdim * total_trial_op_dim;
-      qp_cache.SetSize(residual_size_on_qp * num_qp * nentities);
-      qp_cache.UseDevice(true);
-   }
-
-   void operator()(const std::vector<Vector *> &xe)
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      interpolate(input_to_infd, input_bases, xe, xq);
-
-      const int gnqp_local = gnqp;
-      const int num_qp_local = num_qp;
-      const int trial_vdim_local = trial_vdim;
-      const int total_trial_op_dim_local = total_trial_op_dim;
-      const int residual_size_local = residual_size_on_qp;
-
-      for (int j = 0; j < trial_vdim; j++)
-      {
-         int m_offset = 0;
-         constexpr_for<0, ninputs>([&](auto s)
-         {
-            if (!activity_map[s]) { return; }
-
-            const int input_vdim_s  = get<s>(inputs).vdim;
-            const int input_size_s  = input_size_on_qp_arr[s];
-            const int trial_op_dim_s = input_size_s / input_vdim_s;
-
-            for (int m = 0; m < trial_op_dim_s; m++)
-            {
-               shadow_xq = 0.0;
-               shadow_xq.SyncToBlocks();
-
-               // Set component (j + input_vdim_s * m) to 1 at all QPs
-               const int c_shadow = j + input_vdim_s * m;
-               real_t *shadow_ptr = shadow_xq.GetBlock(s.value).ReadWrite();
-               mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
-               {
-                  shadow_ptr[c_shadow + input_size_s * gq] = 1.0;
-               });
-
-               yq = 0.0;
-               yq.SyncToBlocks();
-
-               if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
-               {
-                  detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t,
-                         inputs_t, outputs_t>(
-                            qfunc, qfunc_shadow, xq, shadow_xq, yq, gnqp,
-                            input_qlayouts, output_qlayouts,
-                            std::make_index_sequence<ninputs> {},
-                            std::make_index_sequence<noutputs> {});
-               }
-               else
-               {
-                  detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
-                     qfunc, xq, shadow_xq, yq, gnqp,
-                     input_qlayouts, output_qlayouts,
-                     std::make_index_sequence<ninputs> {},
-                     std::make_index_sequence<noutputs> {});
-               }
-
-               real_t *cache_d = qp_cache.ReadWrite();
-
-               // Write yq into the cache column
-               const int m_global = m + m_offset;
-               const int j_cur    = j;
-               int out_offset = 0;
-               constexpr_for<0, noutputs>([&](auto o)
-               {
-                  const int test_vdim_o   = out_vdim[o];
-                  const int test_op_dim_o = out_op_dim[o];
-                  const int yq_out_size   = test_vdim_o * test_op_dim_o;
-                  const int out_offset_o  = out_offset;
-                  const real_t *yq_d      = yq.GetBlock(o.value).Read();
-
-                  // The cache is (q, cache_idx, e) with the quadrature index
-                  // fastest, so gq is the fastest-varying thread index to keep
-                  // the stores coalesced.
-                  mfem::forall(gnqp_local * yq_out_size, [=] MFEM_HOST_DEVICE(int idx)
-                  {
-                     const int gq       = idx % gnqp_local;
-                     const int c_out    = idx / gnqp_local;
-                     const int q        = gq % num_qp_local;
-                     const int entity   = gq / num_qp_local;
-                     const int out_comp = out_offset_o + c_out;
-                     const int cache_idx =
-                        out_comp * trial_vdim_local * total_trial_op_dim_local +
-                        j_cur * total_trial_op_dim_local +
-                        m_global;
-                     cache_d[q + num_qp_local *
-                               (cache_idx + residual_size_local * entity)] =
-                                yq_d[c_out + yq_out_size * gq];
-                  });
-                  out_offset += yq_out_size;
-               });
-            }
-            m_offset += trial_op_dim_s;
-         });
-      }
-   }
-
-   IntegratorContext ctx;
-   qfunc_t qfunc;
-   qfunc_shadow_t qfunc_shadow;
-   inputs_t inputs;
-   outputs_t outputs;
-   Vector &qp_cache;
-
-   std::array<size_t, ninputs> input_to_infd;
-   std::array<size_t, noutputs> output_to_outfd;
-
-   std::array<FieldBasis, ninputs> input_bases;
-   std::array<FieldBasis, noutputs> output_bases;
-
-   std::array<std::vector<int>, ninputs>  input_qlayouts;
-   std::array<std::vector<int>, noutputs> output_qlayouts;
-
-   int gnqp = 0;
-   int num_qp = 0;
-   int nentities = 0;
-
-   Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
-   mutable BlockVector xq, shadow_xq, yq;
-
-   int total_out_size_on_qp = 0;
-   int trial_vdim = 0;
-   int total_trial_op_dim = 0;
-   int residual_size_on_qp = 0;
-
-   std::array<int, noutputs> out_vdim {};
-   std::array<int, noutputs> out_op_dim {};
-   std::array<int, ninputs>  input_size_on_qp_arr {};
-   std::array<bool, ninputs> activity_map {};
-};
-
-} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/prelude.hpp

@@ -1,219 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "action.hpp"
-#include "derivative_action.hpp"
-#include "derivative_setup.hpp"
-
-#include "derivative_apply_transpose.hpp"
-
-#include "../local_qf/derivative_apply.hpp"
-#include "../local_qf/derivative_assemble.hpp"
-#include "../local_qf/derivative_assemble_diagonal.hpp"
-
-#include "../scratch_bank.hpp"
-
-namespace mfem::future
-{
-
-namespace detail
-{
-
-template <typename T>
-struct LocalQFShapeArg
-{
-   using type = std::remove_const_t<T>&;
-};
-
-template <typename scalar_t, int ndims, int... tensor_sizes>
-struct LocalQFShapeArg<tensor_ndarray<scalar_t, ndims, tensor_sizes...>>
-{
-   using scalar_type = std::remove_const_t<scalar_t>;
-   using type = std::conditional_t<
-                sizeof...(tensor_sizes) == 0,
-                scalar_type,
-                tensor<scalar_type, tensor_sizes...>>&;
-};
-
-template <typename scalar_t, int... tensor_sizes>
-struct LocalQFShapeArg<tensor<scalar_t, tensor_sizes...>>
-{
-   using scalar_type = std::remove_const_t<scalar_t>;
-   using type = std::conditional_t<
-                sizeof...(tensor_sizes) == 0,
-                scalar_type,
-                tensor<scalar_type, tensor_sizes...>>&;
-};
-
-template <typename qf_param_ts>
-struct LocalQFShapeFunction;
-
-template <typename... qf_param_ts>
-struct LocalQFShapeFunction<tuple<qf_param_ts...>>
-{
-   void operator()(
-      typename LocalQFShapeArg<qf_param_decay_t<qf_param_ts>>::type...) const;
-};
-
-template <typename qfunc_t>
-using LocalQFShapeFunctionFor = LocalQFShapeFunction<
-                                typename get_function_signature<qfunc_t>::type::parameter_ts>;
-
-} // namespace detail
-
-struct GlobalQFBackend
-{
-   /**
-    * @brief Make an action for a global Q-function.
-    *
-    * @param ctx The integrator context.
-    * @param args The arguments to the action.
-    * @return The action.
-    */
-   template<
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeAction(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      return GlobalQFImpl::Action(ctx, qfunc, inputs, outputs);
-   }
-
-   /**
-    * @brief Make a derivative action for a global Q-function.
-    *
-    * @tparam derivative_id The id of the derivative.
-    * @param ctx The integrator context.
-    * @param args The arguments to the derivative action.
-    * @return The derivative action.
-    */
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeAction(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      return GlobalQFImpl::DerivativeAction<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs);
-   }
-
-   /**
-    * @brief Make a derivative setup for a global Q-function.
-    *
-    * @tparam derivative_id The id of the derivative.
-    * @param ctx The integrator context.
-    * @param args The arguments to the derivative setup.
-    * @return The derivative setup.
-    */
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeSetup(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      Vector &qp_cache)
-   {
-      return GlobalQFImpl::DerivativeSetup<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeApply(
-      const IntegratorContext &ctx,
-      const qfunc_t & /*qfunc*/,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeApply<
-             derivative_id,
-             detail::LocalQFShapeFunctionFor<qfunc_t>,
-             inputs_t,
-             outputs_t>(ctx,
-                        detail::LocalQFShapeFunctionFor<qfunc_t> {},
-                        inputs,
-                        outputs,
-                        qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeApplyTranspose(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return GlobalQFImpl::DerivativeApplyTranspose<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeAssemble(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeAssemble<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeAssembleDiagonal(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeAssembleDiagonal<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs, qp_cache);
-   }
-};
-
-} // namespace mfem::future

fem/dfem/backends/local_qf/action.hpp

@@ -1,658 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../fieldoperator.hpp"
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include "../../integrator_ctx.hpp"
-#include "../util.hpp"
-
-#include <utility>
-
-namespace mfem::future::LocalQFImpl
-{
-
-template <typename outputs_t, typename index_seq>
-struct action_outputs_direct_impl;
-
-template <typename outputs_t, std::size_t... Is>
-struct action_outputs_direct_impl<outputs_t, std::index_sequence<Is...>>
-{
-   static constexpr bool value =
-      ((is_identity_fop_v<tuple_element_t<Is, outputs_t>> ||
-        is_functionalvalue_fop_v<tuple_element_t<Is, outputs_t>>) && ...);
-};
-
-template <typename outputs_t>
-constexpr bool action_outputs_direct_v = action_outputs_direct_impl<outputs_t,
-               std::make_index_sequence<tuple_size<outputs_t>::value>>::value;
-
-template <typename qfunc_t, typename inputs_t, typename outputs_t,
-          typename index_seq>
-struct action_outputs_direct_value_impl;
-
-template <typename qfunc_t, typename inputs_t, typename outputs_t,
-          std::size_t... Is>
-struct action_outputs_direct_value_impl<qfunc_t, inputs_t, outputs_t,
-          std::index_sequence<Is...>>
-{
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr bool value =
-      ((is_identity_fop_v<tuple_element_t<Is, outputs_t>> ||
-        is_functionalvalue_fop_v<tuple_element_t<Is, outputs_t>>) && ...)
-      && ((!qf_param_uses_dual_v<
-           typename qf_param_slot<qfunc_t, n_inputs + Is>::qf_decay_param_t>)
-          && ...);
-};
-
-template <typename qfunc_t, typename inputs_t, typename outputs_t>
-constexpr bool action_outputs_direct_value_v =
-   action_outputs_direct_value_impl<qfunc_t, inputs_t, outputs_t,
-   std::make_index_sequence<tuple_size<outputs_t>::value>>::value;
-
-template<typename qfunc_t, typename inputs_t, typename outputs_t>
-class Action
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-   using args_tuple_t = decay_tuple<qf_param_ts>;
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
-                 "LocalQF: q-function arity must match inputs + outputs");
-
-   template<typename backend_t, std::size_t I, typename RArgs, typename InXEs>
-   static MFEM_HOST_DEVICE decltype(auto) direct_input_arg(
-      RArgs &rargs,
-      const InXEs &in_XE,
-      const int qx,
-      const int qy,
-      const int qz,
-      const int e)
-   {
-      const auto &XE = in_XE[I];
-      using FOP = tuple_element_t<I, inputs_t>;
-      using ARG = typename qf_param_slot<qfunc_t, I>::qf_reg_param_t;
-      if constexpr (is_identity_fop_v<FOP>)
-      {
-         using DT = typename qf_param_slot<qfunc_t, I>::qf_decay_param_t;
-         if constexpr (qf_param_uses_dual_v<DT>)
-         {
-            return backend_t::template identity_qp_pull_dual<DT>(
-               false, XE, XE, qx, qy, qz, e);
-         }
-         else
-         {
-            return as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-         }
-      }
-      else if constexpr (is_weight_fop_v<FOP>)
-      {
-         return XE(qx, qy, qz, 0, 0);
-      }
-      else if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-      {
-         return backend_t::template qp_pull<ARG>(get<I>(rargs), qx, qy, qz);
-      }
-      else
-      {
-         static_assert(false, "Unsupported");
-      }
-   }
-
-   template<std::size_t I, typename OutYEs>
-   static MFEM_HOST_DEVICE decltype(auto) direct_output_arg(
-      const OutYEs &out_YE,
-      const int qx,
-      const int qy,
-      const int qz,
-      const int e)
-   {
-      constexpr size_t o = n_inputs + I;
-      const auto &YE = out_YE[I];
-      using DT = typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-      using ARG = typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
-      if constexpr (std::is_same_v<DT, real_t>)
-      {
-         return YE(0, qx, qy, qz, e);
-      }
-      else
-      {
-         return as_tensor<ARG>(&YE(0, qx, qy, qz, e));
-      }
-   }
-
-   template<typename backend_t, typename RArgs, typename InXEs, typename OutYEs,
-            std::size_t... InIs, std::size_t... OutIs>
-   static MFEM_HOST_DEVICE void call_qfunc_direct(
-      const qfunc_t &qfunc,
-      RArgs &rargs,
-      const InXEs &in_XE,
-      const OutYEs &out_YE,
-      const int qx,
-      const int qy,
-      const int qz,
-      const int e,
-      std::index_sequence<InIs...>,
-      std::index_sequence<OutIs...>)
-   {
-      qfunc(direct_input_arg<backend_t, InIs>(rargs, in_XE, qx, qy, qz, e)...,
-            direct_output_arg<OutIs>(out_YE, qx, qy, qz, e)...);
-   }
-
-   const qfunc_t qfunc;
-   const inputs_t inputs;
-   const outputs_t outputs;
-   const IntegratorContext ctx;
-   const std::vector<const DofToQuad *> dtqs;
-   // inputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_inputs> input_dtq;
-   const std::array<size_t, n_inputs> input_idx; // input to field
-   const std::array<const real_t *, n_inputs> input_B, input_G;
-   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
-   // outputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_outputs> output_dtq;
-   const std::array<size_t, n_outputs> output_idx; // output to field
-   const std::array<const real_t *, n_outputs> output_B, output_G;
-   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
-   // other constants
-   const int dim, ne, nq, q1d;
-
-public:
-   ////////////////////////////////////////////////////////
-   Action() = delete;
-
-   Action(IntegratorContext ctx,
-          qfunc_t qfunc,
-          inputs_t inputs,
-          outputs_t outputs):
-      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
-      dtqs(make_dtqs(ctx)),
-      // inputs: dtq, idx, B, G, d1d, q1d, vdim
-      input_dtq(create_dtq_maps<Entity::Element>(
-                   inputs,
-                   dtqs,
-                   create_union_field_map_for_dtq(ctx, inputs),
-                   ctx.unionfds,
-                   ctx.ir)),
-      input_idx(create_input_vector_map(ctx, inputs)),
-      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
-      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
-      input_vdim(get_vdim(inputs)),
-      // outputs: dtq, idx, B, G, d1d, q1d, vdim
-      output_dtq(create_dtq_maps<Entity::Element>(
-                    outputs,
-                    dtqs,
-                    create_union_field_map_for_dtq(ctx, outputs),
-                    ctx.unionfds,
-                    ctx.ir)),
-      output_idx(create_output_vector_map(ctx, outputs)),
-      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
-      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
-      output_vdim(get_vdim(outputs)),
-      // other constants
-      dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
-      q1d(tensor_1d_size(nq, dim))
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-   }
-
-   template<typename Backend>
-   void run_kernels(const std::vector<Vector *> &xe,
-                    std::vector<Vector *> &ye) const
-   {
-      Backend::Run(dim,
-                   q1d,
-                   // arguments
-                   ctx,
-                   qfunc,
-                   // inputs
-                   input_idx,
-                   input_B,
-                   input_G,
-                   input_vdim,
-                   input_d1d,
-                   input_q1d,
-                   // outputs
-                   output_idx,
-                   output_B,
-                   output_G,
-                   output_vdim,
-                   output_d1d,
-                   output_q1d,
-                   // input and output vectors
-                   xe,
-                   ye,
-                   // fallback arguments
-                   dim,
-                   q1d);
-   }
-
-   void operator()(const std::vector<Vector *> &xe,
-                   std::vector<Vector *> &ye) const
-   {
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<ActionLO>(xe, ye);
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<ActionHO>(xe, ye);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-   }
-
-   ////////////////////////////////////////////////////////
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void
-   action_callback(const IntegratorContext &ctx,
-                   const qfunc_t &qfunc,
-                   // inputs: idx, B, G, vdim, d1d, q1d
-                   const std::array<size_t, n_inputs> &in_idx,
-                   const std::array<const real_t *, n_inputs> in_B,
-                   const std::array<const real_t *, n_inputs> in_G,
-                   const std::array<int, n_inputs> &in_vdim,
-                   const std::array<int, n_inputs> &in_d1d,
-                   const std::array<int, n_inputs> &in_q1d,
-                   // outputs: idx, B, G, vdim, d1d, q1d
-                   const std::array<size_t, n_outputs> &out_idx,
-                   const std::array<const real_t *, n_outputs> out_B,
-                   const std::array<const real_t *, n_outputs> out_G,
-                   const std::array<int, n_outputs> &out_vdim,
-                   const std::array<int, n_outputs> &out_d1d,
-                   const std::array<int, n_outputs> &out_q1d,
-                   const std::vector<Vector *> &xe,
-                   std::vector<Vector *> &ye,
-                   // fallback arguments
-                   const int dim,
-                   const int q1d)
-   {
-      if (ctx.attr.Size() == 0) { return; }
-      MFEM_CONTRACT_VAR(dim);
-      MFEM_ASSERT(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-
-      static constexpr auto B2D = backend_t::DIM == 2;
-      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::Q1D;
-      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
-
-      const int ne = ctx.nentities;
-
-      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
-
-      // --------------------------------------------------
-      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = in_idx[i];
-         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else if constexpr (is_weight_fop_v<FOP>)
-         {
-            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
-                        "tensor-product IR expected");
-            in_XE[i] = Reshape(
-                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      // --------------------------------------------------
-      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
-      for_constexpr<n_outputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = out_idx[i];
-         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
-         using FOP = tuple_element_t<i, outputs_t>;
-         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
-         {
-            MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP> || is_functionalvalue_fop_v<FOP>)
-         {
-            MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else
-         {
-            static_assert(false, "Unsupported FieldOperator");
-         }
-      });
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      dfem::forall<MTPB>(
-         [=] MFEM_HOST_DEVICE(const int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         // -----------------------------------------------
-         // Inputs and outputs argument registers
-         // -----------------------------------------------
-         action_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
-
-         // -----------------------------------------------
-         // Shared memory
-         // -----------------------------------------------
-         MFEM_SHARED typename backend_t::Shared smem;
-
-         // -----------------------------------------------
-         // Load inputs
-         // -----------------------------------------------
-         for_constexpr<n_inputs>([&](auto ic)
-         {
-            constexpr size_t i = ic.value;
-            const auto &XE = in_XE[i];
-            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
-            ;
-            const real_t *B = in_B[i], *G = in_G[i];
-            auto &rarg = get<i>(rargs);
-            using FOP = tuple_element_t<i, inputs_t>;
-            if constexpr (is_value_fop<FOP>::value)
-            {
-               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               using XE_t = decltype(XE);
-               using rarg_t = decltype(rarg);
-               using qf_param_t =
-                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
-               backend_t::template LoadGradient<RNK, rarg_t, XE_t, qf_param_t>(
-                  smem, e, d, q, q1d, B, G, XE, rarg);
-            }
-            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
-            {
-               // qp values are read directly from in_XE / IR
-            }
-            else
-            {
-               static_assert(false, "Unsupported");
-            }
-         });
-
-         // -----------------------------------------------
-         // Evaluate the quadrature function
-         // Warning: no 'DIRECT' on the 'Z' direction,
-         // as one backend may need to iterate over it.
-         // -----------------------------------------------
-         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-               {
-                  if constexpr (action_outputs_direct_value_v<qfunc_t, inputs_t,
-                                outputs_t>)
-                  {
-                     call_qfunc_direct<backend_t>(
-                        qfunc, rargs, in_XE, out_YE, qx, qy, qz, e,
-                        std::make_index_sequence<n_inputs> {},
-                        std::make_index_sequence<n_outputs> {});
-                  }
-                  else
-                  {
-                     args_tuple_t qargs;
-
-                     // --------------------------------------
-                     // Pulling arguments from registers to qargs tuple
-                     // --------------------------------------
-                     for_constexpr<n_inputs>([&](auto ic)
-                     {
-                        constexpr size_t i = ic.value;
-                        auto &qarg = get<i>(qargs);
-                        const auto &XE = in_XE[i];
-                        using FOP = tuple_element_t<i, inputs_t>;
-                        using ARG =
-                           typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
-                        if constexpr (is_identity_fop_v<FOP>)
-                        {
-                           using DT =
-                              typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-                           if constexpr (qf_param_uses_dual_v<DT>)
-                           {
-                              qarg = backend_t::template identity_qp_pull_dual<DT>(
-                                 false, XE, XE, qx, qy, qz, e);
-                           }
-                           else
-                           {
-                              qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-                           }
-                        }
-                        else if constexpr (is_weight_fop_v<FOP>)
-                        {
-                           qarg = XE(qx, qy, qz, 0, 0);
-                        }
-                        else if constexpr (is_value_fop_v<FOP> ||
-                                           is_gradient_fop_v<FOP>)
-                        {
-                           qarg = backend_t::template qp_pull<ARG>(
-                              get<i>(rargs), qx, qy, qz);
-                        }
-                        else
-                        {
-                           static_assert(false, "Unsupported");
-                        }
-                     });
-
-                     // --------------------------------------
-                     // Call the quadrature function
-                     // --------------------------------------
-                     call_qfunc_no_move(qfunc, qargs);
-
-                     // --------------------------------------
-                     // Pushing arguments from qargs tuple to registers
-                     // --------------------------------------
-                     for_constexpr<n_outputs>([&](auto ic)
-                     {
-                        constexpr size_t i = ic.value, o = n_inputs + i;
-                        const auto &qarg = get<o>(qargs);
-                        const auto &YE = out_YE[i];
-                        using FOP = tuple_element_t<i, outputs_t>;
-                        using ARG =
-                           typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
-                        if constexpr (is_identity_fop_v<FOP> ||
-                                      is_functionalvalue_fop_v<FOP>)
-                        {
-                           using DT =
-                              typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-                           if constexpr (qf_param_uses_dual_v<DT>)
-                           {
-                              backend_t::identity_qp_write_value(
-                                 YE, qx, qy, qz, e, qarg);
-                           }
-                           else
-                           {
-                              as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
-                           }
-                        }
-                        else if constexpr (is_value_fop_v<FOP> ||
-                                           is_gradient_fop_v<FOP>)
-                        {
-                           auto &rarg = get<o>(rargs);
-                           backend_t::template qp_push<ARG>(
-                              rarg, qx, qy, qz, qarg);
-                        }
-                        else
-                        {
-                           static_assert(false, "Unsupported");
-                        }
-                     });
-                  }
-               }
-            }
-         }
-         if constexpr (!action_outputs_direct_v<outputs_t>)
-         {
-            MFEM_SYNC_THREAD;
-         }
-
-         // -----------------------------------------------
-         // Integrate outputs
-         // -----------------------------------------------
-         if constexpr (!action_outputs_direct_v<outputs_t>)
-         {
-            for_constexpr<n_outputs>([&](auto ic)
-            {
-               constexpr size_t i = ic.value, o = n_inputs + i;
-               const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
-               const auto B = out_B[i], G = out_G[i];
-               const auto &YE = out_YE[i];
-               auto &rarg = get<o>(rargs);
-               using FOP = tuple_element_t<i, outputs_t>;
-               if constexpr (is_value_fop_v<FOP>)
-               {
-                  backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
-               }
-               else if constexpr (is_gradient_fop_v<FOP>)
-               {
-                  using YE_t = decltype(YE);
-                  using rarg_t = decltype(rarg);
-                  // Both the rank and the extents have to come from the output
-                  // parameter slot o, not from the output index i.
-                  using qf_param_t =
-                     typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-                  constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
-                  backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
-                     smem, e, d, q, Q1D, B, G, YE, rarg);
-               }
-               else if constexpr (is_identity_fop_v<FOP> ||
-                                  is_functionalvalue_fop_v<FOP>)
-               {
-                  // nothing to do
-               }
-               else
-               {
-                  static_assert(false, "Unsupported");
-               }
-            });
-         }
-      },
-      ne,
-      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
-                                  q1d, in_d1d, out_d1d)),
-      0,
-      nullptr);
-   }
-   using KernelType = decltype(&Action::action_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionLO, KernelType, (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionHO, KernelType, (int, int) );
-};
-
-// Low Order kernels
-template<typename qfunc_t, typename inputs_t, typename outputs_t>
-template<int DIM, int Q1D>
-inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
-Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Kernel()
-{
-   static_assert(Q1D <= LocalQFLOBackend<DIM>::MQ1);
-   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
-   if constexpr (DIM == 3 && Q1D == LocalQFLOBackendMQ1() &&
-                 action_outputs_direct_value_v<qfunc_t, inputs_t, outputs_t>)
-   {
-      return action_t::template action_callback<LocalQFLOBackend<DIM, Q1D, Q1D / 2>>;
-   }
-   else
-   {
-      return action_t::template action_callback<LocalQFLOBackend<DIM, Q1D>>;
-   }
-}
-
-// Low Order fallback
-template<typename qfunc_t, typename inputs_t, typename outputs_t>
-inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
-Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Fallback(int dim, int q1d)
-{
-   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
-   using ActionLO = typename action_t::ActionLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<ActionLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<ActionLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-// High Order kernels
-template<typename qfunc_t, typename inputs_t, typename outputs_t>
-template<int DIM, int Q1D>
-inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
-Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Kernel()
-{
-   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
-   return action_t::template action_callback<LocalQFHOBackend<DIM>, Q1D>;
-}
-
-// High Order fallback
-template<typename qfunc_t, typename inputs_t, typename outputs_t>
-inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
-Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Fallback(int dim, int q1d)
-{
-   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
-   using ActionHO = typename action_t::ActionHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<ActionHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<ActionHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_action.hpp

@@ -1,926 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include "../util.hpp"
-
-namespace mfem::future::LocalQFImpl
-{
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-class DerivativeAction
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-   using args_tuple_t = decay_tuple<qf_param_ts>;
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
-                 "LocalQF: q-function arity must match inputs + outputs");
-
-   /// Which inputs carry a tangent, i.e. are attached to the field being
-   /// differentiated against. This is a property of `inputs_t` and
-   /// `derivative_id` alone, so it is available at compile time: it decides the
-   /// Enzyme activity of every q-function parameter, which loads the tangent
-   /// pass has to do, and how large the shadow register bank has to be. The
-   /// runtime `input_is_dependent` below holds the same values and is kept for
-   /// the host-side sizing checks.
-   static constexpr auto input_activity =
-      mfem::future::detail::make_activity_map <
-   static_cast<std::size_t>(derivative_id) > (inputs_t {});
-   static_assert(input_activity.size() == n_inputs);
-
-   /// Shadow register bank: only the active input slots are materialized.
-   template <typename backend_t, int MQ1, std::size_t... Is>
-   static auto shadow_bank_type(std::index_sequence<Is...>)
-#ifdef MFEM_USE_ENZYME
-   -> masked_input_args_reg_t<backend_t, qfunc_t, MQ1, input_activity[Is]...>;
-#else
-   // The dual-number path pulls through every input slot unconditionally.
-   -> input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1>;
-#endif
-
-   template <typename backend_t, int MQ1>
-   using shadow_bank_t = decltype(shadow_bank_type<backend_t, MQ1>(
-   std::make_index_sequence<n_inputs> {}));
-
-   /// Per-quadrature-point shadow argument tuple. Only the active inputs and
-   /// the outputs are materialized where supported; the rest are `enzyme_const`
-   /// and their shadow slots are never addressed. This is the innermost live
-   /// state of the kernel, and on device it shares a per-thread register budget
-   /// capped by the launch bounds, so the dead slots are worth removing
-   /// explicitly rather than hoping the optimizer splits the tuple.
-#ifdef _MSC_VER
-   using shadow_args_t = args_tuple_t;
-#else
-   template <std::size_t... Is>
-   static auto shadow_tuple_type(std::index_sequence<Is...>)
-   -> masked_args_tuple_t < args_tuple_t,
-   (Is<n_inputs ? input_activity[Is] : true)... >;
-
-   using shadow_args_t = decltype(shadow_tuple_type(
-   std::make_index_sequence<n_inputs + n_outputs> {}));
-#endif
-
-#ifdef MFEM_USE_ENZYME
-   /// Forward-mode call with the activity of every q-function parameter fixed
-   /// at compile time. Outputs are always active; inputs follow
-   /// `input_activity`, so an inactive input (the mesh nodes and the quadrature
-   /// weight, for a derivative w.r.t. the trial field) is marked `enzyme_const`
-   /// rather than dup'd with a zero tangent. Without this Enzyme differentiates
-   /// everything those inputs feed - for a diffusion q-function the whole
-   /// inv(J) / det(J) chain - to produce a tangent that is structurally zero.
-   // `qf_t` is deduced because the kernel captures the q-function by value into
-   // a const lambda, so it arrives here as `const qfunc_t`.
-   template <typename qf_t, std::size_t... Is>
-   MFEM_FUTURE_ALWAYS_INLINE
-   MFEM_HOST_DEVICE static void call_fwddiff(qf_t &qfunc,
-                                             args_tuple_t &primal_args,
-                                             shadow_args_t &shadow_args,
-                                             std::index_sequence<Is...>)
-   {
-      mfem::future::call_enzyme_fwddiff_active <
-      (Is < n_inputs ? input_activity[Is] : true)... > (
-         qfunc, primal_args, shadow_args);
-   }
-#endif
-
-   qfunc_t qfunc;
-   const inputs_t inputs;
-   const outputs_t outputs;
-   const IntegratorContext ctx;
-   const std::vector<const DofToQuad *> dtqs;
-   // inputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_inputs> input_dtq;
-   const std::array<size_t, n_inputs> input_idx;
-   const std::array<const real_t *, n_inputs> input_B, input_G;
-   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
-   // outputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_outputs> output_dtq;
-   const std::array<size_t, n_outputs> output_idx;
-   const std::array<const real_t *, n_outputs> output_B, output_G;
-   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
-   // other constants
-   const int dim, ne, nq, q1d;
-
-   std::array<bool, n_inputs> input_is_dependent;
-   FieldDescriptor direction_fd;
-   mutable Vector direction_e;
-   mutable RestrictionCache<Entity::Element> direction_rcache;
-
-public:
-   //////////////////////////////////////////////////////////////////
-   DerivativeAction() = delete;
-
-   DerivativeAction(IntegratorContext ctx,
-                    qfunc_t qfunc,
-                    inputs_t inputs,
-                    outputs_t outputs):
-      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
-      dtqs(make_dtqs(ctx)),
-      // inputs: dtq, idx, B, G, d1d, q1d, vdim
-      input_dtq(create_dtq_maps<Entity::Element>(
-                   inputs,
-                   dtqs,
-                   create_union_field_map_for_dtq(ctx, inputs),
-                   ctx.unionfds,
-                   ctx.ir)),
-      input_idx(create_input_vector_map(ctx, inputs)),
-      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
-      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
-      input_vdim(get_vdim(inputs)),
-      // outputs: dtq, idx, B, G, d1d, q1d, vdim
-      output_dtq(create_dtq_maps<Entity::Element>(
-                    outputs,
-                    dtqs,
-                    create_union_field_map_for_dtq(ctx, outputs),
-                    ctx.unionfds,
-                    ctx.ir)),
-      output_idx(create_output_vector_map(ctx, outputs)),
-      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
-      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
-      output_vdim(get_vdim(outputs)),
-      // other constants
-      dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
-      q1d(tensor_1d_size(nq, dim))
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-
-      // Determine which inputs are dependent on the derivative direction
-      auto dependency_map = make_dependency_map(inputs);
-      auto it = dependency_map.find(derivative_id);
-      MFEM_ASSERT(it != dependency_map.end(),
-                  "Derivative ID not found in dependency map");
-      input_is_dependent = it->second;
-
-      // Find direction field index
-      int direction_field_idx = -1;
-      for (size_t uf = 0; uf < nfields; uf++)
-      {
-         if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
-         {
-            direction_field_idx = static_cast<int>(uf);
-            break;
-         }
-      }
-      MFEM_ASSERT(
-         direction_field_idx != -1,
-         "LocalQFBackend: derivative direction field not found in unionfds");
-      direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename Backend>
-   void run_kernels(const std::vector<Vector *> &xe,
-                    std::vector<Vector *> &ye)
-   {
-      Backend::Run(dim,
-                   q1d,
-                   // arguments
-                   ctx,
-                   qfunc,
-                   // inputs
-                   input_idx,
-                   input_B,
-                   input_G,
-                   input_vdim,
-                   input_d1d,
-                   input_q1d,
-                   // outputs
-                   output_idx,
-                   output_B,
-                   output_G,
-                   output_vdim,
-                   output_d1d,
-                   output_q1d,
-                   // input and output vectors
-                   xe,
-                   ye,
-                   input_is_dependent,
-                   direction_e,
-                   // fallback arguments
-                   dim,
-                   q1d);
-   }
-
-   //////////////////////////////////////////////////////////////////
-   void operator()(const std::vector<Vector *> &xe,
-                   const Vector *direction_l,
-                   std::vector<Vector *> &ye)
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      MFEM_ASSERT(direction_l != nullptr,
-                  "LocalQF DerivativeAction: direction vector is null");
-
-      restriction(direction_fd,
-                  direction_rcache,
-                  *direction_l,
-                  direction_e,
-                  ElementDofOrdering::LEXICOGRAPHIC);
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<DerivativeActionLO>(xe, ye);
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<DerivativeActionHO>(xe, ye);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-   }
-
-
-   template<typename backend_t, int T_Q1D>
-   struct DerivativeActionKernelData
-   {
-      qfunc_t qfunc;
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
-      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
-      std::array<const real_t *, n_inputs> in_B;
-      std::array<const real_t *, n_inputs> in_G;
-      std::array<int, n_inputs> in_d1d;
-      std::array<int, n_inputs> in_q1d;
-      std::array<const real_t *, n_outputs> out_B;
-      std::array<const real_t *, n_outputs> out_G;
-      std::array<int, n_outputs> out_d1d;
-      std::array<int, n_outputs> out_q1d;
-      std::array<bool, n_inputs> input_dep;
-      const int *d_attr;
-      bool has_attr;
-      const int *d_elem_attr;
-      int q1d;
-   };
-
-   template<typename backend_t, int T_Q1D>
-   MFEM_FUTURE_ALWAYS_INLINE MFEM_HOST_DEVICE static void
-   derivative_action_kernel_body(
-      const DerivativeActionKernelData<backend_t, T_Q1D> &data, const int e)
-   {
-      static constexpr auto B2D = backend_t::DIM == 2;
-      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
-      auto &qfunc = data.qfunc;
-      const auto &in_XE = data.in_XE;
-      const auto &in_XE_dir = data.in_XE_dir;
-      const auto &out_YE = data.out_YE;
-      const auto &in_B = data.in_B;
-      const auto &in_G = data.in_G;
-      const auto &in_d1d = data.in_d1d;
-      const auto &in_q1d = data.in_q1d;
-      const auto &out_B = data.out_B;
-      const auto &out_G = data.out_G;
-      const auto &out_d1d = data.out_d1d;
-      const auto &out_q1d = data.out_q1d;
-      const auto &input_dep = data.input_dep;
-      MFEM_CONTRACT_VAR(input_dep);
-      const auto d_attr = data.d_attr;
-      const bool has_attr = data.has_attr;
-      const auto d_elem_attr = data.d_elem_attr;
-      const int q1d = data.q1d;
-      if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-      // -----------------------------------------------
-      // Inputs and outputs argument registers
-      // -----------------------------------------------
-      args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
-      shadow_bank_t<backend_t, MQ1> sargs; // shadow, active inputs only
-
-      // -----------------------------------------------
-      // Shared memory
-      // -----------------------------------------------
-      MFEM_SHARED typename backend_t::Shared smem;
-
-      // -----------------------------------------------
-      // Load primal inputs (rargs)
-      // -----------------------------------------------
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const auto &XE = in_XE[i];
-         const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
-         const real_t *B = in_B[i], *G = in_G[i];
-         auto &rarg = get<i>(rargs);
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (is_value_fop<FOP>::value)
-         {
-            backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
-         }
-         else if constexpr (is_gradient_fop_v<FOP>)
-         {
-            constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
-            using FieldParamT =
-               typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-            backend_t::template LoadGradient<RNK,
-                                             decltype(rarg),
-                                             decltype(XE),
-                                             FieldParamT>(
-                                                smem, e, d, q, Q1D, B, G, XE, rarg);
-         }
-         else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
-         {
-            // qp values are read directly from in_XE / IR
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      // -----------------------------------------------
-      // Load tangent directions (sargs)
-      // -----------------------------------------------
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (input_activity[i] &&
-                       (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>))
-         {
-            const auto &XE = in_XE_dir[i];
-            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
-            const real_t *B = in_B[i], *G = in_G[i];
-            auto &sarg = get<i>(sargs); // shadow argument register
-            if constexpr (is_value_fop_v<FOP>)
-            {
-               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
-            }
-            else
-            {
-               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
-               using FieldParamT =
-                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-               backend_t::template LoadGradient<RNK,
-                                                decltype(sarg),
-                                                decltype(XE),
-                                                FieldParamT>(
-                                                   smem, e, d, q, Q1D, B, G, XE, sarg);
-            }
-         }
-         else
-         {
-            // Inactive input, or an input read straight from quadrature
-            // point data (weight / identity): nothing to interpolate.
-            static_assert(!input_activity[i] || is_weight_fop_v<FOP> ||
-                          is_identity_fop_v<FOP>, "Unsupported");
-         }
-      });
-
-      // -----------------------------------------------
-      // Evaluate the quadrature function
-      // Warning: no 'DIRECT' on the 'Z' direction,
-      // as one backend may need to iterate over it.
-      // -----------------------------------------------
-      MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-            {
-#ifdef MFEM_USE_ENZYME
-               args_tuple_t primal_args {};
-               shadow_args_t shadow_args {};
-
-               // --------------------------------------
-               // Pulling arguments from registers to primal and shadow
-               // tuples
-               // --------------------------------------
-               for_constexpr<n_inputs>([&](auto ic)
-               {
-                  constexpr size_t i = ic.value;
-                  auto &parg = get<i>(primal_args);
-                  auto &targ = get<i>(shadow_args);
-                  const auto &XE = in_XE[i];
-                  const auto &XEd = in_XE_dir[i];
-                  using FOP = tuple_element_t<i, inputs_t>;
-                  using ARG =
-                     typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
-                  MFEM_CONTRACT_VAR(targ);
-                  MFEM_CONTRACT_VAR(XEd);
-                  if constexpr (is_identity_fop_v<FOP>)
-                  {
-                     parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-                     if constexpr (input_activity[i])
-                     {
-                        targ = as_tensor<ARG>(&XEd(0, qx, qy, qz, e));
-                     }
-                  }
-                  else if constexpr (is_weight_fop_v<FOP>)
-                  {
-                     parg = XE(qx, qy, qz, 0, 0);
-                  }
-                  else if constexpr (is_value_fop_v<FOP> ||
-                                     is_gradient_fop_v<FOP>)
-                  {
-                     parg = backend_t::template qp_pull<ARG>(
-                        get<i>(rargs), qx, qy, qz);
-                     if constexpr (input_activity[i])
-                     {
-                        targ = backend_t::template qp_pull<ARG>(
-                           get<i>(sargs), qx, qy, qz);
-                     }
-                  }
-                  else
-                  {
-                     static_assert(false, "Unsupported");
-                  }
-               });
-
-               // --------------------------------------
-               // Call the quadrature function. Inactive inputs are
-               // enzyme_const, so their shadow slots are never read and are
-               // deliberately left unset above.
-               // --------------------------------------
-               call_fwddiff(qfunc, primal_args, shadow_args,
-                            std::make_index_sequence<n_inputs + n_outputs> {});
-
-               // --------------------------------------
-               // Pushing arguments from enzyme_shadow tuple to registers
-               // --------------------------------------
-               for_constexpr<n_outputs>([&](auto ic)
-               {
-                  constexpr size_t i = ic.value, o = n_inputs + i;
-                  const auto &qout = get<o>(shadow_args);
-                  auto &YE = out_YE[i];
-                  using FOP = tuple_element_t<i, outputs_t>;
-                  using ARG =
-                     typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
-                  if constexpr (is_identity_fop_v<FOP>)
-                  {
-                     as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qout;
-                  }
-                  else if constexpr (is_value_fop_v<FOP> ||
-                                     is_gradient_fop_v<FOP>)
-                  {
-                     auto &rarg = get<o>(rargs);
-                     backend_t::template qp_push_tangent<ARG>(
-                        rarg, qx, qy, qz, qout);
-                  }
-                  else
-                  {
-                     static_assert(false, "Unsupported");
-                  }
-               });
-#else  // MFEM_USE_ENZYME
-               args_tuple_t qargs;
-
-               // --------------------------------------
-               // Pulling arguments from registers to qargs tuple
-               // --------------------------------------
-               for_constexpr<n_inputs>([&](auto ic)
-               {
-                  constexpr size_t i = ic.value;
-                  auto &qarg = get<i>(qargs);
-                  const auto &XE = in_XE[i];
-                  const auto &XEd = in_XE_dir[i];
-                  using FOP = tuple_element_t<i, inputs_t>;
-                  using ARG =
-                     typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
-                  if constexpr (is_identity_fop_v<FOP>)
-                  {
-                     using DT =
-                        typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-                     if constexpr (qf_param_uses_dual_v<DT>)
-                     {
-                        qarg = backend_t::template identity_qp_pull_dual<DT>(
-                           input_dep[i], XE, XEd, qx, qy, qz, e);
-                     }
-                     else
-                     {
-                        qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-                     }
-                  }
-                  else if constexpr (is_weight_fop_v<FOP>)
-                  {
-                     qarg = XE(qx, qy, qz, 0, 0);
-                  }
-                  else if constexpr (is_value_fop_v<FOP> ||
-                                     is_gradient_fop_v<FOP>)
-                  {
-                     qarg = backend_t::template qp_pull_directional<ARG>(
-                        get<i>(rargs),
-                        get<i>(sargs),
-                        qx,
-                        qy,
-                        qz,
-                        input_dep[i]);
-                  }
-                  else
-                  {
-                     static_assert(false, "Unsupported");
-                  }
-               });
-
-               // --------------------------------------
-               // Call the quadrature function
-               // --------------------------------------
-               call_qfunc_no_move(qfunc, qargs);
-
-               // --------------------------------------
-               // Pushing arguments from qargs tuple to registers
-               // --------------------------------------
-               for_constexpr<n_outputs>([&](auto ic)
-               {
-                  constexpr size_t i = ic.value, o = n_inputs + i;
-                  const auto &qarg = get<o>(qargs);
-                  const auto &YE = out_YE[i];
-                  using FOP = tuple_element_t<i, outputs_t>;
-                  using ARG =
-                     typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
-                  if constexpr (is_identity_fop_v<FOP>)
-                  {
-                     using DT =
-                        typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-                     if constexpr (qf_param_uses_dual_v<DT>)
-                     {
-                        backend_t::identity_qp_write_tangent(
-                           YE, qx, qy, qz, e, qarg);
-                     }
-                     else
-                     {
-                        as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
-                     }
-                  }
-                  else if constexpr (is_value_fop_v<FOP> ||
-                                     is_gradient_fop_v<FOP>)
-                  {
-                     auto &rarg = get<o>(rargs);
-                     backend_t::template qp_push_tangent<ARG>(
-                        rarg, qx, qy, qz, qarg);
-                  }
-                  else
-                  {
-                     static_assert(false, "Unsupported");
-                  }
-               });
-#endif // MFEM_USE_ENZYME
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // -----------------------------------------------
-      // Integrate outputs
-      // -----------------------------------------------
-      for_constexpr<n_outputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value, o = n_inputs + i;
-         const int d = out_d1d[i], q = out_q1d[i];
-         const auto B = out_B[i], G = out_G[i];
-         auto &YE = out_YE[i];
-         auto &rarg = get<o>(rargs);
-         using FOP = tuple_element_t<i, outputs_t>;
-         if constexpr (is_value_fop_v<FOP>)
-         {
-            backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
-         }
-         else if constexpr (is_gradient_fop_v<FOP>)
-         {
-            using YE_t = decltype(YE);
-            using rarg_t = decltype(rarg);
-            using qf_param_t =
-               typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-            constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
-            backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
-               smem, e, d, q, q1d, B, G, YE, rarg);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            // nothing to do
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-   }
-
-   template<typename backend_t, int T_Q1D>
-   struct DerivativeActionKernelBody
-   {
-      MFEM_FUTURE_ALWAYS_INLINE MFEM_HOST_DEVICE static void run(
-         const DerivativeActionKernelData<backend_t, T_Q1D> &data, const int e)
-      {
-         derivative_action_kernel_body<backend_t, T_Q1D>(data, e);
-      }
-   };
-
-   //////////////////////////////////////////////////////////////////
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void
-   derivative_action_callback(const IntegratorContext &ctx,
-                              qfunc_t &qfunc,
-                              // inputs: idx, B, G, vdim, d1d, q1d
-                              const std::array<size_t, n_inputs> &in_idx,
-                              const std::array<const real_t *, n_inputs> in_B,
-                              const std::array<const real_t *, n_inputs> in_G,
-                              const std::array<int, n_inputs> &in_vdim,
-                              const std::array<int, n_inputs> &in_d1d,
-                              const std::array<int, n_inputs> &in_q1d,
-                              // outputs: idx, B, G, vdim, d1d, q1d
-                              const std::array<size_t, n_outputs> &out_idx,
-                              const std::array<const real_t *, n_outputs> out_B,
-                              const std::array<const real_t *, n_outputs> out_G,
-                              const std::array<int, n_outputs> &out_vdim,
-                              const std::array<int, n_outputs> &out_d1d,
-                              const std::array<int, n_outputs> &out_q1d,
-                              const std::vector<Vector *> &xe,
-                              std::vector<Vector *> &ye,
-                              const std::array<bool, n_inputs> &input_dep,
-                              const Vector &direction_e,
-                              // fallback arguments
-                              const int dim,
-                              const int q1d)
-   {
-      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-      // Dependency is resolved at compile time through `input_activity`; the
-      // runtime array is only carried for the non-Enzyme dual-number path.
-      MFEM_CONTRACT_VAR(input_dep);
-
-      if (ctx.attr.Size() == 0) { return; }
-
-      static constexpr auto B2D = backend_t::DIM == 2;
-
-      const int ne = ctx.nentities;
-
-      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
-
-      // --------------------------------------------------
-      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = in_idx[i];
-         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else if constexpr (is_weight_fop_v<FOP>)
-         {
-            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
-                        "tensor-product IR expected");
-            in_XE[i] = Reshape(
-                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      const auto d_direction = direction_e.Read();
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = in_idx[i];
-         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            if constexpr (input_activity[i])
-            {
-               MFEM_ASSERT(direction_e.Size() == xe[k]->Size(),
-                           "direction E-vector size mismatch for input " << i);
-               in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
-            }
-            else
-            {
-               in_XE_dir[i] = in_XE[i];
-            }
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            if constexpr (input_activity[i])
-            {
-               MFEM_VERIFY(direction_e.Size() == xe[k]->Size(),
-                           "direction E-vector size mismatch (identity input) "
-                           << i);
-               in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
-            }
-            else
-            {
-               in_XE_dir[i] = in_XE[i];
-            }
-         }
-         else if constexpr (is_weight_fop_v<FOP>) { in_XE_dir[i] = in_XE[i]; }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      // --------------------------------------------------
-      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
-      for_constexpr<n_outputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = out_idx[i];
-         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
-         using FOP = tuple_element_t<i, outputs_t>;
-         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
-         {
-            MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else
-         {
-            static_assert(false, "Unsupported FieldOperator");
-         }
-      });
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      DerivativeActionKernelData<backend_t, T_Q1D> data
-      {
-         qfunc,
-         in_XE,
-         in_XE_dir,
-         out_YE,
-         in_B,
-         in_G,
-         in_d1d,
-         in_q1d,
-         out_B,
-         out_G,
-         out_d1d,
-         out_q1d,
-         input_dep,
-         d_attr,
-         has_attr,
-         d_elem_attr,
-         q1d
-      };
-
-      const auto blocks = backend_t::thread_blocks(
-                             compute_kernel_thread_1d<inputs_t, outputs_t>(q1d, in_d1d, out_d1d));
-
-      if (Device::Allows(Backend::CUDA_MASK) ||
-          Device::Allows(Backend::HIP_MASK))
-      {
-         dfem::forall_data<backend_t::MAX_THREADS_PER_BLOCK(),
-              DerivativeActionKernelBody<backend_t, T_Q1D>>(
-                 data, ne, blocks);
-      }
-      else if (Device::Allows(Backend::CPU_MASK))
-      {
-         for (int e = 0; e < ne; e++)
-         {
-            derivative_action_kernel_body<backend_t, T_Q1D>(data, e);
-         }
-      }
-      else
-      {
-         MFEM_ABORT("no compute backend available");
-      }
-   }
-
-   using DerivativeKernelType =
-      decltype(&DerivativeAction::derivative_action_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionLO,
-                                     DerivativeKernelType,
-                                     (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionHO,
-                                     DerivativeKernelType,
-                                     (int, int) );
-};
-
-// Low Order kernels
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeKernelType
-DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeActionLO::Kernel()
-{
-   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
-   using derivative_action_t =
-      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return derivative_action_t::template derivative_action_callback<
-             LocalQFLOBackend<DIM, Q1D>>;
-}
-
-// Low Order fallback
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeKernelType
-DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeActionLO::Fallback(int dim, int q1d)
-{
-   using derivative_action_t =
-      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeActionLO =
-      typename derivative_action_t::DerivativeActionLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<DerivativeActionLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<DerivativeActionLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-// High Order kernels
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeKernelType
-DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeActionHO::Kernel()
-{
-   using derivative_action_t =
-      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return derivative_action_t::
-          template derivative_action_callback<LocalQFHOBackend<DIM>, Q1D>;
-}
-
-// High Order fallback
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeKernelType
-DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeActionHO::Fallback(int dim, int q1d)
-{
-   using derivative_action_t =
-      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeActionHO = typename derivative_action_t::DerivativeActionHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<DerivativeActionHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<DerivativeActionHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_apply.hpp

@@ -1,738 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include <array>
-
-namespace mfem::future::LocalQFImpl
-{
-
-// Cached Jacobian apply: J·v from qp_cache filled by DerivativeSetup
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-class DerivativeApply
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-   using args_tuple_t = decay_tuple<qf_param_ts>;
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
-                 "LocalQF: q-function arity must match inputs + outputs");
-
-   const inputs_t inputs;
-   const outputs_t outputs;
-   const IntegratorContext ctx;
-   const Vector &qp_cache;
-   const std::vector<const DofToQuad *> dtqs;
-   // inputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_inputs> input_dtq;
-   const std::array<size_t, n_inputs> input_idx;
-   const std::array<const real_t *, n_inputs> input_B, input_G;
-   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
-   // outputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_outputs> output_dtq;
-   const std::array<size_t, n_outputs> output_idx;
-   const std::array<const real_t *, n_outputs> output_B, output_G;
-   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
-   // Jacobian cache metadata
-   const std::array<bool, n_inputs> input_is_dependent;
-   const std::array<int, n_inputs> input_size_on_qp;
-   const std::array<int, n_outputs> out_op_dim;
-   const std::array<int, n_outputs> out_offsets;
-   const int output_size_on_qp;
-   const int trial_vdim;
-   const int total_trial_op_dim;
-   const int residual_size_on_qp;
-   // other constants
-   const int dim, ne, nq, q1d;
-   FieldDescriptor direction_fd;
-   mutable Vector direction_e;
-   mutable RestrictionCache<Entity::Element> direction_rcache;
-
-   template <std::size_t slot>
-   static constexpr int ParamRank()
-   {
-      using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
-      return qf_param_shape<param_t>::rank;
-   }
-
-   template <std::size_t slot, int dim_idx>
-   static constexpr int ParamExtent()
-   {
-      using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
-      return qf_param_shape<param_t>::extents[dim_idx];
-   }
-
-   template <typename fop_t, std::size_t slot>
-   static constexpr int StaticVDim()
-   {
-      constexpr int rank = ParamRank<slot>();
-      if constexpr (is_gradient_fop_v<fop_t>)
-      {
-         if constexpr (rank <= 1) { return 1; }
-         else { return ParamExtent<slot, 0>(); }
-      }
-      else
-      {
-         if constexpr (rank == 0) { return 1; }
-         else { return ParamExtent<slot, 0>(); }
-      }
-   }
-
-   template <typename fop_t, std::size_t slot>
-   static constexpr int StaticOpDim()
-   {
-      constexpr int rank = ParamRank<slot>();
-      if constexpr (is_gradient_fop_v<fop_t>)
-      {
-         if constexpr (rank == 0) { return 1; }
-         else if constexpr (rank == 1) { return ParamExtent<slot, 0>(); }
-         else { return ParamExtent<slot, 1>(); }
-      }
-      else
-      {
-         if constexpr (rank <= 1) { return 1; }
-         else { return ParamExtent<slot, 1>(); }
-      }
-   }
-
-   template <std::size_t input_slot>
-   static constexpr bool StaticInputDep()
-   {
-      using fop_t = tuple_element_t<input_slot, inputs_t>;
-      return fop_t::GetFieldId() == derivative_id;
-   }
-
-   template <std::size_t input_slot>
-   static constexpr int StaticInputVDim()
-   {
-      using fop_t = tuple_element_t<input_slot, inputs_t>;
-      return StaticVDim<fop_t, input_slot>();
-   }
-
-   template <std::size_t input_slot>
-   static constexpr int StaticInputOpDim()
-   {
-      using fop_t = tuple_element_t<input_slot, inputs_t>;
-      return StaticOpDim<fop_t, input_slot>();
-   }
-
-   template <std::size_t output_slot>
-   static constexpr int StaticOutputVDim()
-   {
-      using fop_t = tuple_element_t<output_slot, outputs_t>;
-      return StaticVDim<fop_t, n_inputs + output_slot>();
-   }
-
-   template <std::size_t output_slot>
-   static constexpr int StaticOutputOpDim()
-   {
-      using fop_t = tuple_element_t<output_slot, outputs_t>;
-      return StaticOpDim<fop_t, n_inputs + output_slot>();
-   }
-
-   template <std::size_t output_slot>
-   static constexpr int StaticOutputOffset()
-   {
-      int offset = 0;
-      for_constexpr<output_slot>([&](auto oc)
-      {
-         constexpr size_t o = oc.value;
-         offset += StaticOutputVDim<o>() * StaticOutputOpDim<o>();
-      });
-      return offset;
-   }
-
-   template <std::size_t input_slot>
-   static constexpr int StaticInputOpOffset()
-   {
-      int offset = 0;
-      for_constexpr<input_slot>([&](auto sc)
-      {
-         constexpr size_t s = sc.value;
-         if constexpr (StaticInputDep<s>()) { offset += StaticInputOpDim<s>(); }
-      });
-      return offset;
-   }
-
-   static constexpr int StaticTrialVDim()
-   {
-      int vdim = 1;
-      for_constexpr<n_inputs>([&](auto sc)
-      {
-         constexpr size_t s = sc.value;
-         if constexpr (StaticInputDep<s>()) { vdim = StaticInputVDim<s>(); }
-      });
-      return vdim;
-   }
-
-   static constexpr int StaticTotalTrialOpDim()
-   {
-      int op_dim = 0;
-      for_constexpr<n_inputs>([&](auto sc)
-      {
-         constexpr size_t s = sc.value;
-         if constexpr (StaticInputDep<s>()) { op_dim += StaticInputOpDim<s>(); }
-      });
-      return op_dim;
-   }
-
-public:
-   DerivativeApply() = delete;
-
-   DerivativeApply(IntegratorContext ctx,
-                   qfunc_t /*qfunc*/,
-                   inputs_t inputs,
-                   outputs_t outputs,
-                   const Vector &qp_cache_in):
-      inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache_in),
-      dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
-                                         inputs,
-                                         dtqs,
-                                         create_union_field_map_for_dtq(ctx, inputs),
-                                         ctx.unionfds,
-                                         ctx.ir)),
-      input_idx(create_input_vector_map(ctx, inputs)),
-      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
-      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
-      input_vdim(get_vdim(inputs)),
-      output_dtq(create_dtq_maps<Entity::Element>(
-                    outputs,
-                    dtqs,
-                    create_union_field_map_for_dtq(ctx, outputs),
-                    ctx.unionfds,
-                    ctx.ir)),
-      output_idx(create_output_vector_map(ctx, outputs)),
-      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
-      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
-      output_vdim(get_vdim(outputs)),
-      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
-      input_size_on_qp(
-         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
-                           out_op_dim(compute_out_op_dim(outputs)),
-                           out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
-                           output_size_on_qp(
-                              [&]
-   {
-      int s = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      { s += get<o>(outputs).size_on_qp; });
-      return s;
-   }()),
-   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
-   total_trial_op_dim(compute_total_trial_op_dim(
-                         inputs, input_is_dependent, input_size_on_qp)),
-   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
-   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
-   q1d(tensor_1d_size(nq, dim))
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-
-      int direction_field_idx = -1;
-      for (size_t uf = 0; uf < nfields; uf++)
-      {
-         if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
-         {
-            direction_field_idx = static_cast<int>(uf);
-            break;
-         }
-      }
-      MFEM_ASSERT(direction_field_idx != -1,
-                  "DerivativeApply: derivative direction field not found");
-
-      direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename Backend>
-   void run_kernels(std::vector<Vector *> &ye) const
-   {
-      Backend::Run(dim,
-                   q1d,
-                   ctx,
-                   qp_cache,
-                   // inputs
-                   input_idx,
-                   input_B,
-                   input_G,
-                   input_vdim,
-                   input_d1d,
-                   input_q1d,
-                   input_size_on_qp,
-                   input_is_dependent,
-                   // outputs
-                   output_idx,
-                   output_B,
-                   output_G,
-                   output_vdim,
-                   output_d1d,
-                   output_q1d,
-                   out_op_dim,
-                   out_offsets,
-                   trial_vdim,
-                   total_trial_op_dim,
-                   residual_size_on_qp,
-                   output_size_on_qp,
-                   direction_e,
-                   ye,
-                   // fallback arguments
-                   dim,
-                   q1d);
-   }
-
-   //////////////////////////////////////////////////////////////////
-   void operator()(const std::vector<Vector *> &,
-                   const Vector *direction_l,
-                   std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      MFEM_ASSERT(direction_l != nullptr,
-                  "LocalQF DerivativeApply: direction vector is null");
-
-      restriction(direction_fd,
-                  direction_rcache,
-                  *direction_l,
-                  direction_e,
-                  ElementDofOrdering::LEXICOGRAPHIC);
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<DerivativeApplyLO>(ye);
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<DerivativeApplyHO>(ye);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void
-   derivative_apply_callback(const IntegratorContext &ctx,
-                             const Vector &qp_cache,
-                             // inputs: idx, B, G, vdim, d1d, q1d
-                             const std::array<size_t, n_inputs> & /*in_idx*/,
-                             const std::array<const real_t *, n_inputs> in_B,
-                             const std::array<const real_t *, n_inputs> in_G,
-                             const std::array<int, n_inputs> &in_vdim,
-                             const std::array<int, n_inputs> &in_d1d,
-                             const std::array<int, n_inputs> &in_q1d,
-                             const std::array<int, n_inputs> &in_size_on_qp,
-                             const std::array<bool, n_inputs> &input_dep,
-                             // outputs: idx, B, G, vdim, d1d, q1d
-                             const std::array<size_t, n_outputs> &out_idx,
-                             const std::array<const real_t *, n_outputs> out_B,
-                             const std::array<const real_t *, n_outputs> out_G,
-                             const std::array<int, n_outputs> &out_vdim,
-                             const std::array<int, n_outputs> &out_d1d,
-                             const std::array<int, n_outputs> &out_q1d,
-                             const std::array<int, n_outputs> &out_op_dim,
-                             const std::array<int, n_outputs> &out_offsets,
-                             const int trial_vdim,
-                             const int total_trial_op_dim,
-                             const int residual_size_on_qp,
-                             const int output_size_on_qp,
-                             const Vector &direction_e,
-                             std::vector<Vector *> &ye,
-                             // fallback arguments
-                             const int dim,
-                             const int q1d)
-   {
-      MFEM_CONTRACT_VAR(input_dep);
-      MFEM_CONTRACT_VAR(in_size_on_qp);
-      MFEM_CONTRACT_VAR(out_vdim);
-      MFEM_CONTRACT_VAR(out_op_dim);
-      MFEM_CONTRACT_VAR(out_offsets);
-      MFEM_CONTRACT_VAR(trial_vdim);
-      MFEM_CONTRACT_VAR(total_trial_op_dim);
-      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-      if (ctx.attr.Size() == 0) { return; }
-
-      static constexpr auto B2D = backend_t::DIM == 2;
-      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
-      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
-
-      const int ne = ctx.nentities;
-      const int nq = ctx.ir.GetNPoints();
-      MFEM_CONTRACT_VAR(output_size_on_qp);
-
-      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
-
-      // --------------------------------------------------
-      // DIRECTION (trial): XE_dir for the dependent inputs
-      // --------------------------------------------------
-      const auto d_direction = direction_e.Read();
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
-         using FOP = tuple_element_t<i, inputs_t>;
-         if (!input_dep[i]) { return; }
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
-         }
-         else if constexpr (is_weight_fop_v<FOP>) { /* never a direction */ }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      // --------------------------------------------------
-      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
-      for_constexpr<n_outputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = out_idx[i];
-         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
-         using FOP = tuple_element_t<i, outputs_t>;
-         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
-         {
-            MFEM_VERIFY(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            MFEM_VERIFY(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else
-         {
-            static_assert(false, "Unsupported FieldOperator");
-         }
-      });
-
-      auto cache_tensor = DeviceTensor<3, const real_t>(
-                             qp_cache.Read(), nq, residual_size_on_qp, ne);
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      dfem::forall<MTPB>(
-         [=] MFEM_HOST_DEVICE(const int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         // -----------------------------------------------
-         // Output integration registers, trial direction (shadow) registers
-         // and shared memory. `rargs` only ever holds test-function data, so
-         // it is an output-only bank: slot `o` is q-function parameter
-         // `n_inputs + o`.
-         // -----------------------------------------------
-         output_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
-         input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> sargs;
-         MFEM_SHARED typename backend_t::Shared smem;
-
-         // -----------------------------------------------
-         // Load trial direction (sargs) for the dependent inputs
-         // -----------------------------------------------
-         for_constexpr<n_inputs>([&](auto ic)
-         {
-            constexpr size_t i = ic.value;
-            if constexpr (!StaticInputDep<i>()) { return; }
-            const auto &XE = in_XE_dir[i];
-            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
-            const real_t *B = in_B[i], *G = in_G[i];
-            auto &sarg = get<i>(sargs);
-            using FOP = tuple_element_t<i, inputs_t>;
-            if constexpr (is_value_fop<FOP>::value)
-            {
-               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
-               using FieldParamT =
-                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-               backend_t::template LoadGradient<RNK,
-                                                decltype(sarg),
-                                                decltype(XE),
-                                                FieldParamT>(
-                                                   smem, e, d, q, Q1D, B, G, XE, sarg);
-            }
-            else if constexpr (is_identity_fop_v<FOP> || is_weight_fop_v<FOP>)
-            {
-               // identity read at qp; weight is never a trial direction
-            }
-            else
-            {
-               static_assert(false, "Unsupported");
-            }
-         });
-         MFEM_SYNC_THREAD;
-
-         // -----------------------------------------------
-         // Contract the cached Jacobian with the trial direction at each
-         // quadrature point and push the result into the test registers.
-         // -----------------------------------------------
-         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-
-                  // The trial direction at this quadrature point is the same
-                  // for every test row (i, k), so pull each dependent input
-                  // slot out of the register bank once, here, instead of once
-                  // per row inside the contraction below.
-                  args_tuple_t dvecs {};
-                  for_constexpr<n_inputs>([&](auto sc)
-                  {
-                     constexpr size_t s = sc.value;
-                     if constexpr (StaticInputDep<s>())
-                     {
-                        using SARG =
-                           typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
-                        get<s>(dvecs) = backend_t::template qp_pull<SARG>(
-                           get<s>(sargs), qx, qy, qz);
-                     }
-                  });
-
-                  for_constexpr<n_outputs>([&](auto oc)
-                  {
-                     constexpr size_t o = oc.value, ao = n_inputs + o;
-                     using FOP = tuple_element_t<o, outputs_t>;
-                     using ARG =
-                        typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
-                     constexpr int tv = StaticOutputVDim<o>();
-                     constexpr int to = StaticOutputOpDim<o>();
-                     constexpr int offset_o = StaticOutputOffset<o>();
-                     constexpr int trial_vdim_ct = StaticTrialVDim();
-                     constexpr int total_trial_op_dim_ct = StaticTotalTrialOpDim();
-
-                     ARG fhat{};
-                     MFEM_UNROLL(tv)
-                     for (int i = 0; i < tv; i++)
-                     {
-                        MFEM_UNROLL(to)
-                        for (int k = 0; k < to; k++)
-                        {
-                           const int row = offset_o + i * to + k;
-                           const int cache_row =
-                              row * trial_vdim_ct * total_trial_op_dim_ct;
-                           real_t sum = 0.0;
-                           for_constexpr<n_inputs>([&](auto sc)
-                           {
-                              constexpr size_t s = sc.value;
-                              if constexpr (StaticInputDep<s>())
-                              {
-                                 constexpr int op_dim_s = StaticInputOpDim<s>();
-                                 constexpr int m_offset = StaticInputOpOffset<s>();
-                                 const auto &dvec = get<s>(dvecs);
-                                 MFEM_UNROLL(trial_vdim_ct)
-                                 for (int j = 0; j < trial_vdim_ct; j++)
-                                 {
-                                    MFEM_UNROLL(op_dim_s)
-                                    for (int m = 0; m < op_dim_s; m++)
-                                    {
-                                       const int cache_idx =
-                                          cache_row + j * total_trial_op_dim_ct +
-                                          (m + m_offset);
-                                       sum += cache_tensor(q, cache_idx, e) *
-                                              qf_value_at(dvec, j, m);
-                                    }
-                                 }
-                              }
-                           });
-                           qf_set_value_at(fhat, i, k, sum);
-                        }
-                     }
-
-                     auto &YE = out_YE[o];
-                     if constexpr (is_identity_fop_v<FOP>)
-                     {
-                        MFEM_UNROLL(tv)
-                        for (int i = 0; i < tv; i++)
-                        {
-                           MFEM_UNROLL(to)
-                           for (int k = 0; k < to; k++)
-                           {
-                              YE(i + tv * k, qx, qy, qz, e) =
-                                 qf_value_at(fhat, i, k);
-                           }
-                        }
-                     }
-                     else
-                     {
-                        backend_t::template qp_push<ARG>(
-                           get<o>(rargs), qx, qy, qz, fhat);
-                     }
-                  });
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // -----------------------------------------------
-         // Integrate value / gradient outputs to the test dofs
-         // -----------------------------------------------
-         for_constexpr<n_outputs>([&](auto ic)
-         {
-            constexpr size_t i = ic.value, o = n_inputs + i;
-            const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
-            const auto B = out_B[i], G = out_G[i];
-            auto &YE = out_YE[i];
-            auto &rarg = get<i>(rargs);
-            using FOP = tuple_element_t<i, outputs_t>;
-            if constexpr (is_value_fop_v<FOP>)
-            {
-               backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, rarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               using YE_t = decltype(YE);
-               using rarg_t = decltype(rarg);
-               using qf_param_t =
-                  typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-               constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
-               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
-                  smem, e, d, q, Q1D, B, G, YE, rarg);
-            }
-            else if constexpr (is_identity_fop_v<FOP>) { /* written at qp */ }
-            else
-            {
-               static_assert(false, "Unsupported");
-            }
-         });
-      },
-      ne,
-      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
-                                  q1d, in_d1d, out_d1d)),
-      0,
-      nullptr);
-   }
-
-   using ApplyKernelType =
-      decltype(&DerivativeApply::derivative_apply_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyLO,
-                                     ApplyKernelType,
-                                     (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyHO,
-                                     ApplyKernelType,
-                                     (int, int) );
-};
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-ApplyKernelType
-DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeApplyLO::Kernel()
-{
-   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
-   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return apply_t::template derivative_apply_callback<
-             LocalQFLOBackend<DIM, Q1D>>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-ApplyKernelType
-DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeApplyLO::Fallback(int dim, int q1d)
-{
-   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeApplyLO = typename apply_t::DerivativeApplyLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<DerivativeApplyLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<DerivativeApplyLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-ApplyKernelType
-DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeApplyHO::Kernel()
-{
-   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return apply_t::template derivative_apply_callback<LocalQFHOBackend<DIM>,
-                                                      Q1D>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-ApplyKernelType
-DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeApplyHO::Fallback(int dim, int q1d)
-{
-   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeApplyHO = typename apply_t::DerivativeApplyHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<DerivativeApplyHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<DerivativeApplyHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_apply_transpose.hpp

@@ -1,637 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include <array>
-
-namespace mfem::future::LocalQFImpl
-{
-
-// Cached transposed Jacobian apply: Jᵀ·w from the qp_cache
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-class DerivativeApplyTranspose
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-   using args_tuple_t = decay_tuple<qf_param_ts>;
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
-                 "LocalQF: q-function arity must match inputs + outputs");
-
-   // Input tuple slot referencing the derivative field (compile-time)
-   static constexpr size_t deriv_input_idx_ct = []() constexpr
-   {
-      size_t idx = SIZE_MAX;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         using FOP = tuple_element_t<i, inputs_t>;
-         if (FOP::GetFieldId() == derivative_id) { idx = i; }
-      });
-      return idx;
-   }();
-   static_assert(deriv_input_idx_ct < n_inputs,
-                 "DerivativeApplyTranspose: derivative input slot not found");
-
-   const inputs_t inputs;
-   const outputs_t outputs;
-   const IntegratorContext ctx;
-   const Vector &qp_cache; // Jacobian cache from DerivativeSetup
-   const std::vector<const DofToQuad *> dtqs;
-   // inputs: dtq, B, G, d1d, q1d, vdim (trial / derivative fields)
-   const std::array<DofToQuadMap, n_inputs> input_dtq;
-   const std::array<const real_t *, n_inputs> input_B, input_G;
-   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
-   // outputs: dtq, idx, B, G, d1d, q1d, vdim (test / cotangent fields)
-   const std::array<DofToQuadMap, n_outputs> output_dtq;
-   const std::array<size_t, n_outputs> output_idx;
-   const std::array<const real_t *, n_outputs> output_B, output_G;
-   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
-   // Jacobian cache metadata
-   const std::array<bool, n_inputs> input_is_dependent;
-   const std::array<int, n_inputs> input_size_on_qp;
-   const std::array<int, n_outputs> out_op_dim;
-   const std::array<int, n_outputs> out_offsets;
-   const int output_size_on_qp;
-   const int trial_vdim;
-   const int total_trial_op_dim;
-   const int residual_size_on_qp;
-   // other constants
-   const int dim, ne, nq, q1d;
-   const size_t deriv_infd_idx; // index of the derivative field in ye
-   // output cotangent restriction workspace (blocked by element)
-   std::array<int, n_outputs> out_elem_dof_size;
-   mutable Vector dir_out_e;
-   /// One restriction cache per output field, resolved on first use.
-   mutable std::array<RestrictionCache<Entity::Element>, n_outputs>
-   out_rcaches;
-
-public:
-   //////////////////////////////////////////////////////////////////
-   DerivativeApplyTranspose() = delete;
-
-   DerivativeApplyTranspose(IntegratorContext ctx,
-                            qfunc_t /*qfunc*/,
-                            inputs_t inputs,
-                            outputs_t outputs,
-                            const Vector &qp_cache):
-      inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache),
-      dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
-                                         inputs,
-                                         dtqs,
-                                         create_union_field_map_for_dtq(ctx, inputs),
-                                         ctx.unionfds,
-                                         ctx.ir)),
-      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
-      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
-      input_vdim(get_vdim(inputs)),
-      output_dtq(create_dtq_maps<Entity::Element>(
-                    outputs,
-                    dtqs,
-                    create_union_field_map_for_dtq(ctx, outputs),
-                    ctx.unionfds,
-                    ctx.ir)),
-      output_idx(create_output_vector_map(ctx, outputs)),
-      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
-      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
-      output_vdim(get_vdim(outputs)),
-      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
-      input_size_on_qp(
-         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
-                           out_op_dim(compute_out_op_dim(outputs)),
-                           out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
-                           output_size_on_qp(
-                              [&]
-   {
-      int s = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      { s += get<o>(outputs).size_on_qp; });
-      return s;
-   }()),
-   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
-   total_trial_op_dim(compute_total_trial_op_dim(
-                         inputs, input_is_dependent, input_size_on_qp)),
-   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
-   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
-   q1d(tensor_1d_size(nq, dim)),
-   deriv_infd_idx(find_infd_index(ctx, derivative_id)), out_elem_dof_size{}
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-      MFEM_ASSERT(
-         deriv_infd_idx != SIZE_MAX,
-         "DerivativeApplyTranspose: derivative field not found in infds");
-
-      // Size the workspace that holds the output cotangent(s) in element
-      // layout.
-      int total_dir_e_size = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         const int elem_sz = compute_element_dof_sz(
-            ctx.outfds[output_idx[o]], ne, ElementDofOrdering::LEXICOGRAPHIC);
-         out_elem_dof_size[o] = elem_sz;
-         total_dir_e_size += elem_sz;
-      });
-      dir_out_e.SetSize(total_dir_e_size * ne);
-      dir_out_e.UseDevice(true);
-      dir_out_e.Read();
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename Backend>
-   void run_kernels(std::vector<Vector *> &ye) const
-   {
-      Backend::Run(dim,
-                   q1d,
-                   ctx,
-                   qp_cache,
-                   dir_out_e,
-                   // inputs (integration target metadata)
-                   input_B,
-                   input_G,
-                   input_vdim,
-                   input_d1d,
-                   input_q1d,
-                   input_size_on_qp,
-                   input_is_dependent,
-                   // outputs (direction interpolation metadata)
-                   output_B,
-                   output_G,
-                   output_vdim,
-                   output_d1d,
-                   output_q1d,
-                   out_op_dim,
-                   out_offsets,
-                   trial_vdim,
-                   total_trial_op_dim,
-                   residual_size_on_qp,
-                   output_size_on_qp,
-                   deriv_infd_idx,
-                   ye,
-                   // fallback arguments
-                   dim,
-                   q1d);
-   }
-
-   //////////////////////////////////////////////////////////////////
-   void operator()(const std::vector<Vector *> & /*xe*/,
-                   const Vector *direction_l,
-                   std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      MFEM_ASSERT(direction_l != nullptr,
-                  "LocalQF DerivativeApplyTranspose: direction vector is null");
-
-      // Restrict output cotangent from L-vectors into element layout
-      // (dir_out_e).
-      int l_offset = 0;
-      int e_offset = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         const size_t outfd = output_idx[o];
-         const auto &fd = ctx.outfds[outfd];
-         const int l_size = GetVSize(fd);
-         Vector dir_o_l(*const_cast<Vector *>(direction_l), l_offset, l_size);
-         dir_o_l.UseDevice(true);
-         const int elem_sz = out_elem_dof_size[o];
-         Vector dir_o_e(dir_out_e, e_offset, elem_sz * ne);
-         dir_o_e.UseDevice(true);
-         restriction(fd, out_rcaches[o], dir_o_l, dir_o_e,
-                     ElementDofOrdering::LEXICOGRAPHIC);
-         l_offset += l_size;
-         e_offset += elem_sz * ne;
-      });
-
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<DerivativeApplyTransposeLO>(ye);
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<DerivativeApplyTransposeHO>(ye);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void derivative_apply_transpose_callback(
-      const IntegratorContext &ctx,
-      const Vector &qp_cache,
-      const Vector &dir_e, // restricted, concatenated output cotangents
-      // inputs (integration target metadata)
-      const std::array<const real_t *, n_inputs> in_B,
-      const std::array<const real_t *, n_inputs> in_G,
-      const std::array<int, n_inputs> &in_vdim,
-      const std::array<int, n_inputs> &in_d1d,
-      const std::array<int, n_inputs> &in_q1d,
-      const std::array<int, n_inputs> &in_size_on_qp,
-      const std::array<bool, n_inputs> &input_dep,
-      // outputs (direction interpolation metadata)
-      const std::array<const real_t *, n_outputs> out_B,
-      const std::array<const real_t *, n_outputs> out_G,
-      const std::array<int, n_outputs> &out_vdim,
-      const std::array<int, n_outputs> &out_d1d,
-      const std::array<int, n_outputs> &out_q1d,
-      const std::array<int, n_outputs> &out_op_dim,
-      const std::array<int, n_outputs> &out_offsets,
-      const int trial_vdim,
-      const int total_trial_op_dim,
-      const int residual_size_on_qp,
-      const int output_size_on_qp,
-      const size_t deriv_infd_idx,
-      std::vector<Vector *> &ye,
-      // fallback arguments
-      const int dim,
-      const int q1d)
-   {
-      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-      if (ctx.attr.Size() == 0) { return; }
-
-      static constexpr auto B2D = backend_t::DIM == 2;
-      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
-      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
-
-      const int ne = ctx.nentities;
-      const int nq = ctx.ir.GetNPoints();
-      MFEM_CONTRACT_VAR(output_size_on_qp);
-      MFEM_CONTRACT_VAR(in_q1d);
-
-      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
-
-      // --------------------------------------------------
-      // DIRECTION (test cotangent): out_XE_dir, concatenated per output
-      // --------------------------------------------------
-      const auto d_dir = dir_e.Read();
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_outputs> out_XE_dir;
-      int e_offset = 0;
-      for_constexpr<n_outputs>([&](auto oc)
-      {
-         constexpr size_t o = oc.value;
-         const int d = out_d1d[o], q = out_q1d[o], v = out_vdim[o];
-         using FOP = tuple_element_t<o, outputs_t>;
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            out_XE_dir[o] = Reshape(d_dir + e_offset, d, d, B2D ? 1 : d, v, ne);
-            e_offset += k_dim(d) * v * ne;
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            out_XE_dir[o] = Reshape(d_dir + e_offset, v, q, q, B2D ? 1 : q, ne);
-            e_offset += k_dim(q) * v * ne;
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      // --------------------------------------------------
-      // DERIVATIVE TRIAL FIELD: ye_XE (accumulates Jᵀ w)
-      // --------------------------------------------------
-      const int d_in = in_d1d[deriv_input_idx_ct];
-      const int v_in = in_vdim[deriv_input_idx_ct];
-      auto ye_XE = Reshape(
-                      ye[deriv_infd_idx]->ReadWrite(), d_in, d_in, B2D ? 1 : d_in, v_in, ne);
-
-      auto cache_tensor = DeviceTensor<3, const real_t>(
-                             qp_cache.Read(), nq, residual_size_on_qp, ne);
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      dfem::forall<MTPB>(
-         [=] MFEM_HOST_DEVICE(const int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         // -----------------------------------------------
-         // Output cotangent (direction) registers live in the output slots;
-         // the trial integration data is pushed into the input slots.
-         // -----------------------------------------------
-         args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
-         MFEM_SHARED typename backend_t::Shared smem;
-
-         // -----------------------------------------------
-         // Interpolate the test cotangent to quadrature points (output slots)
-         // -----------------------------------------------
-         for_constexpr<n_outputs>([&](auto oc)
-         {
-            constexpr size_t o = oc.value, ao = n_inputs + o;
-            using FOP = tuple_element_t<o, outputs_t>;
-            const auto &XE = out_XE_dir[o];
-            const int d = out_d1d[o], q = out_q1d[o], Q1D = q1d;
-            const real_t *B = out_B[o], *G = out_G[o];
-            auto &oarg = get<ao>(rargs);
-            if constexpr (is_value_fop_v<FOP>)
-            {
-               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, oarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               constexpr auto RNK = qf_param_slot<qfunc_t, ao>::extents.size();
-               using FieldParamT =
-                  typename qf_param_slot<qfunc_t, ao>::qf_decay_param_t;
-               backend_t::template LoadGradient<RNK,
-                                                decltype(oarg),
-                                                decltype(XE),
-                                                FieldParamT>(
-                                                   smem, e, d, q, Q1D, B, G, XE, oarg);
-            }
-            else if constexpr (is_identity_fop_v<FOP>)
-            {
-               // identity cotangent is read directly at qp from out_XE_dir
-            }
-            else
-            {
-               static_assert(false, "Unsupported");
-            }
-         });
-         MFEM_SYNC_THREAD;
-
-         // -----------------------------------------------
-         // Contract the transposed cached Jacobian with the test cotangent at
-         // each quadrature point and push the trial result into the dependent
-         // input registers.
-         // -----------------------------------------------
-         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-
-                  // The test cotangent at this quadrature point is the same
-                  // for every trial column (j, m), so pull each interpolated
-                  // output slot out of the register bank once, here, instead
-                  // of once per column inside the contraction below. Identity
-                  // outputs have no register bank and are read from
-                  // out_XE_dir at the point of use.
-                  args_tuple_t wvecs {};
-                  for_constexpr<n_outputs>([&](auto oc)
-                  {
-                     constexpr size_t o = oc.value, ao = n_inputs + o;
-                     using OFOP = tuple_element_t<o, outputs_t>;
-                     if constexpr (is_value_fop_v<OFOP> ||
-                                   is_gradient_fop_v<OFOP>)
-                     {
-                        using OARG =
-                           typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
-                        get<ao>(wvecs) = backend_t::template qp_pull<OARG>(
-                           get<ao>(rargs), qx, qy, qz);
-                     }
-                  });
-
-                  int m_offset = 0;
-                  for_constexpr<n_inputs>([&](auto sc)
-                  {
-                     constexpr size_t s = sc.value;
-                     if (!input_dep[s]) { return; }
-                     using SARG =
-                        typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
-                     const int vdim_s = in_vdim[s];
-                     const int op_dim_s = in_size_on_qp[s] / vdim_s;
-
-                     SARG fhat{};
-                     for (int j = 0; j < trial_vdim; j++)
-                     {
-                        for (int m = 0; m < op_dim_s; m++)
-                        {
-                           const int col =
-                              j * total_trial_op_dim + (m + m_offset);
-                           real_t sum = 0.0;
-                           for_constexpr<n_outputs>([&](auto oc)
-                           {
-                              constexpr size_t o = oc.value, ao = n_inputs + o;
-                              using OFOP = tuple_element_t<o, outputs_t>;
-                              const int tv = out_vdim[o], to = out_op_dim[o];
-                              const auto offset_o = out_offsets[o];
-                              const auto &cache = cache_tensor;
-                              if constexpr (is_value_fop_v<OFOP> ||
-                                            is_gradient_fop_v<OFOP>)
-                              {
-                                 const auto &wvec = get<ao>(wvecs);
-                                 for (int i = 0; i < tv; i++)
-                                 {
-                                    for (int k = 0; k < to; k++)
-                                    {
-                                       const int row = offset_o + i * to + k;
-                                       const int cache_idx =
-                                          row * trial_vdim *
-                                          total_trial_op_dim +
-                                          col;
-                                       sum += cache(q, cache_idx, e) *
-                                              qf_value_at(wvec, i, k);
-                                    }
-                                 }
-                              }
-                              else if constexpr (is_identity_fop_v<OFOP>)
-                              {
-                                 const auto &XEo = out_XE_dir[o];
-                                 for (int i = 0; i < tv; i++)
-                                 {
-                                    for (int k = 0; k < to; k++)
-                                    {
-                                       const int row = offset_o + i * to + k;
-                                       const int cache_idx =
-                                          row * trial_vdim *
-                                          total_trial_op_dim +
-                                          col;
-                                       sum += cache(q, cache_idx, e) *
-                                              XEo(i + tv * k, qx, qy, qz, e);
-                                    }
-                                 }
-                              }
-                           });
-                           qf_set_value_at(fhat, j, m, sum);
-                        }
-                     }
-                     backend_t::template qp_push<SARG>(
-                        get<s>(rargs), qx, qy, qz, fhat);
-                     m_offset += op_dim_s;
-                  });
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // -----------------------------------------------
-         // Integrate the trial result into the derivative field dofs. Multiple
-         // dependent input slots (e.g. value and gradient of the same field)
-         // accumulate into ye_XE via the writers' '+=' semantics.
-         // -----------------------------------------------
-         for_constexpr<n_inputs>([&](auto sc)
-         {
-            constexpr size_t s = sc.value;
-            if (!input_dep[s]) { return; }
-            using FOP = tuple_element_t<s, inputs_t>;
-            const int d = in_d1d[s], q = in_q1d[s], Q1D = q1d;
-            const real_t *B = in_B[s], *G = in_G[s];
-            auto &sarg = get<s>(rargs);
-            auto &YE = ye_XE;
-            if constexpr (is_value_fop_v<FOP>)
-            {
-               backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, sarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               using YE_t = decltype(YE);
-               using rarg_t = decltype(sarg);
-               using qf_param_t =
-                  typename qf_param_slot<qfunc_t, s>::qf_decay_param_t;
-               constexpr auto RNK = qf_param_slot<qfunc_t, s>::extents.size();
-               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
-                  smem, e, d, q, Q1D, B, G, YE, sarg);
-            }
-            else
-            {
-               // identity / weight derivative targets are not produced here
-            }
-         });
-      },
-      ne,
-      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
-                                  q1d, in_d1d, out_d1d)),
-      0,
-      nullptr);
-   }
-
-   using TransposeKernelType =
-      decltype(&DerivativeApplyTranspose::
-               derivative_apply_transpose_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeLO,
-                                     TransposeKernelType,
-                                     (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeHO,
-                                     TransposeKernelType,
-                                     (int, int) );
-};
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeApplyTranspose<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::TransposeKernelType
-       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeApplyTransposeLO::Kernel()
-{
-   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
-   using transpose_t =
-      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return transpose_t::template derivative_apply_transpose_callback<
-             LocalQFLOBackend<DIM, Q1D>>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeApplyTranspose<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::TransposeKernelType
-       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeApplyTransposeLO::Fallback(int dim, int q1d)
-{
-   using transpose_t =
-      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeApplyTransposeLO =
-      typename transpose_t::DerivativeApplyTransposeLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeApplyTranspose<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::TransposeKernelType
-       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeApplyTransposeHO::Kernel()
-{
-   using transpose_t =
-      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return transpose_t::
-          template derivative_apply_transpose_callback<LocalQFHOBackend<DIM>, Q1D>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeApplyTranspose<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::TransposeKernelType
-       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeApplyTransposeHO::Fallback(int dim, int q1d)
-{
-   using transpose_t =
-      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeApplyTransposeHO =
-      typename transpose_t::DerivativeApplyTransposeHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_assemble.hpp

@@ -1,987 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "../../../kernels.hpp"
-
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include <array>
-#include <type_traits>
-
-namespace ker = mfem::kernels::internal;
-
-namespace mfem::future::LocalQFImpl
-{
-
-namespace detail
-{
-
-template<int DIM>
-MFEM_HOST_DEVICE inline int tensor_idx(int x, int y, int z, int N)
-{
-   static_assert(DIM == 2 || DIM == 3);
-   if constexpr (DIM == 2) { assert(z == 0); }
-   return x + N * (y + N * z);
-}
-
-template<int DIM>
-MFEM_HOST_DEVICE inline real_t
-trial_basis_weight_value(const DeviceTensor<3, const real_t> &B,
-                         const int qx,
-                         const int qy,
-                         const int qz,
-                         const int Jx,
-                         const int Jy,
-                         const int Jz)
-{
-   static_assert(DIM == 2 || DIM == 3);
-   return B(qx, 0, Jx) * B(qy, 0, Jy) * ((DIM == 3) ? B(qz, 0, Jz) : 1.0);
-}
-
-template<int DIM>
-MFEM_HOST_DEVICE inline real_t
-trial_basis_weight_gradient(const DeviceTensor<3, const real_t> &B,
-                            const DeviceTensor<3, const real_t> &G,
-                            const int m,
-                            const int qx,
-                            const int qy,
-                            const int qz,
-                            const int Jx,
-                            const int Jy,
-                            const int Jz)
-{
-   const auto Gx = G(qx, 0, Jx), Gy = G(qy, 0, Jy);
-   const auto Bx = B(qx, 0, Jx), By = B(qy, 0, Jy);
-   if constexpr (DIM == 2)
-   {
-      MFEM_CONTRACT_VAR(qz & Jz);
-      return (m == 0) ? Gx * By : Bx * Gy;
-   }
-   else
-   {
-      const auto Bz = B(qz, 0, Jz), Gz = G(qz, 0, Jz);
-      return (m == 0)   ? Gx * By * Bz
-             : (m == 1) ? Bx * Gy * Bz
-             : (m == 2) ? Bx * By * Gz
-             : (assert(false), 0.0);
-   }
-}
-
-template<int DIM, int MQ1, typename Shared, typename output_t>
-MFEM_HOST_DEVICE void
-map_quadrature_data_to_fields(DeviceTensor<2, real_t> &y,
-                              const DeviceTensor<3, real_t> &f,
-                              const output_t &output,
-                              const DofToQuadMap &dtq,
-                              Shared &s,
-                              const int tv_dof = -1)
-{
-   using output_fop_t = std::decay_t<output_t>;
-   const auto B = dtq.B, G = dtq.G;
-   const bool f_slab = (tv_dof >= 0);
-   const int vdim = output.vdim;
-   const int vd_begin = f_slab ? tv_dof : 0;
-   const int vd_end = f_slab ? tv_dof + 1 : vdim;
-
-   if constexpr (is_value_fop_v<output_fop_t>)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      MFEM_CONTRACT_VAR(unused);
-      const int test_dim = output.size_on_qp / vdim;
-      MFEM_CONTRACT_VAR(test_dim);
-      const int f_vdim = f_slab ? 1 : vdim;
-
-      if constexpr (DIM == 2)
-      {
-         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d);
-         auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
-         ker::LoadMatrix(d1d, q1d, B, s.B);
-         ker::s_regs2d_t<MQ1> r_qp, Y;
-         for (int vd = vd_begin; vd < vd_end; vd++)
-         {
-            const int fi = f_slab ? 0 : vd;
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            { r_qp[qy][qx] = fqp(fi, 0, qx, qy); }
-            MFEM_SYNC_THREAD;
-            ker::Eval2d<MQ1, true>(d1d, q1d, s.M, s.B, r_qp, Y);
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            MFEM_FOREACH_THREAD(dx, x, d1d) { yd(dx, dy, vd) += Y[dy][dx]; }
-            MFEM_SYNC_THREAD;
-         }
-      }
-      else
-      {
-         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d, q1d);
-         auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
-         ker::LoadMatrix(d1d, q1d, B, s.B);
-         ker::s_regs3d_t<MQ1> f_qp, Y;
-         for (int vd = vd_begin; vd < vd_end; vd++)
-         {
-            const int fi = f_slab ? 0 : vd;
-            for (int qz = 0; qz < q1d; qz++)
-            {
-               MFEM_FOREACH_THREAD(qy, y, q1d)
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               { f_qp[qz][qy][qx] = fqp(fi, 0, qx, qy, qz); }
-            }
-            MFEM_SYNC_THREAD;
-            ker::Eval3d<MQ1, true>(d1d, q1d, s.M, s.B, f_qp, Y);
-            for (int dz = 0; dz < d1d; dz++)
-            {
-               MFEM_FOREACH_THREAD(dy, y, d1d)
-               MFEM_FOREACH_THREAD(dx, x, d1d)
-               { yd(dx, dy, dz, vd) += Y[dz][dy][dx]; }
-            }
-            MFEM_SYNC_THREAD;
-         }
-      }
-   }
-   else if constexpr (is_gradient_fop_v<output_fop_t>)
-   {
-      const auto [q1d, unused, d1d] = G.GetShape();
-      MFEM_CONTRACT_VAR(unused);
-      const int test_dim = output.size_on_qp / vdim;
-      const int f_vdim = f_slab ? 1 : vdim;
-
-      if constexpr (DIM == 2)
-      {
-         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d);
-         auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
-         ker::LoadMatrix(d1d, q1d, B, s.B);
-         ker::LoadMatrix(d1d, q1d, G, s.G);
-         ker::vd_regs2d_t<1, DIM, MQ1> X, Y;
-         for (int vd = vd_begin; vd < vd_end; vd++)
-         {
-            const int fi = f_slab ? 0 : vd;
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            for (int k = 0; k < DIM; k++)
-            {
-               X[0][k][qy][qx] = fqp(fi, k, qx, qy);
-            }
-            MFEM_SYNC_THREAD;
-            ker::Grad2d<1, DIM, MQ1, true>(d1d, q1d, s.M, s.B, s.G, X, Y);
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               real_t u = 0.0;
-               for (int k = 0; k < DIM; k++) { u += Y[0][k][dy][dx]; }
-               yd(dx, dy, vd) += u;
-            }
-            MFEM_SYNC_THREAD;
-         }
-      }
-      else
-      {
-         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d, q1d);
-         auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
-         ker::LoadMatrix(d1d, q1d, B, s.B);
-         ker::LoadMatrix(d1d, q1d, G, s.G);
-         ker::vd_regs3d_t<1, DIM, MQ1> X, Y;
-         for (int vd = vd_begin; vd < vd_end; vd++)
-         {
-            const int fi = f_slab ? 0 : vd;
-            for (int qz = 0; qz < q1d; qz++)
-            {
-               MFEM_FOREACH_THREAD(qy, y, q1d)
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               for (int k = 0; k < DIM; k++)
-               {
-                  X[0][k][qz][qy][qx] = fqp(fi, k, qx, qy, qz);
-               }
-            }
-            MFEM_SYNC_THREAD;
-            ker::Grad3d<1, DIM, MQ1, true>(d1d, q1d, s.M, s.B, s.G, X, Y);
-            for (int dz = 0; dz < d1d; dz++)
-            {
-               MFEM_FOREACH_THREAD(dy, y, d1d)
-               MFEM_FOREACH_THREAD(dx, x, d1d)
-               {
-                  real_t u = 0.0;
-                  for (int k = 0; k < DIM; k++) { u += Y[0][k][dz][dy][dx]; }
-                  yd(dx, dy, dz, vd) += u;
-               }
-            }
-            MFEM_SYNC_THREAD;
-         }
-      }
-   }
-   else if constexpr (is_identity_fop_v<output_fop_t>)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      MFEM_CONTRACT_VAR(unused);
-      MFEM_CONTRACT_VAR(d1d);
-
-      const int f_sq = f_slab ? 1 : output.size_on_qp;
-      const int sq_begin = f_slab ? tv_dof : 0;
-      const int sq_end = f_slab ? tv_dof + 1 : output.size_on_qp;
-
-      if constexpr (DIM == 2)
-      {
-         const auto fqp = Reshape(&f(0, 0, 0), f_sq, q1d, q1d);
-         auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d);
-         for (int sq = sq_begin; sq < sq_end; sq++)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            {
-               int qz = 0;
-               MFEM_CONTRACT_VAR(qz);
-               yqp(sq, qx, qy) = fqp(0, qx, qy);
-            }
-            MFEM_SYNC_THREAD;
-         }
-      }
-      else
-      {
-         const auto fqp = Reshape(&f(0, 0, 0), f_sq, q1d, q1d, q1d);
-         auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
-         for (int sq = sq_begin; sq < sq_end; sq++)
-         {
-            for (int qz = 0; qz < q1d; qz++)
-            {
-               MFEM_FOREACH_THREAD(qy, y, q1d)
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               { yqp(sq, qx, qy, qz) = fqp(0, qx, qy, qz); }
-            }
-            MFEM_SYNC_THREAD;
-         }
-      }
-   }
-   else
-   {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        " for this field descriptor with sum factorization on"
-                        " tensor product elements");
-   }
-}
-
-template<int DIM,
-         int MQ1,
-         typename Shared,
-         typename input_fop_ts,
-         std::size_t n_inputs,
-         typename output_fop_t>
-MFEM_HOST_DEVICE void assemble_element_mat_sumfact(
-   const DeviceTensor<5, real_t> &Ae,
-   const DeviceTensor<5, const real_t> &qpdc,
-   const int e,
-   const DeviceTensor<1, const real_t> &itod,
-   const input_fop_ts &inputs,
-   const output_fop_t &output,
-   const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
-   const DofToQuadMap &output_dtq,
-   const int row_offset,
-   const int test_vdim,
-   const int test_op_dim,
-   const int q1d,
-   const int num_trial_dof_1d,
-   real_t *fhat_storage,
-   Shared &smem)
-{
-   static constexpr int MQN = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
-   // Slab must hold full (test_vdim, test_op_dim, nq) fhat.
-   // It is allocated by the caller, and is shared by every output, so it must be
-   // Before, declaring it here allocated one slab per output and the device
-   // kernel ran out of shared memory once an integrator had more outputs.
-   static constexpr int FHAT_SLAB_MAX = MQN * 4;
-
-   static constexpr bool grad_out = is_gradient_fop_v<output_fop_t>;
-   static constexpr bool ident_out = is_identity_fop_v<output_fop_t>;
-
-   // qpdc shape: (nq, total_trial_op_dim, trial_vdim, output_size_on_qp, ne),
-   // where output_size_on_qp spans every output FieldOperator (multi-output mode).
-   // The rows of  one output start at @a row_offset and are laid out as
-   // i * test_op_dim + k, matching how DerivativeSetup writes the cache.
-   const int trial_vdim = qpdc.GetShape()[2];
-   const int num_test_dof = Ae.GetShape()[0];
-   const int nq = qpdc.GetShape()[0];
-   const int size_on_qp = output.size_on_qp;
-
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-   MFEM_VERIFY(test_op_dim <= DIM,
-               "DerivativeAssemble: test_op_dim exceeds spatial DIM");
-   MFEM_VERIFY(test_op_dim * nq <= FHAT_SLAB_MAX,
-               "DerivativeAssemble: fhat slab exceeds capacity");
-#endif
-
-   const auto &inputs_ref = inputs;
-
-   // Iterate quadrature points using the thread-block mapping
-   const auto foreach_qp = [&](auto &&body)
-   {
-      if constexpr (DIM == 2)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         MFEM_FOREACH_THREAD(qy, y, q1d) { body(qx, qy, 0); }
-      }
-      else
-      {
-         for (int qz = 0; qz < q1d; qz++)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            MFEM_FOREACH_THREAD(qx, x, q1d) { body(qx, qy, qz); }
-         }
-      }
-   };
-
-   const auto zero_slab = [&](const int n_comp)
-   {
-      foreach_qp([&](const int qx, const int qy, const int qz)
-      {
-         const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
-         for (int k = 0; k < n_comp; k++) { fhat_storage[k * nq + q] = 0.0; }
-      });
-      MFEM_SYNC_THREAD;
-   };
-
-   const auto accumulate_tv = [&](const int Jx,
-                                  const int Jy,
-                                  const int Jz,
-                                  const int j,
-                                  const int tv,
-                                  const int tod_only = -1)
-   {
-      int m_offset = 0;
-      for_constexpr<n_inputs>([&](auto inp)
-      {
-         using fop_t = std::decay_t<decltype(get<inp>(inputs_ref))>;
-
-         const int trial_op_dim = static_cast<int>(itod(static_cast<int>(inp)));
-         if (trial_op_dim == 0) { return; }
-
-         const auto &B = input_dtq_maps[inp].B;
-         const auto &G = input_dtq_maps[inp].G;
-
-         if constexpr (is_value_fop<fop_t>::value)
-         {
-            foreach_qp([&](const int qx, const int qy, const int qz)
-            {
-               const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
-               const real_t w =
-                  trial_basis_weight_value<DIM>(B, qx, qy, qz, Jx, Jy, Jz);
-               for (int m = 0; m < trial_op_dim; m++)
-               {
-                  for (int k = 0; k < test_op_dim; k++)
-                  {
-                     if (tod_only >= 0 && k != tod_only) { continue; }
-                     const real_t f = qpdc(q, m + m_offset, j, row_offset + tv * test_op_dim + k, e);
-                     if constexpr (grad_out && !ident_out)
-                     {
-                        fhat_storage[k * nq + q] += f * w;
-                     }
-                     else
-                     {
-                        fhat_storage[q] += f * w;
-                     }
-                  }
-               }
-            });
-         }
-         else if constexpr (is_gradient_fop<fop_t>::value)
-         {
-            foreach_qp([&](const int qx, const int qy, const int qz)
-            {
-               const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
-               for (int m = 0; m < trial_op_dim; m++)
-               {
-                  const real_t w = trial_basis_weight_gradient<DIM>(
-                                      B, G, m, qx, qy, qz, Jx, Jy, Jz);
-                  for (int k = 0; k < test_op_dim; k++)
-                  {
-                     if (tod_only >= 0 && k != tod_only) { continue; }
-                     const real_t f = qpdc(q, m + m_offset, j, row_offset + tv * test_op_dim + k, e);
-                     if constexpr (grad_out && !ident_out)
-                     {
-                        fhat_storage[k * nq + q] += f * w;
-                     }
-                     else
-                     {
-                        fhat_storage[q] += f * w;
-                     }
-                  }
-               }
-            });
-         }
-         else
-         {
-            MFEM_ABORT_KERNEL("sum factorized sparse matrix assemble routine "
-                              "not implemented for field operator");
-         }
-         MFEM_SYNC_THREAD;
-         m_offset += trial_op_dim;
-      });
-   };
-
-   for (int Jz = 0; Jz < ((DIM == 2) ? 1 : num_trial_dof_1d); Jz++)
-   {
-      for (int Jy = 0; Jy < num_trial_dof_1d; Jy++)
-      {
-         for (int Jx = 0; Jx < num_trial_dof_1d; Jx++)
-         {
-            const int J = tensor_idx<DIM>(Jx, Jy, Jz, num_trial_dof_1d);
-            for (int j = 0; j < trial_vdim; j++)
-            {
-               auto bvtfhat =
-                  Reshape(&Ae(0, 0, J, j, e), num_test_dof, test_vdim);
-               const int fhat_size = test_vdim * test_op_dim * nq;
-
-               if (fhat_size <= FHAT_SLAB_MAX)
-               {
-                  auto fhat =
-                     Reshape(&fhat_storage[0], test_vdim, test_op_dim, nq);
-                  for (int tv = 0; tv < test_vdim; tv++)
-                  {
-                     for (int tod = 0; tod < test_op_dim; tod++)
-                     {
-                        foreach_qp([&](const int qx, const int qy, const int qz)
-                        {
-                           const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
-                           fhat(tv, tod, q) = 0.0;
-                        });
-                     }
-                  }
-                  MFEM_SYNC_THREAD;
-
-                  int m_offset = 0;
-                  for_constexpr<n_inputs>([&](auto inp)
-                  {
-                     using fop_t = std::decay_t<decltype(get<inp>(inputs_ref))>;
-
-                     const int trial_op_dim =
-                        static_cast<int>(itod(static_cast<int>(inp)));
-                     if (trial_op_dim == 0) { return; }
-
-                     const auto &B = input_dtq_maps[inp].B;
-                     const auto &G = input_dtq_maps[inp].G;
-
-                     if constexpr (is_value_fop<fop_t>::value)
-                     {
-                        foreach_qp([&](const int qx, const int qy, const int qz)
-                        {
-                           const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
-                           const real_t w = trial_basis_weight_value<DIM>(
-                                               B, qx, qy, qz, Jx, Jy, Jz);
-                           for (int m = 0; m < trial_op_dim; m++)
-                           {
-                              for (int i = 0; i < test_vdim; i++)
-                              {
-                                 for (int k = 0; k < test_op_dim; k++)
-                                 {
-                                    const real_t f =
-                                       qpdc(q, m + m_offset, j, row_offset + i * test_op_dim + k, e);
-                                    fhat(i, k, q) += f * w;
-                                 }
-                              }
-                           }
-                        });
-                     }
-                     else if constexpr (is_gradient_fop<fop_t>::value)
-                     {
-                        foreach_qp([&](const int qx, const int qy, const int qz)
-                        {
-                           const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
-                           for (int m = 0; m < trial_op_dim; m++)
-                           {
-                              const real_t w = trial_basis_weight_gradient<DIM>(
-                                                  B, G, m, qx, qy, qz, Jx, Jy, Jz);
-                              for (int i = 0; i < test_vdim; i++)
-                              {
-                                 for (int k = 0; k < test_op_dim; k++)
-                                 {
-                                    const real_t f =
-                                       qpdc(q, m + m_offset, j, row_offset + i * test_op_dim + k, e);
-                                    fhat(i, k, q) += f * w;
-                                 }
-                              }
-                           }
-                        });
-                     }
-                     else
-                     {
-                        MFEM_ABORT_KERNEL(
-                           "sum factorized sparse matrix assemble routine "
-                           "not implemented for field operator");
-                     }
-                     MFEM_SYNC_THREAD;
-                     m_offset += trial_op_dim;
-                  });
-                  map_quadrature_data_to_fields<DIM, MQ1>(
-                     bvtfhat, fhat, output, output_dtq, smem);
-               }
-               else if constexpr (ident_out)
-               {
-                  for (int sq = 0; sq < size_on_qp; sq++)
-                  {
-                     const int tv = sq / test_op_dim;
-                     const int tod = sq % test_op_dim;
-                     zero_slab(1);
-                     accumulate_tv(Jx, Jy, Jz, j, tv, tod);
-                     auto f_slab = Reshape(&fhat_storage[0], 1, 1, nq);
-                     map_quadrature_data_to_fields<DIM, MQ1>(
-                        bvtfhat, f_slab, output, output_dtq, smem, sq);
-                  }
-               }
-               else if constexpr (grad_out)
-               {
-                  for (int tv = 0; tv < test_vdim; tv++)
-                  {
-                     zero_slab(test_op_dim);
-                     accumulate_tv(Jx, Jy, Jz, j, tv);
-                     auto f_slab =
-                        Reshape(&fhat_storage[0], 1, test_op_dim, nq);
-                     map_quadrature_data_to_fields<DIM, MQ1>(
-                        bvtfhat, f_slab, output, output_dtq, smem, tv);
-                  }
-               }
-               else
-               {
-                  for (int tv = 0; tv < test_vdim; tv++)
-                  {
-                     zero_slab(1);
-                     accumulate_tv(Jx, Jy, Jz, j, tv);
-                     auto f_slab = Reshape(&fhat_storage[0], 1, 1, nq);
-                     map_quadrature_data_to_fields<DIM, MQ1>(
-                        bvtfhat, f_slab, output, output_dtq, smem, tv);
-                  }
-               }
-            }
-         }
-      }
-   }
-}
-
-} // namespace detail
-
-// ────────────────────────────────────────────────────────────────────────────
-// Assemble sparse Jacobian from cached quadrature derivatives (tensor 2D/3D)
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-class DerivativeAssemble
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-
-   const IntegratorContext ctx;
-   const Vector &qp_cache;
-   inputs_t inputs;
-   outputs_t outputs;
-   const bool use_sum_factorization;
-   const std::vector<const DofToQuad *> dtqs;
-   const std::array<DofToQuadMap, n_inputs> input_dtq_maps;
-   const std::array<DofToQuadMap, n_outputs> output_dtq_maps;
-   const std::array<bool, n_inputs> input_is_dependent;
-   const size_t trial_field_uf;
-   const size_t test_field_uf;
-   const ParFiniteElementSpace *test_fes;
-   const ParFiniteElementSpace *trial_fes;
-   const int test_vdim;
-   /// Per-output row geometry of the quadrature point cache. DerivativeSetup
-   /// lays that cache out over every output FieldOperator, so reading it needs
-   /// all of them.
-   const std::array<int, n_outputs> out_vdim;
-   const std::array<int, n_outputs> out_op_dim;
-   const std::array<int, n_outputs> out_offsets;
-   const int output_size_on_qp;
-   const int num_test_dof;
-   const int trial_vdim;
-   const int trial_op_dim;
-   const int num_trial_dof;
-   const int dim, ne, nq, q1d;
-   const int num_trial_dof_1d;
-   const int total_trial_op_dim;
-   mutable Vector inputs_trial_op_dim;
-   mutable Vector Ae_mem;
-
-public:
-   DerivativeAssemble() = delete;
-
-   DerivativeAssemble(IntegratorContext ctx_in,
-                      qfunc_t /*qfunc*/,
-                      inputs_t inputs_in,
-                      outputs_t outputs_in,
-                      const Vector &qp_cache_in):
-      ctx(ctx_in), qp_cache(qp_cache_in), inputs(inputs_in),
-      outputs(outputs_in), use_sum_factorization(
-         [&]
-   {
-      const Element::Type etype =
-         Element::TypeFromGeometry(ctx_in.mesh.GetTypicalElementGeometry());
-      return (etype == Element::QUADRILATERAL || etype == Element::HEXAHEDRON);
-   }()),
-   dtqs(
-      [&]
-   {
-      const DofToQuad::Mode dtq_mode = use_sum_factorization
-      ? DofToQuad::Mode::TENSOR
-      : DofToQuad::Mode::FULL;
-      std::vector<const DofToQuad *> maps;
-      maps.reserve(ctx_in.unionfds.size());
-      for (const auto &field : ctx_in.unionfds)
-      {
-         maps.emplace_back(
-            GetDofToQuad<Entity::Element>(field, ctx_in.ir, dtq_mode));
-      }
-      return maps;
-   }()),
-   input_dtq_maps(create_dtq_maps<Entity::Element>(
-                     inputs,
-                     dtqs,
-                     create_union_field_map_for_dtq(ctx_in, inputs),
-                     ctx_in.unionfds,
-                     ctx_in.ir)),
-   output_dtq_maps(create_dtq_maps<Entity::Element>(
-                      outputs,
-                      dtqs,
-                      create_union_field_map_for_dtq(ctx_in, outputs),
-                      ctx_in.unionfds,
-                      ctx_in.ir)),
-   input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
-   trial_field_uf(find_union_field_index(ctx_in, derivative_id)),
-   test_field_uf(
-      find_union_field_index(ctx_in, get<0>(outputs).GetFieldId())),
-   test_fes(
-      [&]
-   {
-      const auto *fes = std::get_if<const ParFiniteElementSpace *>(
-         &ctx_in.unionfds[test_field_uf].data);
-      MFEM_ASSERT(fes != nullptr && *fes != nullptr,
-                  "LocalQFBackend: test space is not a ParFiniteElementSpace");
-      return *fes;
-   }()),
-   trial_fes(
-      [&]
-   {
-      const auto *fes = std::get_if<const ParFiniteElementSpace *>(
-         &ctx_in.unionfds[trial_field_uf].data);
-      MFEM_ASSERT(fes != nullptr && *fes != nullptr,
-                  "LocalQFBackend: trial space is not a ParFiniteElementSpace");
-      return *fes;
-   }()),
-   // All outputs are attached to the same test field, so vdim is common to
-   // them; only the operator dimension differs, and that lives in out_op_dim.
-   test_vdim(get<0>(outputs).vdim),
-   out_vdim(get_vdim(outputs)),
-   out_op_dim(compute_out_op_dim(outputs)),
-   out_offsets(compute_out_offsets(out_vdim, out_op_dim)),
-   output_size_on_qp(
-      [&]
-   {
-      int s = 0;
-      for_constexpr<n_outputs>([&](auto o) { s += get<o>(outputs).size_on_qp; });
-      return s;
-   }()),
-   num_test_dof(test_fes->GetFE(0)->GetDof()),
-   trial_vdim(compute_trial_vdim(inputs, derivative_id)), trial_op_dim(
-      [&]
-   {
-      int top = 0;
-      for_constexpr<n_inputs>([&](auto i)
-      {
-         if (get<i>(inputs).GetFieldId() == derivative_id)
-         {
-            top = get<i>(inputs).size_on_qp / get<i>(inputs).vdim;
-         }
-      });
-      return top;
-   }()),
-   num_trial_dof(trial_fes->GetFE(0)->GetDof()),
-   dim(ctx_in.mesh.Dimension()), ne(ctx_in.nentities),
-   nq(ctx_in.ir.GetNPoints()), q1d(tensor_1d_size(nq, dim)),
-   num_trial_dof_1d(tensor_1d_size(num_trial_dof, dim)), total_trial_op_dim(
-      [&]
-   {
-      const auto in_qp_sizes =
-      get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs>{});
-      return compute_total_trial_op_dim(
-         inputs, input_is_dependent, in_qp_sizes);
-   }()),
-   inputs_trial_op_dim(), Ae_mem()
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-      MFEM_ASSERT(trial_field_uf != SIZE_MAX,
-                  "DerivativeAssemble: trial field not found in unionfds");
-      MFEM_ASSERT(test_field_uf != SIZE_MAX,
-                  "DerivativeAssemble: test field not found in unionfds");
-
-      MFEM_ASSERT(trial_vdim > 0,
-                  "LocalQFBackend: could not determine trial vdim");
-      MFEM_ASSERT(total_trial_op_dim > 0,
-                  "LocalQFBackend: no dependent inputs found");
-
-      inputs_trial_op_dim.UseDevice(true);
-      inputs_trial_op_dim.SetSize(n_inputs);
-      auto inputs_trial_op_dim_host = inputs_trial_op_dim.HostWrite();
-      for_constexpr<n_inputs>([&](auto i)
-      {
-         inputs_trial_op_dim_host[i] =
-            input_is_dependent[i]
-            ? get<i>(inputs).size_on_qp / get<i>(inputs).vdim
-            : 0;
-      });
-
-      const int elem_mat_size =
-         num_test_dof * test_vdim * num_trial_dof * trial_vdim;
-      Ae_mem.SetSize(elem_mat_size * ne, Device::GetDeviceMemoryType());
-      Ae_mem.UseDevice(true);
-      Ae_mem = 0.0;
-   }
-
-   void operator()(SparseMatrix *&A) const
-   {
-      // Every output is contracted into one element matrix Ae, sized from the
-      // test space of get<0>(outputs), and filled through a single test
-      // ElementRestriction.
-      //
-      // WIP:
-      // This takes care of single-field, multiple-outputs case.
-      // For a multiple fields case, outputs on a second field would need a second
-      // matrix -- the derivative then eould be a block column with one row block per
-      // test space.
-      //
-      // For now we just add a check that all outputs are attached to the same test field, and abort if not.
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         MFEM_VERIFY(get<o>(outputs).GetFieldId() == get<0>(outputs).GetFieldId(),
-                     "DerivativeAssemble: every output FieldOperator has to be "
-                     "attached to the same test field; assembling outputs that "
-                     "span several fields is not supported");
-      });
-
-      if (ctx.attr.Size() == 0) { return; }
-
-      if (!(use_sum_factorization && (dim == 2 || dim == 3)))
-      {
-         MFEM_ABORT("DerivativeAssemble optimized path is implemented "
-                    "for tensor-product 2D/3D elements only");
-      }
-
-      DerivativeAssembleHO::Run(dim,
-                                q1d,
-                                ctx,
-                                qp_cache,
-                                Ae_mem,
-                                inputs,
-                                outputs,
-                                input_dtq_maps,
-                                output_dtq_maps,
-                                out_vdim,
-                                out_op_dim,
-                                out_offsets,
-                                output_size_on_qp,
-                                inputs_trial_op_dim,
-                                test_vdim,
-                                num_test_dof,
-                                num_trial_dof,
-                                num_trial_dof_1d,
-                                trial_vdim,
-                                total_trial_op_dim,
-                                nq,
-                                ne,
-                                q1d,
-                                dim);
-
-      A = new SparseMatrix;
-      A->OverrideSize(test_fes->GetVSize(), trial_fes->GetVSize());
-
-      const auto *test_restr = dynamic_cast<const ElementRestriction *>(
-                                  test_fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC));
-      const auto *trial_restr = dynamic_cast<const ElementRestriction *>(
-                                   trial_fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC));
-      MFEM_VERIFY(test_restr != nullptr && trial_restr != nullptr,
-                  "DerivativeAssemble SparseMatrix assembly requires "
-                  "H1/conforming ElementRestriction spaces");
-
-      test_restr->FillSparseMatrix(Ae_mem, *A, *trial_restr);
-   }
-
-   template<typename backend_t = LocalQFHOBackend<3>, int T_Q1D = 0>
-   static void derivative_assemble_callback(
-      const IntegratorContext &ctx,
-      const Vector &qp_cache,
-      Vector &Ae_mem,
-      const inputs_t &inputs,
-      const outputs_t &outputs,
-      const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
-      const std::array<DofToQuadMap, n_outputs> &output_dtq_maps,
-      const std::array<int, n_outputs> &out_vdim,
-      const std::array<int, n_outputs> &out_op_dim,
-      const std::array<int, n_outputs> &out_offsets,
-      const int output_size_on_qp,
-      const Vector &inputs_trial_op_dim,
-      const int test_vdim,
-      const int num_test_dof,
-      const int num_trial_dof,
-      const int num_trial_dof_1d,
-      const int trial_vdim,
-      const int total_trial_op_dim,
-      const int nq,
-      const int ne,
-      const int q1d,
-      const int dim)
-   {
-      static constexpr int DIM = backend_t::DIM;
-      static constexpr int MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
-      static constexpr int MNQ = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
-
-      MFEM_VERIFY(dim == DIM,
-                  "DerivativeAssemble: mesh dim does not match backend");
-      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-      MFEM_VERIFY(q1d <= MQ1, "q1d exceeds backend MQ1 limit");
-      MFEM_VERIFY(nq <= MNQ,
-                  "DerivativeAssemble: nq exceeds backend quadrature capacity");
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         MFEM_VERIFY(out_op_dim[o] <= DIM,
-                     "DerivativeAssemble: test_op_dim exceeds spatial DIM");
-      });
-      if (ctx.attr.Size() == 0) { return; }
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      const auto qpdc = Reshape(qp_cache.Read(),
-                                nq,
-                                total_trial_op_dim,
-                                trial_vdim,
-                                output_size_on_qp,
-                                ne);
-      const auto itod = Reshape(inputs_trial_op_dim.Read(), n_inputs);
-
-      auto Ae = Reshape(Ae_mem.ReadWrite(),
-                        num_test_dof,
-                        test_vdim,
-                        num_trial_dof,
-                        trial_vdim,
-                        ne);
-
-      dfem::forall(
-         [=] MFEM_HOST_DEVICE(const int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         static constexpr int DIM = backend_t::DIM;
-         static constexpr int MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
-
-         static constexpr int MQN = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
-         static constexpr int fhat_slab_size = MQN * 4;
-
-         MFEM_SHARED typename backend_t::Shared s;
-         // One slab shared by every output. Declaring it inside the templated
-         // per-output kernel allocates one per output instead, and static
-         // shared memory is summed across instantiations on device.
-         MFEM_SHARED real_t fhat_storage[fhat_slab_size];
-
-         // Each output contributes its own rows of the cache, contracted
-         // against its own test basis operation; map_quadrature_data_to_fields
-         // accumulates, so the element matrix is the sum over outputs for the
-         // same field.
-         for_constexpr<n_outputs>([&](auto o)
-         {
-            using output_fop_t = std::decay_t<decltype(get<o>(outputs))>;
-
-            if constexpr (!is_identity_fop_v<output_fop_t>)
-            {
-               // The outputs share fhat_storage, so one has to be done with it
-               // before the next zeroes it.
-               MFEM_SYNC_THREAD;
-               detail::assemble_element_mat_sumfact<DIM, MQ1>(Ae,
-                                                              qpdc,
-                                                              e,
-                                                              itod,
-                                                              inputs,
-                                                              get<o>(outputs),
-                                                              input_dtq_maps,
-                                                              output_dtq_maps[o],
-                                                              out_offsets[o],
-                                                              out_vdim[o],
-                                                              out_op_dim[o],
-                                                              q1d,
-                                                              num_trial_dof_1d,
-                                                              fhat_storage,
-                                                              s);
-            }
-         });
-      },
-      ne,
-      backend_t::thread_blocks(q1d),
-      0,
-      nullptr);
-   }
-
-   using AssembleKernelType =
-      decltype(&DerivativeAssemble::derivative_assemble_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleHO,
-                                     AssembleKernelType,
-                                     (int, int));
-};
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline
-typename DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
-AssembleKernelType
-DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeAssembleHO::Kernel()
-{
-   static_assert(DIM == 2 || DIM == 3);
-   using assemble_t =
-      DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return assemble_t::template derivative_assemble_callback<
-             LocalQFHOBackend<DIM, Q1D>>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline
-typename DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
-AssembleKernelType
-DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeAssembleHO::Fallback(int dim, int q1d)
-{
-   using assemble_t =
-      DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeAssembleHO = typename assemble_t::DerivativeAssembleHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<DerivativeAssembleHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<DerivativeAssembleHO, 3, 8>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_assemble_diagonal.hpp

@@ -1,540 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include <array>
-
-namespace mfem::future::LocalQFImpl
-{
-
-// Assemble diagonal of cached Jacobian (square trial == test, tensor 2D/3D)
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-class DerivativeAssembleDiagonal
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-
-   const IntegratorContext ctx;
-   const Vector &qp_cache;
-   inputs_t inputs;
-   outputs_t outputs;
-   const bool use_sum_factorization;
-   const std::vector<const DofToQuad *> dtqs;
-   const std::array<DofToQuadMap, n_inputs> input_dtq_maps;
-   const std::array<DofToQuadMap, n_outputs> output_dtq_maps;
-   const std::array<bool, n_inputs> input_is_dependent;
-   const size_t trial_field_uf;
-   const size_t test_field_uf;
-   const bool is_square;
-   const int test_vdim;
-   const std::array<int, n_outputs> out_vdim;
-   const std::array<int, n_outputs> out_op_dim;
-   const std::array<int, n_outputs> out_offsets;
-   const int output_size_on_qp;
-   const int num_test_dof;
-   const int num_test_dof_1d;
-   const int trial_vdim;
-   const int total_trial_op_dim;
-   const int num_trial_dof_1d;
-   const int residual_size_on_qp;
-   const int dim, ne, nq, q1d;
-   const std::array<int, n_inputs> inputs_trial_op_dim;
-   mutable Vector Ye_mem;
-
-public:
-   DerivativeAssembleDiagonal() = delete;
-
-   DerivativeAssembleDiagonal(IntegratorContext ctx_in,
-                              qfunc_t /*qfunc*/,
-                              inputs_t inputs_in,
-                              outputs_t outputs_in,
-                              const Vector &qp_cache_in):
-      ctx(ctx_in), qp_cache(qp_cache_in), inputs(inputs_in),
-      outputs(outputs_in), use_sum_factorization(
-         [&]
-   {
-      const Element::Type etype =
-         Element::TypeFromGeometry(ctx_in.mesh.GetTypicalElementGeometry());
-      return (etype == Element::QUADRILATERAL || etype == Element::HEXAHEDRON);
-   }()),
-   dtqs(
-      [&]
-   {
-      const DofToQuad::Mode dtq_mode = use_sum_factorization
-      ? DofToQuad::Mode::TENSOR
-      : DofToQuad::Mode::FULL;
-      std::vector<const DofToQuad *> maps;
-      maps.reserve(ctx_in.unionfds.size());
-      for (const auto &field : ctx_in.unionfds)
-      {
-         maps.emplace_back(
-            GetDofToQuad<Entity::Element>(field, ctx_in.ir, dtq_mode));
-      }
-      return maps;
-   }()),
-   input_dtq_maps(create_dtq_maps<Entity::Element>(
-                     inputs,
-                     dtqs,
-                     create_union_field_map_for_dtq(ctx_in, inputs),
-                     ctx_in.unionfds,
-                     ctx_in.ir)),
-   output_dtq_maps(create_dtq_maps<Entity::Element>(
-                      outputs,
-                      dtqs,
-                      create_union_field_map_for_dtq(ctx_in, outputs),
-                      ctx_in.unionfds,
-                      ctx_in.ir)),
-   input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
-   trial_field_uf(find_union_field_index(ctx_in, derivative_id)),
-   test_field_uf(
-      find_union_field_index(ctx_in, get<0>(outputs).GetFieldId())),
-   is_square(
-      [&]
-   {
-      const auto *test_fes = std::get_if<const ParFiniteElementSpace *>(
-         &ctx_in.unionfds[test_field_uf].data);
-      const auto *trial_fes = std::get_if<const ParFiniteElementSpace *>(
-         &ctx_in.unionfds[trial_field_uf].data);
-      return test_fes && trial_fes && *test_fes && *trial_fes &&
-      (*test_fes == *trial_fes);
-   }()),
-   test_vdim(get<0>(outputs).vdim),
-   out_vdim(get_vdim(outputs_in)),
-   out_op_dim(compute_out_op_dim(outputs_in)),
-   out_offsets(compute_out_offsets(out_vdim, out_op_dim)),
-   output_size_on_qp(
-      [&]
-   {
-      int s = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      { s += get<o>(outputs_in).size_on_qp; });
-      return s;
-   }()), num_test_dof(
-      [&]
-   {
-      const auto *test_fes = std::get_if<const ParFiniteElementSpace *>(
-         &ctx_in.unionfds[test_field_uf].data);
-      MFEM_ASSERT(test_fes != nullptr && *test_fes != nullptr,
-                  "LocalQFBackend: test space is not a ParFiniteElementSpace");
-      return (*test_fes)->GetFE(0)->GetDof();
-   }()),
-   num_test_dof_1d(tensor_1d_size(num_test_dof, ctx_in.mesh.Dimension())),
-   trial_vdim(compute_trial_vdim(inputs, derivative_id)), total_trial_op_dim(
-      [&]
-   {
-      const auto input_size_on_qp =
-      get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs>{});
-      return compute_total_trial_op_dim(
-         inputs, input_is_dependent, input_size_on_qp);
-   }()),
-   num_trial_dof_1d(
-      [&]
-   {
-      const auto *trial_fes = std::get_if<const ParFiniteElementSpace *>(
-         &ctx_in.unionfds[trial_field_uf].data);
-      MFEM_ASSERT(trial_fes != nullptr && *trial_fes != nullptr,
-                  "LocalQFBackend: trial space is not a ParFiniteElementSpace");
-      const int num_trial_dof = (*trial_fes)->GetFE(0)->GetDof();
-      return tensor_1d_size(num_trial_dof, ctx_in.mesh.Dimension());
-   }()),
-   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
-   dim(ctx_in.mesh.Dimension()), ne(ctx_in.nentities),
-   nq(ctx_in.ir.GetNPoints()), q1d(tensor_1d_size(nq, dim)),
-   inputs_trial_op_dim(
-      [&]
-   {
-      std::array<int, n_inputs> itod{};
-      for_constexpr<n_inputs>([&](auto i)
-      {
-         itod[i] = input_is_dependent[i]
-                   ? get<i>(inputs).size_on_qp / get<i>(inputs).vdim
-                   : 0;
-      });
-      return itod;
-   }()),
-   Ye_mem()
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-      MFEM_ASSERT(
-         trial_field_uf != SIZE_MAX,
-         "DerivativeAssembleDiagonal: trial field not found in unionfds");
-      MFEM_ASSERT(
-         test_field_uf != SIZE_MAX,
-         "DerivativeAssembleDiagonal: test field not found in unionfds");
-      MFEM_ASSERT(trial_vdim > 0,
-                  "LocalQFBackend: could not determine trial vdim");
-      MFEM_ASSERT(total_trial_op_dim > 0,
-                  "LocalQFBackend: no dependent inputs found");
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         MFEM_CONTRACT_VAR(o);
-         MFEM_ASSERT(out_vdim[o] == test_vdim,
-                     "DerivativeAssembleDiagonal: all outputs must share the "
-                     "test field vdim");
-      });
-
-      if (is_square)
-      {
-         Ye_mem.SetSize(num_test_dof * test_vdim * ne);
-         Ye_mem.UseDevice(true);
-      }
-   }
-
-   template<typename Backend>
-   void run_kernels() const
-   {
-      Backend::Run(dim,
-                   q1d,
-                   ctx,
-                   qp_cache,
-                   Ye_mem,
-                   inputs,
-                   outputs,
-                   output_dtq_maps,
-                   input_dtq_maps,
-                   test_vdim,
-                   out_op_dim,
-                   out_offsets,
-                   output_size_on_qp,
-                   num_test_dof,
-                   num_test_dof_1d,
-                   trial_vdim,
-                   total_trial_op_dim,
-                   residual_size_on_qp,
-                   inputs_trial_op_dim,
-                   nq,
-                   ne,
-                   q1d,
-                   dim);
-   }
-
-   void operator()(Vector &diag_e) const
-   {
-      if (!is_square) { return; }
-      if (ctx.attr.Size() == 0) { return; }
-
-      if (!(use_sum_factorization && (dim == 2 || dim == 3)))
-      {
-         MFEM_ABORT("DerivativeAssembleDiagonal optimized path is implemented "
-                    "for tensor-product 2D/3D elements only");
-      }
-      MFEM_VERIFY(num_test_dof_1d == num_trial_dof_1d,
-                  "DerivativeAssembleDiagonal requires matching tensor dofs");
-      MFEM_VERIFY(num_test_dof_1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-      MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-      Ye_mem = 0.0;
-
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<DerivativeAssembleDiagonalLO>();
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<DerivativeAssembleDiagonalHO>();
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-
-      diag_e += Ye_mem;
-   }
-
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void derivative_assemble_diagonal_callback(
-      const IntegratorContext &ctx,
-      const Vector &qp_cache,
-      Vector &Ye_mem,
-      const inputs_t &inputs,
-      const outputs_t &outputs,
-      const std::array<DofToQuadMap, n_outputs> &output_dtq_maps,
-      const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
-      const int test_vdim,
-      const std::array<int, n_outputs> &out_op_dim,
-      const std::array<int, n_outputs> &out_offsets,
-      const int output_size_on_qp,
-      const int num_test_dof,
-      const int num_test_dof_1d,
-      const int trial_vdim,
-      const int total_trial_op_dim,
-      const int residual_size_on_qp,
-      const std::array<int, n_inputs> &inputs_trial_op_dim,
-      const int nq,
-      const int ne,
-      const int q1d,
-      const int dim)
-   {
-      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-      if (ctx.attr.Size() == 0) { return; }
-
-      static constexpr bool B2D = backend_t::DIM == 2;
-      static constexpr int MTPB = backend_t::MAX_THREADS_PER_BLOCK();
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      auto cache_tensor = DeviceTensor<3, const real_t>(
-                             qp_cache.Read(), nq, residual_size_on_qp, ne);
-      const int num_dofs_per_elem = num_test_dof * test_vdim;
-      auto Ye = Reshape(Ye_mem.ReadWrite(), num_dofs_per_elem, ne);
-
-      dfem::forall<MTPB>(
-         [=] MFEM_HOST_DEVICE(const int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         // The cache is written with the quadrature index fastest, then the
-         // trial op index, then the (test vdim, test op) rows of all outputs
-         // stacked via out_offsets.
-         auto qpdc = Reshape(&cache_tensor(0, 0, e),
-                             nq,
-                             total_trial_op_dim,
-                             trial_vdim,
-                             output_size_on_qp);
-
-         // Backend-owned shared scratch for the sum-factorized contraction.
-         MFEM_SHARED typename backend_t::Shared s_diag;
-         const int nz_dof = B2D ? 1 : num_test_dof_1d;
-
-         for (int vd = 0; vd < test_vdim; vd++)
-         {
-            auto Y = Reshape(&Ye(vd * num_test_dof, e),
-                             num_test_dof_1d,
-                             num_test_dof_1d,
-                             nz_dof);
-
-            MFEM_FOREACH_THREAD(dz_t, z, nz_dof)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(dy_t, y, num_test_dof_1d)
-               {
-                  MFEM_FOREACH_THREAD_DIRECT(dx_t, x, num_test_dof_1d)
-                  { Y(dx_t, dy_t, dz_t) = 0.0; }
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            // Accumulate every (output o, test op k, dependent input s,
-            // trial op m) block of the cached Jacobian into the diagonal via
-            // the backend driver.
-            for_constexpr<n_outputs>([&](auto o)
-            {
-               using test_fop_t = std::decay_t<decltype(get<o>(outputs))>;
-               const auto &out_dtq = output_dtq_maps[o];
-               const int test_op_dim = out_op_dim[static_cast<int>(o)];
-
-               // Test-basis factor along a spatial axis
-               const auto eval_test =
-                  [&](const int k, const int axis, const int q, const int d)
-               {
-                  const auto &B = out_dtq.B;
-                  const auto &G = out_dtq.G;
-                  if constexpr (is_value_fop<test_fop_t>::value)
-                  {
-                     return (k == 0) ? B(q, 0, d) : 0.0;
-                  }
-                  else if constexpr (is_gradient_fop<test_fop_t>::value)
-                  {
-                     return (k == axis) ? G(q, 0, d) : B(q, 0, d);
-                  }
-                  else
-                  {
-                     return 0.0;
-                  }
-               };
-
-               for (int k = 0; k < test_op_dim; k++)
-               {
-                  const int row =
-                     out_offsets[static_cast<int>(o)] + vd * test_op_dim + k;
-                  int m_offset = 0;
-                  for_constexpr<n_inputs>([&](auto s)
-                  {
-                     using fop_t = std::decay_t<decltype(get<s>(inputs))>;
-                     const int trial_op_dim =
-                        inputs_trial_op_dim[static_cast<int>(s)];
-                     if (trial_op_dim == 0) { return; }
-
-                     const auto &in_dtq = input_dtq_maps[s];
-                     const auto eval_input =
-                        [&](const int m, const int axis, const int q,
-                            const int d)
-                     {
-                        if constexpr (is_value_fop<fop_t>::value)
-                        {
-                           return (m == 0) ? in_dtq.B(q, 0, d) : 0.0;
-                        }
-                        else if constexpr (is_gradient_fop<fop_t>::value)
-                        {
-                           return (m == axis) ? in_dtq.G(q, 0, d)
-                                  : in_dtq.B(q, 0, d);
-                        }
-                        else
-                        {
-                           return 0.0;
-                        }
-                     };
-
-                     for (int m = 0; m < trial_op_dim; m++)
-                     {
-                        const int col = m_offset + m;
-                        backend_t::DiagContract(
-                           s_diag,
-                           num_test_dof_1d,
-                           q1d,
-                           nz_dof,
-                           [&](int axis, int q, int d)
-                        { return eval_test(k, axis, q, d); },
-                        [&](int axis, int q, int d)
-                        { return eval_input(m, axis, q, d); },
-                        [&](int q) { return qpdc(q, col, vd, row); },
-                        [&](int dx, int dy, int dz, real_t u)
-                        { Y(dx, dy, dz) += u; });
-                     }
-                     m_offset += trial_op_dim;
-                  });
-               }
-            });
-         }
-      },
-      ne,
-      backend_t::thread_blocks(std::max(q1d, num_test_dof_1d)),
-      0,
-      nullptr);
-   }
-
-   using DiagonalKernelType =
-      decltype(&DerivativeAssembleDiagonal::
-               derivative_assemble_diagonal_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalLO,
-                                     DiagonalKernelType,
-                                     (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalHO,
-                                     DiagonalKernelType,
-                                     (int, int) );
-};
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeAssembleDiagonal<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::DiagonalKernelType
-       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeAssembleDiagonalLO::Kernel()
-{
-   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
-   using diag_t =
-      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return diag_t::template derivative_assemble_diagonal_callback<
-             LocalQFLOBackend<DIM, Q1D>>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeAssembleDiagonal<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::DiagonalKernelType
-       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeAssembleDiagonalLO::Fallback(int dim, int q1d)
-{
-   using diag_t =
-      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeAssembleDiagonalLO =
-      typename diag_t::DerivativeAssembleDiagonalLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeAssembleDiagonal<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::DiagonalKernelType
-       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeAssembleDiagonalHO::Kernel()
-{
-   using diag_t =
-      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return diag_t::template derivative_assemble_diagonal_callback<
-             LocalQFHOBackend<DIM>,
-             Q1D>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeAssembleDiagonal<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::DiagonalKernelType
-       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeAssembleDiagonalHO::Fallback(int dim, int q1d)
-{
-   using diag_t =
-      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeAssembleDiagonalHO =
-      typename diag_t::DerivativeAssembleDiagonalHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_setup.hpp

@@ -1,600 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include <array>
-
-namespace mfem::future::LocalQFImpl
-{
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-class DerivativeSetup
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-   using args_tuple_t = decay_tuple<qf_param_ts>;
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
-                 "LocalQF: q-function arity must match inputs + outputs");
-
-   qfunc_t qfunc;
-   const inputs_t inputs;
-   const outputs_t outputs;
-   const IntegratorContext ctx;
-   Vector &qp_cache;
-   const std::vector<const DofToQuad *> dtqs;
-   // inputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_inputs> input_dtq;
-   const std::array<size_t, n_inputs> input_idx;
-   const std::array<const real_t *, n_inputs> input_B, input_G;
-   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
-   // Jacobian cache metadata
-   const std::array<bool, n_inputs> input_is_dependent;
-   const std::array<int, n_inputs> input_size_on_qp;
-   const std::array<int, n_outputs> out_vdim;
-   const std::array<int, n_outputs> out_op_dim;
-   const std::array<int, n_outputs> out_offsets;
-   const int output_size_on_qp;
-   const int trial_vdim;
-   const int total_trial_op_dim;
-   const int residual_size_on_qp;
-   // other constants
-   const int dim, ne, nq, q1d;
-
-public:
-   //////////////////////////////////////////////////////////////////
-   DerivativeSetup() = delete;
-
-   DerivativeSetup(IntegratorContext ctx,
-                   qfunc_t qfunc,
-                   inputs_t inputs,
-                   outputs_t outputs,
-                   Vector &qp_cache):
-      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
-      qp_cache(qp_cache), dtqs(make_dtqs(ctx)),
-      input_dtq(create_dtq_maps<Entity::Element>(
-                   inputs,
-                   dtqs,
-                   create_union_field_map_for_dtq(ctx, inputs),
-                   ctx.unionfds,
-                   ctx.ir)),
-      input_idx(create_input_vector_map(ctx, inputs)),
-      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
-      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
-      input_vdim(get_vdim(inputs)),
-      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
-      input_size_on_qp(
-         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
-                           out_vdim(get_vdim(outputs)), out_op_dim(compute_out_op_dim(outputs)),
-                           out_offsets(compute_out_offsets(out_vdim, out_op_dim)), output_size_on_qp(
-                              [&]
-   {
-      int s = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      { s += get<o>(outputs).size_on_qp; });
-      return s;
-   }()),
-   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
-   total_trial_op_dim(compute_total_trial_op_dim(
-                         inputs, input_is_dependent, input_size_on_qp)),
-   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
-   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
-   q1d(tensor_1d_size(nq, dim))
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-
-      qp_cache.SetSize(ne * nq * residual_size_on_qp);
-      qp_cache.UseDevice(true);
-   }
-
-   //////////////////////////////////////////////////////////////////
-   void operator()(const std::vector<Vector *> &xe)
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      // Quadrature index is fastest-varying so that adjacent threads (one per
-      // quadrature point) touch adjacent addresses.
-      auto cache_tensor = DeviceTensor<3, real_t>(
-                             qp_cache.ReadWrite(), nq, residual_size_on_qp, ne);
-
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<DerivativeSetupLO>(xe, cache_tensor);
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<DerivativeSetupHO>(xe, cache_tensor);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename Backend>
-   void run_kernels(const std::vector<Vector *> &xe,
-                    DeviceTensor<3, real_t> &cache_tensor)
-   {
-      Backend::Run(dim,
-                   q1d,
-                   ctx,
-                   qfunc,
-                   // inputs
-                   input_idx,
-                   input_B,
-                   input_G,
-                   input_vdim,
-                   input_d1d,
-                   input_q1d,
-                   input_size_on_qp,
-                   input_is_dependent,
-                   // outputs / cache metadata
-                   out_vdim,
-                   out_op_dim,
-                   out_offsets,
-                   trial_vdim,
-                   total_trial_op_dim,
-                   residual_size_on_qp,
-                   // vectors
-                   xe,
-                   cache_tensor,
-                   // fallback arguments
-                   dim,
-                   q1d);
-   }
-
-   //////////////////////////////////////////////////////////////////
-   /// Zeroes the q-function *output* slots of an argument tuple.
-   ///
-   /// The argument tuple is built once per quadrature point and reused for
-   /// every trial seed. The q-function writes through its output parameters,
-   /// so those slots have to be restored before each call to give every seed
-   /// the same starting state a freshly value-initialized tuple would.
-   static MFEM_HOST_DEVICE inline void reset_output_args(args_tuple_t &args)
-   {
-      for_constexpr<n_outputs>([&](auto oc)
-      {
-         constexpr size_t ao = n_inputs + oc.value;
-         auto &oarg = get<ao>(args);
-         oarg = std::remove_reference_t<decltype(oarg)> {};
-      });
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void
-   derivative_setup_callback(const IntegratorContext &ctx,
-                             qfunc_t &qfunc,
-                             // inputs: idx, B, G, vdim, d1d, q1d
-                             const std::array<size_t, n_inputs> &in_idx,
-                             const std::array<const real_t *, n_inputs> in_B,
-                             const std::array<const real_t *, n_inputs> in_G,
-                             const std::array<int, n_inputs> &in_vdim,
-                             const std::array<int, n_inputs> &in_d1d,
-                             const std::array<int, n_inputs> &in_q1d,
-                             const std::array<int, n_inputs> &in_size_on_qp,
-                             const std::array<bool, n_inputs> &input_dep,
-                             // outputs / cache metadata
-                             const std::array<int, n_outputs> &out_vdim,
-                             const std::array<int, n_outputs> &out_op_dim,
-                             const std::array<int, n_outputs> &out_offsets,
-                             const int trial_vdim,
-                             const int total_trial_op_dim,
-                             const int residual_size_on_qp,
-                             const std::vector<Vector *> &xe,
-                             DeviceTensor<3, real_t> &cache_tensor,
-                             // fallback arguments
-                             const int dim,
-                             const int q1d)
-   {
-      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-      if (ctx.attr.Size() == 0) { return; }
-
-      static constexpr auto B2D = backend_t::DIM == 2;
-      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
-      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
-
-      const int ne = ctx.nentities;
-      MFEM_CONTRACT_VAR(residual_size_on_qp);
-
-      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
-
-      // --------------------------------------------------
-      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = in_idx[i];
-         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else if constexpr (is_weight_fop_v<FOP>)
-         {
-            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
-                        "tensor-product IR expected");
-            in_XE[i] = Reshape(
-                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      dfem::forall<MTPB>(
-         [=] MFEM_HOST_DEVICE(const int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         // -----------------------------------------------
-         // Inputs argument registers + shared memory
-         // -----------------------------------------------
-         input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
-         MFEM_SHARED typename backend_t::Shared smem;
-
-         // -----------------------------------------------
-         // Load primal inputs (rargs) once for this element
-         // -----------------------------------------------
-         for_constexpr<n_inputs>([&](auto ic)
-         {
-            constexpr size_t i = ic.value;
-            const auto &XE = in_XE[i];
-            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
-            const real_t *B = in_B[i], *G = in_G[i];
-            auto &rarg = get<i>(rargs);
-            using XE_t = decltype(XE);
-            using rarg_t = decltype(rarg);
-            using FOP = tuple_element_t<i, inputs_t>;
-            if constexpr (is_value_fop<FOP>::value)
-            {
-               backend_t::template LoadValue<rarg_t, XE_t>(
-                  smem, e, d, q, Q1D, B, XE, rarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
-               using FieldParamT =
-                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-               backend_t::template LoadGradient<RNK, rarg_t, XE_t, FieldParamT>(
-                  smem, e, d, q, q1d, B, G, XE, rarg);
-            }
-            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
-            {
-               // qp values are read directly from in_XE / IR
-            }
-            else
-            {
-               static_assert(false, "Unsupported");
-            }
-         });
-         MFEM_SYNC_THREAD;
-
-         // -----------------------------------------------
-         // Build the primal arguments once per quadrature point, then, for
-         // each trial seed (j, dependent input s, m), differentiate the
-         // q-function with a unit tangent and store the result row in the
-         // cache. Nothing in the primal pull depends on the seed, so the
-         // thread loop is the outermost one here. The seed loops only touch
-         // per-thread state, hence no barrier inside them.
-         // Warning: no 'DIRECT' on the 'Z' direction,
-         // as one backend may need to iterate over it.
-         // -----------------------------------------------
-         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-#ifdef MFEM_USE_ENZYME
-                  args_tuple_t primal_args {};
-                  for_constexpr<n_inputs>([&](auto ic)
-                  {
-                     constexpr size_t i = ic.value;
-                     auto &parg = get<i>(primal_args);
-                     const auto &XE = in_XE[i];
-                     using FOP = tuple_element_t<i, inputs_t>;
-                     using ARG =
-                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
-                     if constexpr (is_identity_fop_v<FOP>)
-                     {
-                        parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-                     }
-                     else if constexpr (is_weight_fop_v<FOP>)
-                     {
-                        parg = XE(qx, qy, qz, 0, 0);
-                     }
-                     else if constexpr (is_value_fop_v<FOP> ||
-                                        is_gradient_fop_v<FOP>)
-                     {
-                        parg = backend_t::template qp_pull<ARG>(
-                           get<i>(rargs), qx, qy, qz);
-                     }
-                     else
-                     {
-                        static_assert(false, "Unsupported");
-                     }
-                  });
-
-                  for (int j = 0; j < trial_vdim; j++)
-                  {
-                     int m_offset = 0;
-                     for_constexpr<n_inputs>([&](auto sc)
-                     {
-                        constexpr size_t s = sc.value;
-                        if (!input_dep[s]) { return; }
-
-                        const int vdim_s = in_vdim[s];
-                        const int op_dim_s = in_size_on_qp[s] / vdim_s;
-
-                        for (int m = 0; m < op_dim_s; m++)
-                        {
-                           const int col_m = m + m_offset;
-
-                           // Enzyme writes through the output slots of the
-                           // primal tuple, so they are reset per seed.
-                           reset_output_args(primal_args);
-
-                           args_tuple_t shadow_args {};
-                           qf_set_value_at(get<s>(shadow_args), j, m, 1.0);
-
-                           call_enzyme_fwddiff(qfunc, primal_args, shadow_args);
-
-                           for_constexpr<n_outputs>([&](auto oc)
-                           {
-                              constexpr size_t o = oc.value, ao = n_inputs + o;
-                              const auto &tangent = get<ao>(shadow_args);
-                              const int tv = out_vdim[o], to = out_op_dim[o];
-                              for (int i = 0; i < tv; i++)
-                              {
-                                 for (int k = 0; k < to; k++)
-                                 {
-                                    const int row = out_offsets[o] + i * to + k;
-                                    const int cache_idx =
-                                       row * trial_vdim * total_trial_op_dim +
-                                       j * total_trial_op_dim + col_m;
-                                    cache_tensor(q, cache_idx, e) =
-                                       qf_value_at(tangent, i, k);
-                                 }
-                              }
-                           });
-                        }
-                        m_offset += op_dim_s;
-                     });
-                  }
-#else  // MFEM_USE_ENZYME
-                  args_tuple_t qargs {};
-                  for_constexpr<n_inputs>([&](auto ic)
-                  {
-                     constexpr size_t i = ic.value;
-                     auto &qarg = get<i>(qargs);
-                     const auto &XE = in_XE[i];
-                     using FOP = tuple_element_t<i, inputs_t>;
-                     using ARG =
-                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
-                     if constexpr (is_identity_fop_v<FOP>)
-                     {
-                        using DT =
-                           typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-                        if constexpr (qf_param_uses_dual_v<DT>)
-                        {
-                           qarg = backend_t::template identity_qp_pull_dual<DT>(
-                              false, XE, XE, qx, qy, qz, e);
-                        }
-                        else
-                        {
-                           qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-                        }
-                     }
-                     else if constexpr (is_weight_fop_v<FOP>)
-                     {
-                        qarg = XE(qx, qy, qz, 0, 0);
-                     }
-                     else if constexpr (is_value_fop_v<FOP> ||
-                                        is_gradient_fop_v<FOP>)
-                     {
-                        qarg = backend_t::template qp_pull<ARG>(
-                           get<i>(rargs), qx, qy, qz);
-                     }
-                     else
-                     {
-                        static_assert(false, "Unsupported");
-                     }
-                  });
-
-                  for (int j = 0; j < trial_vdim; j++)
-                  {
-                     int m_offset = 0;
-                     for_constexpr<n_inputs>([&](auto sc)
-                     {
-                        constexpr size_t s = sc.value;
-                        if (!input_dep[s]) { return; }
-
-                        const int vdim_s = in_vdim[s];
-                        const int op_dim_s = in_size_on_qp[s] / vdim_s;
-
-                        for (int m = 0; m < op_dim_s; m++)
-                        {
-                           const int col_m = m + m_offset;
-
-                           // The q-function writes through the output slots,
-                           // so they are reset per seed.
-                           reset_output_args(qargs);
-
-                           qf_set_gradient_at(get<s>(qargs), j, m, 1.0);
-
-                           call_qfunc_no_move(qfunc, qargs);
-
-                           for_constexpr<n_outputs>([&](auto oc)
-                           {
-                              constexpr size_t o = oc.value, ao = n_inputs + o;
-                              const auto &tangent = get<ao>(qargs);
-                              const int tv = out_vdim[o], to = out_op_dim[o];
-                              for (int i = 0; i < tv; i++)
-                              {
-                                 for (int k = 0; k < to; k++)
-                                 {
-                                    const int row = out_offsets[o] + i * to + k;
-                                    const int cache_idx =
-                                       row * trial_vdim * total_trial_op_dim +
-                                       j * total_trial_op_dim + col_m;
-                                    cache_tensor(q, cache_idx, e) =
-                                       qf_gradient_at(tangent, i, k);
-                                 }
-                              }
-                           });
-
-                           // Clear the seed so the next direction starts from
-                           // the pristine (zero-tangent) primal state.
-                           qf_set_gradient_at(get<s>(qargs), j, m, 0.0);
-                        }
-                        m_offset += op_dim_s;
-                     });
-                  }
-#endif // MFEM_USE_ENZYME
-               }
-            }
-         }
-      },
-      ne,
-      backend_t::thread_blocks(
-         compute_kernel_thread_1d<inputs_t>(q1d, in_d1d)),
-      0,
-      nullptr);
-   }
-
-   using SetupKernelType =
-      decltype(&DerivativeSetup::derivative_setup_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupLO,
-                                     SetupKernelType,
-                                     (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupHO,
-                                     SetupKernelType,
-                                     (int, int) );
-};
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-SetupKernelType
-DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeSetupLO::Kernel()
-{
-   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
-   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return setup_t::template derivative_setup_callback<
-             LocalQFLOBackend<DIM, Q1D>>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-SetupKernelType
-DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeSetupLO::Fallback(int dim, int q1d)
-{
-   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeSetupLO = typename setup_t::DerivativeSetupLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<DerivativeSetupLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<DerivativeSetupLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-SetupKernelType
-DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeSetupHO::Kernel()
-{
-   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return setup_t::template derivative_setup_callback<LocalQFHOBackend<DIM>,
-                                                      Q1D>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-SetupKernelType
-DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeSetupHO::Fallback(int dim, int q1d)
-{
-   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeSetupHO = typename setup_t::DerivativeSetupHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<DerivativeSetupHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<DerivativeSetupHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/fwddiff_transformer.hpp

@@ -1,277 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../util.hpp"
-
-#ifdef MFEM_USE_ENZYME
-
-namespace mfem::future
-{
-
-template <typename T> struct function_traits;
-
-template <typename R, typename C, typename... Args>
-struct function_traits<R (C::*)(Args...) const>
-{
-   using primal_return_type = R;
-   using args_tuple = std::tuple<Args...>;
-   static constexpr size_t arity = sizeof...(Args);
-};
-
-// Component count and writable counterpart of a per-point argument, which
-// is either a tensor or a plain scalar.
-template <typename Arg> struct qp_traits
-{
-   static_assert(std::is_arithmetic_v<Arg>,
-                 "per-point arguments must be tensors or scalars");
-   using view_type = Arg;
-   static constexpr int components = 1;
-};
-
-template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
-{
-   using view_type = tensor<std::remove_const_t<T>, Sizes...>;
-   static constexpr int components = (Sizes * ... * 1);
-};
-
-// Generic FwdDiff: computes the full gradient of a pointwise qfunction at a
-// single quadrature point.
-//
-// active_input is the index of the argument to differentiate with respect
-// to; active_output is the index of the (scalar) output argument whose
-// derivative is taken.
-//
-// operator()(args...) takes the qfunction's arguments, except that the
-// active output position receives the *gradient*, shaped like the active
-// input (d(output)/d(input component)).
-//
-// Per input component d, one enzyme fwddiff call with the one-hot seed e_d
-// in the input tangent yields gradient entry d, i.e. grad_components enzyme
-// calls per point. The output tangent is not pre-zeroed, so the qfunction
-// must fully write its output (tangent stores overwrite).
-//
-// operator() is MFEM_HOST_DEVICE, stateless and allocation-free, so it is
-// callable inside a GPU kernel: seed, primal scratch and tangent all live
-// on the stack.
-template <typename Func, size_t active_input, size_t active_output>
-struct FwdDiff
-{
-   using traits = function_traits<decltype(&Func::operator())>;
-   using args_tuple = typename traits::args_tuple;
-   static constexpr size_t arity = traits::arity;
-
-   static_assert(std::is_void_v<typename traits::primal_return_type>,
-                 "FwdDiff only supports primal functions with void return type");
-   static_assert(active_input < arity && active_output < arity,
-                 "active argument indices must be within the function arity");
-   static_assert(active_input != active_output,
-                 "active input and output must be different arguments");
-
-   using input_type =
-      std::decay_t<std::tuple_element_t<active_input, args_tuple>>;
-   using output_type =
-      std::decay_t<std::tuple_element_t<active_output, args_tuple>>;
-
-   using grad_type = typename qp_traits<input_type>::view_type;
-   using output_view = typename qp_traits<output_type>::view_type;
-
-   static constexpr int grad_components = qp_traits<input_type>::components;
-
-   static_assert(qp_traits<output_type>::components == 1,
-                 "gradient output requires a scalar output");
-
-   // Signature of the differentiated qfunction: the primal arguments, with
-   // the Active output slot receiving the (writable) gradient instead.
-   // Exposed through create_function_signature below so that
-   // DifferentiableOperator can deduce the parameter types, which it cannot
-   // do from the variadic operator().
-   template <size_t I>
-   using qf_arg_t = std::conditional_t<I == active_output, grad_type &,
-         std::tuple_element_t<I, args_tuple>>;
-
-   template <size_t... Is>
-   static FunctionSignature<void(qf_arg_t<Is>...)>
-   signature_impl(std::index_sequence<Is...>);
-
-   using signature =
-      decltype(signature_impl(std::make_index_sequence<arity> {}));
-
-   // d-th scalar of a per-point argument in flat row-major order, regardless
-   // of rank, built on the native operator[] (tensor has no flat-index
-   // accessor; flatten() returns a copy, so it cannot be written through).
-   MFEM_HOST_DEVICE static double &component(double &t, int) { return t; }
-
-   template <typename T, int n0, int... n>
-   MFEM_HOST_DEVICE static T &component(tensor<T, n0, n...> &t, int d)
-   {
-      if constexpr (sizeof...(n) == 0)
-      {
-         return t[d];
-      }
-      else
-      {
-         constexpr int stride = (n * ... * 1);
-         return component(t[d / stride], d % stride);
-      }
-   }
-
-   // Plain function with the qfunction's exact (reference) signature, so it
-   // can be handed to Enzyme as a function pointer; references are pointers
-   // to Enzyme, so primal arguments and shadows are passed by address below.
-   template <size_t... Is>
-   MFEM_HOST_DEVICE static void
-   static_call(std::tuple_element_t<Is, args_tuple>... args)
-   {
-      Func{}(args...);
-   }
-
-   template <size_t... Is>
-   static constexpr auto fn_ptr(std::index_sequence<Is...>)
-   {
-      return &static_call<Is...>;
-   }
-   static constexpr auto fn()
-   {
-      return fn_ptr(std::make_index_sequence<arity> {});
-   }
-
-   // Writable, zero-initialized scratch with the shape of argument I, used
-   // as its enzyme shadow.
-   template <size_t I>
-   using shadow_t = typename qp_traits<
-                    std::decay_t<std::tuple_element_t<I, args_tuple>>>::view_type;
-
-   template <size_t... Is>
-   MFEM_HOST_DEVICE static auto make_shadows(std::index_sequence<Is...>)
-   {
-      return mfem::future::make_tuple(shadow_t<Is> {}...);
-   }
-
-   template <typename Shadows, size_t... Is>
-   MFEM_HOST_DEVICE static auto make_shadow_ptrs(Shadows &shadows,
-                                                 std::index_sequence<Is...>)
-   {
-      return mfem::future::make_tuple(&mfem::future::get<int(Is)>(shadows)...);
-   }
-
-   // The caller's argument pointers, except the active output slot, which
-   // points to scalar scratch: the caller's slot holds the gradient, while
-   // the primal function writes its scalar output there.
-   template <size_t I, typename Ptrs>
-   MFEM_HOST_DEVICE static auto primal_ptr(Ptrs &ptrs, output_view &primal)
-   {
-      if constexpr (I == active_output) { return &primal; }
-      else { return mfem::future::get<int(I)>(ptrs); }
-   }
-
-   template <typename Ptrs, size_t... Is>
-   MFEM_HOST_DEVICE static auto make_primal_ptrs(Ptrs &ptrs,
-                                                 output_view &primal,
-                                                 std::index_sequence<Is...>)
-   {
-      return mfem::future::make_tuple(primal_ptr<Is>(ptrs, primal)...);
-   }
-
-   // Single flat enzyme call. The activity markers must appear directly in
-   // the __enzyme_fwddiff argument list — Enzyme cannot trace markers that
-   // were forwarded through function parameters (e.g. at -O0, where nothing
-   // is inlined). Every argument is therefore enzyme_dup'd in one sticky
-   // group; Const arguments simply carry a zero tangent, which is equivalent
-   // to marking them enzyme_const.
-   //
-   // always_inline is load-bearing: when FwdDiff is itself differentiated
-   // (second derivatives, forward-over-forward), Enzyme only recognizes this
-   // nested __enzyme_fwddiff call if it sits at most one call level below
-   // the function handed to the outer __enzyme_fwddiff. Without inlining
-   // (-O0) it sits two levels down (wrapper -> operator() -> call_enzyme)
-   // and the outer pass treats it as a regular call: the activity marker
-   // ints then receive undef shadows, which misaligns the argument pairing
-   // (observed as "cannot compute with global variable that doesn't have
-   // marked shadow global" at compile time or null-shadow segfaults at
-   // runtime). The always-inliner runs even at -O0, hoisting this call into
-   // operator() where the nested handling applies.
-   template <typename PrimalPtrs, typename ShadowPtrs, size_t... Is>
-   MFEM_FUTURE_ALWAYS_INLINE
-   MFEM_HOST_DEVICE static void call_enzyme(PrimalPtrs &primal_ptrs,
-                                            ShadowPtrs &shadow_ptrs,
-                                            std::index_sequence<Is...>)
-   {
-      __enzyme_fwddiff<void>(fn(), enzyme_dup,
-                             mfem::future::get<int(Is)>(primal_ptrs)...,
-                             enzyme_interleave,
-                             mfem::future::get<int(Is)>(shadow_ptrs)...,
-                             enzyme_runtime_activity);
-   }
-
-   template <typename... Args>
-   MFEM_HOST_DEVICE void operator()(Args &&...args) const
-   {
-      static_assert(sizeof...(Args) == arity, "Wrong number of arguments");
-      auto ptrs = mfem::future::make_tuple(&args...);
-
-      auto &grad = *mfem::future::get<int(active_output)>(ptrs);
-      static_assert(std::is_same_v<std::decay_t<decltype(grad)>, grad_type>,
-                    "gradient argument must be shaped like the Active input "
-                    "(with writable scalars)");
-
-      constexpr auto seq = std::make_index_sequence<arity> {};
-
-      output_view primal{};
-      auto primal_ptrs = make_primal_ptrs(ptrs, primal, seq);
-
-      auto shadows = make_shadows(seq);
-      auto shadow_ptrs = make_shadow_ptrs(shadows, seq);
-      auto &seed = mfem::future::get<int(active_input)>(shadows);
-      auto &tangent = mfem::future::get<int(active_output)>(shadows);
-
-      // One enzyme call per input component d: seed e_d in the input tangent
-      // and read gradient entry d off the output tangent.
-      for (int d = 0; d < grad_components; d++)
-      {
-         component(seed, d) = 1.0;
-         call_enzyme(primal_ptrs, shadow_ptrs, seq);
-         component(grad, d) = component(tangent, 0);
-         component(seed, d) = 0.0;
-      }
-   }
-
-   static void print() { print_impl(std::make_index_sequence<arity> {}); }
-
-   template <size_t... Is> static void print_impl(std::index_sequence<Is...>)
-   {
-      mfem::out << "for d in [0, " << grad_components
-                << "): __enzyme_fwddiff<void>(fptr, enzyme_dup";
-      ((mfem::out << ", "
-        << get_type_name<std::tuple_element_t<Is, args_tuple>>()),
-       ...);
-      mfem::out << ", enzyme_interleave";
-      (([&]
-      {
-         if constexpr (Is == active_input) { mfem::out << ", e_d seed"; }
-         else if constexpr (Is == active_output) { mfem::out << ", tangent out"; }
-         else { mfem::out << ", zero tangent"; }
-      }()),
-      ...);
-      mfem::out << ")\n";
-   }
-};
-
-template <typename Func, size_t active_input, size_t active_output>
-struct create_function_signature<FwdDiff<Func, active_input, active_output>>
-{
-   using type =
-      typename FwdDiff<Func, active_input, active_output>::signature;
-};
-
-} // namespace mfem::future
-
-#endif // MFEM_USE_ENZYME

fem/dfem/backends/local_qf/kernels.cpp

@@ -1,83 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-// Explicit instantiation of the local Q-function backend device code
-
-#include "../../../../config/config.hpp"
-
-#ifdef MFEM_USE_MPI
-
-#include "kernels.hpp"
-
-namespace mfem::future
-{
-
-// ────────────────────────────────────────────────────────────────────────────
-// Low-order backends instantiations for (DIM, Q1D)
-// ────────────────────────────────────────────────────────────────────────────
-template struct lo_ker_backend<2, 2>;
-template struct lo_ker_backend<2, 3>;
-template struct lo_ker_backend<2, 4>;
-template struct lo_ker_backend<2, 5>;
-template struct lo_ker_backend<2, 6>;
-template struct lo_ker_backend<2, 7>;
-template struct lo_ker_backend<2, 8>;
-
-template struct lo_ker_backend<3, 2>;
-template struct lo_ker_backend<3, 3>;
-template struct lo_ker_backend<3, 4>;
-template struct lo_ker_backend<3, 5>;
-template struct lo_ker_backend<3, 6>;
-template struct lo_ker_backend<3, 7>;
-template struct lo_ker_backend<3, 8>;
-
-template struct LocalQFLOBackend<2, 2>;
-template struct LocalQFLOBackend<2, 3>;
-template struct LocalQFLOBackend<2, 4>;
-template struct LocalQFLOBackend<2, 5>;
-template struct LocalQFLOBackend<2, 6>;
-template struct LocalQFLOBackend<2, 7>;
-template struct LocalQFLOBackend<2, 8>;
-
-template struct LocalQFLOBackend<3, 2>;
-template struct LocalQFLOBackend<3, 3>;
-template struct LocalQFLOBackend<3, 4>;
-template struct LocalQFLOBackend<3, 5>;
-template struct LocalQFLOBackend<3, 6>;
-template struct LocalQFLOBackend<3, 7>;
-template struct LocalQFLOBackend<3, 8>;
-
-// ────────────────────────────────────────────────────────────────────────────
-// High-order backends instantiations for (DIM, Q1D)
-// ────────────────────────────────────────────────────────────────────────────
-template struct ho_ker_backend<2, 8>;
-template struct ho_ker_backend<2, 10>;
-template struct ho_ker_backend<2, 12>;
-template struct ho_ker_backend<2, 16>;
-
-template struct ho_ker_backend<3, 8>;
-template struct ho_ker_backend<3, 10>;
-template struct ho_ker_backend<3, 12>;
-template struct ho_ker_backend<3, 16>;
-
-template struct LocalQFHOBackend<2, 8>;
-template struct LocalQFHOBackend<2, 10>;
-template struct LocalQFHOBackend<2, 12>;
-template struct LocalQFHOBackend<2, 16>;
-
-template struct LocalQFHOBackend<3, 8>;
-template struct LocalQFHOBackend<3, 10>;
-template struct LocalQFHOBackend<3, 12>;
-template struct LocalQFHOBackend<3, 16>;
-
-} // namespace mfem::future
-
-#endif // MFEM_USE_MPI

fem/dfem/backends/local_qf/kernels.hpp

@@ -1,77 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "kernels_lo.hpp" // IWYU pragma: export
-#include "kernels_ho.hpp" // IWYU pragma: export
-
-namespace mfem::future
-{
-
-// ────────────────────────────────────────────────────────────────────────────
-// Low-order backends instantiations for (DIM, Q1D)
-// ────────────────────────────────────────────────────────────────────────────
-extern template struct lo_ker_backend<2, 2>;
-extern template struct lo_ker_backend<2, 3>;
-extern template struct lo_ker_backend<2, 4>;
-extern template struct lo_ker_backend<2, 5>;
-extern template struct lo_ker_backend<2, 6>;
-extern template struct lo_ker_backend<2, 7>;
-extern template struct lo_ker_backend<2, 8>;
-
-extern template struct lo_ker_backend<3, 2>;
-extern template struct lo_ker_backend<3, 3>;
-extern template struct lo_ker_backend<3, 4>;
-extern template struct lo_ker_backend<3, 5>;
-extern template struct lo_ker_backend<3, 6>;
-extern template struct lo_ker_backend<3, 7>;
-extern template struct lo_ker_backend<3, 8>;
-
-extern template struct LocalQFLOBackend<2, 2>;
-extern template struct LocalQFLOBackend<2, 3>;
-extern template struct LocalQFLOBackend<2, 4>;
-extern template struct LocalQFLOBackend<2, 5>;
-extern template struct LocalQFLOBackend<2, 6>;
-extern template struct LocalQFLOBackend<2, 7>;
-extern template struct LocalQFLOBackend<2, 8>;
-
-extern template struct LocalQFLOBackend<3, 2>;
-extern template struct LocalQFLOBackend<3, 3>;
-extern template struct LocalQFLOBackend<3, 4>;
-extern template struct LocalQFLOBackend<3, 5>;
-extern template struct LocalQFLOBackend<3, 6>;
-extern template struct LocalQFLOBackend<3, 7>;
-extern template struct LocalQFLOBackend<3, 8>;
-
-// ────────────────────────────────────────────────────────────────────────────
-// High-order backends instantiations for (DIM, Q1D)
-// ────────────────────────────────────────────────────────────────────────────
-extern template struct ho_ker_backend<2, 8>;
-extern template struct ho_ker_backend<2, 10>;
-extern template struct ho_ker_backend<2, 12>;
-extern template struct ho_ker_backend<2, 16>;
-
-extern template struct ho_ker_backend<3, 8>;
-extern template struct ho_ker_backend<3, 10>;
-extern template struct ho_ker_backend<3, 12>;
-extern template struct ho_ker_backend<3, 16>;
-
-extern template struct LocalQFHOBackend<2, 8>;
-extern template struct LocalQFHOBackend<2, 10>;
-extern template struct LocalQFHOBackend<2, 12>;
-extern template struct LocalQFHOBackend<2, 16>;
-
-extern template struct LocalQFHOBackend<3, 8>;
-extern template struct LocalQFHOBackend<3, 10>;
-extern template struct LocalQFHOBackend<3, 12>;
-extern template struct LocalQFHOBackend<3, 16>;
-
-} // namespace mfem::future

fem/dfem/backends/local_qf/kernels_ho.hpp

@@ -1,803 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../../kernels.hpp"
-namespace ker = mfem::kernels::internal;
-
-#include "../../util.hpp" // for ThreadBlocks
-#include "util.hpp"
-
-namespace mfem::future
-{
-
-// ────────────────────────────────────────────────────────────────────────────
-inline constexpr int LocalQFHOBackendMQ1() { return 16; }
-
-// ────────────────────────────────────────────────────────────────────────────
-/// Register type for one HO q-function parameter
-template<typename KerOps, typename T, int rank = qf_param_shape<T>::rank>
-struct ho_qreg;
-
-template<typename KerOps, typename T>
-struct ho_qreg<KerOps, T, 0>
-{
-   using type = typename KerOps::template val_reg_t<1>;
-};
-
-template<typename KerOps, typename T>
-struct ho_qreg<KerOps, T, 1>
-{
-   static constexpr int e0 = qf_param_shape<T>::extents[0];
-   using type = typename KerOps::template val_reg_t<e0>;
-};
-
-template<typename KerOps, typename T>
-struct ho_qreg<KerOps, T, 2>
-{
-   static constexpr int VDIM = qf_param_shape<T>::extents[0];
-   static constexpr int SDIM = qf_param_shape<T>::extents[1];
-   using type = typename KerOps::template del_reg_t<VDIM, SDIM>;
-};
-
-template<typename KerOps, typename T>
-using ho_qreg_t = typename ho_qreg<KerOps, T>::type;
-
-// ────────────────────────────────────────────────────────────────────────────
-namespace hok
-{
-
-/// Load one quadrature-point value
-template<int DIM, typename T, typename Reg>
-MFEM_HOST_DEVICE inline auto load_at(Reg &reg, int qx, int qy, int qz)
-{
-   static_assert(DIM == 2 || DIM == 3);
-   constexpr int RNK = qf_param_shape<T>::rank;
-   if constexpr (DIM == 2)
-   {
-      MFEM_CONTRACT_VAR(qz);
-      if constexpr (RNK == 0) { return T{ reg(0, qy, qx) }; }
-      else if constexpr (RNK == 1)
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         T t{};
-         MFEM_UNROLL(e0)
-         for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qy, qx); }
-         return t;
-      }
-      else
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         constexpr int e1 = qf_param_shape<T>::extents[1];
-         T t;
-         MFEM_UNROLL(e0)
-         for (int i = 0; i < e0; ++i)
-         {
-            MFEM_UNROLL(e1)
-            for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qy, qx); }
-         }
-         return t;
-      }
-   }
-   else
-   {
-      if constexpr (RNK == 0) { return T{ reg(0, qz, qy, qx) }; }
-      else if constexpr (RNK == 1)
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         T t{};
-         MFEM_UNROLL(e0)
-         for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qz, qy, qx); }
-         return t;
-      }
-      else
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         constexpr int e1 = qf_param_shape<T>::extents[1];
-         T t;
-         MFEM_UNROLL(e0)
-         for (int i = 0; i < e0; ++i)
-         {
-            MFEM_UNROLL(e1)
-            for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qz, qy, qx); }
-         }
-         return t;
-      }
-   }
-}
-
-template<bool tangent, typename U>
-MFEM_HOST_DEVICE inline auto qp_store(const U &v)
-{
-   if constexpr (tangent) { return qf_store_gradient(v); }
-   else
-   {
-      return qf_store_value(v);
-   }
-}
-
-// Store primal value or dual tangent at one quadrature point
-template<int DIM, typename T, typename Reg, bool tangent>
-MFEM_HOST_DEVICE inline void
-store_at(Reg &reg, int qx, int qy, int qz, const T &out)
-{
-   static_assert(DIM == 2 || DIM == 3);
-   constexpr int RNK = qf_param_shape<T>::rank;
-   if constexpr (DIM == 2)
-   {
-      MFEM_CONTRACT_VAR(qz);
-      if constexpr (RNK == 0) { reg(0, qy, qx) = qp_store<tangent>(out); }
-      else if constexpr (RNK == 1)
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         MFEM_UNROLL(e0)
-         for (int dd = 0; dd < e0; ++dd)
-         {
-            reg(dd, qy, qx) = qp_store<tangent>(out(dd));
-         }
-      }
-      else
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         constexpr int e1 = qf_param_shape<T>::extents[1];
-         MFEM_UNROLL(e0)
-         for (int i = 0; i < e0; ++i)
-         {
-            MFEM_UNROLL(e1)
-            for (int j = 0; j < e1; ++j)
-            {
-               reg(i, j, qy, qx) = qp_store<tangent>(out(i, j));
-            }
-         }
-      }
-   }
-   else
-   {
-      if constexpr (RNK == 0) { reg(0, qz, qy, qx) = qp_store<tangent>(out); }
-      else if constexpr (RNK == 1)
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         MFEM_UNROLL(e0)
-         for (int dd = 0; dd < e0; ++dd)
-         {
-            reg(dd, qz, qy, qx) = qp_store<tangent>(out(dd));
-         }
-      }
-      else
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         constexpr int e1 = qf_param_shape<T>::extents[1];
-         MFEM_UNROLL(e0)
-         for (int i = 0; i < e0; ++i)
-         {
-            MFEM_UNROLL(e1)
-            for (int j = 0; j < e1; ++j)
-            {
-               reg(i, j, qz, qy, qx) = qp_store<tangent>(out(i, j));
-            }
-         }
-      }
-   }
-}
-
-// Pull primal/tangent pair into a dual q-function argument
-template<int DIM, typename T, typename Reg>
-MFEM_HOST_DEVICE inline auto
-pull_directional(Reg &preg, Reg &sreg, int qx, int qy, int qz, bool dependent)
-{
-   if constexpr (!qf_param_uses_dual_v<T>)
-   {
-      return load_at<DIM, T>(preg, qx, qy, qz);
-   }
-   else
-   {
-      if (!dependent) { return load_at<DIM, T>(preg, qx, qy, qz); }
-      constexpr int RNK = qf_param_shape<T>::rank;
-      if constexpr (DIM == 2)
-      {
-         MFEM_CONTRACT_VAR(qz);
-         if constexpr (RNK == 0)
-         {
-            return T{ preg(0, qy, qx), sreg(0, qy, qx) };
-         }
-         else if constexpr (RNK == 1)
-         {
-            constexpr int e0 = qf_param_shape<T>::extents[0];
-            T t{};
-            MFEM_UNROLL(e0)
-            for (int dd = 0; dd < e0; ++dd)
-            {
-               t(dd) = { preg(dd, qy, qx), sreg(dd, qy, qx) };
-            }
-            return t;
-         }
-         else
-         {
-            constexpr int e0 = qf_param_shape<T>::extents[0];
-            constexpr int e1 = qf_param_shape<T>::extents[1];
-            T t;
-            MFEM_UNROLL(e0)
-            for (int i = 0; i < e0; ++i)
-            {
-               MFEM_UNROLL(e1)
-               for (int j = 0; j < e1; ++j)
-               {
-                  t(i, j) = { preg(i, j, qy, qx), sreg(i, j, qy, qx) };
-               }
-            }
-            return t;
-         }
-      }
-      else
-      {
-         if constexpr (RNK == 0)
-         {
-            return T{ preg(0, qz, qy, qx), sreg(0, qz, qy, qx) };
-         }
-         else if constexpr (RNK == 1)
-         {
-            constexpr int e0 = qf_param_shape<T>::extents[0];
-            T t{};
-            MFEM_UNROLL(e0)
-            for (int dd = 0; dd < e0; ++dd)
-            {
-               t(dd) = { preg(dd, qz, qy, qx), sreg(dd, qz, qy, qx) };
-            }
-            return t;
-         }
-         else
-         {
-            constexpr int e0 = qf_param_shape<T>::extents[0];
-            constexpr int e1 = qf_param_shape<T>::extents[1];
-            T t;
-            MFEM_UNROLL(e0)
-            for (int i = 0; i < e0; ++i)
-            {
-               MFEM_UNROLL(e1)
-               for (int j = 0; j < e1; ++j)
-               {
-                  t(i, j) = { preg(i, j, qz, qy, qx), sreg(i, j, qz, qy, qx) };
-               }
-            }
-            return t;
-         }
-      }
-   }
-}
-
-} // namespace hok
-
-// ────────────────────────────────────────────────────────────────────────────
-/// HO tensor-product kernels
-template<int T_DIM, int MQ1>
-struct ho_ker_backend
-{
-   static constexpr int DIM = T_DIM;
-   static_assert(DIM == 2 || DIM == 3);
-
-   template<int VDIM>
-   using val_reg_t = std::conditional_t<(DIM == 2),
-         ker::v_regs2d_t<VDIM, MQ1>,
-         ker::v_regs3d_t<VDIM, MQ1>>;
-
-   template<int VDIM, int SDIM>
-   using del_reg_t = std::conditional_t<(DIM == 2),
-         ker::vd_regs2d_t<VDIM, SDIM, MQ1>,
-         ker::vd_regs3d_t<VDIM, SDIM, MQ1>>;
-
-   struct Shared
-   {
-      real_t M[MQ1][MQ1], B[MQ1][MQ1], G[MQ1][MQ1];
-   };
-
-   template<typename XE_t, typename Dofs>
-   static MFEM_HOST_DEVICE void
-   load_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
-   {
-      if constexpr (DIM == 2) { ker::LoadDofs2d(e, d, XE, dofs); }
-      else
-      {
-         ker::LoadDofs3d(e, d, XE, dofs);
-      }
-   }
-
-   template<int VDIM, int SDIM, typename XE_t, typename Dofs>
-   static MFEM_HOST_DEVICE void
-   load_grad_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
-   {
-      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
-      load_dofs(e, d, XE, dofs);
-   }
-
-   template<typename Smem, typename Dofs, typename ArgReg>
-   static MFEM_HOST_DEVICE void
-   eval_value(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
-   {
-      if constexpr (DIM == 2) { ker::Eval2d(d, q, s.M, s.B, dofs, rarg); }
-      else
-      {
-         ker::Eval3d(d, q, s.M, s.B, dofs, rarg);
-      }
-   }
-
-   template<int VDIM, int SDIM, typename Smem, typename Dofs, typename ArgReg>
-   static MFEM_HOST_DEVICE void
-   grad(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
-   {
-      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
-      if constexpr (DIM == 2) { ker::Grad2d(d, q, s.M, s.B, s.G, dofs, rarg); }
-      else
-      {
-         ker::Grad3d(d, q, s.M, s.B, s.G, dofs, rarg);
-      }
-   }
-
-   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
-   static MFEM_HOST_DEVICE void write_value(const int d,
-                                            const int q,
-                                            const int e,
-                                            Smem &s,
-                                            ArgReg &rarg,
-                                            Dofs &dofs,
-                                            YE_t &YE)
-   {
-      if constexpr (DIM == 2)
-      {
-         ker::EvalTranspose2d(d, q, s.M, s.B, rarg, dofs);
-         ker::WriteDofs2d(e, d, dofs, YE);
-      }
-      else
-      {
-         ker::EvalTranspose3d(d, q, s.M, s.B, rarg, dofs);
-         ker::WriteDofs3d(e, d, dofs, YE);
-      }
-   }
-
-   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
-   static MFEM_HOST_DEVICE void write_gradient_2d(const int d,
-                                                  const int q,
-                                                  const int e,
-                                                  Smem &s,
-                                                  ArgReg &rarg,
-                                                  Dofs &dofs,
-                                                  YE_t &YE)
-   {
-      ker::GradTranspose2d(d, q, s.M, s.B, s.G, rarg, dofs);
-      ker::WriteDofs2d(e, d, dofs, YE);
-   }
-
-   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
-   static MFEM_HOST_DEVICE void write_gradient_3d(const int d,
-                                                  const int q,
-                                                  const int e,
-                                                  Smem &s,
-                                                  ArgReg &rarg,
-                                                  Dofs &dofs,
-                                                  YE_t &YE)
-   {
-      ker::GradTranspose3d(d, q, s.M, s.B, s.G, rarg, dofs);
-      ker::WriteDofs3d(e, d, dofs, YE);
-   }
-
-   template<int VDIM,
-            int SDIM,
-            typename Smem,
-            typename Dofs,
-            typename ArgReg,
-            typename YE_t>
-   static MFEM_HOST_DEVICE void write_gradient(const int d,
-                                               const int q,
-                                               const int e,
-                                               Smem &s,
-                                               ArgReg &rarg,
-                                               Dofs &dofs,
-                                               YE_t &YE)
-   {
-      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
-      if constexpr (DIM == 2) { write_gradient_2d(d, q, e, s, rarg, dofs, YE); }
-      else
-      {
-         write_gradient_3d(d, q, e, s, rarg, dofs, YE);
-      }
-   }
-};
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int T_DIM, int T_Q1D = LocalQFHOBackendMQ1()>
-struct LocalQFHOBackend
-{
-   // ─────────────────────────────────────────────────────
-   static constexpr int DIM = T_DIM, MQ1 = T_Q1D, Q1D = T_Q1D;
-   static_assert(DIM == 2 || DIM == 3);
-
-   // ─────────────────────────────────────────────────────
-   static inline ThreadBlocks thread_blocks(const int q1d)
-   {
-      MFEM_ASSERT(q1d <= Q1D, "q1d must be <= " << Q1D);
-      return { q1d, q1d, 1 };
-   }
-
-   // ─────────────────────────────────────────────────────
-   static inline constexpr int MAX_THREADS_PER_BLOCK() { return Q1D * Q1D; }
-
-   // ─────────────────────────────────────────────────────
-   using backend_t = ho_ker_backend<DIM, Q1D>;
-
-   // ─────────────────────────────────────────────────────
-   using Shared = typename backend_t::Shared;
-
-   // ─────────────────────────────────────────────────────
-   template<typename WT, typename WI, typename Cache, typename AddY>
-   static MFEM_HOST_DEVICE inline void DiagContract(Shared &s,
-                                                    const int num_dof_1d,
-                                                    const int q1d,
-                                                    const int nz_dof,
-                                                    WT wt,
-                                                    WI wi,
-                                                    Cache cache,
-                                                    AddY add_y)
-   {
-      MFEM_CONTRACT_VAR(nz_dof);
-      const int nqz = (DIM == 3) ? q1d : 1;
-      const int ndz = (DIM == 3) ? num_dof_1d : 1;
-
-      ker::s_regs3d_t<MQ1> rz, ry;
-      auto &smem = s.M;
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            for (int dz = 0; dz < ndz; dz++)
-            {
-               real_t u = 0.0;
-               for (int qz = 0; qz < nqz; qz++)
-               {
-                  const int q = qx + (qy + qz * q1d) * q1d;
-                  const real_t wz =
-                     (DIM == 3) ? (wt(2, qz, dz) * wi(2, qz, dz)) : real_t(1);
-                  u += wz * cache(q);
-               }
-               rz[dz][qy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      for (int dz = 0; dz < ndz; dz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-            { smem[qy][qx] = rz[dz][qy][qx]; }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < q1d; qy++)
-               {
-                  u += wt(1, qy, dy) * wi(1, qy, dy) * smem[qy][qx];
-               }
-               ry[dz][dy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-
-      for (int dz = 0; dz < ndz; dz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-            { smem[dy][qx] = ry[dz][dy][qx]; }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, num_dof_1d)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < q1d; qx++)
-               {
-                  u += wt(0, qx, dx) * wi(0, qx, dx) * smem[dy][qx];
-               }
-               add_y(dx, dy, dz, u);
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename T>
-   using QReg = ho_qreg_t<backend_t, T>;
-
-   // ─────────────────────────────────────────────────────
-   template<typename ArgRegT, typename XE_T>
-   static inline MFEM_HOST_DEVICE void LoadValue(Shared &s,
-                                                 const int e,
-                                                 const int d,
-                                                 const int q,
-                                                 const int,
-                                                 const real_t *B,
-                                                 const XE_T &XE,
-                                                 ArgRegT &rarg)
-   {
-      ker::LoadMatrix(d, q, B, s.B);
-      std::remove_reference_t<ArgRegT> dofs;
-      backend_t::load_dofs(e, d, XE, dofs);
-      backend_t::eval_value(d, q, s, dofs, rarg);
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<int RNK,
-            typename ArgRegT,
-            typename XE_T,
-            typename FieldParamT = ArgRegT>
-   static inline MFEM_HOST_DEVICE void LoadGradient(Shared &s,
-                                                    const int e,
-                                                    const int d,
-                                                    const int q,
-                                                    const int,
-                                                    const real_t *B,
-                                                    const real_t *G,
-                                                    const XE_T &XE,
-                                                    ArgRegT &rarg)
-   {
-      ker::LoadMatrix(d, q, B, s.B);
-      ker::LoadMatrix(d, q, G, s.G);
-      static_assert(RNK == 1 || RNK == 2);
-      static constexpr int VDIM =
-         (RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
-      static constexpr int SDIM = (RNK == 1)
-                                  ? qf_param_shape<FieldParamT>::extents[0]
-                                  : qf_param_shape<FieldParamT>::extents[1];
-      if constexpr (SDIM == DIM)
-      {
-         typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
-         if constexpr (RNK == 1) { backend_t::load_dofs(e, d, XE, dofs); }
-         else
-         {
-            backend_t::template load_grad_dofs<VDIM, SDIM>(e, d, XE, dofs);
-         }
-         backend_t::template grad<VDIM, SDIM>(d, q, s, dofs, rarg);
-      }
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename T>
-   static MFEM_HOST_DEVICE inline auto
-   qp_pull(QReg<T> &reg, int qx, int qy, int qz)
-   { return hok::load_at<DIM, T>(reg, qx, qy, qz); }
-
-   // ─────────────────────────────────────────────────────
-   template<typename T>
-   static MFEM_HOST_DEVICE inline auto qp_pull_directional(
-      QReg<T> &preg, QReg<T> &sreg, int qx, int qy, int qz, bool dependent)
-   { return hok::pull_directional<DIM, T>(preg, sreg, qx, qy, qz, dependent); }
-
-   // ─────────────────────────────────────────────────────
-   template<typename DT, typename XE_T>
-   static MFEM_HOST_DEVICE inline DT identity_qp_pull_dual(bool dependent,
-                                                           const XE_T &XP,
-                                                           const XE_T &XD,
-                                                           int qx,
-                                                           int qy,
-                                                           int qz,
-                                                           int e)
-   {
-      constexpr int RNK = qf_param_shape<DT>::rank;
-      if constexpr (RNK == 0)
-      {
-         DT t{};
-         t.value = XP(0, qx, qy, qz, e);
-         t.gradient = dependent ? XD(0, qx, qy, qz, e) : 0.0;
-         return t;
-      }
-      else if constexpr (RNK == 1)
-      {
-         constexpr int e0 = qf_param_shape<DT>::extents[0];
-         DT t{};
-         MFEM_UNROLL(e0)
-         for (int dd = 0; dd < e0; ++dd)
-         {
-            t(dd).value = XP(dd, qx, qy, qz, e);
-            t(dd).gradient = dependent ? XD(dd, qx, qy, qz, e) : 0.0;
-         }
-         return t;
-      }
-      else if constexpr (RNK == 2)
-      {
-         constexpr int e0 = qf_param_shape<DT>::extents[0];
-         constexpr int e1 = qf_param_shape<DT>::extents[1];
-         DT t{};
-         MFEM_UNROLL(e0)
-         for (int i = 0; i < e0; ++i)
-         {
-            MFEM_UNROLL(e1)
-            for (int j = 0; j < e1; ++j)
-            {
-               t(i, j).value = XP(i + e0 * j, qx, qy, qz, e);
-               t(i, j).gradient =
-                  dependent ? XD(i + e0 * j, qx, qy, qz, e) : 0.0;
-            }
-         }
-         return t;
-      }
-      else
-      {
-         static_assert(false, "Unsupported");
-      }
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename T>
-   static MFEM_HOST_DEVICE inline void
-   qp_push(QReg<T> &reg, int qx, int qy, int qz, const T &out)
-   { hok::store_at<DIM, T, decltype(reg), false>(reg, qx, qy, qz, out); }
-
-   // ─────────────────────────────────────────────────────
-   template<typename T>
-   static MFEM_HOST_DEVICE inline void
-   qp_push_tangent(QReg<T> &reg, int qx, int qy, int qz, const T &out)
-   {
-      hok::store_at<DIM, T, decltype(reg), qf_param_uses_dual_v<T>>(
-                                                                    reg, qx, qy, qz, out);
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename DT, typename YE_T>
-   static MFEM_HOST_DEVICE inline void identity_qp_write_value(
-      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
-   {
-      constexpr int RNK = qf_param_shape<DT>::rank;
-      if constexpr (qf_param_uses_dual_v<DT>)
-      {
-         if constexpr (RNK == 0)
-         {
-            YE(0, qx, qy, qz, e) = qf_store_value(qout);
-         }
-         else if constexpr (RNK == 1)
-         {
-            constexpr int e0 = qf_param_shape<DT>::extents[0];
-            MFEM_UNROLL(e0)
-            for (int dd = 0; dd < e0; ++dd)
-            {
-               YE(dd, qx, qy, qz, e) = qf_store_value(qout(dd));
-            }
-         }
-         else if constexpr (RNK == 2)
-         {
-            constexpr int e0 = qf_param_shape<DT>::extents[0];
-            constexpr int e1 = qf_param_shape<DT>::extents[1];
-            MFEM_UNROLL(e0)
-            for (int i = 0; i < e0; ++i)
-            {
-               MFEM_UNROLL(e1)
-               for (int j = 0; j < e1; ++j)
-               {
-                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_value(qout(i, j));
-               }
-            }
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      }
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename DT, typename YE_T>
-   static MFEM_HOST_DEVICE inline void identity_qp_write_tangent(
-      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
-   {
-      constexpr int RNK = qf_param_shape<DT>::rank;
-      if constexpr (qf_param_uses_dual_v<DT>)
-      {
-         if constexpr (RNK == 0)
-         {
-            YE(0, qx, qy, qz, e) = qf_store_gradient(qout);
-         }
-         else if constexpr (RNK == 1)
-         {
-            constexpr int e0 = qf_param_shape<DT>::extents[0];
-            MFEM_UNROLL(e0)
-            for (int dd = 0; dd < e0; ++dd)
-            {
-               YE(dd, qx, qy, qz, e) = qf_store_gradient(qout(dd));
-            }
-         }
-         else if constexpr (RNK == 2)
-         {
-            constexpr int e0 = qf_param_shape<DT>::extents[0];
-            constexpr int e1 = qf_param_shape<DT>::extents[1];
-            MFEM_UNROLL(e0)
-            for (int i = 0; i < e0; ++i)
-            {
-               MFEM_UNROLL(e1)
-               for (int j = 0; j < e1; ++j)
-               {
-                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_gradient(qout(i, j));
-               }
-            }
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      }
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename ArgRegT, typename YE_T>
-   static inline MFEM_HOST_DEVICE void WriteValue(Shared &s,
-                                                  const int e,
-                                                  const int d,
-                                                  const int q,
-                                                  const int,
-                                                  const real_t *B,
-                                                  YE_T &YE,
-                                                  ArgRegT &rarg)
-   {
-      ker::LoadMatrix(d, q, B, s.B);
-      std::remove_reference_t<ArgRegT> dofs;
-      backend_t::write_value(d, q, e, s, rarg, dofs, YE);
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<int RNK,
-            typename ArgRegT,
-            typename YE_T,
-            typename FieldParamT = ArgRegT>
-   static inline MFEM_HOST_DEVICE void WriteGradient(Shared &s,
-                                                     const int e,
-                                                     const int d,
-                                                     const int q,
-                                                     const int,
-                                                     const real_t *B,
-                                                     const real_t *G,
-                                                     YE_T &YE,
-                                                     ArgRegT &rarg)
-   {
-      ker::LoadMatrix(d, q, B, s.B);
-      ker::LoadMatrix(d, q, G, s.G);
-      static_assert(RNK == 1 || RNK == 2);
-      static constexpr int VDIM =
-         (RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
-      static constexpr int SDIM = (RNK == 1)
-                                  ? qf_param_shape<FieldParamT>::extents[0]
-                                  : qf_param_shape<FieldParamT>::extents[1];
-      if constexpr (SDIM == DIM)
-      {
-         typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
-         backend_t::template write_gradient<VDIM, SDIM>(
-            d, q, e, s, rarg, dofs, YE);
-      }
-   }
-};
-
-/// @brief Dispatch to a compile-time HO kernel with MQ1 >= runtime @a q1d.
-template <typename HOKernelTable, int DIM, int MQ1 = LocalQFHOBackendMQ1()>
-inline typename HOKernelTable::KernelSignature
-DispatchHOKernelByQ1D(int q1d)
-{
-   MFEM_VERIFY(q1d >= 2 && q1d <= MQ1,
-               "Unsupported HO quadrature order: " << q1d);
-   return HOKernelTable::template Kernel<DIM, MQ1>();
-}
-
-} // namespace mfem::future

fem/dfem/backends/local_qf/prelude.hpp

@@ -1,242 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "action.hpp"
-#include "derivative_action.hpp"
-#include "derivative_setup.hpp"
-#include "derivative_apply.hpp"
-#include "derivative_assemble.hpp"
-#include "derivative_assemble_diagonal.hpp"
-#include "derivative_apply_transpose.hpp"
-
-namespace mfem::future
-{
-
-struct LocalQFBackend
-{
-   /**
-    * @brief Make an action for a local Q-function backend.
-    *
-    * @param ctx The integrator context.
-    * @param args The arguments to the action.
-    * @return The action.
-    */
-   template<typename qfunc_t, typename inputs_t, typename outputs_t>
-   static auto MakeAction(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      return LocalQFImpl::Action<qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs);
-   }
-
-   /**
-    * @brief Make a derivative action for a local Q-function backend.
-    *
-    * @tparam derivative_id The id of the derivative.
-    * @param ctx The integrator context.
-    * @param args The arguments to the derivative action.
-    * @return The derivative action.
-    */
-   template<int id, typename qfunc_t, typename inputs_t, typename outputs_t>
-   static auto MakeDerivativeAction(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      return LocalQFImpl::DerivativeAction<id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   static auto MakeDerivativeSetup(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeSetup<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
-                                                          outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   static auto MakeDerivativeApply(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeApply<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
-                                                          outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   static auto MakeDerivativeApplyTranspose(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeApplyTranspose<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
-                                                          outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   static auto MakeDerivativeAssemble(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeAssemble<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
-                                                          outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   static auto MakeDerivativeAssembleDiagonal(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeAssembleDiagonal<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
-                                                          outputs, qp_cache);
-   }
-};
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, typename QT, typename IT, typename OT>
-inline void AddAction()
-{
-   using ker = LocalQFImpl::Action<QT, IT, OT>;
-   if constexpr (Q1D <= 8)
-   {
-      ker::ActionLO::template Specialization<DIM, Q1D>::Add();
-   }
-   else
-   {
-      ker::ActionHO::template Specialization<DIM, Q1D>::Add();
-   }
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
-inline void AddDerivativeAction()
-{
-   using ker = LocalQFImpl::DerivativeAction<DID, QT, IT, OT>;
-   if constexpr (Q1D <= 8)
-   {
-      ker::DerivativeActionLO::template Specialization<DIM, Q1D>::Add();
-   }
-   else
-   {
-      ker::DerivativeActionHO::template Specialization<DIM, Q1D>::Add();
-   }
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
-inline void AddDerivativeSetup()
-{
-   using ker = LocalQFImpl::DerivativeSetup<DID, QT, IT, OT>;
-   if constexpr (Q1D <= 8)
-   {
-      ker::DerivativeSetupLO::template Specialization<DIM, Q1D>::Add();
-   }
-   else
-   {
-      ker::DerivativeSetupHO::template Specialization<DIM, Q1D>::Add();
-   }
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
-inline void AddDerivativeApply()
-{
-   using ker = LocalQFImpl::DerivativeApply<DID, QT, IT, OT>;
-   if constexpr (Q1D <= 8)
-   {
-      ker::DerivativeApplyLO::template Specialization<DIM, Q1D>::Add();
-   }
-   else
-   {
-      ker::DerivativeApplyHO::template Specialization<DIM, Q1D>::Add();
-   }
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
-inline void AddDerivativeApplyTranspose()
-{
-   using ker = LocalQFImpl::DerivativeApplyTranspose<DID, QT, IT, OT>;
-   if constexpr (Q1D <= 8)
-   {
-      ker::DerivativeApplyTransposeLO::template Specialization<DIM, Q1D>::Add();
-   }
-   else
-   {
-      ker::DerivativeApplyTransposeHO::template Specialization<DIM, Q1D>::Add();
-   }
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, typename QT, typename IT, typename OT,
-         typename derivative_ids_t = std::index_sequence<>>
-inline void AddLocalSpecializations()
-{
-   AddAction<DIM, Q1D, QT, IT, OT>();
-
-   for_constexpr([&](auto i)
-   {
-      using derivative_id = decltype(i);
-      AddDerivativeAction<DIM, Q1D, derivative_id::value, QT, IT, OT>();
-      AddDerivativeSetup<DIM, Q1D, derivative_id::value, QT, IT, OT>();
-      AddDerivativeApply<DIM, Q1D, derivative_id::value, QT, IT, OT>();
-      AddDerivativeApplyTranspose<DIM, Q1D, derivative_id::value, QT, IT, OT>();
-   }, derivative_ids_t{});
-}
-
-}

fem/dfem/backends/local_qf/revdiff_transformer.hpp

@@ -1,606 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../util.hpp"
-#include "util.hpp"
-#include "../../util.hpp"
-
-namespace mfem::future
-{
-
-template <typename T> struct function_traits;
-
-template <typename R, typename C, typename... Args>
-struct function_traits<R (C::*)(Args...) const>
-{
-   using primal_return_type = R;
-   using args_tuple = tuple<Args...>;
-   static constexpr size_t arity = sizeof...(Args);
-};
-
-/// Scalar-level view of a per-point argument: the plain value type it stores
-/// and the dual type used to carry a first-order tangent alongside it.
-template <typename T>
-struct qp_scalar_traits
-{
-   using view_type = T;
-   using dual_type = dual<T, T>;
-};
-
-template <typename V, typename G>
-struct qp_scalar_traits<dual<V, G>>
-{
-   using view_type = V;
-   using dual_type = dual<V, G>;
-};
-
-// Component count and writable counterpart of a per-point argument, which
-// is either a tensor or a plain scalar. `view_type` keeps the argument's own
-// scalar type; `dual_type` is the same shape with a dual scalar, used for the
-// gradient blocks of the native dual-number backend.
-template <typename Arg> struct qp_traits
-{
-   static_assert(std::is_arithmetic_v<Arg> || is_dual_number<Arg>::value,
-                 "per-point arguments must be tensors or scalars");
-   using scalar_type = std::remove_const_t<Arg>;
-   using view_type = scalar_type;
-   using dual_type = typename qp_scalar_traits<scalar_type>::dual_type;
-   static constexpr int components = 1;
-};
-
-template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
-{
-   using scalar_type = std::remove_const_t<T>;
-   using view_type = tensor<scalar_type, Sizes...>;
-   using dual_type =
-      tensor<typename qp_scalar_traits<scalar_type>::dual_type, Sizes...>;
-   static constexpr int components = (Sizes * ... * 1);
-};
-
-template <typename... T1s, typename... T2s>
-constexpr tuple<T1s..., T2s...> concat_tuples(tuple<T1s...>, tuple<T2s...>);
-
-///////////////////////////////////////////////////////////////////////////////
-/// Nested ("hyper") dual utilities, used for second derivatives on the native
-/// dual-number backend.
-///
-/// A second derivative taken with plain duals would have to reuse the single
-/// gradient slot that already carries the incoming direction. Lifting the
-/// scalar to `dual<dual<V,G>, dual<V,G>>` adds a second, independent slot:
-///
-///   dual(a, b) -> ((a, c), (b, d))
-///
-/// `a`/`b` stay the incoming primal/direction, `c` is seeded per component and
-/// `d` returns the second-order result.
-template <typename T>
-struct make_nested_qp_type
-{
-   using type = T;
-};
-
-template <typename V, typename G>
-struct make_nested_qp_type<dual<V, G>>
-{
-   using type = dual<dual<V, G>, dual<V, G>>;
-};
-
-template <typename S, int... Sizes>
-struct make_nested_qp_type<tensor<S, Sizes...>>
-{
-   using type = tensor<typename make_nested_qp_type<S>::type, Sizes...>;
-};
-
-template <typename T>
-using make_nested_qp_type_t = typename make_nested_qp_type<T>::type;
-
-using native_dual_t = typename qp_scalar_traits<real_t>::dual_type;
-using nested_native_dual_t = make_nested_qp_type_t<native_dual_t>;
-
-/// Rebinds a q-function's scalar template parameter so its arguments are
-/// nested duals. Only the leading scalar parameter is rebound; any remaining
-/// non-type parameters (e.g. `dim`) are carried through unchanged. This
-/// requires q-functions of the form `QFunc<scalar_t>` or `QFunc<scalar_t,
-/// Params...>`; `supported` reports whether that shape was matched, so callers
-/// can fail with a readable static_assert.
-template <typename qfunc_t, typename nested_scalar_t, typename = void>
-struct rebind_qfunc_scalar
-{
-   static constexpr bool supported = false;
-};
-
-template <template <typename> class qfunc_template_t,
-          typename old_scalar_t,
-          typename nested_scalar_t>
-struct rebind_qfunc_scalar<qfunc_template_t<old_scalar_t>, nested_scalar_t>
-{
-   static constexpr bool supported = true;
-   using type = qfunc_template_t<nested_scalar_t>;
-};
-
-template <template <typename, auto...> class qfunc_template_t,
-          typename old_scalar_t,
-          auto... Params,
-          typename nested_scalar_t>
-struct rebind_qfunc_scalar<qfunc_template_t<old_scalar_t, Params...>,
-          nested_scalar_t,
-       std::enable_if_t<(sizeof...(Params) > 0)>>
-       {
-          static constexpr bool supported = true;
-          using type = qfunc_template_t<nested_scalar_t, Params...>;
-       };
-
-template <typename qfunc_t, typename nested_scalar_t>
-using rebind_qfunc_scalar_t =
-   typename rebind_qfunc_scalar<qfunc_t, nested_scalar_t>::type;
-
-/// Copies a q-function argument into its nested-dual counterpart, mapping
-/// dual(a, b) -> ((a, 0), (b, 0)). The inner gradients stay zero; the caller
-/// seeds one of them per component.
-template <typename Dst, typename Src>
-MFEM_HOST_DEVICE void lift_to_nested_arg(const Src &src, Dst &dst)
-{
-   using dst_t = std::decay_t<Dst>;
-   constexpr bool dst_uses_dual = is_dual_number<dst_t>::value ||
-                                  qf_param_uses_dual_v<dst_t> ||
-                                  is_nested_dual_number<dst_t>::value ||
-                                  qf_param_uses_nested_dual_v<dst_t>;
-   if constexpr (dst_uses_dual)
-   {
-      constexpr int ncomp = qp_traits<dst_t>::components;
-      for (int component = 0; component < ncomp; component++)
-      {
-         qf_set_flat_value(dst, component, qf_flat_value(src, component));
-         qf_set_flat_gradient(dst, component, qf_flat_gradient(src, component));
-      }
-   }
-   else
-   {
-      // Destination carries no derivative slots: a plain copy is the lift.
-      dst = src;
-   }
-}
-
-// RevDiff: computes the full gradient of a pointwise qfunction at a single
-// quadrature point using one Enzyme reverse-mode (autodiff) call.
-//
-// operator()(args...) takes all qfunction input primals followed by one
-// writable gradient output per Active input (same shape as that input).
-// The qfunction's own output is not passed; Enzyme writes it to stack
-// scratch (enzyme_dupnoneed).
-//
-// A single __enzyme_autodiff call with the output adjoint seeded to 1
-// yields all gradient blocks simultaneously — O(1) calls per point
-// regardless of input size, vs O(ncomp) for forward mode.
-//
-// operator() is MFEM_HOST_DEVICE and allocation-free.
-//
-// Without Enzyme the same interface is served by a forward-mode dual-number
-// fallback (`call_dual_rev`), which seeds one component at a time. `mode`
-// selects what that fallback is being asked for: `Eval` is the plain gradient,
-// `Derivative` is a gradient taken inside an outer derivative, which lifts the
-// q-function to nested duals so seeding does not clobber the outer direction.
-// With Enzyme both modes use the single reverse-mode call and `mode` is inert.
-enum class RevDiffDualMode
-{
-   Eval,
-   Derivative
-};
-
-// Number of Active inputs and their argument indices, in ascending order.
-// A qfunction may have several Active inputs at once: e.g. a field's value
-// u and its gradient dudx both feed the output and both must be
-// differentiated (the chain-rule contraction with the value/gradient shape
-// functions then happens at the FE-operator level). We produce one gradient
-// block, d(output)/d(input), per Active input — each computed with the other
-// Active inputs frozen, so they come out as isolated partials.
-template <typename activity_t, size_t num_inputs, size_t... Is>
-constexpr size_t count_active_inputs(std::index_sequence<Is...>)
-{
-   return ((Is < num_inputs && qf_param_is_active_v<activity_t, Is>
-            ? size_t{1} : size_t{0}) + ...);
-}
-
-template <typename activity_t, size_t num_inputs, size_t num_active,
-          size_t... Is>
-constexpr std::array<size_t, num_active>
-collect_active_inputs(std::index_sequence<Is...>)
-{
-   std::array<size_t, num_active> idx{};
-   size_t j = 0;
-   (((Is < num_inputs && qf_param_is_active_v<activity_t, Is>)
-     ? (idx[j++] = Is) : size_t{0}), ...);
-   return idx;
-}
-
-template <typename Func, typename InputActivityTuple,
-          typename OutputActivityTuple,
-          RevDiffDualMode mode = RevDiffDualMode::Eval>
-struct RevDiff
-{
-   using traits = function_traits<decltype(&Func::operator())>;
-   using args_tuple = typename traits::args_tuple;
-   using activity =
-      decltype(concat_tuples(InputActivityTuple{}, OutputActivityTuple{}));
-   static constexpr size_t arity = traits::arity;
-   static constexpr size_t num_inputs = tuple_size<InputActivityTuple>::value;
-
-   static_assert(std::is_void_v<typename traits::primal_return_type>,
-                 "RevDiff only supports primal functions with void return type");
-   static_assert(tuple_size<activity>::value == arity,
-                 "Number of input and output activity tags must match function "
-                 "arity");
-
-   static constexpr size_t num_active_inputs =
-      count_active_inputs<activity, num_inputs>(
-         std::make_index_sequence<arity> {});
-
-   static constexpr auto active_inputs =
-      collect_active_inputs<activity, num_inputs, num_active_inputs>(
-   std::make_index_sequence<arity> {});
-
-   // Slot index of argument I in the active_inputs array (compile-time).
-   template <size_t I>
-   static constexpr size_t slot_of()
-   {
-      for (size_t s = 0; s < num_active_inputs; s++)
-         if (active_inputs[s] == I) { return s; }
-      return num_active_inputs;
-   }
-
-   static constexpr size_t active_output =
-      find_single_active_qparam<activity, num_inputs, arity>();
-   static_assert(active_output < arity,
-                 "gradient mode requires exactly one Active output");
-   static_assert(num_active_inputs >= 1,
-                 "gradient mode requires at least one Active input");
-   static_assert(tuple_size<OutputActivityTuple>::value == 1,
-                 "gradient mode requires exactly one (scalar) output");
-
-   using output_type =
-      std::decay_t<tuple_element_t<active_output, args_tuple>>;
-
-   using output_view = typename qp_traits<output_type>::view_type;
-
-   static_assert(qp_traits<output_type>::components == 1,
-                 "gradient output requires a scalar output");
-
-   template <size_t I>
-   using primal_arg_t = tuple_element_t<I, args_tuple>;
-
-   // True when reverse mode is served by the dual-number fallback rather than
-   // Enzyme. Everything below that widens a type to a dual is gated on this, so
-   // an Enzyme build sees exactly the types it saw before nested duals existed.
-#ifdef MFEM_USE_ENZYME
-   static constexpr bool native_dual_backend = false;
-#else
-   static constexpr bool native_dual_backend = true;
-#endif
-
-   static constexpr bool use_native_dual_derivative =
-      native_dual_backend && (mode == RevDiffDualMode::Derivative);
-
-   // Under the native-dual second-derivative path the active primals arrive
-   // carrying the outer direction, so they must be dual-typed.
-   template <size_t I>
-   using derivative_arg_t =
-      std::conditional_t<use_native_dual_derivative &&
-      qf_param_is_active_v<activity, I>,
-      typename qp_traits<std::decay_t<tuple_element_t<I, args_tuple>>>::dual_type,
-      primal_arg_t<I>>;
-
-   template <size_t S>
-   using active_arg_decay_t =
-      std::decay_t<tuple_element_t<active_inputs[S], args_tuple>>;
-
-   template <size_t S>
-   static constexpr bool active_arg_uses_dual()
-   {
-      return native_dual_backend &&
-             (is_dual_number<active_arg_decay_t<S>>::value ||
-              qf_param_uses_dual_v<active_arg_decay_t<S>>);
-   }
-
-   // A gradient block mirrors its active input's shape. It needs a dual scalar
-   // whenever the fallback has to return a value and a tangent through it.
-   template <size_t S>
-   using grad_arg_t =
-      std::conditional_t<use_native_dual_derivative || active_arg_uses_dual<S>(),
-      typename qp_traits<active_arg_decay_t<S>>::dual_type,
-      typename qp_traits<active_arg_decay_t<S>>::view_type>
-      &;
-
-   template <typename qfunc_type>
-   using qfunc_args_tuple_t =
-      decay_tuple<typename function_traits<decltype(&qfunc_type::operator())>::args_tuple>;
-
-   template <size_t... Is, size_t... Ss>
-   static FunctionSignature<void(derivative_arg_t<Is>..., grad_arg_t<Ss>...)>
-   signature_impl(std::index_sequence<Is...>, std::index_sequence<Ss...>);
-
-   using signature = decltype(signature_impl(std::make_index_sequence<num_inputs> {},
-                                             std::make_index_sequence<num_active_inputs> {}));
-
-   Func func {};
-
-   RevDiff() = default;
-   MFEM_HOST_DEVICE explicit RevDiff(const Func &func_) : func(func_) { }
-
-   // Plain function with the qfunction's exact (reference) signature, plus the
-   // configured qfunction instance, so it can be handed to Enzyme as a function
-   // pointer without default-constructing away runtime qfunction state.
-   template <size_t... Is>
-   MFEM_HOST_DEVICE static MFEM_FUTURE_ALWAYS_INLINE void
-   static_call(Func *func, tuple_element_t<Is, args_tuple>... args)
-   {
-      (*func)(args...);
-   }
-
-   template <size_t... Is>
-   static constexpr auto fn_ptr(std::index_sequence<Is...>)
-   {
-      return &static_call<Is...>;
-   }
-   static constexpr auto fn()
-   {
-      return fn_ptr(std::make_index_sequence<arity> {});
-   }
-
-   // Load primal inputs from the pointer tuple into a local qargs copy. Dual
-   // gradient parts are implicitly zero because qargs is value-initialized.
-   template <typename QArgs, typename AllPtrs, size_t... Is>
-   MFEM_HOST_DEVICE static MFEM_FUTURE_ALWAYS_INLINE void load_qargs(
-      QArgs &qargs, AllPtrs &ptrs, std::index_sequence<Is...>)
-   {
-      ((mfem::future::get<int(Is)>(qargs) =
-           *mfem::future::get<int(Is)>(ptrs)), ...);
-   }
-
-   template <typename QArgs, typename AllPtrs, size_t... Is>
-   MFEM_HOST_DEVICE static MFEM_FUTURE_ALWAYS_INLINE void
-   lift_qargs_to_nested_dual(QArgs &qargs, AllPtrs &ptrs,
-                             std::index_sequence<Is...>)
-   {
-      (lift_to_nested_arg(*mfem::future::get<int(Is)>(ptrs),
-                          mfem::future::get<Is>(qargs)), ...);
-   }
-
-   // The nested-dual q-function is a *different* type — its scalar template
-   // parameter is rebound — so a configured instance cannot simply be copied
-   // over. Runtime q-function state must still survive, or the second
-   // derivative would silently be taken of a differently-parameterised energy.
-   //
-   // Three cases, in order:
-   //   * the rebound type converts from this one: use that conversion;
-   //   * no state at all: nothing to carry;
-   //   * same size and trivially copyable: none of the members depend on the
-   //     rebound scalar, so the two are layout-identical and the state copies
-   //     over bytewise. A member that *did* depend on the scalar would change
-   //     the size and land in the static_assert below instead.
-   template <typename nested_func_t>
-   MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE nested_func_t
-   make_nested_func() const
-   {
-      if constexpr (std::is_constructible_v<nested_func_t, const Func &>)
-      {
-         return nested_func_t(func);
-      }
-      else if constexpr (std::is_empty_v<Func>)
-      {
-         return nested_func_t {};
-      }
-      else
-      {
-         static_assert(std::is_trivially_copyable_v<Func> &&
-                       std::is_trivially_copyable_v<nested_func_t> &&
-                       sizeof(Func) == sizeof(nested_func_t),
-                       "second derivatives on the native dual backend rebind "
-                       "the q-function's scalar type; a q-function whose state "
-                       "depends on that scalar must be constructible from its "
-                       "rebound form");
-         nested_func_t nested {};
-         const auto *src = reinterpret_cast<const unsigned char *>(&func);
-         auto *dst = reinterpret_cast<unsigned char *>(&nested);
-         for (size_t b = 0; b < sizeof(Func); b++) { dst[b] = src[b]; }
-         return nested;
-      }
-   }
-
-   // Seed the s-th Active input one component at a time and read the resulting
-   // gradient block back out. This is the forward-mode dual-number stand-in for
-   // one reverse-mode call: O(ncomp) evaluations instead of O(1).
-   template <size_t S, typename AllPtrs>
-   MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void seed_active_input(
-      AllPtrs &ptrs) const
-   {
-      constexpr size_t input_idx = active_inputs[S];
-      using active_arg_t = std::decay_t<tuple_element_t<input_idx, args_tuple>>;
-      constexpr int ncomp = qp_traits<active_arg_t>::components;
-
-      for (int component = 0; component < ncomp; component++)
-      {
-         if constexpr (mode == RevDiffDualMode::Eval)
-         {
-            // Fresh value-initialized qargs: primals loaded below, all dual
-            // gradient parts start at zero, so no explicit clear is needed.
-            qfunc_args_tuple_t<Func> qargs {};
-            load_qargs(qargs, ptrs, std::make_index_sequence<num_inputs> {});
-
-            auto &grad = *mfem::future::get<num_inputs + S>(ptrs);
-
-            qf_set_flat_gradient(mfem::future::get<input_idx>(qargs), component,
-                                 1.0);
-
-            call_qfunc_no_move(func, qargs);
-
-            auto &out = mfem::future::get<active_output>(qargs);
-
-            qf_set_flat_value(grad, component, qf_flat_gradient(out, 0));
-         }
-         else
-         {
-            // Lift the incoming dual (a, b) to ((a, c), (b, d)): b is the outer
-            // Hessian-action direction, c is this loop's component seed. After
-            // evaluating E the nested output holds ((E, dE/dx_i), (E'[b],
-            // H_i[b])), and we hand dfem back (dE/dx_i, H_i[b]).
-            static_assert(rebind_qfunc_scalar<Func, nested_native_dual_t>::supported,
-                          "RevDiff native-dual derivative mode requires "
-                          "q-function types of the form QFunc<scalar_t> so they "
-                          "can be rebound to nested dual scalars");
-            using nested_func_t = rebind_qfunc_scalar_t<Func, nested_native_dual_t>;
-            qfunc_args_tuple_t<nested_func_t> nested_qargs {};
-
-            lift_qargs_to_nested_dual(nested_qargs, ptrs,
-                                      std::make_index_sequence<num_inputs> {});
-
-            qf_set_flat_value_gradient(
-               mfem::future::get<input_idx>(nested_qargs), component, 1.0);
-
-            call_qfunc_no_move(make_nested_func<nested_func_t>(), nested_qargs);
-
-            auto &out = mfem::future::get<active_output>(nested_qargs);
-            auto &grad = *mfem::future::get<num_inputs + S>(ptrs);
-
-            qf_set_flat_value(grad, component, qf_flat_value_gradient(out, 0));
-            qf_set_flat_gradient(grad, component,
-                                 qf_flat_gradient_gradient(out, 0));
-         }
-      }
-   }
-
-   // Dual-number fallback for the whole reverse-mode call: one seeded sweep per
-   // Active input.
-   template <typename AllPtrs>
-   MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void call_dual_rev(
-      AllPtrs &ptrs) const
-   {
-      for_constexpr<num_active_inputs>([&](auto s)
-      {
-         seed_active_input<decltype(s)::value>(ptrs);
-      });
-   }
-
-#ifdef MFEM_USE_ENZYME
-   // Recursive builder of the per-argument reverse-mode enzyme call.
-   template <size_t I = 0, typename AllPtrs, typename... Built>
-   MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void
-   call_enzyme_rev(AllPtrs &ptrs, output_view &scratch, output_view &adjoint,
-                   Built... built) const
-   {
-      if constexpr (I == arity)
-      {
-         __enzyme_autodiff<void>(fn(), enzyme_const, const_cast<Func *>(&func),
-                                 built...);
-      }
-      else if constexpr (I == active_output)
-      {
-         // Output: primal written to scratch (unused), adjoint seeded to 1.
-         call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
-                                enzyme_dupnoneed, &scratch, &adjoint);
-      }
-      else if constexpr (qf_param_is_active_v<activity, I>)
-      {
-         // Active input: gradient accumulates into its grad-output slot.
-         call_enzyme_rev<I + 1>(
-            ptrs, scratch, adjoint, built..., enzyme_dup,
-            mfem::future::get<int(I)>(ptrs),
-            mfem::future::get<int(num_inputs + slot_of<I>())>(ptrs));
-      }
-      else
-      {
-         // Const input: primal only, no shadow.
-         call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
-                                enzyme_const, mfem::future::get<int(I)>(ptrs));
-      }
-   }
-#endif // MFEM_USE_ENZYME
-
-   // Zero all gradient outputs before the enzyme call (Enzyme accumulates).
-   template <typename AllPtrs, size_t... Ss>
-   MFEM_HOST_DEVICE static
-   MFEM_FUTURE_ALWAYS_INLINE void zero_grads(
-      AllPtrs &ptrs,
-      std::index_sequence<Ss...>)
-   {
-      ((*mfem::future::get<int(num_inputs + Ss)>(ptrs) =
-           std::decay_t<decltype(*mfem::future::get<int(num_inputs + Ss)>(ptrs))> {}),
-       ...);
-   }
-
-   // Called once per quadrature point. Arguments are, in order:
-   //   * the primal value of every qfunction input (active and const), then
-   //   * one gradient output per Active input (ascending index order), each
-   //     shaped like its Active input.
-   // The qfunction's own output slot is not passed; Enzyme writes it to stack
-   // scratch (enzyme_dupnoneed). A single __enzyme_autodiff call yields all
-   // gradient blocks simultaneously.
-   template <typename... Args>
-   MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void operator()(
-      Args &&...args) const
-   {
-      static_assert(sizeof...(Args) == num_inputs + num_active_inputs,
-                    "expected one primal per input plus one gradient output per "
-                    "Active input");
-      auto ptrs = mfem::future::make_tuple(&args...);
-      zero_grads(ptrs, std::make_index_sequence<num_active_inputs> {});
-#ifdef MFEM_USE_ENZYME
-      output_view out_scratch {};
-      output_view out_adjoint{1.0}; // seed: d(output)/d(output) = 1
-      call_enzyme_rev(ptrs, out_scratch, out_adjoint);
-#else
-      call_dual_rev(ptrs);
-#endif
-   }
-
-   static MFEM_FUTURE_ALWAYS_INLINE void print() { print_impl(std::make_index_sequence<arity> {}); }
-
-   template <size_t... Is> static MFEM_FUTURE_ALWAYS_INLINE void print_impl(
-      std::index_sequence<Is...>)
-   {
-      mfem::out << "__enzyme_autodiff<void>(fptr";
-      (([&]
-      {
-         auto name = get_type_name<tuple_element_t<Is, args_tuple>>();
-         if constexpr (Is == active_output)
-            mfem::out << ", enzyme_dupnoneed, " << name << ", adjoint=1";
-         else if constexpr (qf_param_is_active_v<activity, Is>)
-            mfem::out << ", enzyme_dup, " << name << ", grad out";
-         else
-         {
-            mfem::out << ", enzyme_const, " << name;
-         }
-      }()),
-      ...);
-      mfem::out << ")\n";
-   }
-};
-
-template <typename Func, typename InputActivityTuple,
-          typename OutputActivityTuple, RevDiffDualMode mode>
-struct create_function_signature<RevDiff<Func, InputActivityTuple,
-          OutputActivityTuple, mode>>
-{
-   using type = typename
-                RevDiff<Func, InputActivityTuple, OutputActivityTuple, mode>::signature;
-};
-
-/// Builds the reverse-mode transform of @a f, differentiating the inputs marked
-/// Active in @a activity_t.
-///
-/// A factory rather than a plain declaration of a RevDiff variable for compatibility with MSVC.
-template <typename activity_t, RevDiffDualMode mode = RevDiffDualMode::Eval,
-          typename func_t>
-auto make_revdiff(const func_t &f)
-{
-   return RevDiff<func_t, activity_t, tuple<Active>, mode>(f);
-}
-
-} // namespace mfem::future

fem/dfem/backends/scratch_bank.hpp

@@ -1,327 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-#include "../../../general/error.hpp"
-#include "../../../linalg/vector.hpp"
-#include "../tuple.hpp"
-#include <initializer_list>
-#include <memory>
-#include <type_traits>
-#include <utility>
-#include <vector>
-
-namespace mfem::future
-{
-// Scratch storage and q-function shadow helpers for dFEM backends. The bank
-// supports two scratch kinds:
-// - quadrature-point scratch: real_t buffers sized as NQ * components_per_qp,
-// - global scratch: one tuple of qfunction-local temporaries, independent of
-//   NQ, used for values such as flags, scalars, or small Vector workspaces.
-//
-// @a scalar_t is the scalar the owning q-function uses at a quadrature point.
-// With Enzyme this is real_t and the tangent lives in a separate shadow bank.
-// Without Enzyme the q-function is evaluated on native duals, which carry the
-// tangent inside the value itself; the bank then widens its backing storage
-// accordingly so that a scratch entry can round-trip a dual without dropping
-// the gradient. Backing storage stays a real_t Vector in both cases, so the
-// device and shadow plumbing is unchanged.
-template <typename scalar_t, typename... GlobalScratchTypes>
-struct ScratchBank
-{
-   static_assert(sizeof(scalar_t) % sizeof(real_t) == 0,
-                 "scratch scalar must be a whole number of real_t");
-
-   /// Number of real_t needed to back one scalar_t scratch entry.
-   static constexpr int scalar_size = sizeof(scalar_t) / sizeof(real_t);
-
-
-   //=================================
-   ///<--- Global scratch utilities.
-   //=================================
-
-   using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
-
-   template <typename T>
-   static T MakeGlobalScratchShadow(const T &)
-   {
-      return T {};
-   }
-
-   static Vector MakeGlobalScratchShadow(const Vector &primal)
-   {
-      Vector shadow(primal.Size());
-      shadow.UseDevice(true);
-      shadow = 0.0;
-      return shadow;
-   }
-
-   template <typename Tuple, size_t... Is>
-   static auto MakeGlobalScratchShadowTuple(const Tuple &primal,
-                                            std::index_sequence<Is...>)
-   {
-      return make_tuple(MakeGlobalScratchShadow(get<Is>(primal))...);
-   }
-
-   template <typename Tuple>
-   static auto MakeGlobalScratchShadowTuple(const Tuple &primal)
-   {
-      return MakeGlobalScratchShadowTuple(
-                primal, std::make_index_sequence<tuple_size<Tuple>::value> {});
-   }
-
-
-   //===========================
-   ///<--- Scratch objects
-   //===========================
-
-   mutable GlobalScratchTuple global;
-
-   int nq = 0;
-   std::vector<int> components;
-   std::vector<int> sizes;
-   std::vector<std::shared_ptr<Vector>> owned;
-   std::vector<real_t *> ptrs;
-
-
-   //===========================
-   ///<--- Setter methods
-   //===========================
-
-   void SetScratch(const int nq_,
-                   std::initializer_list<int> components_per_qp = {1})
-   {
-      SetScratch(nq_, std::vector<int>(components_per_qp));
-   }
-
-   void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
-   {
-      nq = nq_;
-      components.clear();
-      sizes.clear();
-      owned.clear();
-      ptrs.clear();
-      for (int component_count : components_per_qp)
-      {
-         AddScratch(component_count);
-      }
-   }
-
-   void AddScratch(const int components_per_qp = 1)
-   {
-      MFEM_VERIFY(nq > 0, "SetScratch must be called before AddScratch");
-      MFEM_VERIFY(components_per_qp > 0,
-                  "scratch components per quadrature point must be positive");
-      owned.push_back(std::make_shared<Vector>());
-      Vector &scratch = *owned.back();
-      const int size = components_per_qp * nq * scalar_size;
-      scratch.SetSize(size);
-      scratch.UseDevice(true);
-      scratch = 0.0;
-      components.push_back(components_per_qp);
-      sizes.push_back(scratch.Size());
-      ptrs.push_back(scratch.ReadWrite());
-   }
-
-   void SetGlobalScratch(const GlobalScratchTuple &global_)
-   {
-      global = global_;
-   }
-
-
-   //===========================
-   ///<--- Getter methods
-   //===========================
-
-   /// Scratch buffer @a i viewed as the q-function's scalar type.
-   scalar_t *GetScratchPointer(const int i) const
-   {
-      return reinterpret_cast<scalar_t *>(ptrs[i]);
-   }
-
-   scalar_t *operator[](const int i) const { return GetScratchPointer(i); }
-
-   /// Raw real_t backing storage of scratch buffer @a i. Its size is
-   /// scalar_size times the number of scalar_t entries.
-   Vector &GetScratchVector(const int i) const { return *owned[i]; }
-
-   template <int I>
-   auto &GetGlobalScratch() const
-   {
-      return get<I>(global);
-   }
-
-
-   //===========================
-   ///<--- Utils methods
-   //===========================
-
-   void CloneScratchLayoutTo(ScratchBank &shadow) const
-   {
-      shadow.SetScratch(nq, components);
-      shadow.SetGlobalScratch(MakeGlobalScratchShadowTuple(global));
-   }
-
-   int Size() const { return static_cast<int>(ptrs.size()); }
-};
-
-// Shared base for Q-functions that use ScratchBank. Under Enzyme a matching
-// scratch shadow is created for forward differentiation; with native duals the
-// tangent rides along in the scratch entry and no shadow is created.
-template <typename scalar_t, typename... GlobalScratchTypes>
-struct QFWithScratch
-{
-   using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
-   using ScratchScalar = scalar_t;
-
-   /// Number of real_t backing one scratch entry; see ScratchBank.
-   static constexpr int scalar_size =
-      ScratchBank<scalar_t, GlobalScratchTypes...>::scalar_size;
-
-   int nq = 0;
-   ScratchBank<scalar_t, GlobalScratchTypes...> scratch;
-
-   void SetScratch(const int nq_,
-                   std::initializer_list<int> components_per_qp = {1})
-   {
-      nq = nq_;
-      scratch.SetScratch(nq, components_per_qp);
-   }
-
-   void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
-   {
-      nq = nq_;
-      scratch.SetScratch(nq, components_per_qp);
-   }
-
-   void SetScratch(const int nq_, const int num_scratch_elem,
-                   const int components_per_qp = 1)
-   {
-      nq = nq_;
-      scratch.SetScratch(nq,
-                         std::vector<int>(num_scratch_elem, components_per_qp));
-   }
-
-   void SetGlobalScratch(const GlobalScratchTuple &global_scratch_)
-   {
-      scratch.SetGlobalScratch(global_scratch_);
-   }
-
-   Vector &GetScratchVector(const int i) const
-   {
-      return scratch.GetScratchVector(i);
-   }
-
-   scalar_t *GetScratchPointer(const int i) const
-   {
-      return scratch.GetScratchPointer(i);
-   }
-
-   template <int I>
-   auto &GetGlobalScratch() const
-   {
-      return scratch.template GetGlobalScratch<I>();
-   }
-
-   void CloneScratchLayoutTo(QFWithScratch &shadow) const
-   {
-      shadow.nq = nq;
-      scratch.CloneScratchLayoutTo(shadow.scratch);
-   }
-
-   QFWithScratch CreateShadow() const
-   {
-      QFWithScratch shadow;
-      CloneScratchLayoutTo(shadow);
-      return shadow;
-   }
-};
-
-/// Q-function base with quadrature-point scratch only. @a scalar_t is the
-/// scalar the q-function signature uses (real_t under Enzyme, dual otherwise).
-template <typename scalar_t = real_t>
-using QFWithScratchType = QFWithScratch<scalar_t>;
-
-/// Q-function base with quadrature-point scratch and a global scratch tuple.
-template <typename scalar_t = real_t>
-using QFWithGlobalScratchType =
-   QFWithScratch<scalar_t, bool, real_t, Vector>;
-
-namespace detail
-{
-
-template <typename T>
-struct qfunc_uses_scratch
-{
-private:
-   template <typename scalar_t, typename... GlobalScratchTypes>
-   static std::true_type Test(
-      const QFWithScratch<scalar_t, GlobalScratchTypes...> *);
-
-   static std::false_type Test(...);
-
-public:
-   static constexpr bool value = decltype(Test(
-                                             static_cast<std::remove_cv_t<std::remove_reference_t<T>> *>(nullptr)))::value;
-};
-
-template <typename T>
-inline constexpr bool qfunc_uses_scratch_v =
-   qfunc_uses_scratch<T>::value;
-
-struct unused_qfunc_shadow { };
-
-// A separate shadow scratch bank only exists for Enzyme, which writes tangents
-// into shadow memory. The native dual fallback carries the tangent inside the
-// scratch entry itself (see ScratchBank::scalar_size), so a shadow bank would
-// be allocated and never read; it is dropped entirely there.
-template <typename T>
-inline constexpr bool qfunc_needs_shadow_v =
-#ifdef MFEM_USE_ENZYME
-   qfunc_uses_scratch_v<T>;
-#else
-   false;
-#endif
-
-template <typename qfunc_t, bool needs_shadow>
-struct qfunc_shadow_type
-{
-   using type = unused_qfunc_shadow;
-};
-
-template <typename qfunc_t>
-struct qfunc_shadow_type<qfunc_t, true>
-{
-   using type = decltype(std::declval<const qfunc_t &>().CreateShadow());
-};
-
-template <typename qfunc_t>
-using qfunc_shadow_t = typename qfunc_shadow_type<qfunc_t,
-      qfunc_needs_shadow_v<qfunc_t>>::type;
-
-// Create a persistent q-function shadow if one is needed, otherwise return an empty struct.
-template <typename qfunc_t>
-inline qfunc_shadow_t<qfunc_t> MakeQFunctionShadowStorage(
-   const qfunc_t &qfunc)
-{
-   if constexpr (qfunc_needs_shadow_v<qfunc_t>)
-   {
-      return qfunc.CreateShadow();
-   }
-   else
-   {
-      MFEM_CONTRACT_VAR(qfunc);
-      return {};
-   }
-}
-
-} // namespace detail
-
-}

fem/dfem/doperator.cpp

@@ -11,366 +11,44 @@
 
 #include "doperator.hpp"
 
-#include <algorithm>
-
 #ifdef MFEM_USE_MPI
 
 using namespace mfem;
 using namespace mfem::future;
 
-namespace
+void DifferentiableOperator::SetParameters(std::vector<Vector *> p) const
 {
-int GetTotalTrueVSize(const std::vector<FieldDescriptor> &fds)
-{
-   int size = 0;
-   for (const auto &fd : fds)
+   MFEM_ASSERT(parameters.size() == p.size(),
+               "number of parameters doesn't match descriptors");
+   for (size_t i = 0; i < parameters.size(); i++)
    {
-      size += mfem::future::GetTrueVSize(fd);
+      p[i]->Read();
+      parameters_l[i] = *p[i];
    }
-   return size;
-}
-
-template <typename map_t>
-const typename map_t::mapped_type &FindOrDefault(
-   const map_t &map, const typename map_t::key_type &id,
-   const typename map_t::mapped_type &fallback)
-{
-   const auto it = map.find(id);
-   return it == map.end() ? fallback : it->second;
-}
-
-template <typename map_t>
-typename map_t::mapped_type FindOrEmpty(
-   const map_t &map, const typename map_t::key_type &id)
-{
-   const auto it = map.find(id);
-   return it == map.end() ? typename map_t::mapped_type{} : it->second;
-}
-
-const std::vector<derivative_action_t> &SelectActionCallbacks(
-   const std::vector<derivative_action_t> &direct_actions,
-   const DerivativeActionMap &cached_actions,
-   size_t derivative_id,
-   bool use_cached_setup)
-{
-   if (use_cached_setup)
-   {
-      const auto it_apply = cached_actions.find(derivative_id);
-      if (it_apply != cached_actions.end() && !it_apply->second.empty())
-      {
-         return it_apply->second;
-      }
-   }
-
-   return direct_actions;
-}
-
-struct DerivativeCallbackSet
-{
-   const DerivativeActionMap &actions;
-   const DerivativeActionMap &cached_actions;
-   const DerivativeActionMap &transpose_actions;
-   const DerivativeFieldMap &outfds;
-   const SparseAssemblyMap &assemble_sparse;
-   const HypreAssemblyMap &assemble_hypre;
-   const DiagonalAssemblyMap &assemble_diagonal;
-   const DerivativeSetupMap &setup;
-   const char *missing_action_message;
-};
-
-struct SecondDerivativeCallbackSet
-{
-   const SecondDerivativeActionMap &actions;
-   const SecondDerivativeActionMap &cached_actions;
-   const SecondDerivativeActionMap &transpose_actions;
-   const SecondDerivativeFieldMap &outfds;
-   const SecondSparseAssemblyMap &assemble_sparse;
-   const SecondHypreAssemblyMap &assemble_hypre;
-   const SecondDiagonalAssemblyMap &assemble_diagonal;
-   const SecondDerivativeSetupMap &setup;
-   const char *missing_action_message;
-};
-
-template <typename vector_t>
-std::shared_ptr<DerivativeOperator> MakeStatefulDerivativeOperator(
-   size_t derivative_id,
-   const vector_t &x,
-   const std::vector<FieldDescriptor> &infds,
-   const std::vector<FieldDescriptor> &default_outfds,
-   const DerivativeCallbackSet &callbacks,
-   bool use_cached_setup,
-   bool lvector_mode,
-   bool functional_gradient = false)
-{
-   const auto it_action = callbacks.actions.find(derivative_id);
-   MFEM_ASSERT(it_action != callbacks.actions.end(),
-               callbacks.missing_action_message << derivative_id);
-
-   const size_t dfidx = FindIdx(derivative_id, infds);
-   const auto &doutfds =
-      FindOrDefault(callbacks.outfds, derivative_id, default_outfds);
-   const auto &mult_callbacks =
-      SelectActionCallbacks(it_action->second, callbacks.cached_actions,
-                            derivative_id, use_cached_setup);
-
-   return std::make_shared<DerivativeOperator>(
-             GetTotalTrueVSize(doutfds),
-             GetTrueVSize(infds[dfidx]),
-             mult_callbacks,
-             FindOrEmpty(callbacks.transpose_actions, derivative_id),
-             infds[dfidx],
-             x,
-             infds,
-             doutfds,
-             FindOrEmpty(callbacks.assemble_sparse, derivative_id),
-             FindOrEmpty(callbacks.assemble_hypre, derivative_id),
-             FindOrEmpty(callbacks.assemble_diagonal, derivative_id),
-             FindOrEmpty(callbacks.setup, derivative_id),
-             lvector_mode,
-             functional_gradient);
-}
-
-const std::vector<derivative_action_t> &SelectSecondDerivativeActionCallbacks(
-   const std::vector<derivative_action_t> &direct_actions,
-   const SecondDerivativeActionMap &cached_actions,
-   second_derivative_key_t derivative_key,
-   bool use_cached_setup)
-{
-   if (use_cached_setup)
-   {
-      const auto it_apply = cached_actions.find(derivative_key);
-      if (it_apply != cached_actions.end() && !it_apply->second.empty())
-      {
-         return it_apply->second;
-      }
-   }
-
-   return direct_actions;
-}
-
-template <typename vector_t>
-std::shared_ptr<DerivativeOperator> MakeStatefulSecondDerivativeOperator(
-   size_t gradient_id,
-   size_t direction_id,
-   const vector_t &x,
-   const std::vector<FieldDescriptor> &infds,
-   const std::vector<FieldDescriptor> &default_outfds,
-   const SecondDerivativeCallbackSet &callbacks,
-   bool use_cached_setup,
-   bool lvector_mode)
-{
-   const second_derivative_key_t derivative_key{gradient_id, direction_id};
-   const auto it_action = callbacks.actions.find(derivative_key);
-   MFEM_ASSERT(it_action != callbacks.actions.end(),
-               callbacks.missing_action_message << "(" << gradient_id << ", "
-               << direction_id << ")");
-
-   const size_t dfidx = FindIdx(direction_id, infds);
-   const auto &doutfds =
-      FindOrDefault(callbacks.outfds, derivative_key, default_outfds);
-   const auto &mult_callbacks =
-      SelectSecondDerivativeActionCallbacks(
-         it_action->second, callbacks.cached_actions, derivative_key,
-         use_cached_setup);
-
-   return std::make_shared<DerivativeOperator>(
-             GetTotalTrueVSize(doutfds),
-             GetTrueVSize(infds[dfidx]),
-             mult_callbacks,
-             FindOrEmpty(callbacks.transpose_actions, derivative_key),
-             infds[dfidx],
-             x,
-             infds,
-             doutfds,
-             FindOrEmpty(callbacks.assemble_sparse, derivative_key),
-             FindOrEmpty(callbacks.assemble_hypre, derivative_key),
-             FindOrEmpty(callbacks.assemble_diagonal, derivative_key),
-             FindOrEmpty(callbacks.setup, derivative_key),
-             lvector_mode);
-}
 }
 
 DifferentiableOperator::DifferentiableOperator(
-   const std::vector<FieldDescriptor> &infds,
-   const std::vector<FieldDescriptor> &outfds,
+   const std::vector<FieldDescriptor> &solutions,
+   const std::vector<FieldDescriptor> &parameters,
    const ParMesh &mesh) :
-   Operator(),
    mesh(mesh),
-   infds(infds),
-   outfds(outfds)
+   solutions(solutions),
+   parameters(parameters)
 {
-   unionfds.clear();
-   unionfds.insert(unionfds.end(), infds.begin(), infds.end());
-   unionfds.insert(unionfds.end(), outfds.begin(), outfds.end());
-   std::sort(unionfds.begin(), unionfds.end());
-   auto last = std::unique(unionfds.begin(), unionfds.end());
-   unionfds.erase(last, unionfds.end());
+   fields.resize(solutions.size() + parameters.size());
+   fields_e.resize(fields.size());
+   solutions_l.resize(solutions.size());
+   parameters_l.resize(parameters.size());
 
-   infields_l.resize(infds.size());
-   for (size_t i = 0; i < infds.size(); i++)
+   for (size_t i = 0; i < solutions.size(); i++)
    {
-      infields_l[i] = new Vector(GetVSize(infds[i]));
+      fields[i] = solutions[i];
    }
 
-   infields_e.resize(infds.size());
-}
-
-void DifferentiableOperator::SetMultLevel(MultLevel level)
-{
-   mult_level = level;
-}
-
-void DifferentiableOperator::Mult(const Vector &x, Vector &y) const
-{
-   MFEM_ASSERT(!action_callbacks.empty(),
-               "no integrators have been set");
-
-   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&x),
-               "x needs to be a BlockVector");
-
-   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&y),
-               "y needs to be a BlockVector");
-
-   const auto &bx = static_cast<const BlockVector &>(x);
-   auto &by = static_cast<BlockVector &>(y);
-
-   Mult(bx, by);
-}
-
-void DifferentiableOperator::DisableTensorProductStructure(bool disable)
-{
-   use_tensor_product_structure = !disable;
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
-   size_t derivative_id, const Vector &x)
-{
-   return MakeStatefulDerivativeOperator(
-             derivative_id, x, infds, outfds,
+   for (size_t i = 0; i < parameters.size(); i++)
    {
-      derivative_action_callbacks,
-      derivative_apply_callbacks,
-      daction_transpose_callbacks,
-      derivative_outfds,
-      assemble_derivative_sparsematrix_callbacks,
-      assemble_derivative_hypreparmatrix_callbacks,
-      assemble_diagonal_callbacks,
-      derivative_setup_callbacks,
-      "no derivative action has been found for ID "
-   },
-   true,
-   mult_level == MultLevel::LVECTOR,
-   IsFunctionalDerivative(derivative_id));
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
-   size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
-{
-   return MakeStatefulDerivativeOperator(
-             derivative_id, x, infds, outfds,
-   {
-      derivative_action_callbacks,
-      derivative_apply_callbacks,
-      daction_transpose_callbacks,
-      derivative_outfds,
-      assemble_derivative_sparsematrix_callbacks,
-      assemble_derivative_hypreparmatrix_callbacks,
-      assemble_diagonal_callbacks,
-      derivative_setup_callbacks,
-      "no derivative action has been found for ID "
-   },
-   use_cached_setup,
-   mult_level == MultLevel::LVECTOR,
-   IsFunctionalDerivative(derivative_id));
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
-   size_t derivative_id)
-{
-   MFEM_ASSERT(HasFunctionalIntegrator(),
-               "stateless GetDerivative is available only for functionals");
-
-   const auto it_action = derivative_action_callbacks.find(derivative_id);
-   MFEM_ASSERT(it_action != derivative_action_callbacks.end(),
-               "no derivative action has been found for ID " << derivative_id);
-
-   const size_t dfidx = FindIdx(derivative_id, infds);
-   const auto &doutfds =
-      FindOrDefault(derivative_outfds, derivative_id, outfds);
-
-   return std::make_shared<DerivativeOperator>(
-             GetTotalTrueVSize(doutfds),
-             GetTrueVSize(infds[dfidx]),
-             it_action->second,
-             infds,
-             doutfds);
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
-   size_t derivative_id, const Vector &x)
-{
-   return GetSecondDerivative(derivative_id, derivative_id, x);
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
-   size_t gradient_id, size_t direction_id, const Vector &x)
-{
-   MFEM_ASSERT(HasFunctionalIntegrator(),
-               "second derivatives are available only for functionals");
-
-   return MakeStatefulSecondDerivativeOperator(
-             gradient_id, direction_id, x, infds, outfds,
-   {
-      second_derivative_action_callbacks,
-      second_derivative_apply_callbacks,
-      second_daction_transpose_callbacks,
-      second_derivative_outfds,
-      assemble_second_derivative_sparsematrix_callbacks,
-      assemble_second_derivative_hypreparmatrix_callbacks,
-      assemble_second_derivative_diagonal_callbacks,
-      second_derivative_setup_callbacks,
-      "no second derivative action has been found for ID "
-   },
-   false,
-   mult_level == MultLevel::LVECTOR);
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
-   size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
-{
-   return GetSecondDerivative(derivative_id, derivative_id, x, use_cached_setup);
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
-   size_t gradient_id, size_t direction_id, const MultiVector &x,
-   const bool use_cached_setup)
-{
-   MFEM_ASSERT(HasFunctionalIntegrator(),
-               "second derivatives are available only for functionals");
-
-   return MakeStatefulSecondDerivativeOperator(
-             gradient_id, direction_id, x, infds, outfds,
-   {
-      second_derivative_action_callbacks,
-      second_derivative_apply_callbacks,
-      second_daction_transpose_callbacks,
-      second_derivative_outfds,
-      assemble_second_derivative_sparsematrix_callbacks,
-      assemble_second_derivative_hypreparmatrix_callbacks,
-      assemble_second_derivative_diagonal_callbacks,
-      second_derivative_setup_callbacks,
-      "no second derivative action has been found for ID "
-   },
-   use_cached_setup,
-   mult_level == MultLevel::LVECTOR);
-}
-
-bool DifferentiableOperator::HasSecondDerivative(size_t gradient_id,
-                                                 size_t direction_id) const
-{
-   const second_derivative_key_t derivative_key{gradient_id, direction_id};
-   const auto it = second_derivative_action_callbacks.find(derivative_key);
-   return it != second_derivative_action_callbacks.end() && !it->second.empty();
+      fields[i + solutions.size()] = parameters[i];
+   }
 }
 
 #endif // MFEM_USE_MPI

fem/dfem/fielddescriptor.hpp

@@ -1,64 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../fespace.hpp"
-#include "../qspace.hpp"
-#include "parameterspace.hpp"
-
-namespace mfem::future
-{
-
-/// @brief FieldDescriptor struct
-///
-/// This struct is used to store information about a field.
-struct FieldDescriptor
-{
-   using data_variant_t =
-      std::variant<const FiniteElementSpace *,
-      const ParFiniteElementSpace *,
-      const VectorQuadratureSpace *,
-      const ParameterSpace *>;
-
-   /// Field ID
-   std::size_t id;
-
-   /// Field variant
-   data_variant_t data;
-
-   /// Default constructor
-   FieldDescriptor() :
-      id(SIZE_MAX), data(data_variant_t{}) {}
-
-   /// Constructor
-   template <typename T>
-   FieldDescriptor(std::size_t field_id, const T* v) :
-      id(field_id), data(v) {}
-
-   bool operator==(const FieldDescriptor& other) const
-   {
-      return id == other.id;
-   }
-
-   bool operator<(const FieldDescriptor& other) const
-   {
-      return id < other.id;
-   }
-
-   friend void swap(FieldDescriptor& a, FieldDescriptor& b)
-   {
-      using std::swap;
-      swap(a.id, b.id);
-      swap(a.data, b.data);
-   }
-};
-
-}

fem/dfem/fieldoperator.hpp

@@ -10,7 +10,6 @@
 // CONTRIBUTING.md for details.
 #pragma once
 
-#include <ostream>
 #include <type_traits>
 
 namespace mfem::future
@@ -75,15 +74,6 @@ struct is_identity_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_identity_fop<Identity<FIELD_ID>> : std::true_type {};
 
-template <typename T>
-constexpr bool is_identity_fop_v = is_identity_fop<T>::value;
-
-template <int FIELD_ID>
-inline std::ostream& operator<<(std::ostream& out, Identity<FIELD_ID>)
-{
-   return out << "Identity<" << FIELD_ID << ">";
-}
-
 /// @brief Weight FieldOperator.
 ///
 /// This FieldOperator is used to signal that this field contains the quadrature
@@ -100,14 +90,6 @@ struct is_weight_fop : std::false_type {};
 template <>
 struct is_weight_fop<Weight> : std::true_type {};
 
-template <typename T>
-constexpr bool is_weight_fop_v = is_weight_fop<T>::value;
-
-inline std::ostream& operator<<(std::ostream& out, Weight)
-{
-   return out << "Weight";
-}
-
 /// @brief Value FieldOperator.
 ///
 /// This FieldOperator is used to signal that the field contains the
@@ -119,20 +101,11 @@ public:
    constexpr Value() : FieldOperator<FIELD_ID>() {};
 };
 
-template <typename T>
+template< typename T >
 struct is_value_fop : std::false_type {};
 
-template <int T>
-struct is_value_fop<Value<T>> : std::true_type {};
-
-template <typename T>
-constexpr bool is_value_fop_v = is_value_fop<T>::value;
-
 template <int FIELD_ID>
-inline std::ostream& operator<<(std::ostream& out, Value<FIELD_ID>)
-{
-   return out << "Value<" << FIELD_ID << ">";
-}
+struct is_value_fop<Value<FIELD_ID>> : std::true_type {};
 
 /// @brief Gradient FieldOperator.
 ///
@@ -151,15 +124,6 @@ struct is_gradient_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_gradient_fop<Gradient<FIELD_ID>> : std::true_type {};
 
-template <typename T>
-constexpr bool is_gradient_fop_v = is_gradient_fop<T>::value;
-
-template <int FIELD_ID>
-inline std::ostream& operator<<(std::ostream& out, Gradient<FIELD_ID>)
-{
-   return out << "Gradient<" << FIELD_ID << ">";
-}
-
 /// @brief Sum FieldOperator.
 ///
 /// This FieldOperator is commonly used to signal that an output of a quadrature
@@ -177,39 +141,4 @@ struct is_sum_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_sum_fop<Sum<FIELD_ID>> : std::true_type {};
 
-template <typename T>
-constexpr bool is_sum_fop_v = is_sum_fop<T>::value;
-
-template <int FIELD_ID>
-inline std::ostream& operator<<(std::ostream& out, Sum<FIELD_ID>)
-{
-   return out << "Sum<" << FIELD_ID << ">";
-}
-
-/// @brief FunctionalValue FieldOperator.
-///
-/// This FieldOperator is commonly used to signal that an output of a quadrature
-/// function should be summed.
-template <int FIELD_ID = -1>
-class FunctionalValue : public FieldOperator<FIELD_ID>
-{
-public:
-   constexpr FunctionalValue() : FieldOperator<FIELD_ID>() {};
-};
-
-template< typename T >
-struct is_functionalvalue_fop : std::false_type {};
-
-template <int FIELD_ID>
-struct is_functionalvalue_fop<FunctionalValue<FIELD_ID>> : std::true_type {};
-
-template <typename T>
-constexpr bool is_functionalvalue_fop_v = is_functionalvalue_fop<T>::value;
-
-template <int FIELD_ID>
-inline std::ostream& operator<<(std::ostream& out, FunctionalValue<FIELD_ID>)
-{
-   return out << "FunctionalValue<" << FIELD_ID << ">";
-}
-
 } // namespace mfem::future

fem/dfem/integrate.hpp

@@ -0,0 +1,536 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "util.hpp"
+
+namespace mfem::future
+{
+
+template <typename output_t>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields_impl(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const output_t &output,
+   const DofToQuadMap &dtq)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   // assuming the quadrature point residual has to "play nice with
+   // the test function"
+   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [num_qp, cdim, num_dof] = B.GetShape();
+      const int vdim = output.vdim > 0 ? output.vdim : cdim ;
+      for (int dof = 0; dof < num_dof; dof++)
+      {
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            real_t acc = 0.0;
+            for (int qp = 0; qp < num_qp; qp++)
+            {
+               acc += B(qp, 0, dof) * f(vd, 0, qp);
+            }
+            y(dof, vd) += acc;
+         }
+      }
+   }
+   else if constexpr (
+      is_gradient_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [num_qp, dim, num_dof] = G.GetShape();
+      const int vdim = output.vdim;
+      for (int dof = 0; dof < num_dof; dof++)
+      {
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            real_t acc = 0.0;
+            for (int d = 0; d < dim; d++)
+            {
+               for (int qp = 0; qp < num_qp; qp++)
+               {
+                  acc += G(qp, d, dof) * f(vd, d, qp);
+               }
+            }
+            y(dof, vd) += acc;
+         }
+      }
+   }
+   else if constexpr (is_sum_fop<std::decay_t<output_t>>::value)
+   {
+      // This is the "integral over all quadrature points type" applying
+      // B = 1 s.t. B^T * C \in R^1.
+      const auto [num_qp, unused, unused1] = B.GetShape();
+      auto cc = Reshape(&f(0, 0, 0), num_qp);
+      for (int i = 0; i < num_qp; i++)
+      {
+         y(0, 0) += cc(i);
+      }
+   }
+   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [num_qp, unused, num_dof] = B.GetShape();
+      const auto vdim = output.vdim;
+      auto cc = Reshape(&f(0, 0, 0), num_qp * vdim);
+      auto yy = Reshape(&y(0, 0), num_qp * vdim);
+      for (int i = 0; i < num_qp * vdim; i++)
+      {
+         yy(i) = cc(i);
+      }
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
+                        " for this field descriptor");
+   }
+}
+
+template <typename output_t>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields_tensor_impl_1d(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const output_t &output,
+   const DofToQuadMap &dtq,
+   std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, vdim);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dx, x, d1d)
+         {
+            real_t acc = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               acc += fqp(vd, 0, qx) * B(qx, 0, dx);
+            }
+            yd(dx, vd) = acc;
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = G.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, vdim);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dx, x, d1d)
+         {
+            real_t acc = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               acc += fqp(vd, 0, qx) * G(qx, 0, dx);
+            }
+            yd(dx, vd) = acc;
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
+      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
+
+      for (int sq = 0; sq < output.size_on_qp; sq++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            yqp(sq, qx) = fqp(sq, qx);
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
+                        "for this field descriptor with sum factorization on"
+                        " tensor product elements");
+   }
+}
+
+template <typename output_t>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields_tensor_impl_2d(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const output_t &output,
+   const DofToQuadMap &dtq,
+   std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
+
+      auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               real_t acc = 0.0;
+               for (int qx = 0; qx < q1d; qx++)
+               {
+                  acc += fqp(vd, 0, qx, qy) * B(qx, 0, dx);
+               }
+               s0(qy, dx) = acc;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               real_t acc = 0.0;
+               for (int qy = 0; qy < q1d; qy++)
+               {
+                  acc += s0(qy, dx) * B(qy, 0, dy);
+               }
+               yd(dx, dy, vd) += acc;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = G.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
+
+      auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
+      auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               real_t uv[2] = {0.0, 0.0};
+               for (int qx = 0; qx < q1d; qx++)
+               {
+                  uv[0] += fqp(vd, 0, qx, qy) * G(qx, 0, dx);
+                  uv[1] += fqp(vd, 1, qx, qy) * B(qx, 0, dx);
+               }
+               s0(qy, dx) = uv[0];
+               s1(qy, dx) = uv[1];
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               real_t uv[2] = {0.0, 0.0};
+               for (int qy = 0; qy < q1d; qy++)
+               {
+                  uv[0] += s0(qy, dx) * B(qy, 0, dy);
+                  uv[1] += s1(qy, dx) * G(qy, 0, dy);
+               }
+               yd(dx, dy, vd) += uv[0] + uv[1];
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+
+      // // TODO: Check if this is the right fix for all cases
+      // auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
+      // auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
+      // for (int sq = 0; sq < output.size_on_qp; sq++)
+      // {
+      //    MFEM_FOREACH_THREAD(qx, x, q1d)
+      //    {
+      //       yqp(sq, qx) = fqp(sq, qx);
+      //    }
+      //    MFEM_SYNC_THREAD;
+      // }
+
+      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d);
+      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d);
+
+      for (int sq = 0; sq < output.size_on_qp; sq++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               yqp(sq, qx, qy) = fqp(sq, qx, qy);
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
+                        " for this field descriptor with sum factorization on"
+                        " tensor product elements");
+   }
+}
+
+template <typename output_t>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields_tensor_impl_3d(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const output_t &output,
+   const DofToQuadMap &dtq,
+   std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
+
+      auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
+      auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d, d1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               MFEM_FOREACH_THREAD(qz, z, q1d)
+               {
+                  real_t acc = 0.0;
+                  for (int qx = 0; qx < q1d; qx++)
+                  {
+                     acc += fqp(vd, 0, qx, qy, qz) * B(qx, 0, dx);
+                  }
+                  s0(qz, qy, dx) = acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               MFEM_FOREACH_THREAD(qz, z, q1d)
+               {
+                  real_t acc = 0.0;
+                  for (int qy = 0; qy < q1d; qy++)
+                  {
+                     acc += s0(qz, qy, dx) * B(qy, 0, dy);
+                  }
+                  s1(qz, dy, dx) = acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               MFEM_FOREACH_THREAD(dz, z, d1d)
+               {
+                  real_t acc = 0.0;
+                  for (int qz = 0; qz < q1d; qz++)
+                  {
+                     acc += s1(qz, dy, dx) * B(qz, 0, dz);
+                  }
+                  yd(dx, dy, dz, vd) += acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = G.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
+
+      auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
+      auto s1 = Reshape(&scratch_mem[1](0), q1d, q1d, d1d);
+      auto s2 = Reshape(&scratch_mem[2](0), q1d, q1d, d1d);
+      auto s3 = Reshape(&scratch_mem[3](0), q1d, d1d, d1d);
+      auto s4 = Reshape(&scratch_mem[4](0), q1d, d1d, d1d);
+      auto s5 = Reshape(&scratch_mem[5](0), q1d, d1d, d1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qz, z, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD(dx, x, d1d)
+               {
+                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  for (int qx = 0; qx < q1d; qx++)
+                  {
+                     uvw[0] += fqp(vd, 0, qx, qy, qz) * G(qx, 0, dx);
+                     uvw[1] += fqp(vd, 1, qx, qy, qz) * B(qx, 0, dx);
+                     uvw[2] += fqp(vd, 2, qx, qy, qz) * B(qx, 0, dx);
+                  }
+                  s0(qz, qy, dx) = uvw[0];
+                  s1(qz, qy, dx) = uvw[1];
+                  s2(qz, qy, dx) = uvw[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(qz, z, q1d)
+         {
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            {
+               MFEM_FOREACH_THREAD(dx, x, d1d)
+               {
+                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  for (int qy = 0; qy < q1d; qy++)
+                  {
+                     uvw[0] += s0(qz, qy, dx) * B(qy, 0, dy);
+                     uvw[1] += s1(qz, qy, dx) * G(qy, 0, dy);
+                     uvw[2] += s2(qz, qy, dx) * B(qy, 0, dy);
+                  }
+                  s3(qz, dy, dx) = uvw[0];
+                  s4(qz, dy, dx) = uvw[1];
+                  s5(qz, dy, dx) = uvw[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dz, z, d1d)
+         {
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            {
+               MFEM_FOREACH_THREAD(dx, x, d1d)
+               {
+                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  for (int qz = 0; qz < q1d; qz++)
+                  {
+                     uvw[0] += s3(qz, dy, dx) * B(qz, 0, dz);
+                     uvw[1] += s4(qz, dy, dx) * B(qz, 0, dz);
+                     uvw[2] += s5(qz, dy, dx) * G(qz, 0, dz);
+                  }
+                  yd(dx, dy, dz, vd) += uvw[0] + uvw[1] + uvw[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d, q1d);
+      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
+
+      for (int sq = 0; sq < output.size_on_qp; sq++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD(qz, z, q1d)
+               {
+                  yqp(sq, qx, qy, qz) = fqp(sq, qx, qy, qz);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
+                        " for this field descriptor with sum factorization on"
+                        " tensor product elements");
+   }
+}
+
+template <typename output_t>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const output_t &output,
+   const DofToQuadMap &dtq,
+   std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 1)
+      {
+         map_quadrature_data_to_fields_tensor_impl_1d(y, f, output, dtq, scratch_mem);
+      }
+      else if (dimension == 2)
+      {
+         map_quadrature_data_to_fields_tensor_impl_2d(y, f, output, dtq, scratch_mem);
+      }
+      else if (dimension == 3)
+      {
+         map_quadrature_data_to_fields_tensor_impl_3d(y, f, output, dtq, scratch_mem);
+      }
+      else { MFEM_ABORT_KERNEL("dimension not supported"); }
+   }
+   else
+   {
+      map_quadrature_data_to_fields_impl(y, f, output, dtq);
+   }
+}
+
+} // namespace mfem::future

fem/dfem/integrator_ctx.hpp

@@ -1,37 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include <typeindex>
-#include <unordered_map>
-#include <vector>
-
-#include "../../general/array.hpp"
-#include "fielddescriptor.hpp"
-
-namespace mfem::future
-{
-
-struct IntegratorContext
-{
-   const ParMesh &mesh;
-   const Array<int> *elem_attr;
-   Array<int> attr;
-   const int nentities;
-   const std::vector<FieldDescriptor> &infds;
-   const std::vector<FieldDescriptor> &outfds;
-   const std::vector<FieldDescriptor> &unionfds;
-   const IntegrationRule &ir;
-   std::unordered_map<std::type_index, std::vector<int>> &in_qlayouts;
-   std::unordered_map<std::type_index, std::vector<int>> &out_qlayouts;
-};
-
-}

fem/dfem/interpolate.hpp

@@ -0,0 +1,678 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "util.hpp"
+
+namespace mfem::future
+{
+
+template <typename field_operator_t>
+MFEM_HOST_DEVICE inline
+void map_field_to_quadrature_data_tensor_product_3d(
+   DeviceTensor<2> &field_qp,
+   const DofToQuadMap &dtq,
+   const DeviceTensor<1> &field_e,
+   const field_operator_t &input,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
+   {
+      auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d, q1d);
+      auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
+      auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dz, z, d1d)
+         {
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            {
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               {
+                  real_t acc = 0.0;
+                  for (int dx = 0; dx < d1d; dx++)
+                  {
+                     acc += B(qx, 0, dx) * field(dx, dy, dz, vd);
+                  }
+                  s0(dz, dy, qx) = acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dz, z, d1d)
+         {
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               MFEM_FOREACH_THREAD(qy, y, q1d)
+               {
+                  real_t acc = 0.0;
+                  for (int dy = 0; dy < d1d; dy++)
+                  {
+                     acc += s0(dz, dy, qx) * B(qy, 0, dy);
+                  }
+                  s1(dz, qy, qx) = acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(qz, z, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               {
+                  real_t acc = 0.0;
+                  for (int dz = 0; dz < d1d; dz++)
+                  {
+                     acc += s1(dz, qy, qx) * B(qz, 0, dz);
+                  }
+                  fqp(vd, qx, qy, qz) = acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (
+      is_gradient_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const int dim = input.dim;
+      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d, q1d);
+
+      auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
+      auto s1 = Reshape(&scratch_mem[1](0), d1d, d1d, q1d);
+      auto s2 = Reshape(&scratch_mem[2](0), d1d, q1d, q1d);
+      auto s3 = Reshape(&scratch_mem[3](0), d1d, q1d, q1d);
+      auto s4 = Reshape(&scratch_mem[4](0), d1d, q1d, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dz, z, d1d)
+         {
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            {
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               {
+                  real_t uv[2] = {0.0, 0.0};
+                  for (int dx = 0; dx < d1d; dx++)
+                  {
+                     const real_t f = field(dx, dy, dz, vd);
+                     uv[0] += f * B(qx, 0, dx);
+                     uv[1] += f * G(qx, 0, dx);
+                  }
+                  s0(dz, dy, qx) = uv[0];
+                  s1(dz, dy, qx) = uv[1];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dz, z, d1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               {
+                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  for (int dy = 0; dy < d1d; dy++)
+                  {
+                     const real_t s0i = s0(dz, dy, qx);
+                     uvw[0] += s1(dz, dy, qx) * B(qy, 0, dy);
+                     uvw[1] += s0i * G(qy, 0, dy);
+                     uvw[2] += s0i * B(qy, 0, dy);
+                  }
+                  s2(dz, qy, qx) = uvw[0];
+                  s3(dz, qy, qx) = uvw[1];
+                  s4(dz, qy, qx) = uvw[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(qz, z, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               {
+                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  for (int dz = 0; dz < d1d; dz++)
+                  {
+                     uvw[0] += s2(dz, qy, qx) * B(qz, 0, dz);
+                     uvw[1] += s3(dz, qy, qx) * B(qz, 0, dz);
+                     uvw[2] += s4(dz, qy, qx) * G(qz, 0, dz);
+                  }
+                  fqp(vd, 0, qx, qy, qz) = uvw[0];
+                  fqp(vd, 1, qx, qy, qz) = uvw[1];
+                  fqp(vd, 2, qx, qy, qz) = uvw[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   // TODO: Create separate function for clarity
+   else if constexpr (
+      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
+   {
+      const int num_qp = integration_weights.GetShape()[0];
+      // TODO: eeek
+      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
+      auto w = Reshape(&integration_weights[0], q1d, q1d, q1d);
+      auto f = Reshape(&field_qp[0], q1d, q1d, q1d);
+      MFEM_FOREACH_THREAD(qx, x, q1d)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(qz, z, q1d)
+            {
+               f(qx, qy, qz) = w(qx, qy, qz);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const int q1d = B.GetShape()[0];
+      auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d * q1d);
+      field_qp = field;
+   }
+   else
+   {
+      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
+                    "can't map field to quadrature data");
+   }
+}
+
+template <typename field_operator_t>
+MFEM_HOST_DEVICE inline
+void map_field_to_quadrature_data_tensor_product_2d(
+   DeviceTensor<2> &field_qp,
+   const DofToQuadMap &dtq,
+   const DeviceTensor<1> &field_e,
+   const field_operator_t &input,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
+   {
+      auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d);
+      auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               real_t acc = 0.0;
+               for (int dx = 0; dx < d1d; dx++)
+               {
+                  acc += B(qx, 0, dx) * field(dx, dy, vd);
+               }
+               s0(dy, qx) = acc;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               real_t acc = 0.0;
+               for (int dy = 0; dy < d1d; dy++)
+               {
+                  acc += s0(dy, qx) * B(qy, 0, dy);
+               }
+               fqp(vd, qx, qy) = acc;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (
+      is_gradient_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const int dim = input.dim;
+      const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d);
+
+      auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
+      auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               real_t uv[2] = {0.0, 0.0};
+               for (int dx = 0; dx < d1d; dx++)
+               {
+                  const real_t f = field(dx, dy, vd);
+                  uv[0] += f * B(qx, 0, dx);
+                  uv[1] += f * G(qx, 0, dx);
+               }
+               s0(dy, qx) = uv[0];
+               s1(dy, qx) = uv[1];
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               real_t uv[2] = {0.0, 0.0};
+               for (int dy = 0; dy < d1d; dy++)
+               {
+                  const real_t s0i = s0(dy, qx);
+                  uv[0] += s1(dy, qx) * B(qy, 0, dy);
+                  uv[1] += s0i * G(qy, 0, dy);
+               }
+               fqp(vd, 0, qx, qy) = uv[0];
+               fqp(vd, 1, qx, qy) = uv[1];
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   // TODO: Create separate function for clarity
+   else if constexpr (
+      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
+   {
+      const int num_qp = integration_weights.GetShape()[0];
+      // TODO: eeek
+      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
+      auto w = Reshape(&integration_weights[0], q1d, q1d);
+      auto f = Reshape(&field_qp[0], q1d, q1d);
+      MFEM_FOREACH_THREAD(qx, x, q1d)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            f(qx, qy) = w(qx, qy);
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const int q1d = B.GetShape()[0];
+      auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d);
+      field_qp = field;
+   }
+   else
+   {
+      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
+                    "can't map field to quadrature data");
+   }
+}
+
+
+template <typename field_operator_t>
+MFEM_HOST_DEVICE inline
+void map_field_to_quadrature_data_tensor_product_1d(
+   DeviceTensor<2> &field_qp,
+   const DofToQuadMap &dtq,
+   const DeviceTensor<1> &field_e,
+   const field_operator_t &input,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
+   {
+      auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const auto field = Reshape(&field_e[0], d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t acc = 0.0;
+            for (int dx = 0; dx < d1d; dx++)
+            {
+               acc += B(qx, 0, dx) * field(dx, vd);
+            }
+            fqp(vd, qx) = acc;
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (
+      is_gradient_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const int dim = input.dim;
+      const auto field = Reshape(&field_e[0], d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t acc = 0.0;
+            for (int dx = 0; dx < d1d; dx++)
+            {
+               acc += G(qx, 0, dx) * field(dx, vd);
+            }
+            fqp(vd, 0, qx) = acc;
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   // TODO: Create separate function for clarity
+   else if constexpr (
+      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
+   {
+      const int num_qp = integration_weights.GetShape()[0];
+      // TODO: eeek
+      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
+      auto w = Reshape(&integration_weights[0], q1d);
+      auto f = Reshape(&field_qp[0], q1d);
+      MFEM_FOREACH_THREAD(qx, x, q1d)
+      {
+         f(qx) = w(qx);
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const int q1d = B.GetShape()[0];
+      auto field = Reshape(&field_e[0], input.size_on_qp, q1d);
+      field_qp = field;
+   }
+   else
+   {
+      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
+                    "can't map field to quadrature data");
+   }
+}
+
+template <typename field_operator_t>
+MFEM_HOST_DEVICE
+void map_field_to_quadrature_data(
+   DeviceTensor<2> field_qp,
+   const DofToQuadMap &dtq,
+   const DeviceTensor<1> &field_e,
+   const field_operator_t &input,
+   const DeviceTensor<1, const real_t> &integration_weights)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+   if constexpr (is_value_fop<field_operator_t>::value)
+   {
+      auto [num_qp, dim, num_dof] = B.GetShape();
+      const int vdim = input.vdim;
+      const auto field = Reshape(&field_e(0), num_dof, vdim);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         for (int qp = 0; qp < num_qp; qp++)
+         {
+            real_t acc = 0.0;
+            for (int dof = 0; dof < num_dof; dof++)
+            {
+               acc += B(qp, 0, dof) * field(dof, vd);
+            }
+            field_qp(vd, qp) = acc;
+         }
+      }
+   }
+   else if constexpr (is_gradient_fop<field_operator_t>::value)
+   {
+      const auto [num_qp, dim, num_dof] = G.GetShape();
+      const int vdim = input.vdim;
+      const auto field = Reshape(&field_e(0), num_dof, vdim);
+
+      auto f = Reshape(&field_qp[0], vdim, dim, num_qp);
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         for (int qp = 0; qp < num_qp; qp++)
+         {
+            for (int d = 0; d < dim; d++)
+            {
+               real_t acc = 0.0;
+               for (int dof = 0; dof < num_dof; dof++)
+               {
+                  acc += G(qp, d, dof) * field(dof, vd);
+               }
+               f(vd, d, qp) = acc;
+            }
+         }
+      }
+   }
+   else if constexpr (std::is_same_v<field_operator_t, Weight>)
+   {
+      const int num_qp = integration_weights.GetShape()[0];
+      auto f = Reshape(&field_qp[0], num_qp);
+      for (int qp = 0; qp < num_qp; qp++)
+      {
+         f(qp) = integration_weights(qp);
+      }
+   }
+   else if constexpr (is_identity_fop<field_operator_t>::value)
+   {
+      auto [num_qp, unused, num_dof] = B.GetShape();
+      const int size_on_qp = input.size_on_qp;
+      const auto field = Reshape(&field_e[0], size_on_qp * num_qp);
+      auto f = Reshape(&field_qp[0], size_on_qp * num_qp);
+      for (int i = 0; i < size_on_qp * num_qp; i++)
+      {
+         f(i) = field(i);
+      }
+   }
+   else
+   {
+      static_assert(dfem::always_false<field_operator_t>,
+                    "can't map field to quadrature data");
+
+   }
+}
+
+template <typename field_operator_ts, size_t num_inputs, size_t num_fields>
+MFEM_HOST_DEVICE inline
+void map_fields_to_quadrature_data(
+   std::array<DeviceTensor<2>, num_inputs> &fields_qp,
+   const std::array<DeviceTensor<1>, num_fields> &fields_e,
+   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
+   const std::array<size_t, num_inputs> &input_to_field,
+   const field_operator_ts &fops,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const int &dimension,
+   const bool &use_sum_factorization = false)
+{
+   // When the input_to_field map returns -1, this means the requested input
+   // is the integration weight. Weights don't have a user defined field
+   // attached to them and we create a dummy field which is not accessed
+   // inside the functions it is passed to.
+   const auto dummy_field_weight = DeviceTensor<1>(nullptr, 0);
+   for_constexpr<num_inputs>([&](auto i)
+   {
+      const DeviceTensor<1> &field_e =
+         (input_to_field[i] == SIZE_MAX) ? dummy_field_weight :
+         fields_e[input_to_field[i]];
+
+      if (use_sum_factorization)
+      {
+         if (dimension == 1)
+         {
+            map_field_to_quadrature_data_tensor_product_1d(
+               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
+               integration_weights, scratch_mem);
+         }
+         else if (dimension == 2)
+         {
+            map_field_to_quadrature_data_tensor_product_2d(
+               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
+               integration_weights, scratch_mem);
+         }
+         else if (dimension == 3)
+         {
+            map_field_to_quadrature_data_tensor_product_3d(
+               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
+               integration_weights, scratch_mem);
+         }
+         else
+         {
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+            MFEM_ABORT("unsupported dimension");
+#endif
+         }
+      }
+      else
+      {
+         map_field_to_quadrature_data(
+            fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
+            integration_weights);
+      }
+   });
+}
+
+template <typename field_operator_t>
+MFEM_HOST_DEVICE
+void map_field_to_quadrature_data_conditional(
+   DeviceTensor<2> &field_qp,
+   const DeviceTensor<1> &field_e,
+   const DofToQuadMap &dtqmap,
+   field_operator_t &fop,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const bool &condition,
+   const int &dimension,
+   const bool &use_sum_factorization = false)
+{
+   if (condition)
+   {
+      if (use_sum_factorization)
+      {
+         if (dimension == 1)
+         {
+            map_field_to_quadrature_data_tensor_product_1d(
+               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
+
+         }
+         else if (dimension == 2)
+         {
+            map_field_to_quadrature_data_tensor_product_2d(
+               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
+         }
+         else if (dimension == 3)
+         {
+            map_field_to_quadrature_data_tensor_product_3d(
+               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
+         }
+      }
+      else
+      {
+         map_field_to_quadrature_data(
+            field_qp, dtqmap, field_e, fop, integration_weights);
+      }
+   }
+}
+
+template <size_t num_fields, size_t num_inputs, typename field_operator_ts>
+MFEM_HOST_DEVICE
+void map_fields_to_quadrature_data_conditional(
+   std::array<DeviceTensor<2>, num_inputs> &fields_qp,
+   const std::array<DeviceTensor<1, const real_t>, num_fields> &fields_e,
+   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
+   field_operator_ts fops,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const std::array<bool, num_inputs> &conditions,
+   const bool &use_sum_factorization = false)
+{
+   for_constexpr<num_inputs>([&](auto i)
+   {
+      map_field_to_quadrature_data_conditional(
+         fields_qp[i], fields_e[i], dtqmaps[i], get<i>(fops), integration_weights,
+         scratch_mem, conditions[i], use_sum_factorization);
+   });
+}
+
+template <size_t num_inputs, typename field_operator_ts>
+MFEM_HOST_DEVICE
+void map_direction_to_quadrature_data_conditional(
+   std::array<DeviceTensor<2>, num_inputs> &directions_qp,
+   const DeviceTensor<1> &direction_e,
+   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
+   field_operator_ts fops,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const std::array<bool, num_inputs> &conditions,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   for_constexpr<num_inputs>([&](auto i)
+   {
+      if (conditions[i])
+      {
+         if (use_sum_factorization)
+         {
+            if (dimension == 1)
+            {
+               map_field_to_quadrature_data_tensor_product_1d(
+                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
+                  integration_weights, scratch_mem);
+            }
+            else if (dimension == 2)
+            {
+               map_field_to_quadrature_data_tensor_product_2d(
+                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
+                  integration_weights, scratch_mem);
+            }
+            else if (dimension == 3)
+            {
+               map_field_to_quadrature_data_tensor_product_3d(
+                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
+                  integration_weights, scratch_mem);
+            }
+         }
+         else
+         {
+            map_field_to_quadrature_data(
+               directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
+               integration_weights);
+         }
+      }
+   });
+}
+
+}

fem/dfem/parameterspace.hpp

@@ -20,7 +20,7 @@ namespace mfem::future
 class ParameterSpace
 {
 public:
-   ParameterSpace(int vdim = 1) : vdim(vdim) { dtq.FE = nullptr; dtq.IntRule = nullptr; }
+   ParameterSpace(int vdim = 1) : vdim(vdim) {}
 
    /// @brief Get vector dimension at each point
    ///
@@ -43,7 +43,7 @@ public:
    /// Get spatial dimension
    ///
    /// returns always 1.
-   constexpr int Dimension() const
+   int Dimension() const
    {
       return 1;
    }
@@ -65,7 +65,7 @@ public:
    /// It should not be used by a user.
    ///
    /// returns identity by default that is lazy evaluated.
-   virtual const Operator* GetElementRestriction(ElementDofOrdering) const
+   virtual const Operator* GetElementRestriction(ElementDofOrdering o) const
    {
       if (!elem_restr)
       {
@@ -74,14 +74,11 @@ public:
       return elem_restr.get();
    }
 
-   virtual const Operator* GetB() const = 0;
-
-   virtual const Operator* GetBt() const = 0;
-
 protected:
    int vdim;
    DofToQuad dtq;
-   mutable std::unique_ptr<Operator> prolongation, elem_restr, B, Bt;
+   mutable std::unique_ptr<Operator> prolongation;
+   mutable std::unique_ptr<Operator> elem_restr;
 };
 
 /// @brief Uniform parameter space

fem/dfem/qfunction_apply.hpp

@@ -0,0 +1,619 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "util.hpp"
+#include "qfunction_transform.hpp"
+
+namespace mfem::future
+{
+
+/// @brief Call a qfunction with the given parameters.
+///
+/// @param qfunc the qfunction to call.
+/// @param input_shmem the input shared memory.
+/// @param residual_shmem the residual shared memory.
+/// @param rs_qp the size of the residual.
+/// @param num_qp the number of quadrature points.
+/// @param q1d the number of quadrature points in 1D.
+/// @param dimension the spatial dimension.
+/// @param use_sum_factorization whether to use sum factorization.
+/// @tparam qf_param_ts the tuple type of the qfunction parameters.
+template <
+   typename qf_param_ts,
+   typename qfunc_t,
+   std::size_t num_fields>
+MFEM_HOST_DEVICE inline
+void call_qfunction(
+   qfunc_t &qfunc,
+   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
+   DeviceTensor<2> &residual_shmem,
+   const int &rs_qp,
+   const int &num_qp,
+   const int &q1d,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 1)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
+         {
+            auto qf_args = decay_tuple<qf_param_ts> {};
+            auto r = Reshape(&residual_shmem(0, q), rs_qp);
+            apply_kernel(r, qfunc, qf_args, input_shmem, q);
+         }
+      }
+      else if (dimension == 2)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               const int q = qx + q1d * qy;
+               auto qf_args = decay_tuple<qf_param_ts> {};
+               auto r = Reshape(&residual_shmem(0, q), rs_qp);
+               apply_kernel(r, qfunc, qf_args, input_shmem, q);
+            }
+         }
+      }
+      else if (dimension == 3)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+                  auto qf_args = decay_tuple<qf_param_ts> {};
+                  auto r = Reshape(&residual_shmem(0, q), rs_qp);
+                  apply_kernel(r, qfunc, qf_args, input_shmem, q);
+               }
+            }
+         }
+      }
+      else
+      {
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+         MFEM_ABORT("unsupported dimension for sum factorization");
+#endif
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else
+   {
+      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
+      {
+         auto qf_args = decay_tuple<qf_param_ts> {};
+         auto r = Reshape(&residual_shmem(0, q), rs_qp);
+         apply_kernel(r, qfunc, qf_args, input_shmem, q);
+      }
+   }
+}
+
+/// @brief Call a qfunction with the given parameters and
+/// compute it's derivative action.
+///
+/// @param qfunc the qfunction to call.
+/// @param input_shmem the input shared memory.
+/// @param shadow_shmem the shadow shared memory.
+/// @param residual_shmem the residual shared memory.
+/// @param das_qp the size of the derivative action.
+/// @param num_qp the number of quadrature points.
+/// @param q1d the number of quadrature points in 1D.
+/// @param dimension the spatial dimension.
+/// @param use_sum_factorization whether to use sum factorization.
+/// @tparam qf_param_ts the tuple type of the qfunction parameters.
+template <
+   typename qf_param_ts,
+   typename qfunc_t,
+   std::size_t num_fields>
+MFEM_HOST_DEVICE inline
+void call_qfunction_derivative_action(
+   qfunc_t &qfunc,
+   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
+   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   DeviceTensor<2> &residual_shmem,
+   const int &das_qp,
+   const int &num_qp,
+   const int &q1d,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 1)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
+         {
+            auto r = Reshape(&residual_shmem(0, q), das_qp);
+            auto qf_args = decay_tuple<qf_param_ts> {};
+#ifdef MFEM_USE_ENZYME
+            auto qf_shadow_args = decay_tuple<qf_param_ts> {};
+            apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
+                                        shadow_shmem, q);
+#else
+            apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
+#endif
+         }
+      }
+      else if (dimension == 2)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               const int q = qx + q1d * qy;
+               auto r = Reshape(&residual_shmem(0, q), das_qp);
+               auto qf_args = decay_tuple<qf_param_ts> {};
+#ifdef MFEM_USE_ENZYME
+               auto qf_shadow_args = decay_tuple<qf_param_ts> {};
+               apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
+                                           shadow_shmem, q);
+#else
+               apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
+#endif
+            }
+         }
+      }
+      else if (dimension == 3)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+                  auto r = Reshape(&residual_shmem(0, q), das_qp);
+                  auto qf_args = decay_tuple<qf_param_ts> {};
+#ifdef MFEM_USE_ENZYME
+                  auto qf_shadow_args = decay_tuple<qf_param_ts> {};
+                  apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
+                                              shadow_shmem, q);
+#else
+                  apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
+#endif
+               }
+            }
+         }
+      }
+      else
+      {
+         MFEM_ABORT_KERNEL("unsupported dimension");
+      }
+   }
+   else
+   {
+      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
+      {
+         auto r = Reshape(&residual_shmem(0, q), das_qp);
+         auto qf_args = decay_tuple<qf_param_ts> {};
+#ifdef MFEM_USE_ENZYME
+         auto qf_shadow_args = decay_tuple<qf_param_ts> {};
+         apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
+                                     shadow_shmem, q);
+#else
+         apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
+#endif
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+namespace detail
+{
+template <
+   typename qf_param_ts,
+   typename qfunc_t,
+   std::size_t num_fields>
+MFEM_HOST_DEVICE inline
+void call_qfunction_derivative(
+   qfunc_t &qfunc,
+   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
+   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   DeviceTensor<2> &residual_shmem,
+   DeviceTensor<5> &qpdc,
+   const DeviceTensor<1, const real_t> &itod,
+   const int &das_qp,
+   const int &q)
+{
+   const int test_vdim = qpdc.GetShape()[0];
+   const int test_op_dim = qpdc.GetShape()[1];
+   const int trial_vdim = qpdc.GetShape()[2];
+   const int num_qp = qpdc.GetShape()[4];
+   const size_t num_inputs = itod.GetShape()[0];
+
+   for (int j = 0; j < trial_vdim; j++)
+   {
+      int m_offset = 0;
+      for (size_t s = 0; s < num_inputs; s++)
+      {
+         const int trial_op_dim = static_cast<int>(itod(s));
+         if (trial_op_dim == 0)
+         {
+            continue;
+         }
+
+         auto d_qp = Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
+         for (int m = 0; m < trial_op_dim; m++)
+         {
+            d_qp(j, m, q) = 1.0;
+
+            auto r = Reshape(&residual_shmem(0, q), das_qp);
+            auto qf_args = decay_tuple<qf_param_ts> {};
+#ifdef MFEM_USE_ENZYME
+            auto qf_shadow_args = decay_tuple<qf_param_ts> {};
+            apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
+                                        shadow_shmem, q);
+#else
+            apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
+#endif
+            d_qp(j, m, q) = 0.0;
+
+            auto f = Reshape(&r(0), test_vdim, test_op_dim);
+            for (int i = 0; i < test_vdim; i++)
+            {
+               for (int k = 0; k < test_op_dim; k++)
+               {
+                  qpdc(i, k, j, m + m_offset, q) = f(i, k);
+               }
+            }
+         }
+         m_offset += trial_op_dim;
+      }
+   }
+}
+}
+
+/// @brief Call a qfunction with the given parameters and
+/// compute it's derivative represented by the Jacobian on
+/// each quadrature point.
+///
+/// @param qfunc the qfunction to call.
+/// @param input_shmem the input shared memory.
+/// @param shadow_shmem the shadow shared memory.
+/// @param residual_shmem the residual shared memory.
+/// @param qpdc the quadrature point data cache holding the resulting
+/// Jacobians on each quadrature point.
+/// @param itod inputs trial operator dimension.
+/// If input is dependent the value corresponds to the spatial dimension, otherwise
+/// a zero indicates non-dependence on the variable.
+/// @param das_qp the size of the derivative action.
+/// @param q1d the number of quadrature points in 1D.
+/// @param dimension the spatial dimension.
+/// @param use_sum_factorization whether to use sum factorization.
+/// @tparam qf_param_ts the tuple type of the qfunction parameters.
+template <
+   typename qf_param_ts,
+   typename qfunc_t,
+   std::size_t num_fields>
+MFEM_HOST_DEVICE inline
+void call_qfunction_derivative(
+   qfunc_t &qfunc,
+   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
+   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   DeviceTensor<2> &residual_shmem,
+   DeviceTensor<5> &qpdc,
+   const DeviceTensor<1, const real_t> &itod,
+   const int &das_qp,
+   const int &q1d,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 1)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
+         {
+            detail::call_qfunction_derivative<qf_param_ts>(
+               qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
+         }
+      }
+      else if (dimension == 2)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               const int q = qx + q1d * qy;
+               detail::call_qfunction_derivative<qf_param_ts>(
+                  qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
+            }
+         }
+      }
+      else if (dimension == 3)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+                  detail::call_qfunction_derivative<qf_param_ts>(
+                     qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
+               }
+            }
+         }
+      }
+      else
+      {
+         MFEM_ABORT_KERNEL("unsupported dimension");
+      }
+   }
+   else
+   {
+      const int num_qp = qpdc.GetShape()[4];
+      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
+      {
+         detail::call_qfunction_derivative<qf_param_ts>(
+            qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+namespace detail
+{
+
+/// @brief Apply the quadrature point data cache (qpdc) to a vector
+/// (usually a direction) on quadrature point q.
+///
+/// The qpdc consists of compatible data to be used for integration with a test
+/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
+/// function including integration weights and necessesary transformations.
+///
+/// @param fhat the qpdc applied to a vector in shadow_memory.
+/// @param shadow_shmem the shadow shared memory.
+/// @param qpdc the quadrature point data cache holding the resulting
+/// Jacobians on each quadrature point.
+/// @param itod inputs trial operator dimension.
+/// If input is dependent the value corresponds to the spatial dimension, otherwise
+/// a zero indicates non-dependence on the variable.
+/// @param q the current quadrature point index.
+template <size_t num_fields>
+MFEM_HOST_DEVICE inline
+void apply_qpdc(
+   DeviceTensor<3> &fhat,
+   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   const DeviceTensor<5, const real_t> &qpdc,
+   const DeviceTensor<1, const real_t> &itod,
+   const int &q)
+{
+   const int test_vdim = qpdc.GetShape()[0];
+   const int test_op_dim = qpdc.GetShape()[1];
+   const int trial_vdim = qpdc.GetShape()[2];
+   const int num_qp = qpdc.GetShape()[4];
+   const size_t num_inputs = itod.GetShape()[0];
+
+   for (int i = 0; i < test_vdim; i++)
+   {
+      for (int k = 0; k < test_op_dim; k++)
+      {
+         real_t sum = 0.0;
+         int m_offset = 0;
+         for (size_t s = 0; s < num_inputs; s++)
+         {
+            const int trial_op_dim = static_cast<int>(itod(s));
+            if (trial_op_dim == 0)
+            {
+               continue;
+            }
+            const auto d_qp =
+               Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
+            for (int j = 0; j < trial_vdim; j++)
+            {
+               for (int m = 0; m < trial_op_dim; m++)
+               {
+                  sum += qpdc(i, k, j, m + m_offset, q) * d_qp(j, m, q);
+               }
+            }
+            m_offset += trial_op_dim;
+         }
+         fhat(i, k, q) = sum;
+      }
+   }
+}
+}
+
+/// @brief Apply the quadrature point data cache (qpdc) to a vector
+/// (usually a direction).
+///
+/// The qpdc consists of compatible data to be used for integration with a test
+/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
+/// function including integration weights and necessesary transformations.
+///
+/// @param fhat the qpdc applied to a vector in shadow_memory.
+/// @param shadow_shmem the shadow shared memory.
+/// @param qpdc the quadrature point data cache holding the resulting
+/// Jacobians on each quadrature point.
+/// @param itod inputs trial operator dimension.
+/// If input is dependent the value corresponds to the spatial dimension, otherwise
+/// a zero indicates non-dependence on the variable.
+/// @param q1d number of quadrature points in 1D.
+/// @param dimension spatial dimension.
+/// @param use_sum_factorization whether to use sum factorization.
+template <size_t num_fields>
+MFEM_HOST_DEVICE inline
+void apply_qpdc(
+   DeviceTensor<3> &fhat,
+   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   const DeviceTensor<5, const real_t> &qpdc,
+   const DeviceTensor<1, const real_t> &itod,
+   const int &q1d,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 1)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
+         {
+            detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+         }
+      }
+      else if (dimension == 2)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               const int q = qx + q1d * qy;
+               detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+            }
+         }
+      }
+      else if (dimension == 3)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+                  detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+               }
+            }
+         }
+      }
+      else
+      {
+         MFEM_ABORT_KERNEL("unsupported dimension");
+      }
+   }
+   else
+   {
+      const int num_qp = qpdc.GetShape()[4];
+      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
+      {
+         detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+      }
+   }
+}
+
+template <typename qfunc_t, typename args_ts, size_t num_args>
+MFEM_HOST_DEVICE inline
+void apply_kernel(
+   DeviceTensor<1, real_t> &f_qp,
+   const qfunc_t &qfunc,
+   args_ts &args,
+   const std::array<DeviceTensor<2>, num_args> &u,
+   int qp)
+{
+   process_qf_args(u, args, qp);
+   process_qf_result(f_qp, get<0>(apply(qfunc, args)));
+}
+
+template <typename qfunc_t, typename arg_ts, size_t num_args>
+MFEM_HOST_DEVICE inline
+void apply_kernel_native_dual(
+   DeviceTensor<1, real_t> &f_qp,
+   const qfunc_t &qfunc,
+   arg_ts &args,
+   const std::array<DeviceTensor<2>, num_args> &u,
+   const std::array<DeviceTensor<2>, num_args> &v,
+   const int &qp_idx)
+{
+   process_qf_args(u, v, args, qp_idx);
+   auto r = get<0>(apply(qfunc, args));
+   process_derivative_from_native_dual(f_qp, r);
+}
+
+#ifdef MFEM_USE_ENZYME
+
+template <typename func_t, typename... arg_ts>
+MFEM_HOST_DEVICE inline
+auto qfunction_wrapper(const func_t &f, arg_ts &&...args)
+{
+   return f(args...);
+}
+
+// Version for active function arguments only
+//
+// This is an Enzyme regression and can be removed in later versions.
+template <typename qfunc_t, typename arg_ts, std::size_t... Is,
+          typename inactive_arg_ts>
+MFEM_HOST_DEVICE inline
+auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
+                                  arg_ts &&shadow_args,
+                                  std::index_sequence<Is...>,
+                                  inactive_arg_ts &&inactive_args,
+                                  std::index_sequence<>)
+{
+   using qf_return_t = typename create_function_signature<
+                       decltype(&qfunc_t::operator())>::type::return_t;
+   return __enzyme_fwddiff<qf_return_t>(
+             qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...>, enzyme_const,
+             (void *)&qfunc, enzyme_dup, &get<Is>(args)..., enzyme_interleave,
+             &get<Is>(shadow_args)...);
+}
+
+// Interleave function arguments for enzyme
+template <typename qfunc_t, typename arg_ts, std::size_t... Is,
+          typename inactive_arg_ts, std::size_t... Js>
+MFEM_HOST_DEVICE inline
+auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
+                                  arg_ts &&shadow_args,
+                                  std::index_sequence<Is...>,
+                                  inactive_arg_ts &&inactive_args,
+                                  std::index_sequence<Js...>)
+{
+   using qf_return_t = typename create_function_signature<
+                       decltype(&qfunc_t::operator())>::type::return_t;
+   return __enzyme_fwddiff<qf_return_t>(
+             qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...,
+             decltype(get<Js>(inactive_args))...>,
+             enzyme_const, (void *)&qfunc, enzyme_dup, &get<Is>(args)...,
+             enzyme_const, &get<Js>(inactive_args)..., enzyme_interleave,
+             &get<Is>(shadow_args)...);
+}
+
+template <typename qfunc_t, typename arg_ts, typename inactive_arg_ts>
+MFEM_HOST_DEVICE inline
+auto fwddiff_apply_enzyme(qfunc_t &qfunc, arg_ts &&args,
+                          arg_ts &&shadow_args,
+                          inactive_arg_ts &&inactive_args)
+{
+   auto arg_indices = std::make_index_sequence<
+                      tuple_size<std::remove_reference_t<arg_ts>>::value> {};
+
+   auto inactive_arg_indices = std::make_index_sequence<
+                               tuple_size<std::remove_reference_t<inactive_arg_ts>>::value> {};
+
+   return fwddiff_apply_enzyme_indexed(qfunc, args, shadow_args, arg_indices,
+                                       inactive_args, inactive_arg_indices);
+}
+
+template <typename qfunc_t, typename arg_ts, size_t num_args>
+MFEM_HOST_DEVICE inline
+void apply_kernel_fwddiff_enzyme(
+   DeviceTensor<1, real_t> &f_qp,
+   qfunc_t &qfunc,
+   arg_ts &args,
+   arg_ts &shadow_args,
+   const std::array<DeviceTensor<2>, num_args> &u,
+   const std::array<DeviceTensor<2>, num_args> &v,
+   int qp_idx)
+{
+   process_qf_args(u, args, qp_idx);
+   process_qf_args(v, shadow_args, qp_idx);
+   process_qf_result(f_qp,
+                     get<0>(fwddiff_apply_enzyme(qfunc, args, shadow_args, tuple<> {})));
+}
+#endif // MFEM_USE_ENZYME
+
+} // namespace mfem::future

fem/dfem/qfunction_transform.hpp

@@ -0,0 +1,346 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "util.hpp"
+#include "../../linalg/tensor.hpp"
+
+namespace mfem::future
+{
+
+template <typename T0, typename T1, typename T2>
+MFEM_HOST_DEVICE
+void process_qf_arg(const T0 &, const T1 &, T2 &)
+{
+   static_assert(dfem::always_false<T0, T1, T2>,
+                 "process_qf_arg not implemented for arg type");
+}
+
+template <typename T>
+MFEM_HOST_DEVICE
+void process_qf_arg(
+   const DeviceTensor<1, T> &u,
+   const DeviceTensor<1, T> &v,
+   T &arg)
+{
+   arg = u(0);
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   tensor<dual<T, T>, n, m> &arg)
+{
+   for (int i = 0; i < m; i++)
+   {
+      for (int j = 0; j < n; j++)
+      {
+         arg(j, i).value = u((i * n) + j);
+      }
+   }
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   dual<T, T> &arg)
+{
+   arg.value = u(0);
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   const DeviceTensor<1> &v,
+   dual<T, T> &arg)
+{
+   arg.value = u(0);
+   arg.gradient = v(0);
+}
+
+template <typename T, int n>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   const DeviceTensor<1> &v,
+   tensor<dual<T, T>, n> &arg)
+{
+   for (int i = 0; i < n; i++)
+   {
+      arg(i).value = u(i);
+      arg(i).gradient = v(i);
+   }
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   const DeviceTensor<1> &v,
+   tensor<dual<T, T>, n, m> &arg)
+{
+   for (int i = 0; i < m; i++)
+   {
+      for (int j = 0; j < n; j++)
+      {
+         arg(j, i).value = u((i * n) + j);
+         arg(j, i).gradient = v((i * n) + j);
+      }
+   }
+}
+
+template <typename T, int n>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const tensor<dual<T, T>, n> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      r(i) = x(i).value;
+   }
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const tensor<dual<T, T>, n, m> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      for (size_t j = 0; j < m; j++)
+      {
+         r(i + n * j) = x(i, j).value;
+      }
+   }
+}
+
+template <typename arg_type>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<2> &u,
+   const DeviceTensor<2> &v,
+   arg_type &arg,
+   const int &qp)
+{
+   const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
+   const auto v_qp = Reshape(&v(0, qp), v.GetShape()[0]);
+   process_qf_arg(u_qp, v_qp, arg);
+}
+
+template <size_t num_fields, typename qf_args>
+MFEM_HOST_DEVICE inline
+void process_qf_args(
+   const std::array<DeviceTensor<2>, num_fields> &u,
+   const std::array<DeviceTensor<2>, num_fields> &v,
+   qf_args &args,
+   const int &qp)
+{
+   for_constexpr<tuple_size<qf_args>::value>([&](auto i)
+   {
+      process_qf_arg(u[i], v[i], get<i>(args), qp);
+   });
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_derivative_from_native_dual(
+   DeviceTensor<1, T> &r,
+   const tensor<dual<T, T>, n, m> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      for (size_t j = 0; j < m; j++)
+      {
+         r(i + n * j) = x(i, j).gradient;
+      }
+   }
+}
+
+template <typename T, int n>
+MFEM_HOST_DEVICE inline
+void process_derivative_from_native_dual(
+   DeviceTensor<1, T> &r,
+   const tensor<dual<T, T>, n> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      r(i) = x(i).gradient;
+   }
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_derivative_from_native_dual(
+   DeviceTensor<1, T> &r,
+   const dual<T, T> &x)
+{
+   r(0) = x.gradient;
+}
+
+template <typename T0, typename T1>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(const T0 &, T1 &)
+{
+   static_assert(dfem::always_false<T0, T1>,
+                 "process_qf_arg not implemented for arg type");
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1, T> &u,
+   T &arg)
+{
+   arg = u(0);
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1, T> &u,
+   tensor<T> &arg)
+{
+   arg(0) = u(0);
+}
+
+template <typename T, int n>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   tensor<T, n> &arg)
+{
+   for (int i = 0; i < n; i++)
+   {
+      arg(i) = u(i);
+   }
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   tensor<T, n, m> &arg)
+{
+   for (int i = 0; i < m; i++)
+   {
+      for (int j = 0; j < n; j++)
+      {
+         arg(j, i) = u((i * n) + j);
+      }
+   }
+}
+
+template <typename arg_type>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(const DeviceTensor<2> &u, arg_type &arg, int qp)
+{
+   const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
+   process_qf_arg(u_qp, arg);
+}
+
+template <size_t num_fields, typename qf_args>
+MFEM_HOST_DEVICE inline
+void process_qf_args(
+   const std::array<DeviceTensor<2>, num_fields> &u,
+   qf_args &args,
+   const int &qp)
+{
+   for_constexpr<tuple_size<qf_args>::value>([&](auto i)
+   {
+      process_qf_arg(u[i], get<i>(args), qp);
+   });
+}
+
+template <typename T0, typename T1>
+MFEM_HOST_DEVICE inline
+Vector process_qf_result(T0, T1)
+{
+   static_assert(dfem::always_false<T0, T1>,
+                 "process_qf_result not implemented for result type");
+   return Vector{};
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const T &x)
+{
+   r(0) = x;
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1> &r,
+   const dual<T, T> &x)
+{
+   r(0) = x.value;
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const tensor<T> &x)
+{
+   r(0) = x(0);
+}
+
+template <typename T, int n>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const tensor<T, n> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      r(i) = x(i);
+   }
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const tensor<T, n, m> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      for (size_t j = 0; j < m; j++)
+      {
+         r(i + n * j) = x(i, j);
+      }
+   }
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1, T> &u,
+   const DeviceTensor<1, T> &v,
+   tensor<T, n, m> &arg)
+{
+   for (int i = 0; i < m; i++)
+   {
+      for (int j = 0; j < n; j++)
+      {
+         arg(j, i) = u((i * n) + j);
+      }
+   }
+}
+
+} // namespace mfem::future

fem/dfem/tensor_functions.hpp

@@ -1,264 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-/**
- * @file tensor_functions.hpp
- *
- * @brief Differentiable functions of tensors
- */
-
-#pragma once
-
-#include <cmath>
-
-#include "../../linalg/dual.hpp"
-#include "../../linalg/tensor.hpp"
-#include "tuple.hpp"
-#include "util.hpp"
-
-// Force-inline every tensor operation under clang
-#if defined(__clang__)
-#pragma clang attribute push (__attribute__((always_inline)), apply_to = function)
-#endif
-
-namespace mfem
-{
-namespace future
-{
-
-/**
- * @brief Differentiable approximation of maximum eigenvale of a symmetric tensor
- *
- * Estimates the maximum eigenvalue using
- * $$
- *     smooth_max_eigenvalue(A) = \frac{1}{\beta} \log\Big( \mathrm{tr}\big(\exp(\beta A) \big) \Big)
- * $$
- * which is equivalent to using the log-sum-exp function on the eigenvalues of A.
- *
- * @param A The input tensor
- * @param beta Sharpness parameter. Must be > 0. Larger values makes the approximation sharper.
- * @return Approximate maximum eigenvalue of A
- */
-template <int n> MFEM_HOST_DEVICE
-real_t smooth_max_eigenvalue_symm(const tensor<real_t, n, n>& A, real_t beta)
-{
-   auto [lambda, V] = eig_symm(A);
-   real_t lambda_max = lambda[n - 1];
-   real_t sum = 0;
-   for (int i = 0; i < n - 1; i++)
-   {
-      sum += std::exp(beta*(lambda[i] - lambda_max));
-   }
-   return lambda_max + std::log1p(sum)/beta;
-}
-
-/**
- * @brief Differentiable approximation of minimum eigenvale of a symmetric tensor
- *
- * Estimates the minimum eigenvalue using
- * $$
- *     smooth_min_eigenvalue(A) = -\frac{1}{\beta} \log\Big( \mathrm{tr}\big(\exp(-\beta A) \big) \Big)
- * $$
- * which is equivalent to using the negated log-sum-exp function on the eigenvalues of -A.
- *
- * @param A The input tensor
- * @param beta Sharpness parameter. Must be > 0. Larger values makes the approximation sharper.
- * @return Approximate minimum eigenvalue of A
- */
-template <int n> MFEM_HOST_DEVICE
-real_t smooth_min_eigenvalue_symm(const tensor<real_t, n, n>& A, real_t beta)
-{
-   return -smooth_max_eigenvalue_symm<n>(-A, beta);
-}
-
-#ifdef MFEM_USE_ENZYME
-
-namespace detail
-{
-
-// Custom forward-mode derivative rule for Enzyme
-template<int n> MFEM_HOST_DEVICE
-dual<real_t, real_t> smooth_max_eigenvalue_symm_fwddiff(
-   const tensor<real_t, n, n>& A, const tensor<real_t, n, n>& A_dot, real_t beta,
-   real_t beta_dot)
-{
-   auto [lambda, V] = eig_symm(A);
-   real_t lambda_max = lambda[n - 1];
-   real_t sum = 0;
-   tensor<real_t, n> eg;
-   tensor<real_t, n> lambda_shifted;
-   for (int i = 0; i < n; i++)
-   {
-      lambda_shifted[i] = lambda[i] - lambda_max;
-      eg[i] = std::exp(beta*lambda_shifted[i]);
-      if (i != n - 1) { sum += eg[i]; }
-   }
-   real_t value = lambda_max + std::log1p(sum)/beta;
-
-   real_t Z = sum + 1.0;
-   real_t derivative{};
-   for (int mu = 0; mu < n; mu++)
-   {
-      real_t w_mu = eg[mu]/Z;
-      for (int i = 0; i < n; i++)
-      {
-         for (int j = 0; j < n; j++)
-         {
-            derivative += w_mu*V[i][mu]*V[j][mu]*A_dot[i][j];
-         }
-      }
-   }
-   derivative += (lambda_max - value + dot(eg, lambda_shifted)/Z)/beta * beta_dot;
-   return {value, derivative};
-}
-
-// Types and functions for Enzyme custom reverse mode derivative
-template <int n>
-struct SmoothMaxEigenvalueSymmTape
-{
-   tensor<real_t, n> lambda;
-   tensor<real_t, n, n> V;
-   tensor<real_t, n> eg;
-   real_t sum;
-   real_t logZ;
-};
-
-template <int n>
-struct SmoothMaxEigenvalueSymmAugmentedReturn
-{
-   void* tape;
-   real_t value;
-};
-
-template <int n> MFEM_HOST_DEVICE
-SmoothMaxEigenvalueSymmAugmentedReturn<n>
-smooth_max_eigenvalue_symm_aug(const tensor<real_t, n, n>* A,
-                               tensor<real_t, n, n>* A_bar,
-                               real_t beta)
-{
-   (void)A_bar; // accumulated in reverse pass
-
-   auto [lambda, V] = eig_symm(*A);
-   const real_t lambda_max = lambda[n - 1];
-
-   tensor<real_t, n> eg;
-   real_t sum = 0;
-   for (int i = 0; i < n; i++)
-   {
-      eg[i] = std::exp(beta*(lambda[i] - lambda_max));
-      if (i != n - 1) { sum += eg[i]; }
-   }
-
-   const real_t logZ = std::log1p(sum);
-   const real_t value = lambda_max + logZ/beta;
-
-   auto* tape = static_cast<SmoothMaxEigenvalueSymmTape<n>*>(
-                   std::malloc(sizeof(SmoothMaxEigenvalueSymmTape<n>)));
-   if (tape)
-   {
-      tape->lambda = lambda;
-      tape->V = V;
-      tape->eg = eg;
-      tape->sum = sum;
-      tape->logZ = logZ;
-   }
-
-   return {static_cast<void*>(tape), value};
-}
-
-template <int n> MFEM_HOST_DEVICE
-real_t smooth_max_eigenvalue_symm_rev(const tensor<real_t, n, n>* A,
-                                      tensor<real_t, n, n>* A_bar,
-                                      real_t beta,
-                                      real_t out_bar,
-                                      void* tape_ptr)
-{
-   (void)A; // all needed info is on the tape
-
-   const auto* tape = static_cast<const SmoothMaxEigenvalueSymmTape<n>*>(tape_ptr);
-   if (!tape)
-   {
-      return 0.0;
-   }
-
-   const real_t Z = tape->sum + 1.0;
-
-   // d/dA = Σ_mu w_mu v_mu v_mu^T, where w_mu = eg[mu]/Z
-   for (int mu = 0; mu < n; mu++)
-   {
-      const real_t w_mu = tape->eg[mu] / Z;
-      for (int i = 0; i < n; i++)
-      {
-         for (int j = 0; j < n; j++)
-         {
-            (*A_bar)[i][j] += out_bar * w_mu * tape->V[i][mu] * tape->V[j][mu];
-         }
-      }
-   }
-
-   // d/dβ = -(log Z)/β^2 + (1/(β Z)) Σ_{i<n-1} exp(β(λ_i-λ_max)) (λ_i-λ_max)
-   real_t dZ_dBeta = 0.0;
-   const real_t& lambda_max = tape->lambda[n - 1];
-   for (int i = 0; i < n - 1; i++)
-   {
-      dZ_dBeta += tape->eg[i] * (tape->lambda[i] - lambda_max);
-   }
-
-   const real_t beta2 = beta * beta;
-   const real_t d_value_dBeta = -(tape->logZ)/beta2 + dZ_dBeta/(beta * Z);
-
-   std::free(const_cast<SmoothMaxEigenvalueSymmTape<n>*>(tape));
-
-   return out_bar * d_value_dBeta;
-}
-
-} // namespace detail
-
-// Register custom derivatives (forward mode) with Enzyme
-__attribute__((used))
-void* __enzyme_register_derivative_smooth_max_eigenvalue_symm_2d[] =
-{
-   reinterpret_cast<void*>(smooth_max_eigenvalue_symm<2>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_fwddiff<2>)
-};
-
-__attribute__((used))
-void* __enzyme_register_derivative_smooth_max_eigenvalue_symm_3d[] =
-{
-   reinterpret_cast<void*>(smooth_max_eigenvalue_symm<3>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_fwddiff<3>)
-};
-
-// Register custom gradients (combined reverse mode) with Enzyme
-__attribute__((used))
-void* __enzyme_register_gradient_smooth_max_eigenvalue_symm_2d[] =
-{
-   reinterpret_cast<void*>(smooth_max_eigenvalue_symm<2>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_aug<2>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_rev<2>)
-};
-
-__attribute__((used))
-void* __enzyme_register_gradient_smooth_max_eigenvalue_symm_3d[] =
-{
-   reinterpret_cast<void*>(smooth_max_eigenvalue_symm<3>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_aug<3>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_rev<3>)
-};
-
-#endif // MFEM_USE_ENZYME
-
-} // namespace future
-} // namespace mfem
-
-#if defined(__clang__)
-#pragma clang attribute pop
-#endif

fem/fe/fe_fixed_order.hpp

@@ -307,12 +307,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (6) */
+       them in the vector shape of dimension Dof (4) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (6 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;
@@ -336,12 +336,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (5) */
+       them in the vector shape of dimension Dof (4) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (5 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;

fem/fe/fe_h1.cpp

@@ -1757,45 +1757,22 @@ H1_BergotPyramidElement::H1_BergotPyramidElement(const int p, const int btype)
       real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       real_t z = ip.z;
 
-      if (std::abs(z - 1.0) < apex_tol)
-      {
-         // Compute the limit of the basis functions as z->1 with x and y on the
-         // line between the center of the base and the apex
-         o = 0;
-         for (int i = 0; i <= p; i++)
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               for (int k = 0; k <= p - maxij; k++)
-                  if (i == 0 && j == 0)
-                  {
-                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
-                  }
-                  else
-                  {
-                     T(o++, m) = 0.;
-                  }
-            }
-      }
-      else
-      {
-         poly1d.CalcLegendre(p, x, shape_x.GetData());
-         poly1d.CalcLegendre(p, y, shape_y.GetData());
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-         o = 0;
-         for (int i = 0; i <= p; i++)
+      o = 0;
+      for (int i = 0; i <= p; i++)
+      {
+         for (int j = 0; j <= p; j++)
          {
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                                z, 1.0, shape_z);
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
 
-               for (int k = 0; k <= p - maxij; k++)
-               {
-                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                              pow(1.0 - ip.z, maxij);
-               }
+            for (int k = 0; k <= p - maxij; k++)
+            {
+               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                           pow(1.0 - ip.z, maxij);
             }
          }
       }
@@ -1816,44 +1793,25 @@ void H1_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    Vector u(dof);
 #endif
 
-   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   const real_t z = ip.z;
+   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the basis functions as z->1 with x and y on the
-      // line between the center of the base and the apex
-      u = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-               if (i == 0 && j == 0)
-               {
-                  u(o) = ((k + 3.) * k + 2.) / 2.;
-               }
-         }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
+   poly1d.CalcLegendre(p, x, shape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData());
+
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z);
+         for (int k = 0; k <= p - maxij; k++)
+            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                     pow(1.0 - ip.z, maxij);
+      }
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
-            for (int k = 0; k <= p - maxij; k++)
-               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                        pow(1.0 - ip.z, maxij);
-         }
-   }
    Ti.Mult(u, shape);
 }
 
@@ -1872,68 +1830,37 @@ void H1_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    Vector dshape_z(order+1);
    Vector dshape_z_dt(order+1);
 #endif
-   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   const real_t z = ip.z;
+   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the gradients of the basis functions as
-      // z->1 with x and y on the line between the center of the base and the
-      // apex
-      du = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
+   poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z, dshape_z, dshape_z_dt);
+
+         for (int k = 0; k <= p - maxij; k++, o++)
          {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               if (i == 0 && j == 0)
-               {
-                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
-               }
-               else if (i == 1 && j == 0)
-               {
-                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-               else if (i == 0 && j == 1)
-               {
-                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-            }
+            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                      pow(1.0 - ip.z, maxij) +
+                      (ip.x * dshape_x(i) * shape_y(j) +
+                       ip.y * shape_x(i) * dshape_y(j)) *
+                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                      maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
          }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+      }
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0,
-                                             shape_z, dshape_z, dshape_z_dt);
-
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                         pow(1.0 - ip.z, maxij) +
-                         (ip.x * dshape_x(i) * shape_y(j) +
-                          ip.y * shape_x(i) * dshape_y(j)) *
-                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
-            }
-         }
-   }
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_h1.hpp

@@ -208,8 +208,6 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
-   static constexpr real_t apex_tol = 1e-8;
-
 public:
    H1_BergotPyramidElement(const int p,
                            const int btype = BasisType::GaussLobatto);

fem/fe/fe_l2.cpp

@@ -1106,16 +1106,9 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
          {
             const real_t wik = op[i] + op[k] + op[p-i-k];
             const real_t w = wik * wjk * op[p-k];
-            if (std::abs(w) < apex_tol)
-            {
-               Nodes.IntPoint(o++).Set3(0.,0.,1.);
-            }
-            else
-            {
-               Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
-                                        op[j] * (op[i] + op[p-i-k]) / w,
-                                        op[k] * op[p-k] / w);
-            }
+            Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
+                                     op[j] * (op[j] + op[p-j-k]) / w,
+                                     op[k] * op[p-k] / w);
          }
       }
 
@@ -1132,45 +1125,22 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
       const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       const real_t z = ip.z;
 
-      if (std::abs(z - 1.0) < apex_tol)
-      {
-         // Compute the limit of the basis functions as z->1 with x and y on the
-         // line between the center of the base and the apex
-         o = 0;
-         for (int i = 0; i <= p; i++)
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               for (int k = 0; k <= p - maxij; k++)
-                  if (i == 0 && j == 0)
-                  {
-                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
-                  }
-                  else
-                  {
-                     T(o++, m) = 0.;
-                  }
-            }
-      }
-      else
-      {
-         poly1d.CalcLegendre(p, x, shape_x.GetData());
-         poly1d.CalcLegendre(p, y, shape_y.GetData());
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-         o = 0;
-         for (int i = 0; i <= p; i++)
+      o = 0;
+      for (int i = 0; i <= p; i++)
+      {
+         for (int j = 0; j <= p; j++)
          {
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                                z, 1.0, shape_z);
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
 
-               for (int k = 0; k <= p - maxij; k++)
-               {
-                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                              pow(1.0 - ip.z, maxij);
-               }
+            for (int k = 0; k <= p - maxij; k++)
+            {
+               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                           pow(1.0 - ip.z, maxij);
             }
          }
       }
@@ -1195,41 +1165,26 @@ void L2_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the basis functions as z->1 with x and y on the
-      // line between the center of the base and the apex
-      u = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-               if (i == 0 && j == 0)
-               {
-                  u(o) = ((k + 3.) * k + 2.) / 2.;
-               }
-         }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
+   poly1d.CalcLegendre(p, x, shape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+   {
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z);
 
-            for (int k = 0; k <= p - maxij; k++)
-               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                        pow(1.0 - ip.z, maxij);
+         for (int k = 0; k <= p - maxij; k++)
+         {
+            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                     pow(1.0 - ip.z, maxij);
          }
+      }
    }
+
    Ti.Mult(u, shape);
 }
 
@@ -1253,64 +1208,35 @@ void L2_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the gradients of the basis functions as
-      // z->1 with x and y on the line between the center of the base and the
-      // apex
-      du = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               if (i == 0 && j == 0)
-               {
-                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
-               }
-               else if (i == 1 && j == 0)
-               {
-                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-               else if (i == 0 && j == 1)
-               {
-                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-            }
-         }
-   }
-   else
-   {
-      Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-      Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+   Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+   Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0,
-                                             shape_z, dshape_z, dshape_z_dt);
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+   {
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z, dshape_z, dshape_z_dt);
 
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                         pow(1.0 - ip.z, maxij) +
-                         (ip.x * dshape_x(i) * shape_y(j) +
-                          ip.y * shape_x(i) * dshape_y(j)) *
-                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
-            }
+         for (int k = 0; k <= p - maxij; k++, o++)
+         {
+            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                      pow(1.0 - ip.z, maxij) +
+                      (ip.x * dshape_x(i) * shape_y(j) +
+                       ip.y * shape_x(i) * dshape_y(j)) *
+                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                      ((maxij > 0) ? (maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                                      pow(1.0 - ip.z, maxij - 1)) : 0.0);
          }
+      }
    }
+
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_l2.hpp

@@ -225,8 +225,6 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
-   static constexpr real_t apex_tol = 1e-8;
-
 public:
    /// Construct the L2_PyramidElement of order @a p and BasisType @a btype
    L2_BergotPyramidElement(const int p,

fem/fe/fe_nd.cpp

@@ -1282,49 +1282,12 @@ ND_SegmentElement::ND_SegmentElement(const int p, const int ob_type)
    }
 }
 
-void ND_SegmentElement::CalcShape(const IntegrationPoint &ip,
-                                  Vector &shape) const
-{
-   if (obasis1d.IsIntegratedType()) { obasis1d.ScaleIntegrated(false); }
-   obasis1d.Eval(ip.x, shape);
-}
-
 void ND_SegmentElement::CalcVShape(const IntegrationPoint &ip,
                                    DenseMatrix &shape) const
 {
    Vector vshape(shape.Data(), dof);
 
-   CalcShape(ip, vshape);
-}
-
-void ND_SegmentElement::ProjectIntegrated(VectorCoefficient &vc,
-                                          ElementTransformation &Trans,
-                                          Vector &dofs) const
-{
-   MFEM_ASSERT(obasis1d.IsIntegratedType(), "Not integrated type");
-   real_t vk[Geometry::MaxDim];
-   Vector xk(vk, vc.GetVDim());
-
-   const real_t *cp = poly1d.ClosedPoints(dof, BasisType::GaussLobatto);
-   const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, dof);
-   IntegrationPoint ip;
-
-   for (int i = 0; i < dof; i++)
-   {
-      const real_t h = cp[i+1] - cp[i];
-      real_t val = 0.0;
-
-      for (int q = 0; q < ir.GetNPoints(); q++)
-      {
-         const IntegrationPoint &ip1d = ir.IntPoint(q);
-         ip.x = cp[i] + h*ip1d.x;
-         Trans.SetIntPoint(&ip);
-         vc.Eval(xk, Trans, ip);
-         val += ip1d.weight*Trans.Jacobian().InnerProduct(tk, vk);
-      }
-
-      dofs(i) = val*h;
-   }
+   obasis1d.Eval(ip.x, vshape);
 }
 
 const real_t ND_WedgeElement::tk[15] =

fem/fe/fe_nd.hpp

@@ -303,7 +303,8 @@ public:
    /** @brief Construct the ND_SegmentElement of order @a p and open
        BasisType @a ob_type */
    ND_SegmentElement(const int p, const int ob_type = BasisType::GaussLegendre);
-   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
+   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override
+   { obasis1d.Eval(ip.x, shape); }
    void CalcVShape(const IntegrationPoint &ip,
                    DenseMatrix &shape) const override;
    void CalcVShape(ElementTransformation &Trans,
@@ -324,10 +325,7 @@ public:
    using FiniteElement::Project;
    void Project(VectorCoefficient &vc,
                 ElementTransformation &Trans, Vector &dofs) const override
-   {
-      if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
-      else { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-   }
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    void ProjectMatrixCoefficient(MatrixCoefficient &mc,
                                  ElementTransformation &T,
                                  Vector &dofs) const override
@@ -340,11 +338,6 @@ public:
                     ElementTransformation &Trans,
                     DenseMatrix &grad) const override
    { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
-
-protected:
-   void ProjectIntegrated(VectorCoefficient &vc,
-                          ElementTransformation &Trans,
-                          Vector &dofs) const;
 };
 
 class ND_WedgeElement : public VectorFiniteElement

fem/fe/fe_pyramid.hpp

@@ -17,12 +17,6 @@
 namespace mfem
 {
 
-struct ScalarPyramid
-{
-   // Default basis type for H1 and L2 pyramids
-   static inline int DefaultType = 1; // Bergot(0) or Fuentes(1)
-};
-
 /** Base class for arbitrary order basis functions on pyramid-shaped elements
 
   This base class provides a common class to store temporary vectors,

fem/fe_coll.cpp

@@ -228,19 +228,7 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1_", 3))
    {
-      // Parse pyramid basis type if included in the name
-      const char *pyr = strstr(name, "Pyr");
-      if (pyr == NULL)
-      {
-         // Use default pyramid type elements
-         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
-      }
-      else
-      {
-         // Use specific pyramid type elements
-         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3),
-                                   BasisType::GaussLobatto, atoi(pyr + 3));
-      }
+      fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
    }
    else if (!strncmp(name, "H1Pos_Trace_", 12))
    {
@@ -257,44 +245,26 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1@", 3))
    {
-      // Parse pyramid basis type if included in the name
-      const char *pyr = strstr(name, "Pyr");
-      if (pyr == NULL)
-      {
-         // Use default pyramid type elements
-         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                   BasisType::GetType(name[3]));
-      }
-      else
-      {
-         // Use specific pyramid type elements
-         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                   BasisType::GetType(name[3]),
-                                   atoi(pyr + 3));
-      }
+      fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
+                                BasisType::GetType(name[3]));
    }
-   else if (!strncmp(name, "L2", 2))
+   else if (!strncmp(name, "L2_T", 4))
+      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
+                                atoi(name + 4));
+   else if (!strncmp(name, "L2_", 3))
    {
-      // Parse Map Type
-      const int mtype = strstr(name, "Int") == NULL ?
-                        FiniteElement::VALUE : FiniteElement::INTEGRAL;
-
-      // Parse the base order
-      const int p = atoi(strstr(name, "_P") + 2);
-
-      // Parse the mesh dimension
-      const int dim = atoi(strstr(name, "D") - 1);
-
-      // Parse basis type if specified
-      const char *t = strstr(name, "_T");
-      const int btype = t == NULL ? BasisType::GaussLegendre : atoi(t + 2);
-
-      // Parse the pyramid type if specified
-      const char *pyr = strstr(name, "Pyr");
-      const int ptype = pyr == NULL ? 1 : atoi(pyr + 3);
-
-      // Create collection
-      fec = new L2_FECollection(p, dim, btype, mtype, ptype);
+      fec = new L2_FECollection(atoi(name + 7), atoi(name + 3));
+   }
+   else if (!strncmp(name, "L2Int_T", 7))
+   {
+      fec = new L2_FECollection(atoi(name + 13), atoi(name + 9),
+                                atoi(name + 7), FiniteElement::INTEGRAL);
+   }
+   else if (!strncmp(name, "L2Int_", 6))
+   {
+      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
+                                BasisType::GaussLegendre,
+                                FiniteElement::INTEGRAL);
    }
    else if (!strncmp(name, "RT_Trace_", 9))
    {
@@ -1739,10 +1709,9 @@ const int *RT1_3DFECollection::DofOrderForOrientation(Geometry::Type GeomType,
 
 
 H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
-                                 const int pyr_type)
+                                 const int pyrtype)
    : FiniteElementCollection(p)
    , dim(dim)
-   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 1, "H1_FECollection requires order >= 1.");
    MFEM_VERIFY(dim >= 0 && dim <= 3, "H1_FECollection requires 0 <= dim <= 3.");
@@ -1755,14 +1724,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
    {
       case BasisType::GaussLobatto:
       {
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
-         }
-         else
-         {
-            snprintf(h1_name, 32, "H1_%dD_P%d_Pyr%d", dim, p, pyr_type);
-         }
+         snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
          break;
       }
       case BasisType::Positive:
@@ -1948,11 +1910,11 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
          H1_dof[Geometry::TETRAHEDRON] = (TriDof*pm3)/3;
          H1_dof[Geometry::CUBE] = QuadDof*pm1;
          H1_dof[Geometry::PRISM] = TriDof*pm1;
-         if (pyr_type == 0 || b_type == BasisType::Positive)
+         if (pyrtype == 0 || b_type == BasisType::Positive)
          {
             H1_dof[Geometry::PYRAMID] = pm2*pm1*(2*p-3)/6; // Bergot (JSC)
          }
-         else if (pyr_type == 1)
+         else if (pyrtype == 1)
          {
             H1_dof[Geometry::PYRAMID] = pm1*pm1*pm1; // Fuentes
          }
@@ -1973,15 +1935,13 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
                new H1_TetrahedronElement(p, btype);
             H1_Elements[Geometry::CUBE] = new H1_HexahedronElement(p, btype);
             H1_Elements[Geometry::PRISM] = new H1_WedgeElement(p, btype);
-            if (pyr_type == 0)
+            if (pyrtype == 0)
             {
-               H1_Elements[Geometry::PYRAMID] =
-                  new H1_BergotPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] = new H1_BergotPyramidElement(p, btype);
             }
             else
             {
-               H1_Elements[Geometry::PYRAMID] =
-                  new H1_FuentesPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] = new H1_FuentesPyramidElement(p, btype);
             }
          }
 
@@ -2188,7 +2148,6 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    : FiniteElementCollection(p)
    , dim(dim)
    , m_type(map_type)
-   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 0, "L2_FECollection requires order >= 0.");
 
@@ -2204,25 +2163,10 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    switch (btype)
    {
       case BasisType::GaussLegendre:
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
-         }
-         else
-         {
-            snprintf(d_name, 32, "%s_%dD_P%d_Pyr%d", prefix, dim, p, pyr_type);
-         }
+         snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
          break;
       default:
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
-         }
-         else
-         {
-            snprintf(d_name, 32, "%s_T%d_%dD_P%d_Pyr%d",
-                     prefix, btype, dim, p, pyr_type);
-         }
+         snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
    }
 
    for (int g = 0; g < Geometry::NumGeom; g++)
@@ -2341,13 +2285,11 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
          L2_Elements[Geometry::PRISM] = new L2_WedgeElement(p, btype);
          if (pyr_type == 0)
          {
-            L2_Elements[Geometry::PYRAMID] =
-               new L2_BergotPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] = new L2_BergotPyramidElement(p, btype);
          }
          else
          {
-            L2_Elements[Geometry::PYRAMID] =
-               new L2_FuentesPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] = new L2_FuentesPyramidElement(p, btype);
          }
       }
 

fem/fe_coll.hpp

@@ -100,10 +100,6 @@ public:
       return FiniteElementForGeometry(GeomType);
    }
 
-   /** @brief Returns a collection of the trace elements.
-
-       @note The collection is owned by the caller and is NOT deleted in the
-       destructor. */
    virtual FiniteElementCollection *GetTraceCollection() const;
 
    virtual ~FiniteElementCollection();
@@ -290,7 +286,7 @@ protected:
 class H1_FECollection : public FiniteElementCollection
 {
 protected:
-   int dim, b_type, p_type;
+   int dim, b_type;
    char h1_name[32];
    FiniteElement *H1_Elements[Geometry::NumGeom];
    int H1_dof[Geometry::NumGeom];
@@ -299,7 +295,7 @@ protected:
 public:
    explicit H1_FECollection(const int p, const int dim = 3,
                             const int btype = BasisType::GaussLobatto,
-                            const int pyr_type = ScalarPyramid::DefaultType);
+                            const int pyrtype = 1);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -324,7 +320,7 @@ public:
    const int *GetDofMap(Geometry::Type GeomType, int p) const;
 
    FiniteElementCollection *Clone(int p) const override
-   { return new H1_FECollection(p, dim, b_type, p_type); }
+   { return new H1_FECollection(p, dim, b_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }
@@ -371,7 +367,6 @@ private:
    int dim;
    int b_type; // BasisType
    int m_type; // map type
-   int p_type; // Pyramid type (0 -> Bergot, 1 -> Fuentes)
    char d_name[32];
    ScalarFiniteElement *L2_Elements[Geometry::NumGeom];
    ScalarFiniteElement *Tr_Elements[Geometry::NumGeom];
@@ -384,7 +379,7 @@ public:
    L2_FECollection(const int p, const int dim,
                    const int btype = BasisType::GaussLegendre,
                    const int map_type = FiniteElement::VALUE,
-                   const int pyr_type = ScalarPyramid::DefaultType);
+                   const int pyrtype = 1);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -414,7 +409,7 @@ public:
    int GetBasisType() const { return b_type; }
 
    FiniteElementCollection *Clone(int p) const override
-   { return new L2_FECollection(p, dim, b_type, m_type, p_type); }
+   { return new L2_FECollection(p, dim, b_type, m_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }

fem/fem.hpp

@@ -52,7 +52,7 @@
 #include "bounds.hpp"
 #include "particleset.hpp"
 
-// #include "dfem/doperator.hpp"
+#include "dfem/doperator.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"

fem/integ/bilininteg_elasticity_kernels.cpp

@@ -91,15 +91,15 @@ void ElasticityAddMultPA(const int dim, const int nDofs,
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag)
+                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
    switch (dim)
    {
       case 2:
-         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, ir, diag);
+         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, QVec, diag);
          break;
       case 3:
-         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, ir, diag);
+         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, QVec, diag);
          break;
       default:
          MFEM_ABORT("Only dimensions 2 and 3 supported.");

fem/integ/bilininteg_elasticity_kernels.hpp

@@ -38,6 +38,7 @@
 #include "../../linalg/vector.hpp"
 #include "../../linalg/tensor.hpp"
 #include "../quadinterpolator.hpp"
+#include "../bilininteg.hpp"
 #include "../coefficient.hpp"
 #include "../qfunction.hpp"
 
@@ -132,12 +133,12 @@ void ElasticityAssembleEA(const int dim, const int i_block, const int j_block,
 /// @param[in] mu Quadrature function for second Lame param.
 /// @param[in] geom Geometric factors corresponding to fespace.
 /// @param[in] maps DofToQuad maps for one element (assume elements all same).
-/// @param[in] ir Integration rule.
+/// @param QVec Scratch Q-Vector. nQuad x dim x dim x dim x dim x numEls.
 /// @param[out] diag diagonal of A. nDofs x dim x numEls.
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag);
+                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag);
 
 /// Templated implementation of ElasticityAddMultPA.
 template<int dim, int i_block = -1, int j_block = -1>
@@ -279,67 +280,77 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
 template<int dim>
 void ElasticityAssembleDiagonalPA_(const int nDofs,
                                    const CoefficientVector &lambda,
-                                   const CoefficientVector &mu,
-                                   const GeometricFactors &geom,
-                                   const DofToQuad &maps,
-                                   const IntegrationRule &ir,
-                                   Vector &diag)
+                                   const CoefficientVector &mu, const GeometricFactors &geom,
+                                   const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
+   using future::tensor;
+   using future::make_tensor;
    using future::det;
    using future::inv;
-   using future::make_tensor;
-   using future::tensor;
 
    // Assuming all elements are the same
+   const auto &ir = QVec.GetIntRule(0);
    static constexpr int d = dim;
    const int numPoints = ir.GetNPoints();
-   const int numEls = lambda.Size() / numPoints;
-
+   const int numEls = lambda.Size()/numPoints;
    const auto lamDev = Reshape(lambda.Read(), numPoints, numEls);
    const auto muDev = Reshape(mu.Read(), numPoints, numEls);
    const auto J = Reshape(geom.J.Read(), numPoints, d, d, numEls);
+   auto Q = Reshape(QVec.ReadWrite(), numPoints, d,d, d, numEls);
    const real_t *ipWeights = ir.GetWeights().Read();
-   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
-   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
+   mfem::forall_2D(numEls, numPoints,1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(p, x,numPoints)
+      {
+         auto invJ = inv(make_tensor<d, d>(
+         [&](int i, int j) { return J(p, i, j, e); }));
+         const real_t w = ipWeights[p] /det(invJ);
+         for (int n = 0; n < d; n++)
+         {
+            for (int m = 0; m < d; m++)
+            {
+               for (int q = 0; q < d; q++)
+               {
+                  // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                  // this contraction could be made slightly cheaper using Voigt
+                  // notation, but repeated entries are summed for simplicity.
+                  real_t contraction = 0.;
+                  for (int a = 0; a < d; a++)
+                  {
+                     for (int b = 0; b < d; b++)
+                     {
+                        contraction += ((a == q)*invJ(m,b) + (b==q)*invJ(m,a))*((a == q)
+                                                                                *invJ(n, b) + (b==q)*invJ(n,a));
+                     }
+                  }
+                  // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
+                  // contraction = 4*sym(grad(u))sym(grad(v))
+                  Q(p,m,n,q,e) = w*(lamDev(p, e)*invJ(m,q)*invJ(n,q)
+                                    + 0.5*muDev(p, e)*contraction);
+               }
+            }
+         }
+      }
+   });
 
+   // Reduce quadrature function to an E-Vector
+   const auto QRead = Reshape(QVec.Read(), numPoints, d, d, d, numEls);
+   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
+   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
    mfem::forall_2D(numEls, d, nDofs, [=] MFEM_HOST_DEVICE (int e)
    {
-      MFEM_FOREACH_THREAD_DIRECT(i, y, nDofs)
+      MFEM_FOREACH_THREAD(i, y, nDofs)
       {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, d)
+         MFEM_FOREACH_THREAD(q, x, d)
          {
-            real_t sum = 0.0;
-            for (int p = 0; p < numPoints; p++)
+            real_t sum = 0.;
+            for (int n = 0; n < d; n++)
             {
-               const auto invJ = inv(make_tensor<d, d>([&](int r, int c)
+               for (int m = 0; m < d; m++)
                {
-                  return J(p, r, c, e);
-               }));
-               const real_t w = ipWeights[p] / det(invJ);
-
-               for (int n = 0; n < d; n++)
-               {
-                  for (int m = 0; m < d; m++)
+                  for (int p = 0; p < numPoints; p++ )
                   {
-                     // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-                     // this contraction could be made slightly cheaper using Voigt
-                     // notation, but repeated entries are summed for simplicity.
-                     real_t contraction = 0.0;
-                     for (int a = 0; a < d; a++)
-                     {
-                        for (int b = 0; b < d; b++)
-                        {
-                           contraction +=
-                              ((a == q) * invJ(m, b) + (b == q) * invJ(m, a)) *
-                              ((a == q) * invJ(n, b) + (b == q) * invJ(n, a));
-                        }
-                     }
-                     // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
-                     // contraction = 4*sym(grad(u))sym(grad(v))
-                     const real_t Q =
-                        w * (lamDev(p, e) * invJ(m, q) * invJ(n, q)
-                             + 0.5 * muDev(p, e) * contraction);
-                     sum += Q * G(p, m, i) * G(p, n, i);
+                     sum += QRead(p,m,n,q,e)*G(p,m,i)*G(p,n,i);
                   }
                }
             }

fem/integ/bilininteg_elasticity_pa.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 #include "bilininteg_elasticity_kernels.hpp"
 
@@ -58,8 +59,9 @@ void ElasticityIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
 void ElasticityIntegrator::AssembleDiagonalPA(Vector &diag)
 {
+   q_vec->SetVDim(vdim*vdim*vdim*vdim);
    internal::ElasticityAssembleDiagonalPA(vdim, ndofs, *lambda_quad, *mu_quad,
-                                          *geom, *maps, *IntRule, diag);
+                                          *geom, *maps, *q_vec, diag);
 }
 
 void ElasticityIntegrator::AddMultPA(const Vector &x, Vector &y) const

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -147,16 +147,18 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHcurlMassApply2D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int D1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D)
+void PAHcurlMassApply2D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y)
 {
-   MFEM_ASSERT(D1D == TestD1D,
-               "Trial and Test space must have the same number of dofs");
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(bc.Read(), Q1D, D1D);
    auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
@@ -275,16 +277,18 @@ void PAHcurlMassApply2D(const int NE, const bool symmetric,
    }); // end of element loop
 }
 
-void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int D1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D)
+void PAHcurlMassApply3D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -181,312 +181,228 @@ inline void SmemPAHcurlMassAssembleDiagonal3D(const int d1d,
 }
 
 // PA H(curl) Mass Apply 2D kernel
-void PAHcurlMassApply2D(const int NE, const bool symmetric,
-                        const bool scalar_coeff, const Array<real_t> &bo,
-                        const Array<real_t> &bc, const Array<real_t> &bot,
-                        const Array<real_t> &bct, const Vector &pa_data,
-                        const Vector &x, Vector &y, const int TrialD1D,
-                        const int TestD1D, const int Q1D);
+void PAHcurlMassApply2D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
 
 // PA H(curl) Mass Apply 3D kernel
-void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int TrialD1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D);
+void PAHcurlMassApply3D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
 
 // Shared memory PA H(curl) Mass Apply 3D kernel
-template <int T_D1D = 0, int T_Q1D = 0, int TBATCH = 0, bool ACCUMULATE = true>
-inline void SmemPAHcurlMassApply3D(
-   const int NE, const bool symmetric, [[maybe_unused]] const bool scalar_coeff,
-   const Array<real_t> &bo, const Array<real_t> &bc,
-   [[maybe_unused]] const Array<real_t> &bot,
-   [[maybe_unused]] const Array<real_t> &bct, const Vector &pa_data,
-   const Vector &x, Vector &y, const int d1d = 0,
-   [[maybe_unused]] const int test_d1d = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHcurlMassApply3D(const int d1d,
+                                   const int q1d,
+                                   const int NE,
+                                   const bool symmetric,
+                                   const Array<real_t> &bo,
+                                   const Array<real_t> &bc,
+                                   const Array<real_t> &bot,
+                                   const Array<real_t> &bct,
+                                   const Vector &pa_data,
+                                   const Vector &x,
+                                   Vector &y)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
    MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: d1d > HCURL_MAX_D1D");
    MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
                "Error: q1d > HCURL_MAX_Q1D");
-   MFEM_ASSERT(Q1D >= D1D, "Expected Q1D >= D1D");
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
    const int dataSize = symmetric ? 6 : 9;
 
-   // assume trial space == test space
-   auto Bo = bo.Read();
-   auto Bc = bc.Read();
-   auto op =
-      Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
-   auto X_ = Reshape(x.Read(), 3 * (D1D - 1) * D1D * D1D, NE);
-   auto y_ = y.ReadWrite();
+   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
+   auto X = Reshape(x.Read(), 3*(D1D-1)*D1D*D1D, NE);
+   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
 
-   constexpr int MD_ = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-   constexpr int MQ_ = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-   constexpr int MDQ_ = std::max(MD_, MQ_);
-   constexpr int MB_ = TBATCH ? TBATCH : 1;
-
-   mfem::forall_2D_batch<MDQ_ * MDQ_ * MDQ_ * MB_>(
-      NE, MDQ_ * MDQ_ * MDQ_, 1, MB_, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-      constexpr int nbz = TBATCH ? TBATCH : 1;
-      int tidz = MFEM_THREAD_ID(z);
-#else
-      constexpr int nbz = 1;
-      constexpr int tidz = 0;
-#endif
-
       constexpr int VDIM = 3;
       constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
       constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-      constexpr int MDQ = std::max(MD1D, MQ1D);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
 
-      // nvcc limit work-around: can't have Y_ be captured first in
-      // if constexpr, so capture y_ and construct Y_ locally
-      // only works on GPU
-      auto Y = Reshape(y_, VDIM * (D1D - 1) * D1D * D1D, NE);
+      MFEM_SHARED real_t sBo[MQ1D][MD1D];
+      MFEM_SHARED real_t sBc[MQ1D][MD1D];
 
-      MFEM_SHARED real_t sBo[MDQ * (MD1D - 1)];
-      MFEM_SHARED real_t sBc[MDQ * MD1D];
-      auto BO = Reshape(sBo, Q1D, D1D - 1);
-      auto BC = Reshape(sBc, Q1D, D1D);
+      real_t op9[9];
+      MFEM_SHARED real_t sop[9*MQ1D*MQ1D];
+      MFEM_SHARED real_t mass[MQ1D][MQ1D][3];
 
-      MFEM_SHARED real_t sX[nbz * VDIM * (MD1D - 1) * MD1D * MD1D];
-      MFEM_SHARED real_t sm0[nbz * VDIM * MDQ * MDQ * MDQ];
-      MFEM_SHARED real_t sm1[nbz * VDIM * MDQ * MDQ * MDQ];
+      MFEM_SHARED real_t sX[MD1D][MD1D][MD1D];
 
-      real_t(*X)[nbz][(MD1D - 1) * MD1D * MD1D] =
-         (real_t(*)[nbz][(MD1D - 1) * MD1D * MD1D])(sX);
-      // shapes of buffers always use MQ1D to mitigate shared memory bank
-      // conflicts
-      real_t(*DDQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-      real_t(*DQQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
-      real_t(*QQQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-      real_t(*QQD)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
-      real_t(*QDD)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-
-      // load dofs into smem
-      const int offset = (D1D - 1) * D1D * D1D;
-      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
-         for (int dim = 0; dim < VDIM; ++dim)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
-            X[dim][tidz][ix] = X_(ix + dim * offset, e);
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               for (int i=0; i<dataSize; ++i)
+               {
+                  op9[i] = op(qx,qy,qz,i,e);
+               }
+            }
          }
       }
-      // load basis functions data
+
+      const int tidx = MFEM_THREAD_ID(x);
+      const int tidy = MFEM_THREAD_ID(y);
+      const int tidz = MFEM_THREAD_ID(z);
+
       if (tidz == 0)
       {
-         MFEM_FOREACH_THREAD_DIRECT(ix, x, D1D * Q1D) { sBc[ix] = Bc[ix]; }
-         MFEM_FOREACH_THREAD_DIRECT(ix, x, (D1D - 1) * Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            sBo[ix] = Bo[ix];
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               sBc[q][d] = Bc(q,d);
+               if (d < D1D-1)
+               {
+                  sBo[q][d] = Bo(q,d);
+               }
+            }
          }
       }
+      MFEM_SYNC_THREAD;
 
-      for (int dim0 = 0; dim0 < VDIM; ++dim0)
+      for (int qz=0; qz < Q1D; ++qz)
       {
-         MFEM_SYNC_THREAD;
-         // sum factor to QQQ = Q_{dim0,dim1} B X_{dim1}
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         int osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
          {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
 
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, Q1D, D1Dy, D1Dz,
-                                                 Q1D, Q1D, Q1D)
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
             {
-               real_t u = 0;
-               for (int dx = 0; dx < D1Dx; ++dx)
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
                {
-                  real_t b;
-                  if (dim1 == 0)
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
                   {
-                     b = BO(qx, dx);
+                     sX[dz][dy][dx] = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
                   }
-                  else
-                  {
-                     b = BC(qx, dx);
-                  }
-                  u += X[dim1][tidz][dx + (dy + dz * D1Dy) * D1Dx] * b;
-               }
-               DDQ[dim1][tidz][dz][dy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
-         {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, Q1D, Q1D, D1Dz,
-                                                 Q1D, Q1D, Q1D)
-            {
-               real_t u = 0;
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  real_t b;
-                  if (dim1 == 1)
-                  {
-                     b = BO(qy, dy);
-                  }
-                  else
-                  {
-                     b = BC(qy, dy);
-                  }
-                  u += DDQ[dim1][tidz][dz][dy][qx] * b;
-               }
-               DQQ[dim1][tidz][dz][qy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
-         {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            // const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
-            MFEM_FOREACH_THREAD_DIRECT_3D(qx, qy, qz, x, Q1D, Q1D, Q1D)
-            {
-               real_t u = 0;
-               for (int dz = 0; dz < D1Dz; ++dz)
-               {
-                  real_t b;
-                  if (dim1 == 2)
-                  {
-                     b = BO(qz, dz);
-                  }
-                  else
-                  {
-                     b = BC(qz, dz);
-                  }
-                  u += DQQ[dim1][tidz][dz][qy][qx] * b;
-               }
-               // pa_data is row major
-               int idx;
-               if (symmetric)
-               {
-                  int row;
-                  int col;
-                  if (dim0 > dim1)
-                  {
-                     row = dim1;
-                     col = dim0;
-                  }
-                  else
-                  {
-                     row = dim0;
-                     col = dim1;
-                  }
-                  idx = col + VDIM * row - row * (row + 1) / 2;
-               }
-               else
-               {
-                  idx = dim0 * VDIM + dim1;
-               }
-               QQQ[dim1][tidz][qz][qy][qx] = op(qx, qy, qz, idx, e) * u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         // sum factor back to Y
-         // Assume bot and bct == bo^t and bc^t respectively (i.e. test ==
-         // trial functions), skip loading them again.
-         {
-            const int D1Dz = (dim0 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim0 == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (dim0 == 0) ? D1D - 1 : D1D;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, D1Dz, Q1D, Q1D,
-                                                 Q1D, Q1D, Q1D)
-            {
-               for (int dim1 = 0; dim1 < VDIM; ++dim1)
-               {
-                  real_t u = 0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     real_t b = 0;
-                     if (dim0 == 2)
-                     {
-                        b = BO(qz, dz);
-                     }
-                     else
-                     {
-                        b = BC(qz, dz);
-                     }
-                     u += QQQ[dim1][tidz][qz][qy][qx] * b;
-                  }
-                  QQD[dim1][tidz][qy][qx][dz] = u;
-               }
-            }
-            MFEM_SYNC_THREAD;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, D1Dy, D1Dz, Q1D,
-                                                 Q1D, Q1D, Q1D)
-            {
-               for (int dim1 = 0; dim1 < VDIM; ++dim1)
-               {
-                  real_t u = 0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     real_t b;
-                     if (dim0 == 1)
-                     {
-                        b = BO(qy, dy);
-                     }
-                     else
-                     {
-                        b = BC(qy, dy);
-                     }
-                     u += QQD[dim1][tidz][qy][qx][dz] * b;
-                  }
-                  QDD[dim1][tidz][qx][dz][dy] = u;
                }
             }
             MFEM_SYNC_THREAD;
 
-            MFEM_FOREACH_THREAD_DIRECT_3D(dx, dy, dz, x, D1Dx, D1Dy, D1Dz)
+            if (tidz == qz)
             {
-               int ix = dx + D1Dx * (dy + D1Dy * dz);
-               real_t u = 0;
-               for (int qx = 0; qx < Q1D; ++qx)
+               for (int i=0; i<dataSize; ++i)
                {
-                  real_t b;
-                  if (dim0 == 0)
-                  {
-                     b = BO(qx, dx);
-                  }
-                  else
-                  {
-                     b = BC(qx, dx);
-                  }
-                  for (int dim1 = 0; dim1 < VDIM; ++dim1)
-                  {
-                     u += QDD[dim1][tidz][qx][dz][dy] * b;
-                  }
+                  sop[i + (dataSize*tidx) + (dataSize*Q1D*tidy)] = op9[i];
                }
-               if constexpr (ACCUMULATE)
+
+               MFEM_FOREACH_THREAD(qy,y,Q1D)
                {
-                  Y(ix + dim0 * offset, e) += u;
-               }
-               else
+                  MFEM_FOREACH_THREAD(qx,x,Q1D)
+                  {
+                     real_t u = 0.0;
+
+                     for (int dz = 0; dz < D1Dz; ++dz)
+                     {
+                        const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
+                        for (int dy = 0; dy < D1Dy; ++dy)
+                        {
+                           const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
+                           for (int dx = 0; dx < D1Dx; ++dx)
+                           {
+                              const real_t t = sX[dz][dy][dx];
+                              const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
+                              u += t * wx * wy * wz;
+                           }
+                        }
+                     }
+
+                     mass[qy][qx][c] = u;
+                  } // qx
+               } // qy
+            } // tidz == qz
+
+            osc += D1Dx * D1Dy * D1Dz;
+            MFEM_SYNC_THREAD;
+         } // c
+
+         MFEM_SYNC_THREAD;  // Sync mass[qy][qx][d] and sop
+
+         osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
+         {
+            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
+
+            real_t dxyz = 0.0;
+
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
+            {
+               const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
+
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
                {
-                  Y(ix + dim0 * offset, e) = u;
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
+                  {
+                     for (int qy = 0; qy < Q1D; ++qy)
+                     {
+                        const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
+                        for (int qx = 0; qx < Q1D; ++qx)
+                        {
+                           const int os = (dataSize*qx) + (dataSize*Q1D*qy);
+                           const int id1 = os + ((c == 0) ? 0 : ((c == 1) ? (symmetric ? 1 : 3) :
+                                                                 (symmetric ? 2 : 6))); // O11, O21, O31
+                           const int id2 = os + ((c == 0) ? 1 : ((c == 1) ? (symmetric ? 3 : 4) :
+                                                                 (symmetric ? 4 : 7))); // O12, O22, O32
+                           const int id3 = os + ((c == 0) ? 2 : ((c == 1) ? (symmetric ? 4 : 5) :
+                                                                 (symmetric ? 5 : 8))); // O13, O23, O33
+
+                           const real_t m_c = (sop[id1] * mass[qy][qx][0]) + (sop[id2] * mass[qy][qx][1]) +
+                                              (sop[id3] * mass[qy][qx][2]);
+
+                           const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
+                           dxyz += m_c * wx * wy * wz;
+                        }
+                     }
+                  }
                }
             }
-         }
-      }
+
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
+            {
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
+               {
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
+                  {
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) += dxyz;
+                  }
+               }
+            }
+
+            osc += D1Dx * D1Dy * D1Dz;
+         } // c loop
+      } // qz
    }); // end of element loop
 }
 

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -62,30 +62,6 @@ void PAHcurlHdivMassApply2D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
-/// H(curl) test, H(div) trial
-inline void
-PAHcurlHdivMassApply2D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false,
-                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
-/// H(div) test, H(curl) trial
-inline void
-PAHdivHcurlMassApply2D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true,
-                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
 // PA H(curl)-H(div) Mass Apply 3D kernel
 void PAHcurlHdivMassApply3D(const int D1D,
                             const int D1Dtest,
@@ -102,30 +78,6 @@ void PAHcurlHdivMassApply3D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
-/// H(curl) test, H(div) trial
-inline void
-PAHcurlHdivMassApply3D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false, false, Bo_,
-                          Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
-/// H(div) test, H(curl) trial
-inline void
-PAHdivHcurlMassApply3D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true, false, Bo_,
-                          Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
 // PA H(curl)-H(div) Curl Apply 3D kernel
 template<int T_D1D = 0, int T_D1D_TEST = 0, int T_Q1D = 0>
 inline void PAHcurlHdivApply3D(const int d1d,

fem/integ/bilininteg_hdiv_kernels.cpp

@@ -294,14 +294,61 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int TestD1D, const int Q1D)
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<real_t> &Bo,
+                     const Array<real_t> &Bc,
+                     const Array<real_t> &Bot,
+                     const Array<real_t> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y)
+{
+   const int id = (D1D << 4) | Q1D;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x22: return SmemPAHdivMassApply2D<2,2>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x33: return SmemPAHdivMassApply2D<3,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x44: return SmemPAHdivMassApply2D<4,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x55: return SmemPAHdivMassApply2D<5,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply2D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x23: return SmemPAHdivMassApply3D<2,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x34: return SmemPAHdivMassApply3D<3,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x45: return SmemPAHdivMassApply3D<4,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x56: return SmemPAHdivMassApply3D<5,6>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x67: return SmemPAHdivMassApply3D<6,7>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x78: return SmemPAHdivMassApply3D<7,8>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply3D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+}
+
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
@@ -421,14 +468,18 @@ void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
    }); // end of element loop
 }
 
-void PAHdivMassApply3D(const int NE, const bool symmetric, const bool,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int TestD1D, const int Q1D)
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
                "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,

fem/integ/bilininteg_hdiv_kernels.hpp

@@ -66,29 +66,58 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
                                   const Vector &op_,
                                   Vector &diag_);
 
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<real_t> &Bo,
+                     const Array<real_t> &Bc,
+                     const Array<real_t> &Bot,
+                     const Array<real_t> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y);
+
 // PA H(div) Mass Apply 2D kernel
-void PAHdivMassApply2D(const int NE, const bool symmetric,
-                       const bool scalar_coeff, const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_, const Vector &op_,
-                       const Vector &x_, Vector &y_, const int D1D,
-                       const int TestD1D, const int Q1D);
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
 
 // PA H(div) Mass Apply 3D kernel
-void PAHdivMassApply3D(const int NE, const bool symmetric,
-                       const bool scalar_coeff, const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_, const Vector &op_,
-                       const Vector &x_, Vector &y_, const int D1D,
-                       const int TestD1D, const int Q1D);
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
 
 // Shared memory PA H(div) Mass Apply 2D kernel
-template <int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHdivMassApply2D(
-   const int NE, const bool symmetric, const bool, const Array<real_t> &Bo_,
-   const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-   const Array<real_t> &Bct_, const Vector &op_, const Vector &x_, Vector &y_,
-   const int d1d = 0, const int = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply2D(const int NE,
+                                  const bool symmetric,
+                                  const Array<real_t> &Bo_,
+                                  const Array<real_t> &Bc_,
+                                  const Array<real_t> &Bot_,
+                                  const Array<real_t> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);
@@ -251,13 +280,18 @@ inline void SmemPAHdivMassApply2D(
 }
 
 // Shared memory PA H(div) Mass Apply 3D kernel
-template <int T_D1D = 0, int T_Q1D = 0>
-inline void
-SmemPAHdivMassApply3D(const int NE, const bool symmetric, const bool,
-                      const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                      const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                      const Vector &op_, const Vector &x_, Vector &y_,
-                      const int d1d = 0, const int = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply3D(const int NE,
+                                  const bool symmetric,
+                                  const Array<real_t> &Bo_,
+                                  const Array<real_t> &Bc_,
+                                  const Array<real_t> &Bot_,
+                                  const Array<real_t> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);

fem/integ/bilininteg_vecdiv_pa.hpp

@@ -1,365 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/array.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../linalg/vector.hpp"
-#include "../bilininteg.hpp"
-#include "../kernels.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-// Shared memory PA Divergence Apply 2D kernel
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApply2D(const int NE,
-                                    const Array<real_t> &b_,
-                                    const Array<real_t> &g_,
-                                    const Array<real_t> &bt_,
-                                    const Vector &q_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
-   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, 2, NE);
-   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, 1, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
-      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
-
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs2d(e, TR_D1D, X, g0);
-      kernels::internal::Grad2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            r0[0][qy][qx] =
-               g1[0][0][qy][qx] * Q(qx, qy, 0, 0, e) +
-               g1[0][1][qy][qx] * Q(qx, qy, 1, 0, e) +
-               g1[1][0][qy][qx] * Q(qx, qy, 0, 1, e) +
-               g1[1][1][qy][qx] * Q(qx, qy, 1, 1, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TE_D1D, Q1D, Bt, sB);
-      kernels::internal::EvalTranspose2d(TE_D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs2d(e, TE_D1D, r1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 2D kernel transpose
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApplyTranspose2D(const int NE,
-                                             const Array<real_t> &bt,
-                                             const Array<real_t> &gt,
-                                             const Array<real_t> &b,
-                                             const Vector &q_,
-                                             const Vector &x_,
-                                             Vector &y_,
-                                             const int tr_d1d = 0,
-                                             const int te_d1d = 0,
-                                             const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
-   const auto X  = Reshape(x_.Read(), TE_D1D, TE_D1D, 1, NE);
-   auto Y  = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, 2, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
-      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
-      kernels::internal::LoadDofs2d(e, TE_D1D, X, r0);
-      kernels::internal::Eval2d(TE_D1D, Q1D, smem, sB, r0, r1);
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            g0[0][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 0, e);
-            g0[0][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 0, e);
-            g0[1][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 1, e);
-            g0[1][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 1, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
-      kernels::internal::GradTranspose2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-      kernels::internal::WriteDofs2d(e, TR_D1D, g1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 3D kernel transpose
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApplyTranspose3D(const int NE,
-                                             const Array<real_t> &bt,
-                                             const Array<real_t> &gt,
-                                             const Array<real_t> &b,
-                                             const Vector &q_,
-                                             const Vector &x_,
-                                             Vector &y_,
-                                             int tr_d1d = 0,
-                                             int te_d1d = 0,
-                                             int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
-   const auto X = Reshape(x_.Read(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
-   auto Y = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
-      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
-      kernels::internal::LoadDofs3d(e, TE_D1D, X, r0);
-      kernels::internal::Eval3d(TE_D1D, Q1D, smem, sB, r0, r1);
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const auto r = r1[0][qz][qy][qx];
-               g0[0][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 0, e);
-               g0[0][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 0, e);
-               g0[0][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 0, e);
-
-               g0[1][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 1, e);
-               g0[1][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 1, e);
-               g0[1][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 1, e);
-
-               g0[2][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 2, e);
-               g0[2][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 2, e);
-               g0[2][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 2, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
-      kernels::internal::GradTranspose3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-      kernels::internal::WriteDofs3d(e, TR_D1D, g1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 3D kernel
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApply3D(const int NE,
-                                    const Array<real_t> &b_,
-                                    const Array<real_t> &g_,
-                                    const Array<real_t> &bt_,
-                                    const Vector &q_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3,3, NE);
-   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
-   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
-      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
-
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs3d(e, TR_D1D, X, g0);
-      kernels::internal::Grad3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               r0[0][qz][qy][qx] =
-                  // c = 0
-                  g1[0][0][qz][qy][qx] * Q(qx, qy, qz, 0, 0, e) +
-                  g1[0][1][qz][qy][qx] * Q(qx, qy, qz, 1, 0, e) +
-                  g1[0][2][qz][qy][qx] * Q(qx, qy, qz, 2, 0, e) +
-                  // c = 1
-                  g1[1][0][qz][qy][qx] * Q(qx, qy, qz, 0, 1, e) +
-                  g1[1][1][qz][qy][qx] * Q(qx, qy, qz, 1, 1, e) +
-                  g1[1][2][qz][qy][qx] * Q(qx, qy, qz, 2, 1, e) +
-                  // c = 2
-                  g1[2][0][qz][qy][qx] * Q(qx, qy, qz, 0, 2, e) +
-                  g1[2][1][qz][qy][qx] * Q(qx, qy, qz, 1, 2, e) +
-                  g1[2][2][qz][qy][qx] * Q(qx, qy, qz, 2, 2, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1, true>(TE_D1D, Q1D, Bt, sB);
-      kernels::internal::EvalTranspose3d(TE_D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs3d(e, TE_D1D, r1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
-VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Kernel()
-{
-   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPADivergenceApply2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADivergenceApply3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Fallback
-(int dim, int tr_d1d, int te_d1d, int q1d)
-{
-   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
-   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPADivergenceApply2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPADivergenceApply3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Kernel()
-{
-   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPADivergenceApplyTranspose2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADivergenceApplyTranspose3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Fallback
-(int dim, int tr_d1d, int te_d1d, int q1d)
-{
-   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
-   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPADivergenceApplyTranspose2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPADivergenceApplyTranspose3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/bilininteg_vecmass_pa.cpp

@@ -205,40 +205,157 @@ void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 
 }
 
+template <const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal2D(const int NE,
+                                           const Array<real_t> &b,
+                                           const Vector &pa_data, Vector &diag,
+                                           const int d1d = 0, const int q1d = 0)
+{
+   constexpr int VDIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const auto B = Reshape(b.Read(), Q1D, D1D);
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, VDIM, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t temp[max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            temp[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               temp[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            real_t temp1 = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               temp1 += B(qx, dx) * B(qx, dx) * temp[qx][dy];
+            }
+            Y(dx, dy, 0, e) = temp1;
+            Y(dx, dy, 1, e) = temp1;
+         }
+      }
+   });
+}
+
+template <const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal3D(const int NE,
+                                           const Array<real_t> &B_,
+                                           const Vector &pa_data, Vector &diag,
+                                           const int d1d = 0, const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const auto B = Reshape(B_.Read(), Q1D, D1D);
+   MFEM_VERIFY(pa_data.Size() == Q1D * Q1D * Q1D * NE, "pa_data size error");
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t temp[max_Q1D][max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               temp[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  temp[qx][qy][dz] +=
+                     B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      real_t temp2[max_Q1D][max_D1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               temp2[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  temp2[qx][dy][dz] +=
+                     B(qy, dy) * B(qy, dy) * temp[qx][qy][dz];
+               }
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               real_t temp3 = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  temp3 += B(qx, dx) * B(qx, dx) * temp2[qx][dy][dz];
+               }
+               Y(dx, dy, dz, 0, e) = temp3;
+               Y(dx, dy, dz, 1, e) = temp3;
+               Y(dx, dy, dz, 2, e) = temp3;
+            }
+         }
+      }
+   });
+}
+
+static void PAVectorMassAssembleDiagonal(const int dim, const int D1D,
+                                         const int Q1D, const int NE,
+                                         const Array<real_t> &B,
+                                         const Vector &pa_data,
+                                         Vector &diag)
+{
+   if (dim == 2)
+   {
+      return PAVectorMassAssembleDiagonal2D(NE, B, pa_data, diag, D1D, Q1D);
+   }
+   else if (dim == 3)
+   {
+      return PAVectorMassAssembleDiagonal3D(NE, B, pa_data, diag, D1D, Q1D);
+   }
+   MFEM_ABORT("Dimension not implemented.");
+}
+
 void VectorMassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   if (DeviceCanUseCeed()) { return ceedOp->GetDiagonal(diag); }
-
-   MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
-   MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
-
-   // Add the VectorMassAssembleDiagonalPA specializations
-   static const auto vector_mass_assemble_diagonal_kernel_specializations =
-      ( // 2D
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 2>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 3>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 4>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 5>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 6>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 7>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 8>::Add(),
-         // 3D
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 2>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 3>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 4>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 5>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 6>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 7>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 8>::Add(),
-         true);
-   MFEM_CONTRACT_VAR(vector_mass_assemble_diagonal_kernel_specializations);
-
-   VectorMassAssembleDiagonalPA::Run(dim, quad1D, // templated arguments
-                                     ne, dofs1D, quad1D,
-                                     maps->B.Read(),
-                                     pa_data.Read(),
-                                     diag.ReadWrite());
-
+   if (DeviceCanUseCeed()) { ceedOp->GetDiagonal(diag); }
+   else
+   {
+      MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
+      MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
+      PAVectorMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
+   }
 }
 
 } // namespace mfem

fem/integ/bilininteg_vecmass_pa.hpp

@@ -176,146 +176,8 @@ void SmemPAVectorMassApply3D(const int NE,
    });
 }
 
-template <int T_Q1D = 0, int T_MDQ = 16>
-static void SmemPAVectorMassAssembleDiagonal2D(const int ne,
-                                               const int d1d,
-                                               const int q1d,
-                                               const real_t *b_r,
-                                               const real_t *d_r,
-                                               real_t *y_rw)
-{
-   constexpr int VDIM = 2;
-
-   const int D1D = d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
-
-   const auto B = Reshape(b_r, Q1D, D1D);
-   const auto D = Reshape(d_r, Q1D, Q1D, ne);
-   auto Y = Reshape(y_rw, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(
-      ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
-
-      MFEM_SHARED real_t sm[MQ1][MQ1];
-
-      MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            real_t u = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               u += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-            sm[qx][dy] = u;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-         {
-            real_t u = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               u += B(qx, dx) * B(qx, dx) * sm[qx][dy];
-            }
-            Y(dx, dy, 0, e) += u;
-            Y(dx, dy, 1, e) += u;
-         }
-      }
-   });
-}
-
-// T_MDQ <= 10 so the Q1D^3 thread block stays within the 1024/block GPU limit
-template <int T_Q1D = 0, int T_MDQ = 10>
-static void SmemPAVectorMassAssembleDiagonal3D(const int ne,
-                                               const int d1d,
-                                               const int q1d,
-                                               const real_t *b_r,
-                                               const real_t *d_r,
-                                               real_t *y_rw)
-{
-   constexpr int VDIM = 3;
-
-   const int D1D = d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
-
-   const auto B = Reshape(b_r, Q1D, D1D);
-   const auto D = Reshape(d_r, Q1D, Q1D, Q1D, ne);
-   auto Y = Reshape(y_rw, D1D, D1D, D1D, VDIM, ne);
-
-   mfem::forall_3D<T_Q1D*T_Q1D*T_Q1D>(
-      ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
-
-      MFEM_SHARED real_t sm[2][MQ1][MQ1][MQ1];
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               real_t u = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-               sm[0][dz][qy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += B(qy, dy) * B(qy, dy) * sm[0][dz][qy][qx];
-               }
-               sm[1][dz][dy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += B(qx, dx) * B(qx, dx) * sm[1][dz][dy][qx];
-               }
-               Y(dx, dy, dz, 0, e) += u;
-               Y(dx, dy, dz, 1, e) += u;
-               Y(dx, dy, dz, 2, e) += u;
-            }
-         }
-      }
-   });
-}
-
 } // namespace internal
 
-// AddMultPA kernels
 template<int DIM, int T_D1D, int T_Q1D>
 VectorMassIntegrator::VectorMassAddMultPAType
 VectorMassIntegrator::VectorMassAddMultPA::Kernel()
@@ -332,7 +194,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Kernel()
 }
 
 inline VectorMassIntegrator::VectorMassAddMultPAType
-VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
+VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int d1d, int q1d)
 {
    if (dim == 2)
    {
@@ -342,37 +204,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
    {
       return internal::SmemPAVectorMassApply3D;
    }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-// DiagonalPA kernels
-template<int DIM, int T_Q1D>
-VectorMassIntegrator::VectorMassAssembleDiagonalPAType
-VectorMassIntegrator::VectorMassAssembleDiagonalPA::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal2D<T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal3D<T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorMassIntegrator::VectorMassAssembleDiagonalPAType
-VectorMassIntegrator::VectorMassAssembleDiagonalPA::Fallback(int dim, int)
-{
-   if (dim == 2)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
+   else { MFEM_ABORT("Unsupported kernel"); }
 }
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_vectorfemass_kernels.hpp

@@ -1,113 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
-#define MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
-
-#include "../../config/config.hpp"
-#include "../bilininteg.hpp"
-
-#include "bilininteg_diffusion_kernels.hpp"
-#include "bilininteg_hcurl_kernels.hpp"
-#include "bilininteg_hdiv_kernels.hpp"
-#include "bilininteg_hcurlhdiv_kernels.hpp"
-
-namespace mfem
-{
-/// \cond DO_NOT_DOCUMENT
-namespace internal
-{
-namespace hcurlmass
-{
-constexpr int NBZ3D(int d1d, int q1d)
-{
-   if (d1d <= 1 || q1d <= 0)
-   {
-      return 1;
-   }
-   // assume q1d >= d1d
-   // z dimension is capped at 64 on nvidia and amd gpus
-   int tmp = std::min((128 + q1d * q1d * q1d - 1) / (q1d * q1d * q1d), 64);
-   int smem_req =
-      sizeof(mfem::real_t) *
-      (3 * ((d1d - 1) * d1d * d1d + 2 * q1d * q1d * q1d) * tmp +
-       q1d * (d1d - 1) + q1d * d1d);
-   // assume GPU has at least 48k shared memory
-   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
-}
-} // namespace hcurlmass
-} // namespace internal
-
-template <FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
-          int DIM, int TrialD1D, int TestD1D, int Q1D>
-VectorFEMassIntegrator::ApplyKernelType
-VectorFEMassIntegrator::ApplyPAKernels::Kernel()
-{
-   constexpr bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
-   constexpr bool trial_div = (TrialType == mfem::FiniteElement::DIV);
-   constexpr bool test_curl = (TestType == mfem::FiniteElement::CURL);
-   constexpr bool test_div = (TestType == mfem::FiniteElement::DIV);
-
-   if constexpr (DIM == 3)
-   {
-      if constexpr (trial_curl && test_curl)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            // assume TrialD1D == TestD1D
-            return internal::SmemPAHcurlMassApply3D<
-                   TrialD1D, Q1D, internal::hcurlmass::NBZ3D(TrialD1D, Q1D)>;
-         }
-         else
-         {
-            return internal::PAHcurlMassApply3D;
-         }
-      }
-      else if constexpr (trial_div && test_div)
-      {
-         // assumes TrialD1D == TestD1D
-         return internal::SmemPAHdivMassApply3D<TrialD1D, Q1D>;
-      }
-      else if constexpr (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply3D;
-      }
-      else if constexpr (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply3D;
-      }
-   }
-   else if constexpr (DIM == 2) // 2D
-   {
-      if constexpr (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply2D;
-      }
-      else if constexpr (trial_div && test_div)
-      {
-         // assumes TrialD1D == TestD1D
-         return internal::SmemPAHdivMassApply2D<TrialD1D, Q1D>;
-      }
-      else if constexpr (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply2D;
-      }
-      else if constexpr (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply2D;
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-/// \endcond DO_NOT_DOCUMENT
-}
-
-#endif

fem/integ/bilininteg_vectorfemass_pa.cpp

@@ -10,123 +10,15 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
-#include "bilininteg_vectorfemass_kernels.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "bilininteg_diffusion_kernels.hpp"
+#include "bilininteg_hcurl_kernels.hpp"
+#include "bilininteg_hdiv_kernels.hpp"
+#include "bilininteg_hcurlhdiv_kernels.hpp"
 
 namespace mfem
 {
-/// \cond DO_NOT_DOCUMENT
-VectorFEMassIntegrator::ApplyKernelType
-VectorFEMassIntegrator::ApplyPAKernels::Fallback(
-   FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
-   int dim, int, int, int)
-{
-   const bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
-   const bool trial_div = (TrialType == mfem::FiniteElement::DIV);
-   const bool test_curl = (TestType == mfem::FiniteElement::CURL);
-   const bool test_div = (TestType == mfem::FiniteElement::DIV);
-
-   if (dim == 3)
-   {
-      if (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply3D;
-      }
-      else if (trial_div && test_div)
-      {
-         return internal::PAHdivMassApply3D;
-      }
-      else if (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply3D;
-      }
-      else if (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply3D;
-      }
-   }
-   else if (dim == 2) // 2D
-   {
-      if (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply2D;
-      }
-      else if (trial_div && test_div)
-      {
-         return internal::PAHdivMassApply2D;
-      }
-      else if (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply2D;
-      }
-      else if (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply2D;
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-/// \endcond DO_NOT_DOCUMENT
-
-VectorFEMassIntegrator::Kernels::Kernels()
-{
-   // h(curl), h(curl)
-   // Q = P + 1 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 6>();
-   // Q = P + 2 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 6>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 7>();
-   // Q = P + 4 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 6>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 7>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 8>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 9>();
-   // h(div), h(div)
-   // Q = P (2D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 2, 2, 2>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 3, 3, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 4, 4, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 5, 5, 5>();
-
-   // Q = P + 1 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 2, 2, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 3, 3, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 4, 4, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 5, 5, 6>();
-}
-
-void VectorFEMassIntegrator::Init(Coefficient *q, DiagonalMatrixCoefficient *dq,
-                                  MatrixCoefficient *mq)
-{
-   static Kernels kernels{};
-   Q = q;
-   DQ = dq;
-   MQ = mq;
-}
 
 void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
@@ -175,8 +67,8 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
 
-   trial_fetype = static_cast<FiniteElement::DerivType>(trial_el->GetDerivType());
-   test_fetype = static_cast<FiniteElement::DerivType>(test_el->GetDerivType());
+   trial_fetype = trial_el->GetDerivType();
+   test_fetype = test_el->GetDerivType();
 
    const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
    const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
@@ -323,34 +215,225 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
 
 void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   const bool scalar_coeff = !(DQ || MQ);
-   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
-                       quad1D, ne, symmetric, scalar_coeff, mapsO->B, mapsC->B,
-                       mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y, dofs1D,
-                       dofs1Dtest, quad1D);
+   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
+   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
+   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
+   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
+
+   if (dim == 3)
+   {
+      if (trial_curl && test_curl)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            const int ID = (dofs1D << 4) | quad1D;
+            switch (ID)
+            {
+               case 0x23:
+                  return internal::SmemPAHcurlMassApply3D<2,3>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x34:
+                  return internal::SmemPAHcurlMassApply3D<3,4>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x45:
+                  return internal::SmemPAHcurlMassApply3D<4,5>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x56:
+                  return internal::SmemPAHcurlMassApply3D<5,6>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               default:
+                  return internal::SmemPAHcurlMassApply3D(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+            }
+         }
+         else
+         {
+            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                         mapsO->Bt, mapsC->Bt, pa_data, x, y);
+         }
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                   mapsO->Bt, mapsC->Bt, pa_data, x, y);
+      }
+      else if (trial_curl && test_div)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          true, false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                          mapsCtest->Bt, pa_data, x, y);
+      }
+      else if (trial_div && test_curl)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          false, false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                          mapsCtest->Bt, pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
+   else // 2D
+   {
+      if (trial_curl && test_curl)
+      {
+         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                      mapsO->Bt, mapsC->Bt, pa_data, x, y);
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                   mapsO->Bt,
+                                   mapsC->Bt, pa_data, x, y);
+      }
+      else if ((trial_curl && test_div) || (trial_div && test_curl))
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          trial_curl, false, mapsO->B, mapsC->B,
+                                          mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
 }
 
 void VectorFEMassIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
 {
-   const bool scalar_coeff = !(DQ || MQ);
+   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
+   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
+   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
+   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
 
    Vector abs_pa_data(pa_data);
    abs_pa_data.Abs();
 
    Array<real_t> absBo(mapsO->B);
    Array<real_t> absBc(mapsC->B);
+   Array<real_t> absBto(mapsO->Bt);
+   Array<real_t> absBtc(mapsC->Bt);
    Array<real_t> absBto_t(mapsOtest->Bt);
    Array<real_t> absBtc_t(mapsCtest->Bt);
 
    absBo.Abs();
    absBc.Abs();
+   absBto.Abs();
+   absBtc.Abs();
    absBto_t.Abs();
    absBtc_t.Abs();
 
-   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
-                       quad1D, ne, symmetric, scalar_coeff, absBo, absBc,
-                       absBto_t, absBtc_t, abs_pa_data, x, y, dofs1D,
-                       dofs1Dtest, quad1D);
+   if (dim == 3)
+   {
+      if (trial_curl && test_curl)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            const int ID = (dofs1D << 4) | quad1D;
+            switch (ID)
+            {
+               case 0x23:
+                  return internal::SmemPAHcurlMassApply3D<2,3>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x34:
+                  return internal::SmemPAHcurlMassApply3D<3,4>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x45:
+                  return internal::SmemPAHcurlMassApply3D<4,5>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x56:
+                  return internal::SmemPAHcurlMassApply3D<5,6>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               default:
+                  return internal::SmemPAHcurlMassApply3D(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+            }
+         }
+         else
+         {
+            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric,
+                                         absBo, absBc, absBto, absBtc,
+                                         abs_pa_data, x, y);
+         }
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric,
+                                   absBo, absBc, absBto, absBtc,
+                                   abs_pa_data, x, y);
+      }
+      else if (trial_curl && test_div)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, true, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else if (trial_div && test_curl)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, false, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
+   else // 2D
+   {
+      if (trial_curl && test_curl)
+      {
+         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric,
+                                      absBo, absBc, absBto, absBtc,
+                                      abs_pa_data, x, y);
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric,
+                                   absBo, absBc, absBto, absBtc,
+                                   abs_pa_data, x, y);
+      }
+      else if ((trial_curl && test_div) || (trial_div && test_curl))
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, trial_curl, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
 }
 
 void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,

fem/integ/nonlininteg_vecconvection_pa.cpp

@@ -9,51 +9,21 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
+#include "../../general/forall.hpp"
+#include "../nonlininteg.hpp"
 #include "../ceed/integrators/nlconvection/nlconvection.hpp"
-#include "./nonlininteg_vecconvection_pa.hpp" // IWYU pragma: keep
-#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
-#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
 
 namespace mfem
 {
 
-VectorConvectionNLFIntegrator::Kernels::Kernels()
-{
-   // 2D
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 2>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 3>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 8>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 6, 8>();
-   // 3D
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 3>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 8>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 8>();
-}
-
 void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES,
                "PA Only supports Ordering::byNODES!");
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
-   ElementTransformation &Tr = *mesh->GetTypicalElementTransformation();
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Tr);
-
+   ElementTransformation &T = *mesh->GetTypicalElementTransformation();
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, T);
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
@@ -69,124 +39,769 @@ void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
       }
       return;
    }
-
-   ne = mesh->GetNE();
-   nq = ir->GetNPoints();
    dim = mesh->Dimension();
-   MFEM_VERIFY(dim == 2 || dim == 3, "Dimension not supported");
-
-   const MemoryType mt = pa_mt == MemoryType::DEFAULT
-                         ? Device::GetDeviceMemoryType()
-                         : pa_mt;
-   pa_adj.SetSize(ne * nq * dim * dim, mt);
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
+   ne = fes.GetMesh()->GetNE();
+   nq = ir->GetNPoints();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   d1d = maps->ndof;
-   q1d = maps->nqpt;
-
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
-
-   const int nq1d = q1d * q1d * (dim==3 ? q1d : 1);
-   MFEM_VERIFY(coeff.Size() == 1 || coeff.Size() == nq1d*ne, "Invalid coeff");
-   MFEM_VERIFY(ir->GetWeights().Size() == nq1d, "Invalid weights size");
-
-   const auto w_r = ir->GetWeights().Read();
-   const bool const_coeff = coeff.Size() == 1;
-
+   pa_data.SetSize(ne * nq * dim * dim, Device::GetMemoryType());
+   real_t COEFF = 1.0;
+   if (Q)
+   {
+      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient *>(Q);
+      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
+      COEFF = cQ->constant;
+   }
+   const int NE = ne;
+   const int NQ = nq;
+   auto W = ir->GetWeights().Read();
+   if (dim == 1)
+   {
+      MFEM_ABORT("dim==1 not supported!");
+   }
    if (dim == 2)
    {
-      const int Q1D = q1d;
-      constexpr int VDIM = 2, DIM = 2;
-      const auto W = Reshape(w_r, Q1D, Q1D);
-      const auto C = const_coeff ?
-                     Reshape(coeff.Read(), 1, 1, 1) :
-                     Reshape(coeff.Read(), Q1D, Q1D, ne);
-      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, VDIM, DIM, ne);
-      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, ne);
-
-      mfem::forall_2D(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+      auto J = Reshape(geom->J.Read(), NQ, 2, 2, NE);
+      auto G = Reshape(pa_data.Write(), NQ, 2, 2, NE);
+      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
       {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         for (int q = 0; q < NQ; ++q)
          {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const real_t J11 = J(qx, qy, 0, 0, e), J12 = J(qx, qy, 0, 1, e);
-               const real_t J21 = J(qx, qy, 1, 0, e), J22 = J(qx, qy, 1, 1, e);
-               // adj(J)
-               const real_t A11 = +J22, A12 = -J12;
-               const real_t A21 = -J21, A22 = +J11;
-               // Store w * coeff * adj(J)
-               const real_t w = W(qx, qy);
-               const real_t c = const_coeff ? C(0, 0, 0) : C(qx, qy, e);
-               A(0, 0, qx, qy, e) = w * c * A11;
-               A(1, 0, qx, qy, e) = w * c * A12;
-               A(0, 1, qx, qy, e) = w * c * A21;
-               A(1, 1, qx, qy, e) = w * c * A22;
-            }
+            const real_t J11 = J(q, 0, 0, e);
+            const real_t J12 = J(q, 0, 1, e);
+            const real_t J21 = J(q, 1, 0, e);
+            const real_t J22 = J(q, 1, 1, e);
+            // Store wq * Q * adj(J)
+            G(q, 0, 0, e) = W[q] * COEFF * J22;  // 1,1
+            G(q, 0, 1, e) = W[q] * COEFF * -J12; // 1,2
+            G(q, 1, 0, e) = W[q] * COEFF * -J21; // 2,1
+            G(q, 1, 1, e) = W[q] * COEFF * J11;  // 2,2
          }
       });
    }
-   else if (dim == 3)
+   if (dim == 3)
    {
-      const int Q1D = q1d;
-      constexpr int VDIM = 3, DIM = 3;
-      const auto W = Reshape(w_r, Q1D, Q1D, Q1D);
-      const auto C = const_coeff ?
-                     Reshape(coeff.Read(), 1, 1, 1, 1) :
-                     Reshape(coeff.Read(), Q1D, Q1D, Q1D, ne);
-      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, Q1D, VDIM, DIM, ne);
-      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, Q1D, ne);
-
-      mfem::forall_3D(ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+      auto J = Reshape(geom->J.Read(), NQ, 3, 3, NE);
+      auto G = Reshape(pa_data.Write(), NQ, 3, 3, NE);
+      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
       {
-         MFEM_FOREACH_THREAD_DIRECT(qz, z, Q1D)
+         for (int q = 0; q < NQ; ++q)
          {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            const real_t J11 = J(q, 0, 0, e);
+            const real_t J21 = J(q, 1, 0, e);
+            const real_t J31 = J(q, 2, 0, e);
+            const real_t J12 = J(q, 0, 1, e);
+            const real_t J22 = J(q, 1, 1, e);
+            const real_t J32 = J(q, 2, 1, e);
+            const real_t J13 = J(q, 0, 2, e);
+            const real_t J23 = J(q, 1, 2, e);
+            const real_t J33 = J(q, 2, 2, e);
+            const real_t cw = W[q] * COEFF;
+            // adj(J)
+            const real_t A11 = (J22 * J33) - (J23 * J32);
+            const real_t A12 = (J32 * J13) - (J12 * J33);
+            const real_t A13 = (J12 * J23) - (J22 * J13);
+            const real_t A21 = (J31 * J23) - (J21 * J33);
+            const real_t A22 = (J11 * J33) - (J13 * J31);
+            const real_t A23 = (J21 * J13) - (J11 * J23);
+            const real_t A31 = (J21 * J32) - (J31 * J22);
+            const real_t A32 = (J31 * J12) - (J11 * J32);
+            const real_t A33 = (J11 * J22) - (J12 * J21);
+            // Store wq * Q * adj(J)
+            G(q, 0, 0, e) = cw * A11; // 1,1
+            G(q, 0, 1, e) = cw * A12; // 1,2
+            G(q, 0, 2, e) = cw * A13; // 1,3
+            G(q, 1, 0, e) = cw * A21; // 2,1
+            G(q, 1, 1, e) = cw * A22; // 2,2
+            G(q, 1, 2, e) = cw * A23; // 2,3
+            G(q, 2, 0, e) = cw * A31; // 3,1
+            G(q, 2, 1, e) = cw * A32; // 3,2
+            G(q, 2, 2, e) = cw * A33; // 3,3
+         }
+      });
+   }
+}
+
+// PA Convection NL 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAConvectionNLApply2D(const int NE,
+                                  const Array<real_t> &b,
+                                  const Array<real_t> &g,
+                                  const Array<real_t> &bt,
+                                  const Vector &q_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Q = Reshape(q_.Read(), Q1D * Q1D, 2, 2, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, 2, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t data[max_Q1D][max_Q1D][2];
+      real_t grad0[max_Q1D][max_Q1D][2];
+      real_t grad1[max_Q1D][max_Q1D][2];
+      real_t Z[max_Q1D][max_Q1D][2];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            data[qy][qx][0] = 0.0;
+            data[qy][qx][1] = 0.0;
+            grad0[qy][qx][0] = 0.0;
+            grad0[qy][qx][1] = 0.0;
+            grad1[qy][qx][0] = 0.0;
+            grad1[qy][qx][1] = 0.0;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         real_t dataX[max_Q1D][2];
+         real_t gradX0[max_Q1D][2];
+         real_t gradX1[max_Q1D][2];
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            dataX[qx][0] = 0.0;
+            dataX[qx][1] = 0.0;
+            gradX0[qx][0] = 0.0;
+            gradX0[qx][1] = 0.0;
+            gradX1[qx][0] = 0.0;
+            gradX1[qx][1] = 0.0;
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const real_t s0 = x(dx, dy, 0, e);
+            const real_t s1 = x(dx, dy, 1, e);
+            for (int qx = 0; qx < Q1D; ++qx)
             {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               const real_t Bx = B(qx, dx);
+               const real_t Gx = G(qx, dx);
+               dataX[qx][0] += s0 * Bx;
+               dataX[qx][1] += s1 * Bx;
+               gradX0[qx][0] += s0 * Gx;
+               gradX0[qx][1] += s0 * Bx;
+               gradX1[qx][0] += s1 * Gx;
+               gradX1[qx][1] += s1 * Bx;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const real_t By = B(qy, dy);
+            const real_t Gy = G(qy, dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               data[qy][qx][0] += dataX[qx][0] * By;
+               data[qy][qx][1] += dataX[qx][1] * By;
+               grad0[qy][qx][0] += gradX0[qx][0] * By;
+               grad0[qy][qx][1] += gradX0[qx][1] * Gy;
+               grad1[qy][qx][0] += gradX1[qx][0] * By;
+               grad1[qy][qx][1] += gradX1[qx][1] * Gy;
+            }
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const int q = qx + qy * Q1D;
+            const real_t u1 = data[qy][qx][0];
+            const real_t u2 = data[qy][qx][1];
+            const real_t grad00 = grad0[qy][qx][0];
+            const real_t grad01 = grad0[qy][qx][1];
+            const real_t grad10 = grad1[qy][qx][0];
+            const real_t grad11 = grad1[qy][qx][1];
+            const real_t Dxu1 = grad00 * Q(q, 0, 0, e) + grad01 * Q(q, 1, 0, e);
+            const real_t Dyu1 = grad00 * Q(q, 0, 1, e) + grad01 * Q(q, 1, 1, e);
+            const real_t Dxu2 = grad10 * Q(q, 0, 0, e) + grad11 * Q(q, 1, 0, e);
+            const real_t Dyu2 = grad10 * Q(q, 0, 1, e) + grad11 * Q(q, 1, 1, e);
+            Z[qy][qx][0] = u1 * Dxu1 + u2 * Dyu1;
+            Z[qy][qx][1] = u1 * Dxu2 + u2 * Dyu2;
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         real_t Y[max_D1D][2];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            Y[dx][0] = 0.0;
+            Y[dx][1] = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const real_t Btx = Bt(dx, qx);
+               Y[dx][0] += Btx * Z[qy][qx][0];
+               Y[dx][1] += Btx * Z[qy][qx][1];
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const real_t Bty = Bt(dy, qy);
+               y(dx, dy, 0, e) += Bty * Y[dx][0];
+               y(dx, dy, 1, e) += Bty * Y[dx][1];
+            }
+         }
+      }
+   });
+}
+
+// PA Convection NL 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAConvectionNLApply3D(const int NE,
+                                  const Array<real_t> &b,
+                                  const Array<real_t> &g,
+                                  const Array<real_t> &bt,
+                                  const Vector &q_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Q = Reshape(q_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int VDIM = 3;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t data[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad0[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad1[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad2[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t Z[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               data[qz][qy][qx][0] = 0.0;
+               data[qz][qy][qx][1] = 0.0;
+               data[qz][qy][qx][2] = 0.0;
+
+               grad0[qz][qy][qx][0] = 0.0;
+               grad0[qz][qy][qx][1] = 0.0;
+               grad0[qz][qy][qx][2] = 0.0;
+
+               grad1[qz][qy][qx][0] = 0.0;
+               grad1[qz][qy][qx][1] = 0.0;
+               grad1[qz][qy][qx][2] = 0.0;
+
+               grad2[qz][qy][qx][0] = 0.0;
+               grad2[qz][qy][qx][1] = 0.0;
+               grad2[qz][qy][qx][2] = 0.0;
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         real_t dataXY[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY0[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY1[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY2[max_Q1D][max_Q1D][VDIM];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dataXY[qy][qx][0] = 0.0;
+               dataXY[qy][qx][1] = 0.0;
+               dataXY[qy][qx][2] = 0.0;
+
+               gradXY0[qy][qx][0] = 0.0;
+               gradXY0[qy][qx][1] = 0.0;
+               gradXY0[qy][qx][2] = 0.0;
+
+               gradXY1[qy][qx][0] = 0.0;
+               gradXY1[qy][qx][1] = 0.0;
+               gradXY1[qy][qx][2] = 0.0;
+
+               gradXY2[qy][qx][0] = 0.0;
+               gradXY2[qy][qx][1] = 0.0;
+               gradXY2[qy][qx][2] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            real_t dataX[max_Q1D][VDIM];
+            real_t gradX0[max_Q1D][VDIM];
+            real_t gradX1[max_Q1D][VDIM];
+            real_t gradX2[max_Q1D][VDIM];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dataX[qx][0] = 0.0;
+               dataX[qx][1] = 0.0;
+               dataX[qx][2] = 0.0;
+
+               gradX0[qx][0] = 0.0;
+               gradX0[qx][1] = 0.0;
+               gradX0[qx][2] = 0.0;
+
+               gradX1[qx][0] = 0.0;
+               gradX1[qx][1] = 0.0;
+               gradX1[qx][2] = 0.0;
+
+               gradX2[qx][0] = 0.0;
+               gradX2[qx][1] = 0.0;
+               gradX2[qx][2] = 0.0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const real_t s0 = x(dx, dy, dz, 0, e);
+               const real_t s1 = x(dx, dy, dz, 1, e);
+               const real_t s2 = x(dx, dy, dz, 2, e);
+               for (int qx = 0; qx < Q1D; ++qx)
                {
-                  const real_t J11 = J(qx, qy, qz, 0, 0, e),
-                               J12 = J(qx, qy, qz, 0, 1, e),
-                               J13 = J(qx, qy, qz, 0, 2, e);
-                  const real_t J21 = J(qx, qy, qz, 1, 0, e),
-                               J22 = J(qx, qy, qz, 1, 1, e),
-                               J23 = J(qx, qy, qz, 1, 2, e);
-                  const real_t J31 = J(qx, qy, qz, 2, 0, e),
-                               J32 = J(qx, qy, qz, 2, 1, e),
-                               J33 = J(qx, qy, qz, 2, 2, e);
-                  const real_t c =
-                     const_coeff ? C(0, 0, 0, 0) : C(qx, qy, qz, e);
-                  const real_t cw = W(qx, qy, qz) * c;
-                  // adj(J)
-                  const real_t A11 = (J22 * J33) - (J23 * J32);
-                  const real_t A12 = (J32 * J13) - (J12 * J33);
-                  const real_t A13 = (J12 * J23) - (J22 * J13);
-                  const real_t A21 = (J31 * J23) - (J21 * J33);
-                  const real_t A22 = (J11 * J33) - (J13 * J31);
-                  const real_t A23 = (J21 * J13) - (J11 * J23);
-                  const real_t A31 = (J21 * J32) - (J31 * J22);
-                  const real_t A32 = (J31 * J12) - (J11 * J32);
-                  const real_t A33 = (J11 * J22) - (J12 * J21);
-                  // Store wq * coeff * adj(J)
-                  A(0, 0, qx, qy, qz, e) = cw * A11;
-                  A(1, 0, qx, qy, qz, e) = cw * A12;
-                  A(2, 0, qx, qy, qz, e) = cw * A13;
-                  A(0, 1, qx, qy, qz, e) = cw * A21;
-                  A(1, 1, qx, qy, qz, e) = cw * A22;
-                  A(2, 1, qx, qy, qz, e) = cw * A23;
-                  A(0, 2, qx, qy, qz, e) = cw * A31;
-                  A(1, 2, qx, qy, qz, e) = cw * A32;
-                  A(2, 2, qx, qy, qz, e) = cw * A33;
+                  const real_t Bx = B(qx, dx);
+                  const real_t Gx = G(qx, dx);
+
+                  dataX[qx][0] += s0 * Bx;
+                  dataX[qx][1] += s1 * Bx;
+                  dataX[qx][2] += s2 * Bx;
+
+                  gradX0[qx][0] += s0 * Gx;
+                  gradX0[qx][1] += s0 * Bx;
+                  gradX0[qx][2] += s0 * Bx;
+
+                  gradX1[qx][0] += s1 * Gx;
+                  gradX1[qx][1] += s1 * Bx;
+                  gradX1[qx][2] += s1 * Bx;
+
+                  gradX2[qx][0] += s2 * Gx;
+                  gradX2[qx][1] += s2 * Bx;
+                  gradX2[qx][2] += s2 * Bx;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const real_t By = B(qy, dy);
+               const real_t Gy = G(qy, dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  dataXY[qy][qx][0] += dataX[qx][0] * By;
+                  dataXY[qy][qx][1] += dataX[qx][1] * By;
+                  dataXY[qy][qx][2] += dataX[qx][2] * By;
+
+                  gradXY0[qy][qx][0] += gradX0[qx][0] * By;
+                  gradXY0[qy][qx][1] += gradX0[qx][1] * Gy;
+                  gradXY0[qy][qx][2] += gradX0[qx][2] * By;
+
+                  gradXY1[qy][qx][0] += gradX1[qx][0] * By;
+                  gradXY1[qy][qx][1] += gradX1[qx][1] * Gy;
+                  gradXY1[qy][qx][2] += gradX1[qx][2] * By;
+
+                  gradXY2[qy][qx][0] += gradX2[qx][0] * By;
+                  gradXY2[qy][qx][1] += gradX2[qx][1] * Gy;
+                  gradXY2[qy][qx][2] += gradX2[qx][2] * By;
                }
             }
          }
-      });
-   }
-   else
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            const real_t Bz = B(qz, dz);
+            const real_t Gz = G(qz, dz);
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  data[qz][qy][qx][0] += dataXY[qy][qx][0] * Bz;
+                  data[qz][qy][qx][1] += dataXY[qy][qx][1] * Bz;
+                  data[qz][qy][qx][2] += dataXY[qy][qx][2] * Bz;
+
+                  grad0[qz][qy][qx][0] += gradXY0[qy][qx][0] * Bz;
+                  grad0[qz][qy][qx][1] += gradXY0[qy][qx][1] * Bz;
+                  grad0[qz][qy][qx][2] += gradXY0[qy][qx][2] * Gz;
+
+                  grad1[qz][qy][qx][0] += gradXY1[qy][qx][0] * Bz;
+                  grad1[qz][qy][qx][1] += gradXY1[qy][qx][1] * Bz;
+                  grad1[qz][qy][qx][2] += gradXY1[qy][qx][2] * Gz;
+
+                  grad2[qz][qy][qx][0] += gradXY2[qy][qx][0] * Bz;
+                  grad2[qz][qy][qx][1] += gradXY2[qy][qx][1] * Bz;
+                  grad2[qz][qy][qx][2] += gradXY2[qy][qx][2] * Gz;
+               }
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const int q = qx + Q1D * (qy + qz * Q1D);
+
+               const real_t u1 = data[qz][qy][qx][0];
+               const real_t u2 = data[qz][qy][qx][1];
+               const real_t u3 = data[qz][qy][qx][2];
+
+               const real_t grad00 = grad0[qz][qy][qx][0];
+               const real_t grad01 = grad0[qz][qy][qx][1];
+               const real_t grad02 = grad0[qz][qy][qx][2];
+
+               const real_t grad10 = grad1[qz][qy][qx][0];
+               const real_t grad11 = grad1[qz][qy][qx][1];
+               const real_t grad12 = grad1[qz][qy][qx][2];
+
+               const real_t grad20 = grad2[qz][qy][qx][0];
+               const real_t grad21 = grad2[qz][qy][qx][1];
+               const real_t grad22 = grad2[qz][qy][qx][2];
+
+               const real_t Dxu1 = grad00 * Q(q, 0, 0, e)
+                                   + grad01 * Q(q, 1, 0, e)
+                                   + grad02 * Q(q, 2, 0, e);
+               const real_t Dyu1 = grad00 * Q(q, 0, 1, e)
+                                   + grad01 * Q(q, 1, 1, e)
+                                   + grad02 * Q(q, 2, 1, e);
+               const real_t Dzu1 = grad00 * Q(q, 0, 2, e)
+                                   + grad01 * Q(q, 1, 2, e)
+                                   + grad02 * Q(q, 2, 2, e);
+
+               const real_t Dxu2 = grad10 * Q(q, 0, 0, e)
+                                   + grad11 * Q(q, 1, 0, e)
+                                   + grad12 * Q(q, 2, 0, e);
+               const real_t Dyu2 = grad10 * Q(q, 0, 1, e)
+                                   + grad11 * Q(q, 1, 1, e)
+                                   + grad12 * Q(q, 2, 1, e);
+               const real_t Dzu2 = grad10 * Q(q, 0, 2, e)
+                                   + grad11 * Q(q, 1, 2, e)
+                                   + grad12 * Q(q, 2, 2, e);
+
+               const real_t Dxu3 = grad20 * Q(q, 0, 0, e)
+                                   + grad21 * Q(q, 1, 0, e)
+                                   + grad22 * Q(q, 2, 0, e);
+               const real_t Dyu3 = grad20 * Q(q, 0, 1, e)
+                                   + grad21 * Q(q, 1, 1, e)
+                                   + grad22 * Q(q, 2, 1, e);
+               const real_t Dzu3 = grad20 * Q(q, 0, 2, e)
+                                   + grad21 * Q(q, 1, 2, e)
+                                   + grad22 * Q(q, 2, 2, e);
+
+               Z[qz][qy][qx][0] = u1 * Dxu1 + u2 * Dyu1 + u3 * Dzu1;
+               Z[qz][qy][qx][1] = u1 * Dxu2 + u2 * Dyu2 + u3 * Dzu2;
+               Z[qz][qy][qx][2] = u1 * Dxu3 + u2 * Dyu3 + u3 * Dzu3;
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         real_t opXY[max_D1D][max_D1D][VDIM];
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               opXY[dy][dx][0] = 0.0;
+               opXY[dy][dx][1] = 0.0;
+               opXY[dy][dx][2] = 0.0;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            real_t opX[max_D1D][VDIM];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               opX[dx][0] = 0.0;
+               opX[dx][1] = 0.0;
+               opX[dx][2] = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const real_t Btx = Bt(dx, qx);
+                  opX[dx][0] += Btx * Z[qz][qy][qx][0];
+                  opX[dx][1] += Btx * Z[qz][qy][qx][1];
+                  opX[dx][2] += Btx * Z[qz][qy][qx][2];
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t Bty = Bt(dy, qy);
+                  opXY[dy][dx][0] += Bty * opX[dx][0];
+                  opXY[dy][dx][1] += Bty * opX[dx][1];
+                  opXY[dy][dx][2] += Bty * opX[dx][2];
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t Btz = Bt(dz, qz);
+                  y(dx, dy, dz, 0, e) += Btz * opXY[dy][dx][0];
+                  y(dx, dy, dz, 1, e) += Btz * opXY[dy][dx][1];
+                  y(dx, dy, dz, 2, e) += Btz * opXY[dy][dx][2];
+               }
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, int T_MAX_D1D = 0, int T_MAX_Q1D = 0>
+static void SmemPAConvectionNLApply3D(const int NE,
+                                      const Array<real_t> &b_,
+                                      const Array<real_t> &g_,
+                                      const Vector &d_,
+                                      const Vector &x_,
+                                      Vector &y_,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto g = Reshape(g_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-      MFEM_ABORT("dim " << dim << " not supported!");
-   }
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
+      MFEM_SHARED real_t BG[2][MQ1 * MD1];
+      real_t(*B)[MD1] = (real_t(*)[MD1])(BG + 0);
+      real_t(*G)[MD1] = (real_t(*)[MD1])(BG + 1);
+      real_t(*Bt)[MQ1] = (real_t(*)[MQ1])(BG + 0);
+      MFEM_SHARED real_t U[2][MQ1][MQ1][MQ1];
+      MFEM_SHARED real_t sm0[3][MQ1 * MQ1 * MQ1];
+      MFEM_SHARED real_t sm1[3][MQ1 * MQ1 * MQ1];
+      real_t(*DDQ0)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 0);
+      real_t(*DDQ1)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 1);
+      real_t(*X)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 2);
+      real_t(*DQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 0);
+      real_t(*DQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 1);
+      real_t(*DQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 2);
+      real_t(*QQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 0);
+      real_t(*QQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 1);
+      real_t(*QQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 2);
+      real_t(*QQD0)[MQ1][MD1] = (real_t(*)[MQ1][MD1])(sm1 + 0);
+      real_t(*QDD0)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 0);
+      MFEM_SHARED real_t Z[MQ1][MQ1][MQ1];
+
+      for (int cy = 0; cy < VDIM; ++cy)
+      {
+         if (tidz == 0)
+         {
+            MFEM_FOREACH_THREAD(q, x, Q1D)
+            {
+               MFEM_FOREACH_THREAD(d, y, D1D)
+               {
+                  B[q][d] = b(q, d);
+                  G[q][d] = g(q, d);
+               }
+            }
+         }
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx, x, Q1D) { Z[qz][qy][qx] = 0.0; }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         for (int c = 0; c < VDIM; ++c)
+         {
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(dy, y, D1D)
+               {
+                  MFEM_FOREACH_THREAD(dx, x, D1D)
+                  {
+                     X[dz][dy][dx] = x(dx, dy, dz, cy, e);
+                     U[0][dz][dy][dx] = x(dx, dy, dz, c, e);
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(dy, y, D1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t z = 0.0;
+                     for (int dx = 0; dx < D1D; ++dx)
+                     {
+                        const real_t coord = X[dz][dy][dx];
+                        const real_t value = U[0][dz][dy][dx];
+                        u += coord * B[qx][dx];
+                        v += coord * G[qx][dx];
+                        z += value * B[qx][dx];
+                     }
+                     DDQ0[dz][dy][qx] = u;
+                     DDQ1[dz][dy][qx] = v;
+                     U[1][dz][dy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t w = 0.0;
+                     real_t z = 0.0;
+                     for (int dy = 0; dy < D1D; ++dy)
+                     {
+                        u += DDQ1[dz][dy][qx] * B[qy][dy];
+                        v += DDQ0[dz][dy][qx] * G[qy][dy];
+                        w += DDQ0[dz][dy][qx] * B[qy][dy];
+                        z += U[1][dz][dy][qx] * B[qy][dy];
+                     }
+                     DQQ0[dz][qy][qx] = u;
+                     DQQ1[dz][qy][qx] = v;
+                     DQQ2[dz][qy][qx] = w;
+                     U[0][dz][qy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(qz, z, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t w = 0.0;
+                     real_t z = 0.0;
+                     for (int dz = 0; dz < D1D; ++dz)
+                     {
+                        u += DQQ0[dz][qy][qx] * B[qz][dz];
+                        v += DQQ1[dz][qy][qx] * B[qz][dz];
+                        w += DQQ2[dz][qy][qx] * G[qz][dz];
+                        z += U[0][dz][qy][qx] * B[qz][dz];
+                     }
+                     QQQ0[qz][qy][qx] = u;
+                     QQQ1[qz][qy][qx] = v;
+                     QQQ2[qz][qy][qx] = w;
+                     U[1][qz][qy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(qz, z, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     const int q = qx + (qy + qz * Q1D) * Q1D;
+                     const real_t z = U[1][qz][qy][qx];
+                     const real_t gX = QQQ0[qz][qy][qx];
+                     const real_t gY = QQQ1[qz][qy][qx];
+                     const real_t gZ = QQQ2[qz][qy][qx];
+                     const real_t d = gX * D(q, 0, c, e) + gY * D(q, 1, c, e)
+                                      + gZ * D(q, 2, c, e);
+                     Z[qz][qy][qx] += z * d;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+         } // for each conv component
+         if (tidz == 0)
+         {
+            MFEM_FOREACH_THREAD(d, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(q, x, Q1D) { Bt[d][q] = b(q, d); }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += Z[qz][qy][qx] * Bt[dx][qx];
+                  }
+                  QQD0[qz][qy][dx] = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += QQD0[qz][qy][dx] * Bt[dy][qy];
+                  }
+                  QDD0[qz][dy][dx] = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz, z, D1D)
+         {
+            MFEM_FOREACH_THREAD(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += QDD0[qz][dy][dx] * Bt[dz][qz];
+                  }
+                  Y(dx, dy, dz, cy, e) += u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
 }
 
 void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
@@ -197,13 +812,26 @@ void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
    else
    {
-      AddMultPAKernels::Run(dim, d1d, q1d, ne,
-                            maps->B.Read(),
-                            maps->G.Read(),
-                            pa_adj.Read(),
-                            x.Read(),
-                            y.ReadWrite(),
-                            d1d, q1d);
+      const int NE = ne;
+      const int D1D = maps->ndof;
+      const int Q1D = maps->nqpt;
+      const Vector &QV = pa_data;
+      const Array<real_t> &B = maps->B;
+      const Array<real_t> &G = maps->G;
+      const Array<real_t> &Bt = maps->Bt;
+      if (dim == 2)
+      {
+         return PAConvectionNLApply2D(NE, B, G, Bt, QV, x, y, D1D, Q1D);
+      }
+      if (dim == 3)
+      {
+         constexpr int T_MAX_D1D = 8;
+         constexpr int T_MAX_Q1D = 8;
+         MFEM_VERIFY(D1D <= T_MAX_D1D && Q1D <= T_MAX_Q1D, "Not yet implemented!");
+         return SmemPAConvectionNLApply3D<0, 0, T_MAX_D1D, T_MAX_Q1D>
+                (NE, B, G, QV, x, y, D1D, Q1D);
+      }
+      MFEM_ABORT("Not yet implemented!");
    }
 }
 

fem/integ/nonlininteg_vecconvection_pa.hpp

@@ -1,209 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-// PA Convection NL 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLApply2D(const int NE,
-                                      const real_t *b,
-                                      const real_t *g,
-                                      const real_t *a,
-                                      const real_t *x,
-                                      real_t *y,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
-   const auto X = Reshape(x, D1D, D1D, VDIM, NE);
-   auto Y = Reshape(y, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
-      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::v_regs2d_t<VDIM, MQ1> s0, s1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, X, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
-      kernels::internal::LoadDofs2d(e, D1D, X, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            const future::tensor<real_t, 2> U =
-            {
-               r1[0][qy][qx], r1[1][qy][qx]
-            };
-            const future::tensor<real_t, 2,2> gradU = {{
-                  {g1[0][0][qy][qx], g1[1][0][qy][qx]},
-                  {g1[0][1][qy][qx], g1[1][1][qy][qx]},
-               }
-            };
-            const future::tensor<real_t, 2,2> Q = {{
-                  {A(0,0,qx,qy,e), A(1,0,qx,qy,e)},
-                  {A(0,1,qx,qy,e), A(1,1,qx,qy,e)},
-               }
-            };
-            const future::tensor<real_t, 2> conv = transpose(gradU) * (Q * U);
-            s0[0][qy][qx] = conv[0];
-            s0[1][qy][qx] = conv[1];
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, s0, s1);
-      kernels::internal::WriteDofs2d(e, D1D, s1, Y);
-   });
-}
-
-// PA Convection NL 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLApply3D(const int NE,
-                                      const real_t *b,
-                                      const real_t *g,
-                                      const real_t *a,
-                                      const real_t *x,
-                                      real_t *y,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto X = Reshape(x, D1D, D1D, D1D, VDIM, NE);
-   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::v_regs3d_t<VDIM, MQ1> s0, s1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, X, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
-      kernels::internal::LoadDofs3d(e, D1D, X, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const future::tensor<real_t, 3> U =
-               {
-                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, 3,3> gradU = {{
-                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                  }
-               };
-               const future::tensor<real_t, 3,3> Q = {{
-                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                  }
-               };
-               const future::tensor<real_t, 3> conv = transpose(gradU) * (Q * U);
-               s0[0][qz][qy][qx] = conv[0];
-               s0[1][qz][qy][qx] = conv[1];
-               s0[2][qz][qy][qx] = conv[2];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, s0, s1);
-      kernels::internal::WriteDofs3d(e, D1D, s1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int DIM, int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultPAType
-VectorConvectionNLFIntegrator::AddMultPAKernels::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPAConvectionNLApply2D<T_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPAConvectionNLApply3D<T_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorConvectionNLFIntegrator::AddMultPAType
-VectorConvectionNLFIntegrator::AddMultPAKernels::Fallback
-(int dim, int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPAConvectionNLApply2D<>;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPAConvectionNLApply3D<>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.cpp

@@ -1,50 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../ceed/interface/util.hpp"
-#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
-
-namespace mfem
-{
-
-void VectorConvectionNLFIntegrator::AssembleGradDiagonalPA(Vector &de) const
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   if (dim == 2)
-   {
-      GradDiagPA2D::Run(d1d, q1d, ne,
-                        maps->B.Read(),
-                        maps->G.Read(),
-                        pa_adj.Read(),
-                        pa_u.Read(),
-                        de.ReadWrite(),
-                        d1d, q1d);
-   }
-   else if (dim == 3)
-   {
-      GradDiagPA3D::Run(d1d, q1d, ne,
-                        maps->B.Read(),
-                        maps->G.Read(),
-                        pa_adj.Read(),
-                        pa_u.Read(),
-                        de.ReadWrite(),
-                        d1d, q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-   }
-}
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.hpp

@@ -1,302 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradDiagonal2D(const int NE,
-                                             const real_t *b,
-                                             const real_t *g,
-                                             const real_t *a,
-                                             const real_t *u,
-                                             real_t *de,
-                                             const int d1d,
-                                             const int q1d)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
-   const auto U = Reshape(u, D1D, D1D, VDIM, NE);
-   auto D = Reshape(de, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t sM[3][MQ1][MQ1], sQ[3][MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, U, r0);
-      kernels::internal::Eval2d(D1D, Q1D, sM[0], sB, r0, r1);
-
-      kernels::internal::LoadDofs2d(e, D1D, U, g0);
-      kernels::internal::Grad2d(D1D, Q1D, sM[0], sB, sG, g0, g1);
-
-      for (int v = 0; v < VDIM; ++v)
-      {
-         future::tensor<real_t, VDIM> e_v = {};
-         e_v[v] = real_t(1);
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               const future::tensor<real_t, VDIM> u_val =
-               {
-                  r1[0][qy][qx], r1[1][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> Q_adj =
-               {
-                  {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
-                     { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
-                  }
-               };
-               const future::tensor<real_t, VDIM, DIM> grad_U =
-               {
-                  {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
-                     { g1[0][1][qy][qx], g1[1][1][qy][qx] }
-                  }
-               };
-               const auto one = Q_adj * u_val;
-               const auto two = transpose(grad_U) * (Q_adj * e_v);
-               sQ[0][qx][qy] = one[0];
-               sQ[1][qx][qy] = one[1];
-               sQ[2][qx][qy] = two[v];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               real_t s[3] = {};
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const real_t By = sB[dy][qy], Gy = sG[dy][qy];
-                  s[0] += By * By * sQ[0][qx][qy];
-                  s[1] += Gy * By * sQ[1][qx][qy];
-                  s[2] += By * By * sQ[2][qx][qy];
-               }
-               sM[0][qx][dy] = s[0];
-               sM[1][qx][dy] = s[1];
-               sM[2][qx][dy] = s[2];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-            {
-               real_t d = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
-                  d += Gx * Bx * sM[0][qx][dy] +
-                       Bx * Bx * sM[1][qx][dy] +
-                       Bx * Bx * sM[2][qx][dy];
-               }
-               D(dx, dy, v, e) += d;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradDiagonal3D(const int NE,
-                                             const real_t *b,
-                                             const real_t *g,
-                                             const real_t *a,
-                                             const real_t *u,
-                                             real_t *de,
-                                             const int d1d,
-                                             const int q1d)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, NE);
-   auto D = Reshape(de, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t sM[4][MQ1][MQ1], sQ[4][MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, U, r0);
-      kernels::internal::Eval3d(D1D, Q1D, sM[0], sB, r0, r1);
-
-      kernels::internal::LoadDofs3d(e, D1D, U, g0);
-      kernels::internal::Grad3d(D1D, Q1D, sM[0], sB, sG, g0, g1);
-
-      for (int v = 0; v < VDIM; ++v)
-      {
-         future::tensor<real_t, VDIM> e_v = {};
-         e_v[v] = real_t(1);
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  real_t s[4] = {};
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     const future::tensor<real_t, VDIM> u_val =
-                     {
-                        r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-                     };
-                     const future::tensor<real_t, VDIM, DIM> Q_adj = {{
-                           {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                           {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                           {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                        }
-                     };
-                     const future::tensor<real_t, VDIM, DIM> grad_U = {{
-                           {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                           {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                           {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                        }
-                     };
-                     const auto one = Q_adj * u_val;
-                     const auto two = transpose(grad_U) * (Q_adj * e_v);
-
-                     const real_t Bz = sB[dz][qz], Gz = sG[dz][qz];
-                     s[0] += one[0] * Bz * Bz;
-                     s[1] += one[1] * Bz * Bz;
-                     s[2] += one[2] * Bz * Gz;
-                     s[3] += two[v] * Bz * Bz;
-                  }
-                  sQ[0][qx][qy] = s[0];
-                  sQ[1][qx][qy] = s[1];
-                  sQ[2][qx][qy] = s[2];
-                  sQ[3][qx][qy] = s[3];
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  real_t s[4] = {};
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     const real_t By = sB[dy][qy], Gy = sG[dy][qy];
-                     s[0] += By * By * sQ[0][qx][qy];
-                     s[1] += Gy * By * sQ[1][qx][qy];
-                     s[2] += By * By * sQ[2][qx][qy];
-                     s[3] += By * By * sQ[3][qx][qy];
-                  }
-                  sM[0][dy][qx] = s[0];
-                  sM[1][dy][qx] = s[1];
-                  sM[2][dy][qx] = s[2];
-                  sM[3][dy][qx] = s[3];
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-               {
-                  real_t d = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
-                     d += Gx * Bx * sM[0][dy][qx];
-                     d += Bx * Bx * sM[1][dy][qx];
-                     d += Bx * Bx * sM[2][dy][qx];
-                     d += Bx * Bx * sM[3][dy][qx];
-                  }
-                  D(dx, dy, dz, v, e) += d;
-               }
-            }
-            MFEM_SYNC_THREAD;
-         }
-      }
-   });
-}
-
-} // namespace internal
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA2D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradDiagonal2D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA2D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradDiagonal2D<>;
-}
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA3D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradDiagonal3D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA3D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradDiagonal3D<>;
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.cpp

@@ -1,64 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../ceed/interface/util.hpp"
-#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
-
-namespace mfem
-{
-
-void VectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &u, const FiniteElementSpace &fes)
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   this->pa_u = u;
-   AssemblePA(fes);
-}
-
-void VectorConvectionNLFIntegrator::AddMultGradPA(const Vector &x,
-                                                  Vector &y) const
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   if (dim == 2)
-   {
-      AddMultGradPA2D::Run(d1d, q1d, ne,
-                           maps->B.Read(),
-                           maps->G.Read(),
-                           pa_adj.Read(),
-                           pa_u.Read(),
-                           x.Read(),
-                           y.ReadWrite(),
-                           d1d, q1d);
-   }
-   else if (dim == 3)
-   {
-      AddMultGradPA3D::Run(d1d, q1d, ne,
-                           maps->B.Read(),
-                           maps->G.Read(),
-                           pa_adj.Read(),
-                           pa_u.Read(),
-                           x.Read(),
-                           y.ReadWrite(),
-                           d1d, q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-   }
-}
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.hpp

@@ -1,257 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradApply2D(const int ne,
-                                          const real_t *b,
-                                          const real_t *g,
-                                          const real_t *a,
-                                          const real_t *u,
-                                          const real_t *du,
-                                          real_t *y,
-                                          const int d1d,
-                                          const int q1d)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, ne);
-   const auto U = Reshape(u, D1D, D1D, VDIM, ne);
-   const auto dU = Reshape(du, D1D, D1D, VDIM, ne);
-   auto Y = Reshape(y, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1, g2;
-      kernels::internal::v_regs2d_t<DIM, MQ1> r0, r1, r2;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, dU, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
-
-      kernels::internal::LoadDofs2d(e, D1D, U, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r2); // u value
-
-      kernels::internal::LoadDofs2d(e, D1D, dU, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // δu value
-
-      kernels::internal::LoadDofs2d(e, D1D, U, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            // First part of the Jacobian: u·∇δu
-            const future::tensor<real_t, DIM> u_val =
-            {
-               r2[0][qy][qx], r2[1][qy][qx]
-            };
-            const future::tensor<real_t, VDIM, DIM> Q_adj =
-            {
-               {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
-                  { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
-               }
-            };
-            const future::tensor<real_t, VDIM, DIM> grad_dU =
-            {
-               {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
-                  { g1[0][1][qy][qx], g1[1][1][qy][qx] }
-               }
-            };
-            const auto one = transpose(grad_dU) * (Q_adj * u_val);
-
-            // Second part of the Jacobian: δu·∇u
-            const future::tensor<real_t, DIM> du_val =
-            {
-               r1[0][qy][qx], r1[1][qy][qx]
-            };
-            const future::tensor<real_t, VDIM, DIM> grad_U =
-            {
-               {  { g2[0][0][qy][qx], g2[1][0][qy][qx] },
-                  { g2[0][1][qy][qx], g2[1][1][qy][qx] }
-               }
-            };
-            const auto two = transpose(grad_U) * (Q_adj * du_val);
-
-            // u⋅∇δu + δu⋅∇u
-            r0[0][qy][qx] = one[0] + two[0];
-            r0[1][qy][qx] = one[1] + two[1];
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs2d(e, D1D, r1, Y);
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradApply3D(const int ne,
-                                          const real_t *b,
-                                          const real_t *g,
-                                          const real_t *a,
-                                          const real_t *u,
-                                          const real_t *du,
-                                          real_t *y,
-                                          const int d1d,
-                                          const int q1d)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, ne);
-   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, ne);
-   const auto dU = Reshape(du, D1D, D1D, D1D, VDIM, ne);
-   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1, r2;
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1, g2;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, dU, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
-
-      kernels::internal::LoadDofs3d(e, D1D, U, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r2); // u value
-
-      kernels::internal::LoadDofs3d(e, D1D, dU, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // δu value
-
-      kernels::internal::LoadDofs3d(e, D1D, U, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               // First part of the Jacobian: u·∇δu
-               const future::tensor<real_t, DIM> u_val =
-               {
-                  r2[0][qz][qy][qx],
-                  r2[1][qz][qy][qx],
-                  r2[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> Q_adj = {{
-                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                  }
-               };
-               const future::tensor<real_t, DIM, DIM> grad_dU = {{
-                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                  }
-               };
-               const auto one = transpose(grad_dU) * (Q_adj * u_val);
-
-               // Second part of the Jacobian: δu·∇u
-               const future::tensor<real_t, DIM> du_val =
-               {
-                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> grad_U = {{
-                     {g2[0][0][qz][qy][qx], g2[1][0][qz][qy][qx], g2[2][0][qz][qy][qx]},
-                     {g2[0][1][qz][qy][qx], g2[1][1][qz][qy][qx], g2[2][1][qz][qy][qx]},
-                     {g2[0][2][qz][qy][qx], g2[1][2][qz][qy][qx], g2[2][2][qz][qy][qx]}
-                  }
-               };
-               const auto two = transpose(grad_U) * (Q_adj * du_val);
-
-               // u⋅∇δu + δu⋅∇u
-               r0[0][qz][qy][qx] = one[0] + two[0];
-               r0[1][qz][qy][qx] = one[1] + two[1];
-               r0[2][qz][qy][qx] = one[2] + two[2];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs3d(e, D1D, r1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA2D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradApply2D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA2D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradApply2D<>;
-}
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA3D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradApply3D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA3D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradApply3D<>;
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/intrules.cpp

@@ -542,10 +542,7 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
          return;
    }
 
-#ifdef MFEM_USE_MPFR
-   MFEM_WARNING("MPFR implementation of Gauss-Jacobi quadrature not implemented yet. Falling "
-                "back to double precision implementation...");
-#endif
+#ifndef MFEM_USE_MPFR
 
    const int n = np;
    // common constants for Jacobi polynomials
@@ -614,6 +611,13 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
                                                 ab + 1) / ((1.0 - xi*xi)*pp*pp) / pow(2, ab);
       // map nodes and weights to the interval [0,1]
    }
+
+#else // MFEM_USE_MPFR is defined
+
+   MFEM_ABORT("MPFR implementation of Gauss-Jacobi quadrature not defined yet");
+
+#endif // MFEM_USE_MPFR
+
 }
 
 

fem/kernel_dispatch.hpp

@@ -48,11 +48,7 @@ namespace mfem
 
 #define MFEM_REGISTER_KERNELS(KernelName, KernelType, ...)                     \
    MFEM_EXPAND(MFEM_EXPAND(MFEM_REGISTER_KERNELS_N(__VA_ARGS__,2,1,))          \
-      (MFEM_EXPORT, KernelName, KernelType, __VA_ARGS__))
-
-#define MFEM_REGISTER_KERNELS_HEADER_ONLY(KernelName, KernelType, ...)         \
-   MFEM_EXPAND(MFEM_EXPAND(MFEM_REGISTER_KERNELS_N(__VA_ARGS__,2,1,))          \
-      (, KernelName, KernelType, __VA_ARGS__))
+      (KernelName,KernelType,__VA_ARGS__))
 
 #define MFEM_REGISTER_KERNELS_N(_1, _2, N, ...) MFEM_REGISTER_KERNELS_##N
 
@@ -62,19 +58,19 @@ namespace mfem
 
 // Version of MFEM_REGISTER_KERNELS without any "optional" (non-dispatch)
 // parameters.
-#define MFEM_REGISTER_KERNELS_1(ExportMacro, KernelName, KernelType, Params)   \
-   MFEM_REGISTER_KERNELS_(ExportMacro, KernelName, KernelType, Params, (), Params)
+#define MFEM_REGISTER_KERNELS_1(KernelName, KernelType, Params)                \
+   MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, (), Params)
 
-// Version of MFEM_REGISTER_KERNELS with optional (non-dispatch)
+// Version of MFEM_REGISTER_KERNELS without any optional (non-dispatch)
 // parameters (e.g. NBZ).
-#define MFEM_REGISTER_KERNELS_2(ExportMacro, KernelName, KernelType, Params, OptParams) \
-   MFEM_REGISTER_KERNELS_(ExportMacro, KernelName, KernelType, Params, OptParams,       \
+#define MFEM_REGISTER_KERNELS_2(KernelName, KernelType, Params, OptParams)     \
+   MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, OptParams,           \
                           (MFEM_PARAM_LIST Params, MFEM_PARAM_LIST OptParams))
 
 // P1 are the parameters, P2 are the optional (non-dispatch parameters), and P3
 // is the concatenation of P1 and P2. We need to pass it as a separate argument
 // to avoid a trailing comma in the case that P2 is empty.
-#define MFEM_REGISTER_KERNELS_(ExportMacro, KernelName, KernelType, P1, P2, P3) \
+#define MFEM_REGISTER_KERNELS_(KernelName, KernelType, P1, P2, P3)             \
   class KernelName                                                             \
       : public ::mfem::KernelDispatchTable<                                    \
             KernelName, KernelType,                                            \
@@ -84,8 +80,8 @@ namespace mfem
     const char *kernel_name = MFEM_KERNEL_NAME(KernelName);                    \
     using KernelSignature = KernelType;                                        \
     template <MFEM_PARAM_LIST P3> static KernelSignature Kernel();             \
-    static ExportMacro KernelSignature Fallback(MFEM_PARAM_LIST P1);           \
-    static ExportMacro KernelName &Get() {                                     \
+    static MFEM_EXPORT KernelSignature Fallback(MFEM_PARAM_LIST P1);           \
+    static MFEM_EXPORT KernelName &Get() {                                     \
       static KernelName table;                                                 \
       return table;                                                            \
     }                                                                          \

fem/nonlinearform.hpp

@@ -224,9 +224,6 @@ public:
    /** @see GetGradient(const Vector &) */
    Operator &GetGradient(const Vector &x, bool finalize) const;
 
-   /// Suppress a warning about hiding overloaded virtual function.
-   using Operator::GetGradient;
-
    /// Update the NonlinearForm to propagate updates of the associated FE space.
    /** After calling this method, the essential boundary conditions need to be
        set again. */

fem/nonlinearform_ext.cpp

@@ -100,17 +100,6 @@ PANonlinearFormExtension::Gradient::Gradient(const PANonlinearFormExtension &e):
 
 void PANonlinearFormExtension::Gradient::AssembleGrad(const Vector &g)
 {
-   if (DeviceCanUseCeed())
-   {
-      for (int i = 0; i < ext.dnfi.Size(); ++i)
-      {
-         MFEM_VERIFY(dynamic_cast<VectorConvectionNLFIntegrator *>
-                     (ext.dnfi[i]) == nullptr,
-                     "VectorConvectionNLFIntegrator PA gradients are not supported "
-                     "with the libCEED backend");
-      }
-   }
-
    ext.elemR->Mult(g, ext.xe);
    for (int i = 0; i < ext.dnfi.Size(); ++i)
    {

fem/nonlininteg.cpp

@@ -954,74 +954,4 @@ void SkewSymmetricVectorConvectionNLFIntegrator::AssembleElementGrad(
    }
 }
 
-void ConvectiveVectorConvectionNLFIntegrator::AssemblePA(
-   const FiniteElementSpace &)
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &, const FiniteElementSpace &)
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AddMultPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AddMultGradPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
-   Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssemblePA(
-   const FiniteElementSpace &)
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &, const FiniteElementSpace &)
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AddMultPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AddMultGradPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
-   Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
 }

fem/nonlininteg.hpp

@@ -18,7 +18,6 @@
 #include "fespace.hpp"
 #include "ceed/interface/operator.hpp"
 #include "integrator.hpp"
-#include "kernel_dispatch.hpp"
 
 namespace mfem
 {
@@ -385,17 +384,15 @@ private:
    DenseMatrix dshape, dshapex, EF, gradEF, ELV, elmat_comp;
    Vector shape;
    // PA extension
-   int dim, ne, nq, d1d, q1d;
-   Vector pa_adj, pa_u;
+   Vector pa_data;
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq;
 
 public:
-   struct Kernels { Kernels(); };
+   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { }
 
-   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { static Kernels kernels; }
-
-   VectorConvectionNLFIntegrator() { static Kernels kernels; }
+   VectorConvectionNLFIntegrator() = default;
 
    static const IntegrationRule &GetRule(const FiniteElement &fe,
                                          const ElementTransformation &T);
@@ -414,56 +411,13 @@ public:
 
    void AssemblePA(const FiniteElementSpace &fes) override;
 
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
+   void AssembleMF(const FiniteElementSpace &fes) override;
 
    void AddMultPA(const Vector &x, Vector &y) const override;
 
-   using AddMultPAType =
-      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-              const real_t *x, real_t *y,
-              const int d1d, const int q1d);
-   MFEM_REGISTER_KERNELS(AddMultPAKernels, AddMultPAType, (int, int, int));
-
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-
-   using AddMultGradPAType =
-      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-              const real_t *u, const real_t *x, real_t *y,
-              const int d1d, const int q1d);
-
-   MFEM_REGISTER_KERNELS(AddMultGradPA2D, AddMultGradPAType, (int, int));
-   MFEM_REGISTER_KERNELS(AddMultGradPA3D, AddMultGradPAType, (int, int));
-
-   void AssembleGradDiagonalPA(Vector &) const override;
-
-   using GradDiagPAType =
-      void (*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-               const real_t *u, real_t *y,
-               const int d1d, const int q1d);
-
-   MFEM_REGISTER_KERNELS(GradDiagPA2D, GradDiagPAType, (int, int));
-   MFEM_REGISTER_KERNELS(GradDiagPA3D, GradDiagPAType, (int, int));
-
-   template <int DIM, int D1D, int Q1D>
-   static void AddSpecialization()
-   {
-      AddMultPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      if constexpr (DIM == 2)
-      {
-         AddMultGradPA2D::Specialization<D1D, Q1D>::Add();
-         GradDiagPA2D::Specialization<D1D, Q1D>::Add();
-      }
-      else if constexpr (DIM == 3)
-      {
-         AddMultGradPA3D::Specialization<D1D, Q1D>::Add();
-         GradDiagPA3D::Specialization<D1D, Q1D>::Add();
-      }
-   }
-
-   void AssembleMF(const FiniteElementSpace &fes) override;
-
    void AddMultMF(const Vector &x, Vector &y) const override;
 
+
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
       const FiniteElement& trial_fe,
@@ -476,8 +430,7 @@ protected:
 
 
 /** This class is used to assemble the convective form of the nonlinear term
-    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$.
-    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
+    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$ */
 class ConvectiveVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -495,20 +448,12 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
-
-   using NonlinearFormIntegrator::AssemblePA;
-   void AssemblePA(const FiniteElementSpace &fes) override;
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
-   void AddMultPA(const Vector &x, Vector &y) const override;
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 
 /** This class is used to assemble the skew-symmetric form of the nonlinear term
     arising in the Navier-Stokes equations
-    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$.
-    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
+    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$ */
 class SkewSymmetricVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -526,13 +471,6 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
-
-   using NonlinearFormIntegrator::AssemblePA;
-   void AssemblePA(const FiniteElementSpace &fes) override;
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
-   void AddMultPA(const Vector &x, Vector &y) const override;
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 }

fem/pgridfunc.cpp

@@ -22,20 +22,10 @@ using namespace std;
 namespace mfem
 {
 
-ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
-                                 bool preserve)
+ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf)
 {
    fes = pfes = pf;
    SetDataAndSize(gf->GetData(), gf->Size());
-
-   if (pfes->HaveDofSigns())
-   {
-      MFEM_VERIFY(!preserve, "Differing sign conventions for the serial and "
-                  "parallel grid functions will prevent preserving the serial "
-                  "GridFunctions in this context.");
-
-      pfes->ApplyDofSigns(HostReadWrite());
-   }
 }
 
 ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, HypreParVector *tv)

fem/pgridfunc.hpp

@@ -100,12 +100,8 @@ public:
    /// Construct a ParGridFunction using a GridFunction as external data.
    /** The parallel space @a *pf and the space used by @a *gf should match. The
        data from @a *gf is used as the local data of the ParGridFunction on each
-       processor. The ParGridFunction does not assume ownership of the data.
-       The boolean, @a preserve, indicates that the data stored in @a *gf should
-       remain unchanged. An error will occur if @a preserve is true and
-       construction of a valid ParGridFunction requires the data to change. */
-   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
-                   bool preserve = true);
+       processor. The ParGridFunction does not assume ownership of the data. */
+   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf);
 
    /** @brief Creates grid function on (all) dofs from a given vector on the
        true dofs, i.e. P tv. */

fem/qfunction.hpp

@@ -79,27 +79,6 @@ public:
       Vector::operator=(orig);
    }
 
-   /** @brief Construct a QuadratureFunction on the given
-       VectorQuadratureSpace, @a vqspace.
-
-       After construction, the QuadratureFunction does not need the
-       VectorQuadratureSpace object. Instead, it uses directly its underlying
-       QuadratureSpaceBase object. */
-   QuadratureFunction(VectorQuadratureSpace &vqspace)
-      : QuadratureFunction(*vqspace.GetSpace(), vqspace.GetVDim())
-   { }
-
-   /** @brief Construct a QuadratureFunction on the given
-       VectorQuadratureSpace, @a vqspace, with the given MemoryType, @a mt, used
-       for the underlying Vector object.
-
-       After construction, the QuadratureFunction does not need the
-       VectorQuadratureSpace object. Instead, it uses directly its underlying
-       QuadratureSpaceBase object. */
-   QuadratureFunction(VectorQuadratureSpace &vqspace, MemoryType mt)
-      : QuadratureFunction(*vqspace.GetSpace(), mt, vqspace.GetVDim())
-   { }
-
    /// Read a QuadratureFunction from the stream @a in.
    /** The QuadratureFunction assumes ownership of the read QuadratureSpace. */
    QuadratureFunction(Mesh *mesh, std::istream &in);

fem/qinterp/det.cpp

@@ -21,23 +21,24 @@ namespace quadrature_interpolator
 
 void InitDetKernels()
 {
+   using k = QuadratureInterpolator::DetKernels;
    // 2D
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,2>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,3>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,3,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,3,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,4,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,4,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,5,6>();
+   k::Specialization<2,2,2,2>::Add();
+   k::Specialization<2,2,2,3>::Add();
+   k::Specialization<2,2,2,4>::Add();
+   k::Specialization<2,2,2,6>::Add();
+   k::Specialization<2,2,3,4>::Add();
+   k::Specialization<2,2,3,6>::Add();
+   k::Specialization<2,2,4,4>::Add();
+   k::Specialization<2,2,4,6>::Add();
+   k::Specialization<2,2,5,6>::Add();
    // 3D
-   QuadratureInterpolator::AddDetSpecializations<3,3,2,4>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,3>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,5>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,6>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,4,6>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,4>();
+   k::Specialization<3,3,2,4>::Add();
+   k::Specialization<3,3,3,3>::Add();
+   k::Specialization<3,3,3,5>::Add();
+   k::Specialization<3,3,3,6>::Add();
+   k::Specialization<3,3,4,6>::Add();
+   k::Specialization<3,3,3,4>::Add();
 }
 
 } // namespace quadrature_interpolator
@@ -46,8 +47,8 @@ void InitDetKernels()
 /// @cond Suppress_Doxygen_warnings
 
 QuadratureInterpolator::DetKernelType
-QuadratureInterpolator::DetKernels::Fallback(int DIM, int SDIM, int D1D,
-                                             int Q1D)
+QuadratureInterpolator::DetKernels::Fallback(
+   int DIM, int SDIM, int D1D, int Q1D)
 {
    if (DIM == 1)
    {

fem/qinterp/eval.hpp

@@ -30,18 +30,23 @@ namespace internal
 namespace quadrature_interpolator
 {
 
-template <QVectorLayout Q_LAYOUT, bool Integral>
-static void ImplValues1D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim,
-                         const int d1d, const int q1d)
+template<QVectorLayout Q_LAYOUT>
+static void Values1D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim,
+                     const int d1d,
+                     const int q1d)
 {
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   const auto b = Reshape(b_, q1d, d1d);
+   const auto x = Reshape(x_, d1d, vdim, NE);
+   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
+            Reshape(y_, q1d, vdim, NE):
+            Reshape(y_, vdim, q1d, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto b = Reshape(b_, q1d, d1d);
-      const auto x = Reshape(x_, d1d, vdim, NE);
-      const auto detJ = Reshape(detJ_, q1d, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES ? Reshape(y_, q1d, vdim, NE)
-               : Reshape(y_, vdim, q1d, NE);
       for (int c = 0; c < vdim; c++)
       {
          for (int q = 0; q < q1d; q++)
@@ -51,36 +56,24 @@ static void ImplValues1D(const int NE, const real_t *b_, const real_t *detJ_,
             {
                u += b(q, d) * x(d, c, e);
             }
-            if constexpr (Integral)
-            {
-               u /= detJ(q, e);
-            }
-            if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-            {
-               y(c, q, e) = u;
-            }
-            if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-            {
-               y(q, c, e) = u;
-            }
+            if (Q_LAYOUT == QVectorLayout::byVDIM)  { y(c, q, e) = u; }
+            if (Q_LAYOUT == QVectorLayout::byNODES) { y(q, c, e) = u; }
          }
       }
    });
 }
 
-template <QVectorLayout Q_LAYOUT>
-static void Values1D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim, const int d1d, const int q1d)
-{
-   ImplValues1D<Q_LAYOUT, false>(NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
 // Template compute kernel for Values in 2D: tensor product version.
-template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
-          int T_Q1D = 0, int T_NBZ = 1>
-static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim = 0,
-                         const int d1d = 0, const int q1d = 0)
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int T_NBZ = 1>
+static void Values2D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim = 0,
+                     const int d1d = 0,
+                     const int q1d = 0)
 {
    static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
 
@@ -89,14 +82,13 @@ static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
+            Reshape(y_, Q1D, Q1D, VDIM, NE):
+            Reshape(y_, VDIM, Q1D, Q1D, NE);
 
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
-      const auto detJ = Reshape(detJ_, Q1D, Q1D, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES
-               ? Reshape(y_, Q1D, Q1D, VDIM, NE)
-               : Reshape(y_, VDIM, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -118,33 +110,16 @@ static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
 
       for (int c = 0; c < VDIM; c++)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx, x, D1D)
-            {
-               DD(dx, dy) = x(dx, dy, c, e);
-            }
-         }
-         MFEM_SYNC_THREAD;
+         kernels::internal::LoadX(e,D1D,c,x,DD);
          kernels::internal::EvalX(D1D,Q1D,B,DD,DQ);
          kernels::internal::EvalY(D1D,Q1D,B,DQ,QQ);
          MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
             MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               real_t u = QQ(qx, qy);
-               if constexpr (Integral)
-               {
-                  u /= detJ(qx, qy, e);
-               }
-               if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-               {
-                  y(c, qx, qy, e) = u;
-               }
-               if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-               {
-                  y(qx, qy, c, e) = u;
-               }
+               real_t u = QQ(qx,qy);
+               if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,e) = u; }
+               if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,c,e) = u; }
             }
          }
          MFEM_SYNC_THREAD;
@@ -152,37 +127,29 @@ static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
    });
 }
 
-// Template compute kernel for Values in 2D: tensor product version.
-template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-          int T_NBZ = 1>
-static void Values2D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim = 0, const int d1d = 0,
-                     const int q1d = 0)
-{
-   return ImplValues2D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D, T_NBZ>(
-             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
 // Template compute kernel for Values in 3D: tensor product version.
-template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
-          int T_Q1D = 0>
-static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim = 0,
-                         const int d1d = 0, const int q1d = 0)
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
+static void Values3D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim = 0,
+                     const int d1d = 0,
+                     const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
+            Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE):
+            Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
 
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
-      const auto detJ = Reshape(detJ_, Q1D, Q1D, Q1D, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES
-               ? Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE)
-               : Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -204,17 +171,7 @@ static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
 
       for (int c = 0; c < VDIM; c++)
       {
-         MFEM_FOREACH_THREAD(dz, z, D1D)
-         {
-            MFEM_FOREACH_THREAD(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD(dx, x, D1D)
-               {
-                  DDD(dx, dy, dz) = x(dx, dy, dz, c, e);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
+         kernels::internal::LoadX(e,D1D,c,x,DDD);
          kernels::internal::EvalX(D1D,Q1D,B,DDD,DDQ);
          kernels::internal::EvalY(D1D,Q1D,B,DDQ,DQQ);
          kernels::internal::EvalZ(D1D,Q1D,B,DQQ,QQQ);
@@ -224,19 +181,9 @@ static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
             {
                MFEM_FOREACH_THREAD(qx,x,Q1D)
                {
-                  real_t u = QQQ(qz,qy,qx);
-                  if constexpr (Integral)
-                  {
-                     u /= detJ(qx, qy, qz, e);
-                  }
-                  if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     y(c, qx, qy, qz, e) = u;
-                  }
-                  if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-                  {
-                     y(qx, qy, qz, c, e) = u;
-                  }
+                  const real_t u = QQQ(qz,qy,qx);
+                  if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,qz,e) = u; }
+                  if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,qz,c,e) = u; }
                }
             }
          }
@@ -245,431 +192,14 @@ static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
    });
 }
 
-// Template compute kernel for Values in 3D: tensor product version.
-template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
-static void Values3D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim = 0, const int d1d = 0,
-                     const int q1d = 0)
-{
-   return ImplValues3D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D>(
-             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
-template <bool Integral>
-void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-                const real_t *detJ, const GeometricFactors *geom,
-                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                Vector &q_der, Vector &q_det, const int eval_flags);
-
-inline void Eval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der, Vector &q_det,
-                   const int eval_flags)
-{
-   ImplEval1D<false>(NE, vdim, q_layout, nullptr, geom, maps, e_vec, q_val,
-                     q_der, q_det, eval_flags);
-}
-
-// Template compute kernel for 2D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
-static void ImplEval2D(const int NE, const int vdim,
-                       const QVectorLayout q_layout, const real_t *detJ_,
-                       const GeometricFactors *geom, const DofToQuad &maps,
-                       const Vector &e_vec, Vector &q_val, Vector &q_der,
-                       Vector &q_det, const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
-
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
-   if constexpr(Integral)
-   {
-      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
-                                  QI::DETERMINANTS)),
-                  "Integral FE does not support computing derivatives");
-   }
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
-   const auto E_ = e_vec.Read();
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 2, NE):
-              Reshape(q_der.Write(), VDIM, 2, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const auto E = Reshape(E_, ND, VDIM, NE);
-      const auto detJ = Reshape(detJ_, NQ, NE);
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND2D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM2D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c + d * VDIM] = E(d, c, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++)
-            {
-               ed[c] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  ed[c] += b * s_E[c + d * VDIM];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if constexpr (Integral)
-               {
-                  ed[c] /= detJ(q, e);
-               }
-               if (q_layout == QVectorLayout::byVDIM)
-               {
-                  val(c, q, e) = ed[c];
-               }
-               if (q_layout == QVectorLayout::byNODES)
-               {
-                  val(q, c, e) = ed[c];
-               }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM2D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM2D*2];
-            for (int i = 0; i < 2*VDIM; i++)
-            {
-               D[i] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[4], Jinv[4];
-               Jloc[0] = J(q,0,0,e);
-               Jloc[1] = J(q,1,0,e);
-               Jloc[2] = J(q,0,1,e);
-               Jloc[3] = J(q,1,1,e);
-               kernels::CalcInverse<2>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v;
-                  const real_t JiV = Jinv[2]*u + Jinv[3]*v;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                  }
-               }
-            }
-            if (eval_flags & QI::DETERMINANTS)
-            {
-               if (VDIM == 2)
-               {
-                  det(q, e) = kernels::Det<2>(D);
-               }
-               else
-               {
-                  DeviceTensor<2> j(D, 3, 2);
-                  const real_t dE = j(0,0)*j(0,0) + j(1,0)*j(1,0) + j(2,0)*j(2,0);
-                  const real_t dF = j(0,0)*j(0,1) + j(1,0)*j(1,1) + j(2,0)*j(2,1);
-                  const real_t dG = j(0,1)*j(0,1) + j(1,1)*j(1,1) + j(2,1)*j(2,1);
-                  det(q,e) = std::sqrt(dE*dG - dF*dF);
-               }
-            }
-         }
-      }
-   });
-}
-
-// Template compute kernel for 2D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval2D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der,
-                   Vector &q_det, const int eval_flags)
-{
-   ImplEval2D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
-                                         maps, e_vec, q_val, q_der, q_det,
-                                         eval_flags);
-}
-
-// Template compute kernel for 3D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
-static void ImplEval3D(const int NE, const int vdim,
-                       const QVectorLayout q_layout, const real_t *detJ_,
-                       const GeometricFactors *geom, const DofToQuad &maps,
-                       const Vector &e_vec, Vector &q_val, Vector &q_der,
-                       Vector &q_det, const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
-
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 3, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
-   MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
-   if constexpr(Integral)
-   {
-      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
-                                  QI::DETERMINANTS)),
-                  "Integral FE does not support computing derivatives");
-   }
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
-   auto E_ = e_vec.Read();
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 3, NE):
-              Reshape(q_der.Write(), VDIM, 3, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const auto E = Reshape(E_, ND, VDIM, NE);
-      const auto detJ = Reshape(detJ_, NQ, NE);
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND3D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM3D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c + d * VDIM] = E(d, c, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++)
-            {
-               ed[c] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  ed[c] += b * s_E[c + d * VDIM];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if constexpr (Integral)
-               {
-                  ed[c] /= detJ(q, e);
-               }
-               if (q_layout == QVectorLayout::byVDIM)
-               {
-                  val(c, q, e) = ed[c];
-               }
-               if (q_layout == QVectorLayout::byNODES)
-               {
-                  val(q, c, e) = ed[c];
-               }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM3D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM3D*3];
-            for (int i = 0; i < 3*VDIM; i++)
-            {
-               D[i] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               const real_t wz = G(q,2,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-                  D[c+VDIM*2] += s_e * wz;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                     der(c,2,q,e) = D[c+VDIM*2];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                     der(q,c,2,e) = D[c+VDIM*2];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[9], Jinv[9];
-               for (int col = 0; col < 3; col++)
-               {
-                  for (int row = 0; row < 3; row++)
-                  {
-                     Jloc[row+3*col] = J(q,row,col,e);
-                  }
-               }
-               kernels::CalcInverse<3>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t w = D[c+VDIM*2];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
-                  const real_t JiV = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
-                  const real_t JiW = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                     der(c,2,q,e) = JiW;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                     der(q,c,2,e) = JiW;
-                  }
-               }
-            }
-            if (VDIM == 3 && (eval_flags & QI::DETERMINANTS))
-            {
-               // The check (VDIM == 3) should eliminate this block when VDIM is
-               // known at compile time and (VDIM != 3).
-               det(q,e) = kernels::Det<3>(D);
-            }
-         }
-      }
-   });
-}
-
-// Template compute kernel for 3D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval3D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der,
-                   Vector &q_det, const int eval_flags)
-{
-   ImplEval3D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
-                                         maps, e_vec, q_val, q_der, q_det,
-                                         eval_flags);
-}
-
 } // namespace quadrature_interpolator
 
 } // namespace internal
 
 /// @cond Suppress_Doxygen_warnings
 
-template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
-QuadratureInterpolator::IntTensorEvalKernelType
-QuadratureInterpolator::IntTensorEvalKernels::Kernel()
-{
-   if constexpr (DIM == 1) { return internal::quadrature_interpolator::ImplValues1D<Q_LAYOUT, true>; }
-   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::ImplValues2D<Q_LAYOUT, true, VDIM, D1D, Q1D, NBZ>; }
-   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::ImplValues3D<Q_LAYOUT, true, VDIM, D1D, Q1D>; }
-   MFEM_ABORT("");
-}
-
-template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
+template<int DIM, QVectorLayout Q_LAYOUT,
+         int VDIM, int D1D, int Q1D, int NBZ>
 QuadratureInterpolator::TensorEvalKernelType
 QuadratureInterpolator::TensorEvalKernels::Kernel()
 {
@@ -679,28 +209,6 @@ QuadratureInterpolator::TensorEvalKernels::Kernel()
    MFEM_ABORT("");
 }
 
-template <int DIM, int VDIM, int ND, int NQ>
-QuadratureInterpolator::IntEvalKernelType
-QuadratureInterpolator::IntEvalKernels::Kernel()
-{
-   using namespace internal::quadrature_interpolator;
-   if constexpr (DIM == 1) { return ImplEval1D<true>; }
-   else if constexpr (DIM == 2) { return ImplEval2D<true,VDIM,ND,NQ>; }
-   else if constexpr (DIM == 3) { return ImplEval3D<true,VDIM,ND,NQ>; }
-   MFEM_ABORT("");
-}
-
-template <int DIM, int VDIM, int ND, int NQ>
-QuadratureInterpolator::EvalKernelType
-QuadratureInterpolator::EvalKernels::Kernel()
-{
-   using namespace internal::quadrature_interpolator;
-   if constexpr (DIM == 1) { return Eval1D; }
-   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
-   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
-   MFEM_ABORT("");
-}
-
 /// @endcond
 
 } // namespace mfem

fem/qinterp/eval_by_nodes.cpp

@@ -21,105 +21,63 @@ namespace quadrature_interpolator
 
 void InitEvalByNodesKernels()
 {
+   using k = QuadratureInterpolator::TensorEvalKernels;
+
    // 2D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 4, 4, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,2>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,5>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 7, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,5>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,6>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,7>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 5, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,5,6>::Opt<1>::Add();
 
    // 3D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,8>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 3, 4, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 6, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 6, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 7, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,7>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,8>::Opt<1>::Add();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/eval_by_vdim.cpp

@@ -21,59 +21,36 @@ namespace quadrature_interpolator
 
 void InitEvalByVDimKernels()
 {
+   using k = QuadratureInterpolator::TensorEvalKernels;
    // 2D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 2, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 3, 6, 4>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 4, 8, 2>();
+   k::Specialization<2,QVectorLayout::byVDIM,1,2,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,1,3,6>::Opt<4>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,1,4,8>::Opt<2>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 2, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 3, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 3, 6, 4>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 4, 6, 2>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 4, 8, 2>();
+   k::Specialization<2,QVectorLayout::byVDIM,2,2,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,3,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,3,6>::Opt<4>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,4,6>::Opt<2>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,4,8>::Opt<2>::Add();
    // 3D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 4, 8, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,1,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,1,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,1,4,8>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,8>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 5, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 6, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 7, 7, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 8, 8, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 9, 9, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,3,2,2>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,5,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,6,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,7,7>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,8,8>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,9,9>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 4, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,4>::Opt<1>::Add();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/eval_transpose.cpp

@@ -1,44 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../quadinterpolator.hpp"
-#include "eval_transpose.hpp"
-
-namespace mfem
-{
-
-/// @cond Suppress_Doxygen_warnings
-
-QuadratureInterpolator::TensorEvalTransposeKernelType
-QuadratureInterpolator::TensorEvalTransposeKernels::Fallback(
-   int DIM, QVectorLayout Q_LAYOUT, int, int, int)
-{
-   using namespace internal::quadrature_interpolator;
-
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      if (DIM == 1) { return ValuesTranspose1D<QVectorLayout::byNODES>; }
-      else if (DIM == 2) { return ValuesTranspose2D<QVectorLayout::byNODES>; }
-      else if (DIM == 3) { return ValuesTranspose3D<QVectorLayout::byNODES>; }
-   }
-   else
-   {
-      if (DIM == 1) { return ValuesTranspose1D<QVectorLayout::byVDIM>; }
-      else if (DIM == 2) { return ValuesTranspose2D<QVectorLayout::byVDIM>; }
-      else if (DIM == 3) { return ValuesTranspose3D<QVectorLayout::byVDIM>; }
-   }
-   MFEM_ABORT("Invalid dimension");
-   return nullptr;
-}
-
-/// @endcond
-
-} // namespace mfem