Merge pull request #4983 from dnpham23/master

Methods for extracting boundary edge DOFs from a 3D ParMesh and for synchronizing boundary conditions across processor boundaries

.gitignore

@@ -214,7 +214,6 @@ miniapps/adjoint/adjoint_advection_diffusion
 
 miniapps/dfem/dfem-minimal-surface
 miniapps/dfem/dfem-minimal-surface-output
-miniapps/dfem/dfem-hyperelasticity
 
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla

CHANGELOG

@@ -44,9 +44,6 @@ Discretization improvements
 - Added methods to estimate function extremum using piecewise linear bounds plus
   recursive subdivision.
 
-- Added class VectorQuadratureSpace that represents a vector (multi-component)
-  version of the scalar (single-component) class QuadratureSpaceBase.
-
 - Extend FindPointsGSLIB to support surface meshes.
 
 - Added support for complex-valued mixed bilinear forms via the new classes
@@ -56,6 +53,16 @@ Discretization improvements
   ComplexHypreParMatrix::GetSystemMatrix, which previously assumed equal
   trial and test spaces.
 
+- Added FiniteElementSpace::GetBoundaryLoopEdgeDofs to extract the edge DOFs on
+  the perimeter loop of a set of boundary elements, with a ParFiniteElementSpace
+  overload that reconciles the selection across processor boundaries so the
+  result is partition invariant. This is useful for imposing boundary conditions
+  on boundary edge DOFs.
+
+- Added a MaxAbs reduction to GroupCommunicator that selects the signed value of
+  largest magnitude across a group, keeping its sign. Equal-magnitude ties
+  resolve deterministically to the positive value.
+
 Meshing improvements
 --------------------
 - Added support for nonuniform anisotropic mesh refinement on parallel quad/hex
@@ -82,6 +89,11 @@ Linear and nonlinear solvers
   PRefinement multigrid methods for problems posed on trace spaces (see e.g. the
   DPG miniapps).
 
+- Added new class MultiVector: an array of Vectors of different sizes where each
+  Vector can be allocated independently. Also, added associated methods in class
+  Operator: MultMV, MultTransposeMV, and GetGradientMV, that use MultiVector
+  objects for input and/or output parameters. [PR #5249]
+
 GPU computing
 -------------
 - Improved partial assembly for VectorDivergenceIntegrator with shared-memory
@@ -95,6 +107,22 @@ GPU computing
 
 - Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
 
+- Added partial assembly support for MixedScalarWeakGradientIntegrator.
+
+- Added partial assembly support for MixedDotProductIntegrator.
+
+- Added partial assembly support for MixedScalarCrossProductIntegrator.
+
+- Added partial assembly support for MixedScalarWeakCrossProductIntegrator.
+
+- Added partial assembly support for MixedVectorGradientIntegrator for H1->RT.
+
+- Added support for device partial assembly CurlInterpolator.
+  This supports 2D and 3D variants:
+    2D H1 (out-of-plane) to RT (in-plane)
+    2D ND (in-plane) to Integral L2 (out-of-plane)
+    3D ND to RT
+
 - Added NVIDIA cuDSS library interface. Implementation examples have been
   added to ex1 and ex1p. See https://developer.nvidia.com/cudss for more
   details. Supported versions >= 0.6.0.
@@ -107,6 +135,9 @@ GPU computing
 - Added support for FiniteElement::MapType::INTEGRAL spaces to
   QuadratureInterpolator.
 
+- Added support for FiniteElement::MapType::INTEGRAL spaces to
+  MixedScalarCurlIntegrator.
+
 New and updated examples and miniapps
 -------------------------------------
 - The Lorentz miniapp (in miniapps/electromagnetics) has been updated to
@@ -121,6 +152,20 @@ Miscellaneous
   using the new method ApplyDofSigns() in class ParFiniteElementSpace: the
   method will return immediately if no sign flips are needed.
 
+- Added support for coefficient-weighted LOR transfer in
+  L2ProjectionGridTransfer. The transfer conserves the weighted mass, for
+  example when transferring velocity while conserving density-weighted momentum.
+  This is illustrated in the lor-transfer and plor-transfer miniapps.
+
+- Added support for saving DataCollection output on the node-local storage,
+  instead of requiring that the filesystem is shared among all the ranks.
+
+API changes
+-----------
+- Removed ProjectGrad from 2D RT elements. Users should use ProjectCurl instead.
+  This also fixes a bug where ProjectCurl was returning the negative curl,
+  identical to ProjectGrad.
+
 
 Version 4.9, released on Dec 11, 2025
 =====================================

CMakeLists.txt

@@ -88,18 +88,9 @@ if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
-# Ginkgo requires C++17:
-if ((MFEM_USE_GINKGO) AND ("${CMAKE_CXX_STANDARD}" LESS "17"))
-  set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to use." FORCE)
-# Google Benchmark, SUNDIALS, STRUMPACK, Tribol, RAJA and Umpire require C++14:
-elseif ((MFEM_USE_BENCHMARK OR
-         MFEM_USE_SUNDIALS OR
-         MFEM_USE_STRUMPACK OR
-         MFEM_USE_TRIBOL OR
-         MFEM_USE_RAJA OR
-         MFEM_USE_UMPIRE) AND
-    ("${CMAKE_CXX_STANDARD}" LESS "14"))
-  set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ standard to use." FORCE)
+# RAJA requires C++20:
+if ((MFEM_USE_UMPIRE OR MFEM_USE_RAJA) AND ("${CMAKE_CXX_STANDARD}" LESS "20"))
+  set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard to use." FORCE)
 endif()
 
 # Include xSDK default CMake file.
@@ -184,45 +175,6 @@ if (MFEM_USE_CUDA)
    set(CMAKE_CUDA_EXTENSIONS OFF CACHE BOOL "Enable CUDA standard extensions.")
    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
    find_package(CUDAToolkit REQUIRED)
-   if(CMAKE_CUDA_COMPILER_ID STREQUAL "Clang")
-     set(_real_fatbinary "${CMAKE_CUDA_FATBINARY}")
-     set(_fatbinary_wrapper
-       "${CMAKE_BINARY_DIR}/cmake-fatbinary-cuda13")
-
-     file(WRITE "${_fatbinary_wrapper}"
-"#!/usr/bin/env bash
-real_fatbinary=\"${_real_fatbinary}\"
-"
-[=[
-translated=()
-
-for argument in "$@"; do
-  case "$argument" in
-    -im=profile=sm_*,file=*)
-      value=${argument#-im=profile=sm_}
-      architecture=${value%%,*}
-      filename=${value#*,file=}
-      translated+=(
-        "--image3=kind=elf,sm=${architecture},file=${filename}"
-      )
-      ;;
-    *)
-      translated+=("$argument")
-      ;;
-  esac
-done
-
-exec "$real_fatbinary" "${translated[@]}"
-]=])
-
-    file(CHMOD "${_fatbinary_wrapper}"
-    PERMISSIONS
-      OWNER_READ OWNER_WRITE OWNER_EXECUTE
-      GROUP_READ GROUP_EXECUTE
-      WORLD_READ WORLD_EXECUTE)
-
-    set(CMAKE_CUDA_FATBINARY "${_fatbinary_wrapper}")
-endif()
    set(CUSPARSE_FOUND TRUE)
    set(CUBLAS_FOUND TRUE)
    # Initialize CUSPARSE_LIBRARIES and CUBLAS_LIBRARIES:
@@ -647,13 +599,6 @@ if (MFEM_USE_ENZYME)
     set(ENZYME_INCLUDE_DIRS ${ENZYME_DIR}/include)
 endif()
 
-if (MFEM_USE_PROTEUS)
-    enable_language(C)
-    find_package(proteus REQUIRED PATHS "${PROTEUS_DIR}")
-    message(STATUS "${PROTEUS_DIR}/include")
-    include_directories("${PROTEUS_DIR}/include")
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -790,16 +735,6 @@ mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
-
-if (MFEM_USE_PROTEUS)
-    add_library(ClangProteusFlags INTERFACE IMPORTED)
-    set_target_properties(ClangProteusFlags PROPERTIES
-        INTERFACE_COMPILE_OPTIONS "-fpass-plugin=$<TARGET_FILE:ProteusPass>"
-    )
-    target_link_libraries(mfem PUBLIC ClangProteusFlags)
-    target_link_libraries(mfem PUBLIC proteus)
-endif()
-
 if (TPL_TARGETS)
   add_dependencies(mfem ${TPL_TARGETS})
 endif()
@@ -807,7 +742,7 @@ if (MINGW)
   target_link_libraries(mfem PRIVATE ws2_32)
 endif()
 if (MSVC)
-  target_compile_options(mfem PUBLIC "/wd4819" "/bigobj")
+  target_compile_options(mfem PUBLIC "/wd4819")
 endif()
 message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
 target_include_directories(mfem
@@ -836,7 +771,7 @@ set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
 # If building out-of-source, define MFEM_CONFIG_FILE to point to the config file
 # inside the build directory.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
-  target_compile_definitions(mfem PUBLIC
+  target_compile_definitions(mfem PRIVATE
     "MFEM_CONFIG_FILE=\"${PROJECT_BINARY_DIR}/config/_config.hpp\"")
 endif()
 
@@ -896,15 +831,16 @@ if (MFEM_ENABLE_TESTING)
   add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
   add_subdirectory(tests EXCLUDE_FROM_ALL)
 
-  # Create a target for all benchmarks and, optionally, enable it. Some simple
-  # timer-based benchmarks in tests/benchmarks do not require Google Benchmark.
-  set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
-  add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
-                  ${MFEM_ENABLE_BENCHMARKS})
-  if (MFEM_ENABLE_BENCHMARKS)
-    add_subdirectory(tests/benchmarks) #install benchmarks if enabled
-  else()
-    add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
+  if (MFEM_USE_BENCHMARK)
+    # Create a target for all benchmarks and, optionally, enable it.
+    set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
+    add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
+                    ${MFEM_ENABLE_BENCHMARKS})
+    if (MFEM_ENABLE_BENCHMARKS)
+      add_subdirectory(tests/benchmarks) #install benchmarks if enabled
+    else()
+      add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
+    endif()
   endif()
 endif()
 
@@ -1084,7 +1020,6 @@ install(EXPORT ${PROJECT_NAME_UC}Targets
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
   DESTINATION ${MFEM_INSTALL_DIR} OPTIONAL)
 
-
 #-------------------------------------------------------------------------------
 # Create 'config.mk' from 'config.mk.in' for the build and install locations and
 # define install rules for 'config.mk' and 'test.mk'

config/config.hpp

@@ -18,9 +18,9 @@
 #define MFEM_CONFIG_HPP
 
 #ifdef MFEM_CONFIG_FILE
-#include MFEM_CONFIG_FILE // IWYU pragma: export
+#include MFEM_CONFIG_FILE
 #else
-#include "_config.hpp" // IWYU pragma: export
+#include "_config.hpp"
 #endif
 
 #include <cstdint>

config/defaults.mk

@@ -28,11 +28,8 @@ MPICXX = mpicxx
 BASE_FLAGS  = -std=c++17
 OPTIM_FLAGS = -O3 $(BASE_FLAGS)
 
-# Shadow warnings for clang only; GCC's -Wshadow flags more.
-SHADOW_WARNING_FLAG = $(if $(findstring clang,\
- $(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
-WARNING_FLAGS = -pedantic -Wall $(SHADOW_WARNING_FLAG)
-
+# The variable WARNING_FLAGS depends on which compiler is used, and is defined
+# later in this file.
 DEBUG_FLAGS = $(strip -g $(addprefix $(XCOMPILER),$(WARNING_FLAGS)) $(BASE_FLAGS))
 
 # Prefixes for passing flags to the compiler and linker when using CXX or MPICXX
@@ -52,6 +49,10 @@ SHARED = NO
 #
 # If you set MFEM_USE_ENZYME=YES, must use CUDA_CXX=clang++
 CUDA_CXX = nvcc
+# CUDA compute capability used during compilation, e.g. sm_60. Multiple
+# architectures can be requested as a comma-separated list, e.g. sm_70,sm_80.
+# A single value may also be one of the nvcc special values "all",
+# "all-major", or "native".
 CUDA_ARCH = sm_60
 # Base CUDA install directory, only needed if building with clang+cuda:
 # The default setting is:
@@ -60,11 +61,23 @@ CUDA_ARCH = sm_60
 # 3. Use /usr/local/cuda
 CUDA_DIR = $(or $(CUDA_HOME),$(patsubst %/,%,$(dir \
  $(patsubst %/,%,$(dir $(shell command -v nvcc))))),/usr/local/cuda)
+# Derive nvcc/clang architecture flags from CUDA_ARCH. A comma-separated list
+# expands into one -gencode / --cuda-gpu-arch flag per architecture; otherwise
+# use the -arch / --cuda-gpu-arch shorthand.
+MFEM_COMMA := ,
+CUDA_ARCH_NUMS = $(patsubst sm_%,%,$(subst $(MFEM_COMMA), ,$(CUDA_ARCH)))
+NVCC_ARCH_FLAGS = $(strip $(if $(findstring $(MFEM_COMMA),$(CUDA_ARCH)),\
+ $(foreach arch,$(CUDA_ARCH_NUMS),\
+ -gencode arch=compute_$(arch)$(MFEM_COMMA)code=sm_$(arch)),\
+ -arch=$(CUDA_ARCH)))
+CLANG_ARCH_FLAGS = $(strip $(if $(findstring $(MFEM_COMMA),$(CUDA_ARCH)),\
+ $(foreach arch,$(CUDA_ARCH_NUMS),--cuda-gpu-arch=sm_$(arch)),\
+ --cuda-gpu-arch=$(CUDA_ARCH)))
 # flags for clang+cuda
-CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) --cuda-gpu-arch=$(CUDA_ARCH)
+CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) $(CLANG_ARCH_FLAGS)
 # flags for nvcc
 NVCC_FLAGS = -x=cu --expt-extended-lambda --expt-relaxed-constexpr \
- -arch=$(CUDA_ARCH) -isystem "$(CUDA_DIR)/include"
+ $(NVCC_ARCH_FLAGS) -isystem "$(CUDA_DIR)/include"
 # Prefixes for passing flags to the host compiler and linker when using
 # CUDA_CXX=nvcc
 CUDA_XCOMPILER = -Xcompiler=
@@ -382,7 +395,7 @@ CUDSS_LIBRARY_DIR = $(CUDSS_DIR)/lib
 CUDSS_OPT         = -I$(CUDSS_INCLUDE_DIR)
 CUDSS_LIB         = \
  $(XLINKER)-rpath,$(CUDSS_LIBRARY_DIR) -L$(CUDSS_LIBRARY_DIR) -lcudss
-# The cuDSS communication and threading libraries. 
+# The cuDSS communication and threading libraries.
 MFEM_CUDSS_COMM_LIB = $(abspath $(wildcard $(or $(CUDSS_COMM_LIB),\
    $(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so))))
 MFEM_CUDSS_THREADING_LIB = $(abspath $(wildcard $(or $(CUDSS_THREADING_LIB),\
@@ -665,3 +678,15 @@ VERBOSE = NO
 
 # Optional build tag
 MFEM_BUILD_TAG = $(shell uname -snm)
+
+# Enable -pedantic flag only for gcc or clang. nvcc complains with -pedantic
+# because of line directives.
+PEDANTIC_FLAG = $(if \
+   $(findstring NVIDIA,$(shell $(MFEM_CXX) --version 2>&1)),, \
+   $(if $(or \
+      $(findstring gcc version,$(shell $(MFEM_CXX) -v 2>&1)), \
+      $(findstring clang version,$(shell $(MFEM_CXX) -v 2>&1))),-pedantic,))
+# Enable shadow warnings for clang only; GCC's -Wshadow flags more.
+SHADOW_WARNING_FLAG = $(if $(findstring clang,\
+ $(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
+WARNING_FLAGS = $(PEDANTIC_FLAG) -Wall $(SHADOW_WARNING_FLAG)

data/nested_cubes.geo

@@ -0,0 +1,131 @@
+// Define the cube sizes
+L_outer = 1.0;
+L_inner = 0.5;
+
+// Set mesh size and algorithm
+mesh_size = 0.4;
+Mesh.Algorithm3D = 1; // Delaunay algorithm for 3D mesh
+Mesh.CharacteristicLengthFactor = 1.0;
+Mesh.MshFileVersion = 2.2;
+
+// Define center point for concentric cubes
+cx = 0.5;
+cy = 0.5;
+cz = 0.5;
+
+// Define the points (vertices of the outer cube)
+Point(1) = {cx-L_outer/2, cy-L_outer/2, cz-L_outer/2, mesh_size};
+Point(2) = {cx+L_outer/2, cy-L_outer/2, cz-L_outer/2, mesh_size};
+Point(3) = {cx+L_outer/2, cy+L_outer/2, cz-L_outer/2, mesh_size};
+Point(4) = {cx-L_outer/2, cy+L_outer/2, cz-L_outer/2, mesh_size};
+Point(5) = {cx-L_outer/2, cy-L_outer/2, cz+L_outer/2, mesh_size};
+Point(6) = {cx+L_outer/2, cy-L_outer/2, cz+L_outer/2, mesh_size};
+Point(7) = {cx+L_outer/2, cy+L_outer/2, cz+L_outer/2, mesh_size};
+Point(8) = {cx-L_outer/2, cy+L_outer/2, cz+L_outer/2, mesh_size};
+
+// Define the points (vertices of the inner cube)
+Point(9) = {cx-L_inner/2, cy-L_inner/2, cz-L_inner/2, mesh_size};
+Point(10) = {cx+L_inner/2, cy-L_inner/2, cz-L_inner/2, mesh_size};
+Point(11) = {cx+L_inner/2, cy+L_inner/2, cz-L_inner/2, mesh_size};
+Point(12) = {cx-L_inner/2, cy+L_inner/2, cz-L_inner/2, mesh_size};
+Point(13) = {cx-L_inner/2, cy-L_inner/2, cz+L_inner/2, mesh_size};
+Point(14) = {cx+L_inner/2, cy-L_inner/2, cz+L_inner/2, mesh_size};
+Point(15) = {cx+L_inner/2, cy+L_inner/2, cz+L_inner/2, mesh_size};
+Point(16) = {cx-L_inner/2, cy+L_inner/2, cz+L_inner/2, mesh_size};
+
+// Define the lines (edges of the outer cube)
+Line(1) = {1, 2};
+Line(2) = {2, 3};
+Line(3) = {3, 4};
+Line(4) = {4, 1};
+Line(5) = {5, 6};
+Line(6) = {6, 7};
+Line(7) = {7, 8};
+Line(8) = {8, 5};
+Line(9) = {1, 5};
+Line(10) = {2, 6};
+Line(11) = {3, 7};
+Line(12) = {4, 8};
+
+// Define the lines (edges of the inner cube)
+Line(13) = {9, 10};
+Line(14) = {10, 11};
+Line(15) = {11, 12};
+Line(16) = {12, 9};
+Line(17) = {13, 14};
+Line(18) = {14, 15};
+Line(19) = {15, 16};
+Line(20) = {16, 13};
+Line(21) = {9, 13};
+Line(22) = {10, 14};
+Line(23) = {11, 15};
+Line(24) = {12, 16};
+
+// Define the surfaces (faces of the outer cube)
+Line Loop(1) = {1, 2, 3, 4};
+Plane Surface(1) = {1};
+
+Line Loop(2) = {5, 6, 7, 8};
+Plane Surface(2) = {2};
+
+Line Loop(3) = {9, 5, -10, -1};
+Plane Surface(3) = {3};
+
+Line Loop(4) = {10, 6, -11, -2};
+Plane Surface(4) = {4};
+
+Line Loop(5) = {11, 7, -12, -3};
+Plane Surface(5) = {5};
+
+Line Loop(6) = {12, 8, -9, -4};
+Plane Surface(6) = {6};
+
+// Define the surfaces (faces of the inner cube)
+Line Loop(7) = {13, 14, 15, 16};
+Plane Surface(7) = {7};
+
+Line Loop(8) = {17, 18, 19, 20};
+Plane Surface(8) = {8};
+
+Line Loop(9) = {21, 17, -22, -13};
+Plane Surface(9) = {9};
+
+Line Loop(10) = {22, 18, -23, -14};
+Plane Surface(10) = {10};
+
+Line Loop(11) = {23, 19, -24, -15};
+Plane Surface(11) = {11};
+
+Line Loop(12) = {24, 20, -21, -16};
+Plane Surface(12) = {12};
+
+// Define the volumes
+Surface Loop(1) = {1, 2, 3, 4, 5, 6};
+Surface Loop(2) = {7, 8, 9, 10, 11, 12};
+Volume(1) = {1, 2}; // Outer volume with inner hole
+Volume(2) = {2};    // Inner volume
+
+// Assign physical groups
+Physical Volume(1) = {1}; // Outer volume
+Physical Volume(2) = {2}; // Inner volume
+
+// Outer cube surfaces
+Physical Surface(1) = {1}; // Outer bottom
+Physical Surface(2) = {2}; // Outer top
+Physical Surface(3) = {3}; // Outer front
+Physical Surface(4) = {4}; // Outer right
+Physical Surface(5) = {5}; // Outer back
+Physical Surface(6) = {6}; // Outer left
+
+// Inner cube surfaces
+Physical Surface(7) = {7};  // Inner bottom (-xy)
+Physical Surface(8) = {8};  // Inner top (+xy)
+Physical Surface(9) = {9};  // Inner front (-xz)
+Physical Surface(10) = {10}; // Inner right (+yz)
+Physical Surface(11) = {11}; // Inner back (+xz)
+Physical Surface(12) = {12}; // Inner left (-yz)
+
+// Mesh control
+Mesh.OptimizeNetgen = 1;
+Mesh.Optimize = 1;
+Mesh.ElementOrder = 1;

data/nested_cubes.msh

@@ -0,0 +1,907 @@
+$MeshFormat
+2.2 0 8
+$EndMeshFormat
+$Nodes
+138
+1 0 0 0
+2 1 0 0
+3 1 1 0
+4 0 1 0
+5 0 0 1
+6 1 0 1
+7 1 1 1
+8 0 1 1
+9 0.25 0.25 0.25
+10 0.75 0.25 0.25
+11 0.75 0.75 0.25
+12 0.25 0.75 0.25
+13 0.25 0.25 0.75
+14 0.75 0.25 0.75
+15 0.75 0.75 0.75
+16 0.25 0.75 0.75
+17 0.3333333333325025 0 0
+18 0.6666666666657889 0 0
+19 1 0.3333333333325025 0
+20 1 0.6666666666657889 0
+21 0.6666666666675911 1 0
+22 0.3333333333347203 1 0
+23 0 0.6666666666675911 0
+24 0 0.3333333333347203 0
+25 0.3333333333325025 0 1
+26 0.6666666666657889 0 1
+27 1 0.3333333333325025 1
+28 1 0.6666666666657889 1
+29 0.6666666666675911 1 1
+30 0.3333333333347203 1 1
+31 0 0.6666666666675911 1
+32 0 0.3333333333347203 1
+33 0 0 0.3333333333325025
+34 0 0 0.6666666666657889
+35 1 0 0.3333333333325025
+36 1 0 0.6666666666657889
+37 1 1 0.3333333333325025
+38 1 1 0.6666666666657889
+39 0 1 0.3333333333325025
+40 0 1 0.6666666666657889
+41 0.5000000000003468 0.25 0.25
+42 0.75 0.5000000000003468 0.25
+43 0.5000000000013763 0.75 0.25
+44 0.25 0.5000000000013763 0.25
+45 0.5000000000003468 0.25 0.75
+46 0.75 0.5000000000003468 0.75
+47 0.5000000000013763 0.75 0.75
+48 0.25 0.5000000000013763 0.75
+49 0.25 0.25 0.5000000000003468
+50 0.75 0.25 0.5000000000003468
+51 0.75 0.75 0.5000000000003468
+52 0.25 0.75 0.5000000000003468
+53 0.7113248654055673 0.4999999999991457 0
+54 0.2886751345942123 0.5000000000011557 0
+55 0.5000000000006117 0.7525600817161773 0
+56 0.4999999999993867 0.2474399182839603 0
+57 0.2423197548524782 0.7576802451481532 0
+58 0.757680245147464 0.2423197548520695 0
+59 0.2423197548507857 0.2423197548513912 0
+60 0.7576802451491019 0.7576802451486099 0
+61 0.7113248654055673 0.4999999999991457 1
+62 0.2886751345942123 0.5000000000011557 1
+63 0.5000000000006117 0.7525600817161773 1
+64 0.4999999999993867 0.2474399182839603 1
+65 0.2423197548524782 0.7576802451481532 1
+66 0.757680245147464 0.2423197548520695 1
+67 0.2423197548507857 0.2423197548513912 1
+68 0.7576802451491019 0.7576802451486099 1
+69 0.4999999999993203 0 0.301447615129799
+70 0.4999999999992795 0 0.7028666213189801
+71 0.7525600817158393 0 0.5007190394076877
+72 0.2474399182836191 0 0.5007190394076877
+73 0.7576802451479793 0 0.7576802451479793
+74 0.2423197548517962 0 0.7576802451477375
+75 0.7576802451484569 0 0.2423197548510767
+76 0.2423197548513188 0 0.2423197548513187
+77 1 0.4999999999993203 0.301447615129799
+78 1 0.4999999999992795 0.7028666213189801
+79 1 0.7525600817158394 0.5007190394076877
+80 1 0.2474399182836191 0.5007190394076877
+81 1 0.7576802451479794 0.7576802451479794
+82 1 0.2423197548517962 0.7576802451477376
+83 1 0.7576802451484569 0.2423197548510768
+84 1 0.2423197548513188 0.2423197548513188
+85 0.5000000000008327 1 0.3014476151298047
+86 0.500000000000961 1 0.7028666213191928
+87 0.2474399182842484 1 0.5007190394077241
+88 0.7525600817164384 1 0.5007190394078933
+89 0.2423197548520873 1 0.7576802451480517
+90 0.7576802451481496 1 0.2423197548518761
+91 0.2423197548516099 1 0.2423197548510044
+92 0.7576802451486874 1 0.7576802451481952
+93 0 0.5000000000008327 0.3014476151298047
+94 0 0.500000000000961 0.7028666213191928
+95 0 0.2474399182842484 0.5007190394077241
+96 0 0.7525600817164384 0.5007190394078933
+97 0 0.2423197548520873 0.7576802451480517
+98 0 0.7576802451481496 0.2423197548518761
+99 0 0.2423197548516099 0.2423197548510044
+100 0 0.7576802451486874 0.7576802451481952
+101 0.3968750000003409 0.603125000000244 0.25
+102 0.4374999999998713 0.4375000000001287 0.25
+103 0.5739583333335919 0.5718750000001767 0.25
+104 0.6093749999999631 0.3906250000003402 0.25
+105 0.3968750000003409 0.603125000000244 0.75
+106 0.4374999999998713 0.4375000000001287 0.75
+107 0.5739583333335919 0.5718750000001767 0.75
+108 0.6093749999999631 0.3906250000003402 0.75
+109 0.3806942419826734 0.25 0.3806942419826734
+110 0.5625000000001735 0.25 0.4375000000000001
+111 0.4254282069971791 0.25 0.5712615403304835
+112 0.6093749999998808 0.25 0.6093749999998808
+113 0.75 0.3806942419826734 0.3806942419826734
+114 0.75 0.5625000000001735 0.4375000000000001
+115 0.75 0.4254282069971791 0.5712615403304835
+116 0.75 0.6093749999998808 0.6093749999998808
+117 0.3968750000004991 0.75 0.3968750000002804
+118 0.4375000000000001 0.75 0.5625000000004308
+119 0.5739583333336153 0.75 0.4281250000000707
+120 0.6093749999998166 0.75 0.6093749999993661
+121 0.25 0.3968750000004991 0.3968750000002804
+122 0.25 0.4375000000000001 0.5625000000004308
+123 0.25 0.5739583333336153 0.4281250000000707
+124 0.25 0.6093749999998166 0.6093749999993661
+125 0.4962939304035875 0.5214350017087855 0.4925553109323813
+126 0.3432581985549767 0.6471275530923523 0.3554206606168006
+127 0.6442168181713744 0.5929232373214773 0.3593785632300704
+128 0.625174517421737 0.3491579444372839 0.3604636129462503
+129 0.6130544111091688 0.6576364353993367 0.5046720648819553
+130 0.4281518698369243 0.3632662294430484 0.3548726263205102
+131 0.3639383531355198 0.3520949250221278 0.4978389662613994
+132 0.629585530087249 0.3489878162230438 0.5124654846339122
+133 0.3710853378652663 0.6517586292698121 0.6382643241302075
+134 0.5917018263727056 0.6522525456211955 0.6390764961119443
+135 0.3530810314228338 0.4582477062424107 0.6430239639637545
+136 0.6571010904289998 0.5300774811423468 0.6603970500567977
+137 0.6484596018596915 0.361399676127967 0.6360267588157177
+138 0.4782020887035478 0.3534611476388013 0.6141275027013793
+$EndNodes
+$Elements
+760
+1 2 2 1 1 1 17 59
+2 2 2 1 1 24 1 59
+3 2 2 1 1 18 2 58
+4 2 2 1 1 2 19 58
+5 2 2 1 1 20 3 60
+6 2 2 1 1 3 21 60
+7 2 2 1 1 22 4 57
+8 2 2 1 1 4 23 57
+9 2 2 1 1 17 18 56
+10 2 2 1 1 17 56 59
+11 2 2 1 1 56 18 58
+12 2 2 1 1 19 20 53
+13 2 2 1 1 19 53 58
+14 2 2 1 1 53 20 60
+15 2 2 1 1 21 22 55
+16 2 2 1 1 21 55 60
+17 2 2 1 1 55 22 57
+18 2 2 1 1 23 24 54
+19 2 2 1 1 23 54 57
+20 2 2 1 1 54 24 59
+21 2 2 1 1 54 53 55
+22 2 2 1 1 53 54 56
+23 2 2 1 1 55 53 60
+24 2 2 1 1 53 56 58
+25 2 2 1 1 54 55 57
+26 2 2 1 1 56 54 59
+27 2 2 2 2 5 25 67
+28 2 2 2 2 32 5 67
+29 2 2 2 2 26 6 66
+30 2 2 2 2 6 27 66
+31 2 2 2 2 28 7 68
+32 2 2 2 2 7 29 68
+33 2 2 2 2 30 8 65
+34 2 2 2 2 8 31 65
+35 2 2 2 2 25 26 64
+36 2 2 2 2 25 64 67
+37 2 2 2 2 64 26 66
+38 2 2 2 2 27 28 61
+39 2 2 2 2 27 61 66
+40 2 2 2 2 61 28 68
+41 2 2 2 2 29 30 63
+42 2 2 2 2 29 63 68
+43 2 2 2 2 63 30 65
+44 2 2 2 2 31 32 62
+45 2 2 2 2 31 62 65
+46 2 2 2 2 62 32 67
+47 2 2 2 2 62 61 63
+48 2 2 2 2 61 62 64
+49 2 2 2 2 63 61 68
+50 2 2 2 2 61 64 66
+51 2 2 2 2 62 63 65
+52 2 2 2 2 64 62 67
+53 2 2 3 3 17 1 76
+54 2 2 3 3 1 33 76
+55 2 2 3 3 2 18 75
+56 2 2 3 3 35 2 75
+57 2 2 3 3 5 25 74
+58 2 2 3 3 34 5 74
+59 2 2 3 3 26 6 73
+60 2 2 3 3 6 36 73
+61 2 2 3 3 18 17 69
+62 2 2 3 3 69 17 76
+63 2 2 3 3 18 69 75
+64 2 2 3 3 25 26 70
+65 2 2 3 3 25 70 74
+66 2 2 3 3 70 26 73
+67 2 2 3 3 33 34 72
+68 2 2 3 3 33 72 76
+69 2 2 3 3 72 34 74
+70 2 2 3 3 36 35 71
+71 2 2 3 3 71 35 75
+72 2 2 3 3 36 71 73
+73 2 2 3 3 69 70 71
+74 2 2 3 3 70 69 72
+75 2 2 3 3 69 71 75
+76 2 2 3 3 72 69 76
+77 2 2 3 3 71 70 73
+78 2 2 3 3 70 72 74
+79 2 2 4 4 19 2 84
+80 2 2 4 4 2 35 84
+81 2 2 4 4 3 20 83
+82 2 2 4 4 37 3 83
+83 2 2 4 4 6 27 82
+84 2 2 4 4 36 6 82
+85 2 2 4 4 28 7 81
+86 2 2 4 4 7 38 81
+87 2 2 4 4 20 19 77
+88 2 2 4 4 77 19 84
+89 2 2 4 4 20 77 83
+90 2 2 4 4 27 28 78
+91 2 2 4 4 27 78 82
+92 2 2 4 4 78 28 81
+93 2 2 4 4 35 36 80
+94 2 2 4 4 35 80 84
+95 2 2 4 4 80 36 82
+96 2 2 4 4 38 37 79
+97 2 2 4 4 79 37 83
+98 2 2 4 4 38 79 81
+99 2 2 4 4 77 78 79
+100 2 2 4 4 78 77 80
+101 2 2 4 4 77 79 83
+102 2 2 4 4 80 77 84
+103 2 2 4 4 79 78 81
+104 2 2 4 4 78 80 82
+105 2 2 5 5 21 3 90
+106 2 2 5 5 3 37 90
+107 2 2 5 5 4 22 91
+108 2 2 5 5 39 4 91
+109 2 2 5 5 7 29 92
+110 2 2 5 5 38 7 92
+111 2 2 5 5 30 8 89
+112 2 2 5 5 8 40 89
+113 2 2 5 5 22 21 85
+114 2 2 5 5 85 21 90
+115 2 2 5 5 22 85 91
+116 2 2 5 5 29 30 86
+117 2 2 5 5 29 86 92
+118 2 2 5 5 86 30 89
+119 2 2 5 5 37 38 88
+120 2 2 5 5 37 88 90
+121 2 2 5 5 88 38 92
+122 2 2 5 5 40 39 87
+123 2 2 5 5 87 39 91
+124 2 2 5 5 40 87 89
+125 2 2 5 5 85 86 87
+126 2 2 5 5 86 85 88
+127 2 2 5 5 85 87 91
+128 2 2 5 5 88 85 90
+129 2 2 5 5 87 86 89
+130 2 2 5 5 86 88 92
+131 2 2 6 6 1 24 99
+132 2 2 6 6 33 1 99
+133 2 2 6 6 23 4 98
+134 2 2 6 6 4 39 98
+135 2 2 6 6 32 5 97
+136 2 2 6 6 5 34 97
+137 2 2 6 6 8 31 100
+138 2 2 6 6 40 8 100
+139 2 2 6 6 24 23 93
+140 2 2 6 6 93 23 98
+141 2 2 6 6 24 93 99
+142 2 2 6 6 31 32 94
+143 2 2 6 6 31 94 100
+144 2 2 6 6 94 32 97
+145 2 2 6 6 34 33 95
+146 2 2 6 6 95 33 99
+147 2 2 6 6 34 95 97
+148 2 2 6 6 39 40 96
+149 2 2 6 6 39 96 98
+150 2 2 6 6 96 40 100
+151 2 2 6 6 93 94 95
+152 2 2 6 6 94 93 96
+153 2 2 6 6 93 95 99
+154 2 2 6 6 96 93 98
+155 2 2 6 6 95 94 97
+156 2 2 6 6 94 96 100
+157 2 2 7 7 9 41 102
+158 2 2 7 7 44 9 102
+159 2 2 7 7 41 10 104
+160 2 2 7 7 10 42 104
+161 2 2 7 7 42 11 103
+162 2 2 7 7 11 43 103
+163 2 2 7 7 43 12 101
+164 2 2 7 7 12 44 101
+165 2 2 7 7 102 41 104
+166 2 2 7 7 42 103 104
+167 2 2 7 7 43 101 103
+168 2 2 7 7 101 44 102
+169 2 2 7 7 101 102 103
+170 2 2 7 7 103 102 104
+171 2 2 8 8 13 45 106
+172 2 2 8 8 48 13 106
+173 2 2 8 8 45 14 108
+174 2 2 8 8 14 46 108
+175 2 2 8 8 46 15 107
+176 2 2 8 8 15 47 107
+177 2 2 8 8 47 16 105
+178 2 2 8 8 16 48 105
+179 2 2 8 8 106 45 108
+180 2 2 8 8 46 107 108
+181 2 2 8 8 47 105 107
+182 2 2 8 8 105 48 106
+183 2 2 8 8 105 106 107
+184 2 2 8 8 107 106 108
+185 2 2 9 9 41 9 109
+186 2 2 9 9 9 49 109
+187 2 2 9 9 10 41 110
+188 2 2 9 9 50 10 110
+189 2 2 9 9 13 45 111
+190 2 2 9 9 49 13 111
+191 2 2 9 9 45 14 112
+192 2 2 9 9 14 50 112
+193 2 2 9 9 41 109 110
+194 2 2 9 9 111 45 112
+195 2 2 9 9 109 49 111
+196 2 2 9 9 50 110 112
+197 2 2 9 9 110 109 111
+198 2 2 9 9 110 111 112
+199 2 2 10 10 42 10 113
+200 2 2 10 10 10 50 113
+201 2 2 10 10 11 42 114
+202 2 2 10 10 51 11 114
+203 2 2 10 10 14 46 115
+204 2 2 10 10 50 14 115
+205 2 2 10 10 46 15 116
+206 2 2 10 10 15 51 116
+207 2 2 10 10 42 113 114
+208 2 2 10 10 115 46 116
+209 2 2 10 10 113 50 115
+210 2 2 10 10 51 114 116
+211 2 2 10 10 114 113 115
+212 2 2 10 10 114 115 116
+213 2 2 11 11 43 11 119
+214 2 2 11 11 11 51 119
+215 2 2 11 11 12 43 117
+216 2 2 11 11 52 12 117
+217 2 2 11 11 15 47 120
+218 2 2 11 11 51 15 120
+219 2 2 11 11 47 16 118
+220 2 2 11 11 16 52 118
+221 2 2 11 11 117 43 119
+222 2 2 11 11 47 118 120
+223 2 2 11 11 119 51 120
+224 2 2 11 11 52 117 118
+225 2 2 11 11 118 117 119
+226 2 2 11 11 118 119 120
+227 2 2 12 12 9 44 121
+228 2 2 12 12 49 9 121
+229 2 2 12 12 44 12 123
+230 2 2 12 12 12 52 123
+231 2 2 12 12 48 13 122
+232 2 2 12 12 13 49 122
+233 2 2 12 12 16 48 124
+234 2 2 12 12 52 16 124
+235 2 2 12 12 121 44 123
+236 2 2 12 12 48 122 124
+237 2 2 12 12 49 121 122
+238 2 2 12 12 123 52 124
+239 2 2 12 12 122 121 123
+240 2 2 12 12 122 123 124
+241 4 2 1 1 105 62 106 107
+242 4 2 1 1 102 54 101 103
+243 4 2 1 1 118 86 120 119
+244 4 2 1 1 124 94 123 122
+245 4 2 1 1 52 39 12 96
+246 4 2 1 1 52 12 39 87
+247 4 2 1 1 88 38 15 51
+248 4 2 1 1 79 15 38 51
+249 4 2 1 1 80 14 50 36
+250 4 2 1 1 71 50 14 36
+251 4 2 1 1 120 88 15 51
+252 4 2 1 1 116 15 79 51
+253 4 2 1 1 50 14 112 71
+254 4 2 1 1 52 117 12 87
+255 4 2 1 1 111 69 109 110
+256 4 2 1 1 114 77 115 113
+257 4 2 1 1 124 123 94 96
+258 4 2 1 1 120 86 88 119
+259 4 2 1 1 14 45 64 108
+260 4 2 1 1 103 54 101 55
+261 4 2 1 1 62 105 63 107
+262 4 2 1 1 63 29 15 47
+263 4 2 1 1 14 45 26 64
+264 4 2 1 1 51 11 88 37
+265 4 2 1 1 80 50 10 35
+266 4 2 1 1 49 33 9 72
+267 4 2 1 1 71 10 50 35
+268 4 2 1 1 79 11 51 37
+269 4 2 1 1 49 9 33 95
+270 4 2 1 1 12 123 52 96
+271 4 2 1 1 96 16 52 40
+272 4 2 1 1 49 13 34 72
+273 4 2 1 1 87 52 16 40
+274 4 2 1 1 13 49 34 95
+275 4 2 1 1 43 101 12 55
+276 4 2 1 1 43 12 22 55
+277 4 2 1 1 106 61 108 107
+278 4 2 1 1 102 103 104 53
+279 4 2 1 1 50 115 14 80
+280 4 2 1 1 80 10 50 113
+281 4 2 1 1 9 109 49 72
+282 4 2 1 1 47 15 63 107
+283 4 2 1 1 124 16 52 96
+284 4 2 1 1 49 121 9 95
+285 4 2 1 1 15 28 68 46
+286 4 2 1 1 81 28 15 46
+287 4 2 1 1 92 15 29 47
+288 4 2 1 1 27 82 14 46
+289 4 2 1 1 14 66 27 46
+290 4 2 1 1 26 45 14 73
+291 4 2 1 1 78 115 116 114
+292 4 2 1 1 112 70 111 110
+293 4 2 1 1 79 11 114 51
+294 4 2 1 1 71 50 10 110
+295 4 2 1 1 10 41 104 56
+296 4 2 1 1 77 80 115 113
+297 4 2 1 1 69 109 72 111
+298 4 2 1 1 63 16 30 47
+299 4 2 1 1 25 45 13 64
+300 4 2 1 1 52 16 118 87
+301 4 2 1 1 95 13 49 122
+302 4 2 1 1 57 12 23 44
+303 4 2 1 1 119 11 88 51
+304 4 2 1 1 41 17 9 56
+305 4 2 1 1 18 41 10 56
+306 4 2 1 1 21 11 43 55
+307 4 2 1 1 63 105 16 47
+308 4 2 1 1 13 45 106 64
+309 4 2 1 1 9 102 41 56
+310 4 2 1 1 72 49 13 111
+311 4 2 1 1 11 103 43 55
+312 4 2 1 1 70 112 71 110
+313 4 2 1 1 116 79 78 114
+314 4 2 1 1 32 13 48 67
+315 4 2 1 1 16 89 30 47
+316 4 2 1 1 32 48 13 97
+317 4 2 1 1 48 16 100 31
+318 4 2 1 1 121 123 93 122
+319 4 2 1 1 119 117 118 85
+320 4 2 1 1 10 19 42 84
+321 4 2 1 1 17 9 76 41
+322 4 2 1 1 24 9 59 44
+323 4 2 1 1 42 19 10 58
+324 4 2 1 1 10 41 18 75
+325 4 2 1 1 20 83 11 42
+326 4 2 1 1 90 11 43 21
+327 4 2 1 1 53 104 102 56
+328 4 2 1 1 106 61 64 108
+329 4 2 1 1 87 118 117 85
+330 4 2 1 1 95 121 93 122
+331 4 2 1 1 80 50 115 113
+332 4 2 1 1 109 49 72 111
+333 4 2 1 1 78 27 28 46
+334 4 2 1 1 31 94 48 32
+335 4 2 1 1 19 77 20 42
+336 4 2 1 1 23 24 44 93
+337 4 2 1 1 123 124 52 96
+338 4 2 1 1 88 120 119 51
+339 4 2 1 1 91 43 85 117
+340 4 2 1 1 39 98 12 96
+341 4 2 1 1 12 91 39 87
+342 4 2 1 1 15 92 38 88
+343 4 2 1 1 38 81 15 79
+344 4 2 1 1 36 80 14 82
+345 4 2 1 1 14 71 36 73
+346 4 2 1 1 103 102 54 53
+347 4 2 1 1 61 106 62 107
+348 4 2 1 1 86 29 30 47
+349 4 2 1 1 26 45 70 25
+350 4 2 1 1 21 85 22 43
+351 4 2 1 1 94 93 123 122
+352 4 2 1 1 119 118 86 85
+353 4 2 1 1 12 96 93 123
+354 4 2 1 1 115 78 77 114
+355 4 2 1 1 69 111 70 110
+356 4 2 1 1 28 27 61 46
+357 4 2 1 1 48 62 31 32
+358 4 2 1 1 88 15 92 120
+359 4 2 1 1 79 81 15 116
+360 4 2 1 1 71 14 112 73
+361 4 2 1 1 114 116 79 51
+362 4 2 1 1 112 50 71 110
+363 4 2 1 1 91 12 117 87
+364 4 2 1 1 80 115 14 82
+365 4 2 1 1 63 15 29 68
+366 4 2 1 1 64 26 14 66
+367 4 2 1 1 47 63 105 107
+368 4 2 1 1 43 103 101 55
+369 4 2 1 1 35 10 80 84
+370 4 2 1 1 33 76 9 72
+371 4 2 1 1 37 11 88 90
+372 4 2 1 1 35 71 10 75
+373 4 2 1 1 37 79 11 83
+374 4 2 1 1 9 99 33 95
+375 4 2 1 1 68 63 15 107
+376 4 2 1 1 99 44 93 121
+377 4 2 1 1 18 69 41 17
+378 4 2 1 1 10 35 2 84
+379 4 2 1 1 15 7 28 81
+380 4 2 1 1 3 37 90 11
+381 4 2 1 1 108 64 14 66
+382 4 2 1 1 51 38 79 37
+383 4 2 1 1 12 4 23 98
+384 4 2 1 1 12 39 91 4
+385 4 2 1 1 49 33 72 34
+386 4 2 1 1 10 71 110 75
+387 4 2 1 1 114 11 79 83
+388 4 2 1 1 40 96 16 100
+389 4 2 1 1 34 13 74 72
+390 4 2 1 1 16 87 40 89
+391 4 2 1 1 34 97 13 95
+392 4 2 1 1 20 19 42 53
+393 4 2 1 1 54 23 24 44
+394 4 2 1 1 87 16 118 89
+395 4 2 1 1 95 97 13 122
+396 4 2 1 1 57 22 12 55
+397 4 2 1 1 41 102 104 56
+398 4 2 1 1 64 45 106 108
+399 4 2 1 1 101 57 12 55
+400 4 2 1 1 74 13 5 25
+401 4 2 1 1 13 34 74 5
+402 4 2 1 1 8 40 16 100
+403 4 2 1 1 16 31 65 8
+404 4 2 1 1 82 36 6 14
+405 4 2 1 1 102 9 59 56
+406 4 2 1 1 106 67 13 64
+407 4 2 1 1 119 88 11 90
+408 4 2 1 1 95 49 121 122
+409 4 2 1 1 52 118 117 87
+410 4 2 1 1 124 16 96 100
+411 4 2 1 1 80 10 113 84
+412 4 2 1 1 109 9 76 72
+413 4 2 1 1 72 13 70 111
+414 4 2 1 1 99 9 121 95
+415 4 2 1 1 30 65 16 63
+416 4 2 1 1 13 67 25 64
+417 4 2 1 1 11 55 53 103
+418 4 2 1 1 123 12 44 93
+419 4 2 1 1 56 104 10 58
+420 4 2 1 1 10 18 56 58
+421 4 2 1 1 21 60 11 55
+422 4 2 1 1 17 59 9 56
+423 4 2 1 1 16 63 62 105
+424 4 2 1 1 48 105 16 62
+425 4 2 1 1 111 45 13 70
+426 4 2 1 1 119 86 88 85
+427 4 2 1 1 123 93 94 96
+428 4 2 1 1 115 78 80 77
+429 4 2 1 1 111 72 69 70
+430 4 2 1 1 55 103 54 53
+431 4 2 1 1 63 61 62 107
+432 4 2 1 1 14 27 66 6
+433 4 2 1 1 99 9 1 24
+434 4 2 1 1 49 33 34 95
+435 4 2 1 1 96 52 39 40
+436 4 2 1 1 22 12 91 4
+437 4 2 1 1 20 11 83 3
+438 4 2 1 1 29 15 92 7
+439 4 2 1 1 18 10 75 2
+440 4 2 1 1 36 50 80 35
+441 4 2 1 1 18 17 41 56
+442 4 2 1 1 51 88 38 37
+443 4 2 1 1 62 31 16 48
+444 4 2 1 1 62 16 31 65
+445 4 2 1 1 26 45 25 64
+446 4 2 1 1 104 58 42 10
+447 4 2 1 1 29 63 30 47
+448 4 2 1 1 77 78 79 114
+449 4 2 1 1 71 69 70 110
+450 4 2 1 1 42 103 11 53
+451 4 2 1 1 93 94 95 122
+452 4 2 1 1 118 87 86 85
+453 4 2 1 1 61 62 106 64
+454 4 2 1 1 53 102 54 56
+455 4 2 1 1 78 82 27 46
+456 4 2 1 1 94 48 32 97
+457 4 2 1 1 86 30 89 47
+458 4 2 1 1 78 28 81 46
+459 4 2 1 1 26 70 45 73
+460 4 2 1 1 100 94 48 31
+461 4 2 1 1 86 92 29 47
+462 4 2 1 1 76 69 17 41
+463 4 2 1 1 42 19 77 84
+464 4 2 1 1 77 83 20 42
+465 4 2 1 1 18 41 69 75
+466 4 2 1 1 43 85 90 21
+467 4 2 1 1 91 43 117 12
+468 4 2 1 1 48 62 32 67
+469 4 2 1 1 27 66 61 46
+470 4 2 1 1 68 28 61 46
+471 4 2 1 1 54 24 59 44
+472 4 2 1 1 19 42 53 58
+473 4 2 1 1 23 54 57 44
+474 4 2 1 1 12 93 96 98
+475 4 2 1 1 99 44 121 9
+476 4 2 1 1 42 58 104 53
+477 4 2 1 1 21 43 22 55
+478 4 2 1 1 53 20 11 42
+479 4 2 1 1 53 11 20 60
+480 4 2 1 1 106 67 48 13
+481 4 2 1 1 36 71 50 35
+482 4 2 1 1 52 87 39 40
+483 4 2 1 1 48 67 106 62
+484 4 2 1 1 72 70 13 74
+485 4 2 1 1 9 33 99 1
+486 4 2 1 1 3 37 11 83
+487 4 2 1 1 13 34 5 97
+488 4 2 1 1 12 39 4 98
+489 4 2 1 1 14 36 6 73
+490 4 2 1 1 15 38 92 7
+491 4 2 1 1 8 40 89 16
+492 4 2 1 1 75 35 2 10
+493 4 2 1 1 30 16 65 8
+494 4 2 1 1 14 27 6 82
+495 4 2 1 1 26 14 66 6
+496 4 2 1 1 16 31 8 100
+497 4 2 1 1 32 13 67 5
+498 4 2 1 1 13 67 5 25
+499 4 2 1 1 68 7 28 15
+500 4 2 1 1 29 15 7 68
+501 4 2 1 1 57 4 23 12
+502 4 2 1 1 22 12 4 57
+503 4 2 1 1 10 19 84 2
+504 4 2 1 1 9 59 1 24
+505 4 2 1 1 17 9 59 1
+506 4 2 1 1 18 10 2 58
+507 4 2 1 1 11 21 90 3
+508 4 2 1 1 20 11 3 60
+509 4 2 1 1 16 62 63 65
+510 4 2 1 1 11 53 55 60
+511 4 2 1 1 78 81 79 116
+512 4 2 1 1 71 112 70 73
+513 4 2 1 1 86 88 92 120
+514 4 2 1 1 96 94 124 100
+515 4 2 1 1 94 97 95 122
+516 4 2 1 1 89 118 87 86
+517 4 2 1 1 15 38 7 81
+518 4 2 1 1 102 44 59 9
+519 4 2 1 1 59 44 102 54
+520 4 2 1 1 110 71 69 75
+521 4 2 1 1 79 77 114 83
+522 4 2 1 1 76 69 109 72
+523 4 2 1 1 113 77 80 84
+524 4 2 1 1 93 99 121 95
+525 4 2 1 1 87 117 91 85
+526 4 2 1 1 90 119 43 11
+527 4 2 1 1 80 78 115 82
+528 4 2 1 1 88 119 85 90
+529 4 2 1 1 90 43 119 85
+530 4 2 1 1 46 108 66 61
+531 4 2 1 1 46 66 108 14
+532 4 2 1 1 26 14 6 73
+533 4 2 1 1 44 101 57 12
+534 4 2 1 1 44 57 101 54
+535 4 2 1 1 46 107 68 15
+536 4 2 1 1 46 68 107 61
+537 4 2 1 1 9 33 1 76
+538 4 2 1 1 100 124 48 94
+539 4 2 1 1 100 48 124 16
+540 4 2 1 1 25 70 13 45
+541 4 2 1 1 56 53 104 58
+542 4 2 1 1 61 64 108 66
+543 4 2 1 1 106 62 67 64
+544 4 2 1 1 102 59 54 56
+545 4 2 1 1 32 13 5 97
+546 4 2 1 1 30 16 8 89
+547 4 2 1 1 13 70 25 74
+548 4 2 1 1 97 122 48 13
+549 4 2 1 1 75 110 41 69
+550 4 2 1 1 54 57 101 55
+551 4 2 1 1 97 48 122 94
+552 4 2 1 1 47 118 89 86
+553 4 2 1 1 47 89 118 16
+554 4 2 1 1 17 9 1 76
+555 4 2 1 1 11 21 3 60
+556 4 2 1 1 10 19 2 58
+557 4 2 1 1 75 41 110 10
+558 4 2 1 1 61 63 68 107
+559 4 2 1 1 70 111 45 112
+560 4 2 1 1 78 115 46 116
+561 4 2 1 1 77 114 42 113
+562 4 2 1 1 69 41 109 110
+563 4 2 1 1 107 61 108 46
+564 4 2 1 1 103 42 104 53
+565 4 2 1 1 118 86 47 120
+566 4 2 1 1 94 124 48 122
+567 4 2 1 1 117 119 43 85
+568 4 2 1 1 123 44 121 93
+569 4 2 1 1 101 54 102 44
+570 4 2 1 1 48 105 62 106
+571 4 2 1 1 91 43 12 22
+572 4 2 1 1 91 43 22 85
+573 4 2 1 1 93 12 23 98
+574 4 2 1 1 93 23 12 44
+575 4 2 1 1 46 81 116 15
+576 4 2 1 1 46 116 81 78
+577 4 2 1 1 92 47 120 15
+578 4 2 1 1 120 47 92 86
+579 4 2 1 1 46 82 115 78
+580 4 2 1 1 46 115 82 14
+581 4 2 1 1 73 45 112 70
+582 4 2 1 1 73 112 45 14
+583 4 2 1 1 99 44 9 24
+584 4 2 1 1 99 44 24 93
+585 4 2 1 1 84 42 113 77
+586 4 2 1 1 84 113 42 10
+587 4 2 1 1 76 41 109 69
+588 4 2 1 1 109 41 76 9
+589 4 2 1 1 42 83 114 77
+590 4 2 1 1 42 114 83 11
+591 4 2 2 2 135 13 122 131
+592 4 2 2 2 132 138 125 110
+593 4 2 2 2 138 108 125 106
+594 4 2 2 2 136 108 125 137
+595 4 2 2 2 110 138 125 131
+596 4 2 2 2 136 107 134 125
+597 4 2 2 2 122 135 131 125
+598 4 2 2 2 13 49 122 131
+599 4 2 2 2 136 108 137 46
+600 4 2 2 2 110 138 131 111
+601 4 2 2 2 132 138 110 112
+602 4 2 2 2 106 107 108 125
+603 4 2 2 2 138 112 137 45
+604 4 2 2 2 104 127 125 128
+605 4 2 2 2 113 115 114 125
+606 4 2 2 2 131 122 125 121
+607 4 2 2 2 112 45 14 137
+608 4 2 2 2 110 131 125 130
+609 4 2 2 2 115 116 114 125
+610 4 2 2 2 132 138 112 137
+611 4 2 2 2 50 137 115 132
+612 4 2 2 2 104 127 128 42
+613 4 2 2 2 105 134 125 133
+614 4 2 2 2 129 120 51 119
+615 4 2 2 2 104 130 128 125
+616 4 2 2 2 105 134 133 47
+617 4 2 2 2 135 106 138 125
+618 4 2 2 2 106 105 107 125
+619 4 2 2 2 138 108 106 45
+620 4 2 2 2 104 130 125 102
+621 4 2 2 2 104 127 42 103
+622 4 2 2 2 109 131 49 111
+623 4 2 2 2 106 135 48 105
+624 4 2 2 2 126 101 43 125
+625 4 2 2 2 138 137 125 108
+626 4 2 2 2 104 127 103 125
+627 4 2 2 2 116 114 129 51
+628 4 2 2 2 136 108 46 107
+629 4 2 2 2 136 108 107 125
+630 4 2 2 2 52 124 123 125
+631 4 2 2 2 132 112 110 50
+632 4 2 2 2 44 121 125 123
+633 4 2 2 2 131 122 121 49
+634 4 2 2 2 105 134 47 107
+635 4 2 2 2 106 13 135 138
+636 4 2 2 2 52 133 125 118
+637 4 2 2 2 103 43 101 125
+638 4 2 2 2 117 43 119 125
+639 4 2 2 2 105 134 107 125
+640 4 2 2 2 137 115 132 125
+641 4 2 2 2 50 137 132 112
+642 4 2 2 2 106 135 105 125
+643 4 2 2 2 102 101 44 125
+644 4 2 2 2 104 130 102 41
+645 4 2 2 2 138 45 13 111
+646 4 2 2 2 131 138 135 13
+647 4 2 2 2 138 112 45 111
+648 4 2 2 2 131 138 13 111
+649 4 2 2 2 107 134 15 136
+650 4 2 2 2 118 52 117 125
+651 4 2 2 2 101 126 12 44
+652 4 2 2 2 104 130 41 128
+653 4 2 2 2 138 137 108 45
+654 4 2 2 2 133 118 16 47
+655 4 2 2 2 44 101 126 125
+656 4 2 2 2 133 134 118 47
+657 4 2 2 2 133 134 125 118
+658 4 2 2 2 124 122 123 125
+659 4 2 2 2 129 118 119 125
+660 4 2 2 2 102 104 103 125
+661 4 2 2 2 135 48 13 106
+662 4 2 2 2 123 122 121 125
+663 4 2 2 2 103 127 43 125
+664 4 2 2 2 106 45 13 138
+665 4 2 2 2 103 11 127 42
+666 4 2 2 2 127 51 129 114
+667 4 2 2 2 13 49 131 111
+668 4 2 2 2 137 108 14 46
+669 4 2 2 2 15 47 134 107
+670 4 2 2 2 10 41 128 104
+671 4 2 2 2 16 118 133 52
+672 4 2 2 2 46 107 15 136
+673 4 2 2 2 41 102 9 130
+674 4 2 2 2 128 50 132 110
+675 4 2 2 2 10 41 110 128
+676 4 2 2 2 50 137 112 14
+677 4 2 2 2 130 102 9 44
+678 4 2 2 2 105 133 16 47
+679 4 2 2 2 127 11 103 43
+680 4 2 2 2 128 130 41 110
+681 4 2 2 2 116 134 15 51
+682 4 2 2 2 137 45 14 108
+683 4 2 2 2 12 126 101 43
+684 4 2 2 2 133 48 135 105
+685 4 2 2 2 128 42 10 104
+686 4 2 2 2 131 9 109 49
+687 4 2 2 2 118 117 119 125
+688 4 2 2 2 102 103 101 125
+689 4 2 2 2 129 118 125 134
+690 4 2 2 2 125 129 116 114
+691 4 2 2 2 117 126 43 125
+692 4 2 2 2 126 117 12 52
+693 4 2 2 2 52 126 117 125
+694 4 2 2 2 12 117 126 43
+695 4 2 2 2 127 51 114 11
+696 4 2 2 2 114 129 127 125
+697 4 2 2 2 127 119 43 125
+698 4 2 2 2 119 11 127 43
+699 4 2 2 2 127 113 114 125
+700 4 2 2 2 127 113 42 114
+701 4 2 2 2 128 130 110 125
+702 4 2 2 2 15 47 120 134
+703 4 2 2 2 127 11 114 42
+704 4 2 2 2 120 47 118 134
+705 4 2 2 2 44 130 102 125
+706 4 2 2 2 44 126 12 123
+707 4 2 2 2 123 44 126 125
+708 4 2 2 2 129 118 134 120
+709 4 2 2 2 110 132 128 125
+710 4 2 2 2 128 127 125 113
+711 4 2 2 2 128 127 113 42
+712 4 2 2 2 128 50 110 10
+713 4 2 2 2 113 42 10 128
+714 4 2 2 2 129 134 125 116
+715 4 2 2 2 127 51 11 119
+716 4 2 2 2 129 134 116 51
+717 4 2 2 2 127 51 119 129
+718 4 2 2 2 129 119 127 125
+719 4 2 2 2 110 131 130 109
+720 4 2 2 2 109 130 9 131
+721 4 2 2 2 121 130 9 44
+722 4 2 2 2 122 48 13 135
+723 4 2 2 2 44 121 130 125
+724 4 2 2 2 110 138 111 112
+725 4 2 2 2 131 138 125 135
+726 4 2 2 2 115 137 14 46
+727 4 2 2 2 126 52 12 123
+728 4 2 2 2 123 126 52 125
+729 4 2 2 2 50 137 14 115
+730 4 2 2 2 136 137 115 46
+731 4 2 2 2 121 9 131 49
+732 4 2 2 2 121 131 130 125
+733 4 2 2 2 131 130 9 121
+734 4 2 2 2 113 132 115 125
+735 4 2 2 2 113 50 115 132
+736 4 2 2 2 128 50 10 113
+737 4 2 2 2 132 113 128 125
+738 4 2 2 2 128 50 113 132
+739 4 2 2 2 133 105 135 125
+740 4 2 2 2 52 124 125 133
+741 4 2 2 2 133 48 105 16
+742 4 2 2 2 16 133 124 52
+743 4 2 2 2 136 137 125 115
+744 4 2 2 2 132 138 137 125
+745 4 2 2 2 41 130 9 109
+746 4 2 2 2 134 120 15 51
+747 4 2 2 2 129 118 120 119
+748 4 2 2 2 129 120 134 51
+749 4 2 2 2 135 122 124 125
+750 4 2 2 2 135 48 124 122
+751 4 2 2 2 133 48 16 124
+752 4 2 2 2 133 135 124 125
+753 4 2 2 2 133 48 124 135
+754 4 2 2 2 116 136 134 125
+755 4 2 2 2 115 136 116 125
+756 4 2 2 2 46 115 136 116
+757 4 2 2 2 136 134 15 116
+758 4 2 2 2 46 136 15 116
+759 4 2 2 2 109 41 130 110
+760 4 2 2 2 110 131 109 111
+$EndElements

data/square_in_square.geo

@@ -0,0 +1,77 @@
+// Square-in-square 2D geometry for MFEM
+// Creates concentric squares with different material attributes
+
+// Define the square sizes
+L_outer = 2.0;
+L_inner = 0.5;
+
+// Set mesh size and algorithm
+mesh_size = 1.0;
+Mesh.Algorithm = 6; // Frontal-Delaunay for 2D triangular mesh
+Mesh.CharacteristicLengthFactor = 1.0;
+Mesh.MshFileVersion = 2.2;
+
+// Define center point for concentric squares
+cx = 0.0;
+cy = 0.0;
+
+// Define the points (vertices of the outer square)
+Point(1) = {cx-L_outer/2, cy-L_outer/2, 0, mesh_size}; // bottom-left outer
+Point(2) = {cx+L_outer/2, cy-L_outer/2, 0, mesh_size}; // bottom-right outer
+Point(3) = {cx+L_outer/2, cy+L_outer/2, 0, mesh_size}; // top-right outer
+Point(4) = {cx-L_outer/2, cy+L_outer/2, 0, mesh_size}; // top-left outer
+
+// Define the points (vertices of the inner square)
+Point(5) = {cx-L_inner/2, cy-L_inner/2, 0, mesh_size}; // bottom-left inner
+Point(6) = {cx+L_inner/2, cy-L_inner/2, 0, mesh_size}; // bottom-right inner
+Point(7) = {cx+L_inner/2, cy+L_inner/2, 0, mesh_size}; // top-right inner
+Point(8) = {cx-L_inner/2, cy+L_inner/2, 0, mesh_size}; // top-left inner
+
+// Define the lines (edges of the outer square)
+Line(1) = {1, 2}; // bottom edge
+Line(2) = {2, 3}; // right edge
+Line(3) = {3, 4}; // top edge
+Line(4) = {4, 1}; // left edge
+
+// Define the lines (edges of the inner square)
+Line(5) = {5, 6}; // bottom edge
+Line(6) = {6, 7}; // right edge
+Line(7) = {7, 8}; // top edge
+Line(8) = {8, 5}; // left edge
+
+// Define the surfaces
+// Outer square boundary
+Line Loop(1) = {1, 2, 3, 4};
+
+// Inner square boundary (hole in the outer region)
+Line Loop(2) = {5, 6, 7, 8};
+
+// Define the surface areas
+// Outer region (annular region between squares)
+Plane Surface(1) = {1, 2}; // Outer loop minus inner loop (creates hole)
+
+// Inner region (solid inner square)
+Plane Surface(2) = {2}; // Inner loop only
+
+// Assign physical groups for materials
+Physical Surface(1) = {1}; // Outer material (annular region)
+Physical Surface(2) = {2}; // Inner material (solid square)
+
+// Physical lines for boundary conditions
+// Outer square boundary edges
+Physical Line(1) = {1}; // outer bottom
+Physical Line(2) = {2}; // outer right
+Physical Line(3) = {3}; // outer top
+Physical Line(4) = {4}; // outer left
+
+// Inner square boundary edges
+Physical Line(5) = {5}; // inner bottom
+Physical Line(6) = {6}; // inner right
+Physical Line(7) = {7}; // inner top
+Physical Line(8) = {8}; // inner left
+
+// Mesh control for quality
+Mesh.OptimizeNetgen = 1;
+Mesh.Optimize = 1;
+Mesh.ElementOrder = 1;
+Mesh.RecombineAll = 0; // Keep triangular elements (don't recombine to quads)

data/square_in_square.msh

@@ -0,0 +1,50 @@
+$MeshFormat
+2.2 0 8
+$EndMeshFormat
+$Nodes
+13
+1 -1 -1 0
+2 1 -1 0
+3 1 1 0
+4 -1 1 0
+5 -0.25 -0.25 0
+6 0.25 -0.25 0
+7 0.25 0.25 0
+8 -0.25 0.25 0
+9 -2.752797989558076e-12 -1 0
+10 1 -2.752797989558076e-12 0
+11 2.752797989558076e-12 1 0
+12 -1 2.752797989558076e-12 0
+13 0 0 0
+$EndNodes
+$Elements
+28
+1 1 2 1 1 1 9
+2 1 2 1 1 9 2
+3 1 2 2 2 2 10
+4 1 2 2 2 10 3
+5 1 2 3 3 3 11
+6 1 2 3 3 11 4
+7 1 2 4 4 4 12
+8 1 2 4 4 12 1
+9 1 2 5 5 5 6
+10 1 2 6 6 6 7
+11 1 2 7 7 7 8
+12 1 2 8 8 8 5
+13 2 2 1 1 6 5 9
+14 2 2 1 1 5 8 12
+15 2 2 1 1 7 6 10
+16 2 2 1 1 8 7 11
+17 2 2 1 1 9 5 1
+18 2 2 1 1 5 12 1
+19 2 2 1 1 6 9 2
+20 2 2 1 1 10 6 2
+21 2 2 1 1 7 10 3
+22 2 2 1 1 11 7 3
+23 2 2 1 1 8 11 4
+24 2 2 1 1 8 4 12
+25 2 2 2 2 5 6 13
+26 2 2 2 2 8 5 13
+27 2 2 2 2 6 7 13
+28 2 2 2 2 7 8 13
+$EndElements

doc/CodeDocumentation.conf.in

@@ -1083,7 +1083,8 @@ EXCLUDE_PATTERNS       =
 # ANamespace::AClass, ANamespace::*Test
 
 EXCLUDE_SYMBOLS        = mfem::internal \
-                         mfem::kernels::internal
+                         mfem::kernels::internal \
+                         mfem::future::detail
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include

examples/CMakeLists.txt

@@ -47,7 +47,6 @@ list(APPEND ALL_EXE_SRCS
   ex39.cpp
   ex40.cpp
   ex41.cpp
-  # jitplayground.cpp
   )
 
 if (MFEM_USE_MPI)

examples/jitplayground.cpp

@@ -1,548 +0,0 @@
-#include <mfem.hpp>
-
-#ifdef MFEM_USE_PROTEUS
-
-#include "../fem/dfem/util.hpp"
-
-#include <proteus/CppJitModule.h>
-
-#include "jitplayground.hpp"
-
-#include <algorithm>
-#include <array>
-#include <cctype>
-#include <cmath>
-#include <fstream>
-#include <initializer_list>
-#include <iostream>
-#include <memory>
-#include <sstream>
-#include <string>
-#include <string_view>
-#include <type_traits>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
-#include <vector>
-
-namespace util
-{
-constexpr std::string_view Dirname(std::string_view path)
-{
-   const size_t last_sep = path.find_last_of("/\\");
-   if (last_sep == std::string_view::npos) { return {}; }
-   return path.substr(0, last_sep);
-}
-
-constexpr std::string_view thisFileDir = Dirname(__FILE__);
-}
-
-template <typename T>
-static std::string TypeNameString()
-{
-   return std::string(mfem::future::get_type_name<T>());
-}
-
-template <typename Tuple, size_t... Is>
-static auto ParamTypeStringsImpl(std::index_sequence<Is...>)
-{
-   return std::array<std::string, sizeof...(Is)>
-   {
-      TypeNameString<std::remove_reference_t<decltype(mfem::future::get<Is>(std::declval<Tuple&>()))>>()...
-   };
-}
-
-template <typename Tuple>
-static auto ParamTypeStrings()
-{
-   return ParamTypeStringsImpl<Tuple>(
-             std::make_index_sequence<mfem::future::tuple_size<Tuple>::value> {});
-}
-
-static std::string_view Trim(std::string_view s)
-{
-   size_t begin = 0;
-   while (begin < s.size() && std::isspace(static_cast<unsigned char>(s[begin])))
-   {
-      ++begin;
-   }
-   size_t end = s.size();
-   while (end > begin &&
-          std::isspace(static_cast<unsigned char>(s[end - 1])))
-   {
-      --end;
-   }
-   return s.substr(begin, end - begin);
-}
-
-static bool IsValidIdentifier(std::string_view s)
-{
-   if (s.empty()) { return false; }
-   const unsigned char c0 = static_cast<unsigned char>(s[0]);
-   if (!(std::isalpha(c0) || c0 == '_')) { return false; }
-   for (size_t i = 1; i < s.size(); ++i)
-   {
-      const unsigned char c = static_cast<unsigned char>(s[i]);
-      if (!(std::isalnum(c) || c == '_')) { return false; }
-   }
-   return true;
-}
-
-static bool ParseJitDirective(std::string_view line,
-                              std::string &type,
-                              std::string &var,
-                              std::string &kind)
-{
-   const size_t jit_pos = line.find("$JIT");
-   if (jit_pos == std::string_view::npos) { return false; }
-
-   const size_t open = line.find('[', jit_pos);
-   const size_t close = line.find(']', jit_pos);
-   MFEM_VERIFY(open != std::string_view::npos &&
-               close != std::string_view::npos &&
-               close > open,
-               "malformed $JIT directive (expected brackets): " << line);
-
-   const std::string_view payload = line.substr(open + 1, close - open - 1);
-   const size_t comma1 = payload.find(',');
-   const size_t comma2 = (comma1 == std::string_view::npos)
-                         ? std::string_view::npos
-                         : payload.find(',', comma1 + 1);
-   MFEM_VERIFY(comma1 != std::string_view::npos &&
-               comma2 != std::string_view::npos,
-               "malformed $JIT directive (expected 3 comma-separated fields): "
-               << line);
-
-   const std::string_view f0 = Trim(payload.substr(0, comma1));
-   const std::string_view f1 = Trim(payload.substr(comma1 + 1,
-                                                   comma2 - comma1 - 1));
-   const std::string_view f2 = Trim(payload.substr(comma2 + 1));
-   MFEM_VERIFY(!f0.empty() && !f1.empty() && !f2.empty(),
-               "malformed $JIT directive (empty field): " << line);
-
-   type.assign(f0);
-   var.assign(f1);
-   kind.assign(f2);
-   return true;
-}
-
-static std::string ReadFileOrEmpty(const std::string &fn)
-{
-   std::ifstream file(fn);
-   if (!file.is_open())
-   {
-      std::cerr << "could not open file " << fn << "\n";
-      return {};
-   }
-   std::stringstream buffer;
-   buffer << file.rdbuf();
-   return buffer.str();
-}
-
-static std::vector<std::string> ExtractJitVarNames(const std::string
-                                                   &kernel_code)
-{
-   std::stringstream ss(kernel_code);
-   std::string line;
-   std::vector<std::string> var_names;
-   std::unordered_set<std::string> seen_vars;
-
-   while (std::getline(ss, line))
-   {
-      std::string type, var, kind;
-      if (ParseJitDirective(line, type, var, kind))
-      {
-         MFEM_VERIFY(IsValidIdentifier(var),
-                     "$JIT variable must be a valid identifier: " << var);
-         MFEM_VERIFY(seen_vars.insert(var).second,
-                     "duplicate $JIT variable name: " << var);
-         var_names.push_back(var);
-      }
-   }
-   return var_names;
-}
-
-static std::string RewriteKernelForJit(std::string kernel_code,
-                                       const std::vector<std::string> &jit_values)
-{
-   std::stringstream ss(kernel_code);
-   std::string line;
-
-   std::string out;
-   out.reserve(kernel_code.size() + 128);
-
-   bool have_pending = false;
-   size_t pending_index = 0;
-   std::string pending_type;
-   std::string pending_var;
-   std::unordered_set<std::string> seen_vars;
-
-   while (std::getline(ss, line))
-   {
-      line.push_back('\n');
-
-      if (have_pending)
-      {
-         MFEM_VERIFY(pending_index < jit_values.size(),
-                     "not enough JIT values provided");
-         const size_t indent_end = line.find_first_not_of(" \t");
-         const std::string indent =
-            (indent_end == std::string::npos) ? std::string() :
-            line.substr(0, indent_end);
-         out += indent + "const " + pending_type + " " + pending_var + " = " +
-                jit_values[pending_index] + ";\n";
-         have_pending = false;
-         ++pending_index;
-         continue;
-      }
-
-      std::string type, var, kind;
-      if (ParseJitDirective(line, type, var, kind))
-      {
-         MFEM_VERIFY(IsValidIdentifier(var),
-                     "$JIT variable must be a valid identifier: " << var);
-         MFEM_VERIFY(kind == "generic",
-                     "unsupported $JIT kind: " << kind);
-         MFEM_VERIFY(seen_vars.insert(var).second,
-                     "duplicate $JIT variable name: " << var);
-
-         pending_type = std::move(type);
-         pending_var = std::move(var);
-         have_pending = true;
-         continue; // drop directive line
-      }
-
-      out += line;
-   }
-
-   MFEM_VERIFY(!have_pending,
-               "$JIT directive must annotate a following line");
-   MFEM_VERIFY(jit_values.size() == pending_index,
-               "JIT value count must match number of $JIT directives");
-   return out;
-}
-
-static std::string GeneratedOutputPath(std::string_view original_path)
-{
-   const size_t last_sep = original_path.find_last_of("/\\");
-   const size_t dot = original_path.find_last_of('.');
-   const bool dot_in_filename =
-      (dot != std::string_view::npos) &&
-      (last_sep == std::string_view::npos || dot > last_sep);
-
-   const std::string_view base =
-      dot_in_filename ? original_path.substr(0, dot) : original_path;
-   return std::string(base) + "_generated.hpp";
-}
-
-static void WriteFileOrWarn(const std::string &path,
-                            const std::string &contents)
-{
-   std::ofstream out(path);
-   if (!out.is_open())
-   {
-      std::cerr << "could not write generated file " << path << "\n";
-      return;
-   }
-   out << contents;
-}
-
-class JitQFunction
-{
-public:
-   template <typename ImplT, size_t N>
-   JitQFunction(ImplT, const std::string &fn,
-                const std::array<bool, N> &activity_map)
-   {
-      using qf_signature = typename
-                           mfem::future::get_function_signature<
-                           decltype(&ImplT::operator())>::type;
-      using qf_param_ts = typename qf_signature::parameter_ts;
-      constexpr size_t nparams = mfem::future::tuple_size<qf_param_ts>::value;
-      static_assert(N == nparams, "activity_map size must match qfunc arity");
-
-      this->fn = fn;
-      this->nparams = nparams;
-      this->activity_map.reserve(N);
-      for (size_t i = 0; i < N; ++i)
-      {
-         this->activity_map.push_back(activity_map[i]);
-      }
-      {
-         const auto param_types_arr = ParamTypeStrings<qf_param_ts>();
-         this->param_types.assign(param_types_arr.begin(), param_types_arr.end());
-      }
-      this->return_type = TypeNameString<typename qf_signature::return_t>();
-      this->return_is_void = std::is_same_v<typename qf_signature::return_t, void>;
-      this->impl_type_name = TypeNameString<ImplT>();
-      this->jit_var_names = ExtractJitVarNames(ReadFileOrEmpty(fn));
-   }
-
-   template <typename ReturnT, typename... Args>
-   ReturnT run(std::string_view name,
-               std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
-               Args&&... args)
-   {
-      auto ordered_values = MatchJitValues(jit_values);
-      auto &mod = GetOrCreateModule(ordered_values);
-      auto &instance = mod.instantiate(std::string(name), std::string());
-      return instance.template run<ReturnT>(std::forward<Args>(args)...);
-   }
-
-   template <typename ReturnT, typename... Args>
-   ReturnT run_primal(
-      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
-      Args&&... args)
-   {
-      return run<ReturnT>(qfunc_name, jit_values,
-                          std::forward<Args>(args)...);
-   }
-
-   template <typename ReturnT, typename... Args>
-   ReturnT run_derivative(
-      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
-      Args&&... args)
-   {
-      return run<ReturnT>(qfunc_name + "_fwddiff", jit_values,
-                          std::forward<Args>(args)...);
-   }
-
-private:
-   std::vector<std::string_view> MatchJitValues(
-      std::initializer_list<std::pair<std::string_view, std::string_view>>
-      named_values) const
-   {
-      std::unordered_map<std::string_view, std::string_view> value_map;
-      for (const auto &[name, value] : named_values)
-      {
-         value_map[name] = value;
-      }
-
-      std::vector<std::string_view> ordered_values;
-      ordered_values.reserve(jit_var_names.size());
-      for (const auto &var_name : jit_var_names)
-      {
-         auto it = value_map.find(var_name);
-         MFEM_VERIFY(it != value_map.end(),
-                     "missing JIT value for variable: " << var_name);
-         ordered_values.push_back(it->second);
-      }
-
-      MFEM_VERIFY(ordered_values.size() == named_values.size(),
-                  "provided " << named_values.size() << " JIT values but expected "
-                  << jit_var_names.size());
-      return ordered_values;
-   }
-
-
-   std::string BuildModuleCode(const std::vector<std::string> &jit_values) const
-   {
-      std::string module_code =
-         RewriteKernelForJit(ReadFileOrEmpty(fn), jit_values);
-      module_code += "\n\n";
-      module_code += "// --- generated ---\n";
-      module_code +=
-         "template <typename return_type, typename... Args>\n"
-         "return_type __enzyme_fwddiff(Args...);\n"
-         "\n"
-         "extern int enzyme_const;\n"
-         "extern int enzyme_dup;\n"
-         "\n";
-
-      // Generate a primal wrapper with the requested symbol name, so the kernel
-      // header can just define the qfunc as a functor.
-      //
-      // Note: Proteus instantiates entrypoints via `qfunc_wrapper<>(...)` even
-      // when there are no user template args, so keep the wrapper itself a
-      // template (with a default parameter) while still doing literal `$JIT`
-      // replacements in the kernel code.
-      module_code += "template <typename = void>\n";
-      module_code += return_type + " " +
-                     std::string(qfunc_name) + "(";
-      bool first = true;
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (!first) { module_code += ", "; }
-         first = false;
-         module_code += param_types[i] + " Arg" + std::to_string(i);
-      }
-      module_code += ")\n";
-      module_code += "{\n";
-      module_code += "   " + impl_type_name + " qf;\n";
-      if (return_is_void)
-      {
-         module_code += "   ";
-      }
-      else
-      {
-         module_code += "   return ";
-      }
-      module_code += "qf(";
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (i) { module_code += ", "; }
-         module_code += "Arg" + std::to_string(i);
-      }
-      module_code += ");\n";
-      module_code += "}\n\n";
-
-      module_code += "template <typename = void>\n";
-      module_code += return_type + " " +
-                     std::string(qfunc_name) + "_fwddiff(";
-
-      first = true;
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (!first) { module_code += ", "; }
-         first = false;
-         module_code += param_types[i] + " Arg" + std::to_string(i);
-         if (activity_map[i])
-         {
-            module_code += ", " + param_types[i] + " dArg" + std::to_string(i);
-         }
-      }
-      module_code += ")\n";
-      module_code += "{\n";
-      if (return_is_void)
-      {
-         module_code += "   __enzyme_fwddiff<void>(\n";
-      }
-      else
-      {
-         module_code += "   return __enzyme_fwddiff<" +
-                        return_type + ">(\n";
-      }
-      module_code += "      (void*)" + std::string(qfunc_name) + "<>";
-      module_code += ",\n";
-      for (size_t i = 0; i < nparams; ++i)
-      {
-         if (activity_map[i])
-         {
-            module_code += "      enzyme_dup, Arg" + std::to_string(i) +
-                           ", dArg" + std::to_string(i);
-         }
-         else
-         {
-            module_code += "      enzyme_const, Arg" + std::to_string(i);
-         }
-         module_code += (i + 1 == nparams) ? ");\n" : ",\n";
-      }
-      module_code += "}\n";
-
-      WriteFileOrWarn(GeneratedOutputPath(fn), module_code);
-      return module_code;
-   }
-
-   proteus::CppJitModule &GetOrCreateModule(
-      const std::vector<std::string_view> &jit_values)
-   {
-      std::string key;
-      for (const auto &val : jit_values)
-      {
-         if (!key.empty()) { key += ","; }
-         key += val;
-      }
-
-      auto it = modules.find(key);
-      if (it != modules.end())
-      {
-         return *it->second;
-      }
-
-      std::vector<std::string> values(jit_values.begin(), jit_values.end());
-      std::string code = BuildModuleCode(values);
-      auto mod = std::make_unique<proteus::CppJitModule>("host", code,
-                                                         DefaultExtraArgs());
-      auto [inserted, ok] = modules.emplace(key, std::move(mod));
-      MFEM_VERIFY(ok, "failed to cache JIT module");
-      return *inserted->second;
-   }
-
-   static std::vector<std::string> DefaultExtraArgs()
-   {
-      return {"-fplugin=/Users/andrej1/local/enzyme/lib/ClangEnzyme-20.dylib"};
-   }
-
-   std::string qfunc_name = "qfunc_wrapper";
-   std::string fn;
-   size_t nparams = 0;
-   std::vector<bool> activity_map;
-   std::vector<std::string> param_types;
-   std::string return_type;
-   bool return_is_void = false;
-   std::string impl_type_name;
-   std::vector<std::string> jit_var_names;
-   std::unordered_map<std::string, std::unique_ptr<proteus::CppJitModule>> modules;
-};
-
-int main()
-{
-   const size_t N = 4;
-   const size_t M = 5;
-   const double A = 123.4;
-
-   std::vector<double> X(N);
-   std::vector<double> Y(N);
-   for (size_t i = 0; i < N; ++i)
-   {
-      X[i] = static_cast<double>(i + 1);
-      Y[i] = static_cast<double>(N - i);
-   }
-
-   // // >>> user interface calls
-   // const std::string kernel_path = std::string(util::thisFileDir) +
-   //                                 "/jitplayground.hpp";
-   // JitQFunction qf(daxpy_op{}, kernel_path, std::array{false, true, false});
-   // // <<< user interface calls
-
-   // // this will happen internally in dFEM
-
-   daxpy_op op;
-   printf("\n\nfunction call\n");
-   op(&A, X.data(), Y.data(), &N);
-
-   // reset X for the derivative test
-   for (size_t i = 0; i < N; ++i)
-   {
-      X[i] = static_cast<double>(i + 1);
-      Y[i] = static_cast<double>(N - i);
-   }
-
-   std::vector<double> dX(N, 1.0);
-   printf("\n\nforward diff call\n");
-   daxpy_op_fwddiff(&A, X.data(), dX.data(), Y.data(), &N);
-
-   std::vector<double> dX_manual(N, A);
-
-   printf("\n\nderivative checks\n");
-   std::cout << "dX: ";
-   for (size_t i = 0; i < N; ++i)
-   {
-      std::cout << dX[i] << (i + 1 == N ? '\n' : ' ');
-   }
-
-   std::cout << "dX_manual: ";
-   for (size_t i = 0; i < N; ++i)
-   {
-      std::cout << dX_manual[i] << (i + 1 == N ? '\n' : ' ');
-   }
-
-   double max_abs_err = 0.0;
-   for (size_t i = 0; i < N; ++i)
-   {
-      max_abs_err = std::max(max_abs_err, std::abs(dX[i] - dX_manual[i]));
-   }
-   std::cout << "max |dX - dX_manual| = " << max_abs_err << "\n";
-
-   return 0;
-}
-
-#else
-
-int main()
-{
-   std::cerr << "Proteus is required to run this example.\n";
-   return EXIT_FAILURE;
-}
-
-#endif // MFEM_USE_PROTEUS

examples/jitplayground.hpp

@@ -1,58 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <vector>
-#include <type_traits>
-
-#include "proteus/JitInterface.h"
-
-struct daxpy_op
-{
-   void operator()(
-      const double *a,
-      double *x,
-      const double *y,
-      const size_t *N) const
-   {
-      const size_t n = *N;
-      auto lam = [=, n = proteus::jit_variable(n)]
-                 () __attribute__((annotate("jit")))
-      {
-         printf("N = %zu\n", n);
-         for (size_t i = 0; i < n; ++i)
-         {
-            printf("x[%zu] = %f, y[%zu] = %f\n", i, x[i], i, y[i]);
-            x[i] = *a * x[i] + y[i];
-            printf("updated x[%zu] = %f\n", i, x[i]);
-         }
-      };
-
-      proteus::register_lambda(lam);
-
-      lam();
-   }
-};
-
-template <typename return_type, typename... Args>
-return_type __enzyme_fwddiff(Args...);
-
-extern int enzyme_const;
-extern int enzyme_dup;
-
-void daxpy_op_wrapper(const double * Arg0, double * Arg1,
-                      const double * Arg2, const size_t *Arg3)
-{
-   daxpy_op qf;
-   qf(Arg0, Arg1, Arg2, Arg3);
-}
-
-void daxpy_op_fwddiff(const double * Arg0, double * Arg1,
-                      double * dArg1, const double * Arg2, const size_t *Arg3)
-{
-   __enzyme_fwddiff<void>(
-      (void*)daxpy_op_wrapper,
-      enzyme_const, Arg0,
-      enzyme_dup, Arg1, dArg1,
-      enzyme_const, Arg2,
-      enzyme_const, Arg3);
-}

fem/CMakeLists.txt

@@ -67,7 +67,6 @@ set(SRCS
   dgmassinv.cpp
   doftrans.cpp
   dfem/doperator.cpp
-  dfem/backends/local_qf/kernels.cpp
   eltrans.cpp
   batchitrans.cpp
   estimators.cpp
@@ -124,11 +123,6 @@ set(SRCS
   qinterp/eval_hdiv.cpp
   qinterp/grad_by_nodes.cpp
   qinterp/grad_by_vdim.cpp
-  qinterp/grad_transpose.cpp
-  qinterp/grad_transpose_by_nodes.cpp
-  qinterp/grad_transpose_by_vdim.cpp
-  qinterp/eval_transpose.cpp
-  qinterp/eval_transpose_by_vdim.cpp
   qspace.cpp
   quadinterpolator.cpp
   quadinterpolator_face.cpp
@@ -225,32 +219,13 @@ set(HDRS
   dgmassinv_kernels.hpp
   doftrans.hpp
   dfem/doperator.hpp
-  dfem/fielddescriptor.hpp
   dfem/fieldoperator.hpp
-  dfem/integrator_ctx.hpp
+  dfem/integrate.hpp
   dfem/parameterspace.hpp
-  dfem/tensor_functions.hpp
+  dfem/qfunction_apply.hpp
+  dfem/qfunction_transform.hpp
   dfem/tuple.hpp
   dfem/util.hpp
-  dfem/backends/util.hpp
-  dfem/backends/global_qf/action.hpp
-  dfem/backends/global_qf/derivative_action.hpp
-  dfem/backends/global_qf/derivative_apply.hpp
-  dfem/backends/global_qf/derivative_apply_transpose.hpp
-  dfem/backends/global_qf/derivative_setup.hpp
-  dfem/backends/global_qf/prelude.hpp
-  dfem/backends/local_qf/action.hpp
-  dfem/backends/local_qf/derivative_action.hpp
-  dfem/backends/local_qf/derivative_apply.hpp
-  dfem/backends/local_qf/derivative_apply_transpose.hpp
-  dfem/backends/local_qf/derivative_assemble.hpp
-  dfem/backends/local_qf/derivative_assemble_diagonal.hpp
-  dfem/backends/local_qf/derivative_setup.hpp
-  dfem/backends/local_qf/kernels.hpp
-  dfem/backends/local_qf/kernels_ho.hpp
-  dfem/backends/local_qf/kernels_lo.hpp
-  dfem/backends/local_qf/prelude.hpp
-  dfem/backends/local_qf/util.hpp
   eltrans.hpp
   estimators.hpp
   fe.hpp
@@ -315,10 +290,8 @@ set(HDRS
   qfunction.hpp
   qinterp/det.hpp
   qinterp/eval.hpp
-  qinterp/eval_transpose.hpp
   qinterp/eval_hdiv.hpp
   qinterp/grad.hpp
-  qinterp/grad_transpose.hpp
   qspace.hpp
   quadinterpolator.hpp
   quadinterpolator_face.hpp
@@ -353,36 +326,36 @@ set(HDRS
   )
 
 if (MFEM_USE_SIDRE)
-    list(APPEND SRCS sidredatacollection.cpp)
-    list(APPEND HDRS sidredatacollection.hpp)
+  list(APPEND SRCS sidredatacollection.cpp)
+  list(APPEND HDRS sidredatacollection.hpp)
 endif()
 
 if (MFEM_USE_CONDUIT)
-    list(APPEND SRCS conduitdatacollection.cpp)
-    list(APPEND HDRS conduitdatacollection.hpp)
+  list(APPEND SRCS conduitdatacollection.cpp)
+  list(APPEND HDRS conduitdatacollection.hpp)
 endif()
 
 if (MFEM_USE_ADIOS2)
-    list(APPEND SRCS adios2datacollection.cpp)
-    list(APPEND HDRS adios2datacollection.hpp)
+  list(APPEND SRCS adios2datacollection.cpp)
+  list(APPEND HDRS adios2datacollection.hpp)
 endif()
 
 if (MFEM_USE_FMS)
-    list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
-    list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
+  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
+  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
 endif()
 
 if (MFEM_USE_MPI)
-    list(APPEND SRCS
+  list(APPEND SRCS
     pbilinearform.cpp
     pfespace.cpp
     pgridfunc.cpp
     plinearform.cpp
     pnonlinearform.cpp
     prestriction.cpp)
-    # If this list (HDRS -> HEADERS) is used for install, we probably want the
-    # headers added all the time.
-    list(APPEND HDRS
+  # If this list (HDRS -> HEADERS) is used for install, we probably want the
+  # headers added all the time.
+  list(APPEND HDRS
     pbilinearform.hpp
     pfespace.hpp
     pgridfunc.hpp

fem/bilinearform.cpp

@@ -1255,6 +1255,31 @@ void BilinearForm::Mult(const Vector &x, Vector &y) const
    }
 }
 
+void BilinearForm::AddMult(const Vector &x, Vector &y, const real_t a) const
+{
+   if (ext)
+   {
+      ext->AddMult(x, y, a);
+   }
+   else
+   {
+      mat->AddMult(x, y, a);
+   }
+}
+
+void BilinearForm::AddMultTranspose(const Vector &x, Vector &y,
+                                    const real_t a) const
+{
+   if (ext)
+   {
+      ext->AddMultTranspose(x, y, a);
+   }
+   else
+   {
+      mat->AddMultTranspose(x, y, a);
+   }
+}
+
 void BilinearForm::MultTranspose(const Vector & x, Vector & y) const
 {
    if (ext)

fem/bilinearform.hpp

@@ -307,8 +307,8 @@ public:
    { mat->Mult(x, y); mat_e->AddMult(x, y); }
 
    /// Add the matrix vector multiple to a vector:  $ y += a M x $
-   void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const override
-   { mat -> AddMult (x, y, a); }
+   void AddMult(const Vector &x, Vector &y,
+                const real_t a = 1.0) const override;
 
    /** @brief Add the original uneliminated matrix vector multiple to a vector.
        The original matrix is $ M + Me $ so we have:
@@ -318,8 +318,7 @@ public:
 
    /// Add the matrix transpose vector multiplication:  $ y += a M^T x $
    void AddMultTranspose(const Vector & x, Vector & y,
-                         const real_t a = 1.0) const override
-   { mat->AddMultTranspose(x, y, a); }
+                         const real_t a = 1.0) const override;
 
    /** @brief Add the original uneliminated matrix transpose vector
        multiple to a vector. The original matrix is $ M + M_e $

fem/bilinearform_ext.cpp

@@ -1997,7 +1997,11 @@ void PADiscreteLinearOperatorExtension::Assemble()
    }
    else
    {
-      mfem_error("A real ElementRestriction is required in this setting!");
+      const L2ElementRestriction* l2_elem_restrict =
+         dynamic_cast<const L2ElementRestriction*>(elem_restrict_test);
+      MFEM_VERIFY(l2_elem_restrict,
+                  "A real ElementRestriction is required in this setting!");
+      test_multiplicity = 1.0;
    }
 
    auto tm = test_multiplicity.ReadWrite();
@@ -2036,7 +2040,13 @@ void PADiscreteLinearOperatorExtension::AddMult(
    }
    else
    {
-      mfem_error("In this setting you need a real ElementRestriction!");
+      const L2ElementRestriction* l2_elem_restrict =
+         dynamic_cast<const L2ElementRestriction*>(elem_restrict_test);
+      MFEM_VERIFY(l2_elem_restrict,
+                  "In this setting you need a real ElementRestriction!");
+      tempY.SetSize(y.Size());
+      l2_elem_restrict->MultTranspose(localTest, tempY);
+      y += tempY;
    }
 }
 

fem/coefficient.hpp

@@ -1055,7 +1055,8 @@ public:
 
 typedef VectorCoefficient DiagonalMatrixCoefficient;
 
-/// Base class for Matrix Coefficients that optionally depend on time and space.
+/** Base class for matrix-valued coefficients that optionally depend on time
+    and space. */
 class MatrixCoefficient
 {
 protected:
@@ -1102,6 +1103,9 @@ public:
    /// the quadrature points. The matrix will be transposed or not according to
    /// the boolean argument @a transpose.
    ///
+   /// The stored entries use the same row/column convention as `Eval()`,
+   /// unless `transpose == true`, in which case `K^T` is stored instead.
+   ///
    /// The @a vdim of the QuadratureFunction should be equal to the height times
    /// the width of the matrix.
    virtual void Project(QuadratureFunction &qf, bool transpose=false);

fem/complex_fem.cpp

@@ -588,6 +588,38 @@ SesquilinearForm::AssembleComplexSparseMatrix()
                                   false, false, conv);
 }
 
+void
+SesquilinearForm::BuildComplexOperator(OperatorHandle &A_r,
+                                       OperatorHandle &A_i,
+                                       OperatorHandle &A) const
+{
+   // A = A_r + i A_i
+   A.Clear();
+   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
+   {
+      ComplexSparseMatrix * A_sp =
+         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
+                                 A_i.As<SparseMatrix>(),
+                                 A_r.OwnsOperator(),
+                                 A_i.OwnsOperator(),
+                                 conv);
+      A.Reset<ComplexSparseMatrix>(A_sp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op =
+         new ComplexOperator(A_r.Ptr(),
+                             A_i.Ptr(),
+                             A_r.OwnsOperator(),
+                             A_i.OwnsOperator(),
+                             conv);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+   A_r.SetOperatorOwner(false);
+   A_i.SetOperatorOwner(false);
+}
+
 void
 SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                    Vector &x, Vector &b,
@@ -716,31 +748,7 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
    B_r.SyncAliasMemory(B);
    B_i.SyncAliasMemory(B);
 
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
-   {
-      ComplexSparseMatrix * A_sp =
-         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
-                                 A_i.As<SparseMatrix>(),
-                                 A_r.OwnsOperator(),
-                                 A_i.OwnsOperator(),
-                                 conv);
-      A.Reset<ComplexSparseMatrix>(A_sp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op =
-         new ComplexOperator(A_r.Ptr(),
-                             A_i.Ptr(),
-                             A_r.OwnsOperator(),
-                             A_i.OwnsOperator(),
-                             conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
+   BuildComplexOperator(A_r, A_i, A);
 }
 
 void
@@ -777,31 +785,7 @@ SesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
       }
    }
 
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
-   {
-      ComplexSparseMatrix * A_sp =
-         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
-                                 A_i.As<SparseMatrix>(),
-                                 A_r.OwnsOperator(),
-                                 A_i.OwnsOperator(),
-                                 conv);
-      A.Reset<ComplexSparseMatrix>(A_sp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op =
-         new ComplexOperator(A_r.Ptr(),
-                             A_i.Ptr(),
-                             A_r.OwnsOperator(),
-                             A_i.OwnsOperator(),
-                             conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
+   BuildComplexOperator(A_r, A_i, A);
 }
 
 void
@@ -1893,6 +1877,81 @@ ParSesquilinearForm::ParallelAssemble()
                                     true, true, conv);
 }
 
+void
+ParSesquilinearForm::BuildComplexOperator(OperatorHandle &A_r,
+                                          OperatorHandle &A_i,
+                                          OperatorHandle &A) const
+{
+   // A = A_r + i A_i
+   A.Clear();
+   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
+       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
+   {
+      ComplexHypreParMatrix * A_hyp =
+         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
+                                   A_i.As<HypreParMatrix>(),
+                                   A_r.OwnsOperator(),
+                                   A_i.OwnsOperator(),
+                                   conv);
+      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op =
+         new ComplexOperator(A_r.As<Operator>(),
+                             A_i.As<Operator>(),
+                             A_r.OwnsOperator(),
+                             A_i.OwnsOperator(),
+                             conv);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+   A_r.SetOperatorOwner(false);
+   A_i.SetOperatorOwner(false);
+}
+
+namespace
+{
+struct ZeroDiagonalHypreKernel
+{
+   const int *ess_tdof_list;
+   const HYPRE_Int *diag_i;
+   real_t *diag_data;
+
+   void MFEM_HOST_DEVICE operator()(int k) const
+   {
+      const int j = ess_tdof_list[k];
+      diag_data[diag_i[j]] = 0.0;
+   }
+};
+}
+
+void
+ParSesquilinearForm::SetImaginaryEssentialDiagonalToZero(
+   const Array<int> &ess_tdof_list, OperatorHandle &A)
+{
+   if (A.Type() == Operator::Hypre_ParCSR)
+   {
+      const int n = ess_tdof_list.Size();
+      HypreParMatrix *Ah;
+      A.Get(Ah);
+      hypre_ParCSRMatrix *Aih = *Ah;
+      Ah->HypreReadWrite();
+      const int *d_ess_tdof_list =
+         ess_tdof_list.GetMemory().Read(GetHypreForallMemoryClass(), n);
+      HYPRE_Int *d_diag_i = Aih->diag->i;
+      real_t *d_diag_data = Aih->diag->data;
+      mfem::hypre_forall(n, ZeroDiagonalHypreKernel
+      {
+         d_ess_tdof_list, d_diag_i, d_diag_data
+      });
+   }
+   else
+   {
+      A.As<ConstrainedOperator>()->SetDiagonalPolicy
+      (mfem::Operator::DiagonalPolicy::DIAG_ZERO);
+   }
+}
+
 void
 ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                       Vector &x, Vector &b,
@@ -1993,27 +2052,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       });
       // Modify off-diagonal blocks (imaginary parts of the matrix) to conform
       // with standard essential BC treatment
-      if (A_i.Type() == Operator::Hypre_ParCSR)
-      {
-         HypreParMatrix * Ah;
-         A_i.Get(Ah);
-         hypre_ParCSRMatrix *Aih = *Ah;
-         Ah->HypreReadWrite();
-         const int *d_ess_tdof_list =
-            ess_tdof_list.GetMemory().Read(GetHypreForallMemoryClass(), n);
-         HYPRE_Int *d_diag_i = Aih->diag->i;
-         real_t *d_diag_data = Aih->diag->data;
-         mfem::hypre_forall(n, [=] MFEM_HOST_DEVICE (int k)
-         {
-            const int j = d_ess_tdof_list[k];
-            d_diag_data[d_diag_i[j]] = 0.0;
-         });
-      }
-      else
-      {
-         A_i.As<ConstrainedOperator>()->SetDiagonalPolicy
-         (mfem::Operator::DiagonalPolicy::DIAG_ZERO);
-      }
+      SetImaginaryEssentialDiagonalToZero(ess_tdof_list, A_i);
    }
 
    if (conv == ComplexOperator::BLOCK_SYMMETRIC)
@@ -2032,31 +2071,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
    B_r.SyncAliasMemory(B);
    B_i.SyncAliasMemory(B);
 
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
-   {
-      ComplexHypreParMatrix * A_hyp =
-         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
-                                   A_i.As<HypreParMatrix>(),
-                                   A_r.OwnsOperator(),
-                                   A_i.OwnsOperator(),
-                                   conv);
-      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op =
-         new ComplexOperator(A_r.As<Operator>(),
-                             A_i.As<Operator>(),
-                             A_r.OwnsOperator(),
-                             A_i.OwnsOperator(),
-                             conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
+   BuildComplexOperator(A_r, A_i, A);
 }
 
 void
@@ -2081,50 +2096,10 @@ ParSesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
    {
       // Modify off-diagonal blocks (imaginary parts of the matrix) to conform
       // with standard essential BC treatment
-      if ( A_i.Type() == Operator::Hypre_ParCSR )
-      {
-         int n = ess_tdof_list.Size();
-         HypreParMatrix * Ah;
-         A_i.Get(Ah);
-         hypre_ParCSRMatrix * Aih = *Ah;
-         for (int k = 0; k < n; k++)
-         {
-            int j = ess_tdof_list[k];
-            Aih->diag->data[Aih->diag->i[j]] = 0.0;
-         }
-      }
-      else
-      {
-         A_i.As<ConstrainedOperator>()->SetDiagonalPolicy
-         (mfem::Operator::DiagonalPolicy::DIAG_ZERO);
-      }
+      SetImaginaryEssentialDiagonalToZero(ess_tdof_list, A_i);
    }
 
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
-   {
-      ComplexHypreParMatrix * A_hyp =
-         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
-                                   A_i.As<HypreParMatrix>(),
-                                   A_r.OwnsOperator(),
-                                   A_i.OwnsOperator(),
-                                   conv);
-      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op =
-         new ComplexOperator(A_r.As<Operator>(),
-                             A_i.As<Operator>(),
-                             A_r.OwnsOperator(),
-                             A_i.OwnsOperator(),
-                             conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
+   BuildComplexOperator(A_r, A_i, A);
 }
 
 void

fem/complex_fem.hpp

@@ -392,6 +392,9 @@ private:
    bool RealInteg();
    bool ImagInteg();
 
+   void BuildComplexOperator(OperatorHandle &A_r, OperatorHandle &A_i,
+                             OperatorHandle &A) const;
+
 public:
    SesquilinearForm(FiniteElementSpace *fes,
                     ComplexOperator::Convention
@@ -986,6 +989,12 @@ private:
    bool RealInteg();
    bool ImagInteg();
 
+   void SetImaginaryEssentialDiagonalToZero(
+      const Array<int> &ess_tdof_list, OperatorHandle &A);
+
+   void BuildComplexOperator(OperatorHandle &A_r, OperatorHandle &A_i,
+                             OperatorHandle &A) const;
+
 public:
    ParSesquilinearForm(ParFiniteElementSpace *pf,
                        ComplexOperator::Convention

fem/datacollection.cpp

@@ -38,9 +38,24 @@ int DataCollection::create_directory(const std::string &dir_name,
    // create directories recursively
    const char path_delim = '/';
    std::string::size_type pos = 0;
-   int err_flag;
+   int err_flag = 0;
 #ifdef MFEM_USE_MPI
    const ParMesh *pmesh = dynamic_cast<const ParMesh*>(mesh);
+   // In addition to the global root, let the lowest rank on each shared-memory
+   // node create the directory too, so that node-local (non-shared) filesystems
+   // get it on every node rather than only where the global root lives. On a
+   // shared filesystem the extra mkdir() hits EEXIST and is tolerated below.
+   bool node_root = true;
+   if (pmesh)
+   {
+      MPI_Comm node_comm;
+      MPI_Comm_split_type(pmesh->GetComm(), MPI_COMM_TYPE_SHARED, myid,
+                          MPI_INFO_NULL, &node_comm);
+      int node_rank;
+      MPI_Comm_rank(node_comm, &node_rank);
+      node_root = (node_rank == 0);
+      MPI_Comm_free(&node_comm);
+   }
 #endif
 
    do
@@ -52,7 +67,7 @@ int DataCollection::create_directory(const std::string &dir_name,
       err_flag = mkdir(subdir.c_str(), 0777);
       err_flag = (err_flag && (errno != EEXIST)) ? 1 : 0;
 #else
-      if (myid == 0 || pmesh == NULL)
+      if (node_root || pmesh == NULL)
       {
          err_flag = mkdir(subdir.c_str(), 0777);
          err_flag = (err_flag && (errno != EEXIST)) ? 1 : 0;
@@ -64,7 +79,8 @@ int DataCollection::create_directory(const std::string &dir_name,
 #ifdef MFEM_USE_MPI
    if (pmesh)
    {
-      MPI_Bcast(&err_flag, 1, MPI_INT, 0, pmesh->GetComm());
+      MPI_Allreduce(MPI_IN_PLACE, &err_flag, 1, MPI_INT, MPI_MAX,
+                    pmesh->GetComm());
    }
 #endif
 

fem/dfem/assemble.hpp

@@ -0,0 +1,403 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "util.hpp"
+
+namespace mfem::future
+{
+
+/// @brief Assemble element matrix for three dimensional data.
+///
+/// Note: In the below layouts, total_trial_op_dim is > 1 if
+/// there are more than one inputs dependent on the derivative variable.
+///
+/// @param A Memory for one element matrix with layout
+/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
+/// @param fhat Memory to hold the residual computation with layout
+/// [test_vdim, test_op_dim, nqp].
+/// @param qpdc The quadrature point data cache with data layout
+/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
+/// @param itod Input Trial Operator Dimension array. If the trial
+/// operator is not dependent, the dimension is 0 to indicate that.
+/// @param inputs The input field operator types.
+/// @param output The output field operator types.
+/// @param input_dtqmaps The input DofToQuad maps.
+/// @param output_dtqmap The output DofToQuad maps.
+/// @param scratch_shmem Scratch shared memory for computations.
+/// @param q1d The number of quadrature points in one dimension.
+/// @param td1d The number of trial dofs in one dimension.
+template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
+MFEM_HOST_DEVICE void assemble_element_mat_t3d(
+   const DeviceTensor<4, real_t>& A,
+   const DeviceTensor<3, real_t>& fhat,
+   const DeviceTensor<5, const real_t>& qpdc,
+   const DeviceTensor<1, const real_t>& itod,
+   const input_fop_ts& inputs,
+   const output_fop_t& output,
+   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
+   const DofToQuadMap& output_dtqmap,
+   std::array<DeviceTensor<1>, 6>& scratch_shmem,
+   const int& q1d,
+   const int& td1d)
+{
+   constexpr int dimension = 3;
+
+   // [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
+   const int test_vdim = qpdc.GetShape()[0];
+   const int test_op_dim = qpdc.GetShape()[1];
+   const int trial_vdim = qpdc.GetShape()[2];
+
+   // [num_test_dof, ...]
+   const auto num_test_dof = A.GetShape()[0];
+
+   for (int Jx = 0; Jx < td1d; Jx++)
+   {
+      for (int Jy = 0; Jy < td1d; Jy++)
+      {
+         for (int Jz = 0; Jz < td1d; Jz++)
+         {
+            const int J = Jx + td1d * (Jy + td1d * Jz);
+
+            for (int j = 0; j < trial_vdim; j++)
+            {
+               for (int tv = 0; tv < test_vdim; tv++)
+               {
+                  for (int tod = 0; tod < test_op_dim; tod++)
+                  {
+                     MFEM_FOREACH_THREAD(qx, x, q1d)
+                     {
+                        MFEM_FOREACH_THREAD(qy, y, q1d)
+                        {
+                           MFEM_FOREACH_THREAD(qz, z, q1d)
+                           {
+                              const int q = qx + q1d * (qy + q1d * qz);
+                              fhat(tv, tod, q) = 0.0;
+                           }
+                        }
+                     }
+                  }
+               }
+
+               // MSVC lambda capture workaround
+               [[maybe_unused]] const auto& inputs_ref = inputs;
+
+               int m_offset = 0;
+               for_constexpr<num_inputs>([&](auto s)
+               {
+                  using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
+
+                  const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
+                  if (trial_op_dim == 0)
+                  {
+                     // This is inside a lambda so we have to return
+                     // instead of idiomatic 'continue'.
+                     return;
+                  }
+
+                  auto& B = input_dtqmaps[s].B;
+                  auto& G = input_dtqmaps[s].G;
+
+                  if constexpr (is_value_fop<fop_t>::value)
+                  {
+                     MFEM_FOREACH_THREAD(qx, x, q1d)
+                     {
+                        MFEM_FOREACH_THREAD(qy, y, q1d)
+                        {
+                           MFEM_FOREACH_THREAD(qz, z, q1d)
+                           {
+                              const int q = qx + q1d * (qy + q1d * qz);
+
+                              for (int m = 0; m < trial_op_dim; m++)
+                              {
+                                 for (int i = 0; i < test_vdim; i++)
+                                 {
+                                    for (int k = 0; k < test_op_dim; k++)
+                                    {
+                                       const real_t f = qpdc(i, k, j, m + m_offset, q);
+                                       fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
+                                    }
+                                 }
+                              }
+                           }
+                        }
+                     }
+                  }
+                  else if constexpr (is_gradient_fop<fop_t>::value)
+                  {
+                     MFEM_FOREACH_THREAD(qx, x, q1d)
+                     {
+                        MFEM_FOREACH_THREAD(qy, y, q1d)
+                        {
+                           MFEM_FOREACH_THREAD(qz, z, q1d)
+                           {
+                              const int q = qx + q1d * (qy + q1d * qz);
+                              for (int m = 0; m < trial_op_dim; m++)
+                              {
+                                 for (int i = 0; i < test_vdim; i++)
+                                 {
+                                    for (int k = 0; k < test_op_dim; k++)
+                                    {
+                                       const real_t f = qpdc(i, k, j, m + m_offset, q);
+                                       if (m == 0)
+                                       {
+                                          fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
+                                       }
+                                       else if (m == 1)
+                                       {
+                                          fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy) * B(qz, 0, Jz);
+                                       }
+                                       else if (m == 2)
+                                       {
+                                          fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * G(qz, 0, Jz);
+                                       }
+                                    }
+                                 }
+                              }
+                           }
+                        }
+                     }
+                  }
+                  else
+                  {
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+                     MFEM_ABORT("sum factorized sparse matrix assemble routine "
+                                "not implemented for field operator");
+#endif
+                  }
+                  MFEM_SYNC_THREAD;
+                  m_offset += trial_op_dim;
+               });
+
+               auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
+               map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
+                                             scratch_shmem, dimension, true);
+            }
+         }
+      }
+   }
+}
+
+/// @brief Assemble element matrix for two dimensional data.
+///
+/// Note: In the below layouts, total_trial_op_dim is > 1 if
+/// there are more than one inputs dependent on the derivative variable.
+///
+/// @param A Memory for one element matrix with layout
+/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
+/// @param fhat Memory to hold the residual computation with layout
+/// [test_vdim, test_op_dim, nqp].
+/// @param qpdc The quadrature point data cache with data layout
+/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
+/// @param itod Input Trial Operator Dimension array. If the trial
+/// operator is not dependent, the dimension is 0 to indicate that.
+/// @param inputs The input field operator types.
+/// @param output The output field operator types.
+/// @param input_dtqmaps The input DofToQuad maps.
+/// @param output_dtqmap The output DofToQuad maps.
+/// @param scratch_shmem Scratch shared memory for computations.
+/// @param q1d The number of quadrature points in one dimension.
+/// @param td1d The number of trial dofs in one dimension.
+template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
+MFEM_HOST_DEVICE void assemble_element_mat_t2d(
+   const DeviceTensor<4, real_t>& A,
+   const DeviceTensor<3, real_t>& fhat,
+   const DeviceTensor<5, const real_t>& qpdc,
+   const DeviceTensor<1, const real_t>& itod,
+   const input_fop_ts& inputs,
+   const output_fop_t& output,
+   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
+   const DofToQuadMap& output_dtqmap,
+   std::array<DeviceTensor<1>, 6>& scratch_shmem,
+   const int& q1d,
+   const int& td1d)
+{
+   constexpr int dimension = 2;
+
+   // [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
+   const int test_vdim = qpdc.GetShape()[0];
+   const int test_op_dim = qpdc.GetShape()[1];
+   const int trial_vdim = qpdc.GetShape()[2];
+
+   // [num_test_dof, ...]
+   const auto num_test_dof = A.GetShape()[0];
+
+   for (int Jx = 0; Jx < td1d; Jx++)
+   {
+      for (int Jy = 0; Jy < td1d; Jy++)
+      {
+         const int J = Jy + Jx * td1d;
+
+         for (int j = 0; j < trial_vdim; j++)
+         {
+            for (int tv = 0; tv < test_vdim; tv++)
+            {
+               for (int tod = 0; tod < test_op_dim; tod++)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, q1d)
+                  {
+                     MFEM_FOREACH_THREAD(qy, y, q1d)
+                     {
+                        const int q = qy + qx * q1d;
+                        fhat(tv, tod, q) = 0.0;
+                     }
+                  }
+               }
+            }
+
+            // MSVC lambda capture workaround
+            [[maybe_unused]] const auto& inputs_ref = inputs;
+
+            int m_offset = 0;
+            for_constexpr<num_inputs>([&](auto s)
+            {
+               using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
+
+               const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
+               if (trial_op_dim == 0)
+               {
+                  // This is inside a lambda so we have to return
+                  // instead of idiomatic 'continue'.
+                  return;
+               }
+
+               auto& B = input_dtqmaps[s].B;
+               auto& G = input_dtqmaps[s].G;
+
+               if constexpr (is_value_fop<fop_t>::value)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, q1d)
+                  {
+                     MFEM_FOREACH_THREAD(qy, y, q1d)
+                     {
+                        const int q = qy + qx * q1d;
+                        for (int m = 0; m < trial_op_dim; m++)
+                        {
+                           for (int i = 0; i < test_vdim; i++)
+                           {
+                              for (int k = 0; k < test_op_dim; k++)
+                              {
+                                 const real_t f = qpdc(i, k, j, m + m_offset, q);
+                                 fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy);
+                              }
+                           }
+                        }
+                     }
+                  }
+               }
+               else if constexpr (is_gradient_fop<fop_t>::value)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, q1d)
+                  {
+                     MFEM_FOREACH_THREAD(qy, y, q1d)
+                     {
+                        const int q = qy + qx * q1d;
+                        for (int m = 0; m < trial_op_dim; m++)
+                        {
+                           for (int i = 0; i < test_vdim; i++)
+                           {
+                              for (int k = 0; k < test_op_dim; k++)
+                              {
+                                 const real_t f = qpdc(i, k, j, m + m_offset, q);
+                                 if (m == 0)
+                                 {
+                                    fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy);
+                                 }
+                                 else
+                                 {
+                                    fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy);
+                                 }
+                              }
+                           }
+                        }
+                     }
+                  }
+               }
+               else
+               {
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+                  MFEM_ABORT("sum factorized sparse matrix assemble routine "
+                             "not implemented for field operator");
+#endif
+               }
+               MFEM_SYNC_THREAD;
+               m_offset += trial_op_dim;
+            });
+
+            auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
+            map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
+                                          scratch_shmem, dimension, true);
+         }
+      }
+   }
+}
+
+/// @brief Assemble element matrix for two or three dimensional data.
+///
+/// Note: In the below layouts, total_trial_op_dim is > 1 if
+/// there are more than one inputs dependent on the derivative variable.
+///
+/// @param A Memory for one element matrix with layout
+/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
+/// @param fhat Memory to hold the residual computation with layout
+/// [test_vdim, test_op_dim, nqp].
+/// @param qpdc The quadrature point data cache with data layout
+/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
+/// @param itod Input Trial Operator Dimension array. If the trial
+/// operator is not dependent, the dimension is 0 to indicate that.
+/// @param inputs The input field operator types.
+/// @param output The output field operator types.
+/// @param input_dtqmaps The input DofToQuad maps.
+/// @param output_dtqmap The output DofToQuad maps.
+/// @param scratch_shmem Scratch shared memory for computations.
+/// @param dimension The spatial dimension.
+/// @param q1d The number of quadrature points in one dimension.
+/// @param td1d The number of trial dofs in one dimension.
+/// @param use_sum_factorization Indicator if sum factorization is used.
+template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
+MFEM_HOST_DEVICE void assemble_element_mat_naive(
+   const DeviceTensor<4, real_t>& A,
+   const DeviceTensor<3, real_t>& fhat,
+   const DeviceTensor<5, const real_t>& qpdc,
+   const DeviceTensor<1, const real_t>& itod,
+   const input_fop_ts& inputs,
+   const output_fop_t& output,
+   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
+   const DofToQuadMap& output_dtqmap,
+   std::array<DeviceTensor<1>, 6>& scratch_shmem,
+   const int& dimension,
+   const int& q1d,
+   const int& td1d,
+   const bool& use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 2)
+      {
+         assemble_element_mat_t2d(A, fhat, qpdc, itod, inputs, output,
+                                  input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
+      }
+      else if (dimension == 3)
+      {
+         assemble_element_mat_t3d(A, fhat, qpdc, itod, inputs, output,
+                                  input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
+      }
+   }
+   else
+   {
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+      MFEM_ABORT("element matrix assemble not implemented for non tensor "
+                 "product basis");
+#endif
+   }
+}
+
+} // namespace mfem::future

fem/dfem/backends/global_qf/action.hpp

@@ -1,122 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../util.hpp"
-#include "../../integrator_ctx.hpp"
-
-#include <utility>
-
-namespace mfem::future
-{
-
-namespace GlobalQFImpl
-{
-
-template<
-   typename qfunc_t,
-   typename inputs_t,
-   typename outputs_t,
-   size_t ninputs = tuple_size<inputs_t>::value,
-   size_t noutputs = tuple_size<outputs_t>::value>
-struct Action
-{
-   Action(
-      IntegratorContext ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs) :
-      ctx(ctx),
-      qfunc(qfunc),
-      inputs(inputs),
-      outputs(outputs)
-   {
-      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(inputs, input_to_infd, ctx.infds);
-      check_consistency(outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
-      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
-
-      const int nqp = ctx.ir.GetNPoints();
-      gnqp = nqp * ctx.nentities;
-
-      // prepare xq and yq BlockVectors
-      xq_offsets.SetSize(ninputs + 1);
-      xq_offsets[0] = 0;
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         const auto input = get<i>(inputs);
-         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
-      });
-      xq_offsets.PartialSum();
-      InitBlockVector(xq, xq_offsets);
-
-      yq_offsets.SetSize(noutputs + 1);
-      yq_offsets[0] = 0;
-      constexpr_for<0, noutputs>([&](auto i)
-      {
-         const auto output = get<i>(outputs);
-         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
-      });
-      yq_offsets.PartialSum();
-      InitBlockVector(yq, yq_offsets);
-   }
-
-   void operator()(
-      const std::vector<Vector *> &xe,
-      std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      // E -> Q
-      interpolate(input_to_infd, input_bases, xe, xq);
-
-      // Q -> Q
-      static_assert(
-         detail::supports_tensor_array_qfunc<qfunc_t, inputs_t, outputs_t>::value,
-         "qfunc signature not supported by default backend Action");
-
-      detail::call_qfunc(
-         qfunc, xq, yq, gnqp, input_qlayouts, output_qlayouts,
-         std::make_index_sequence<ninputs> {},
-         std::make_index_sequence<noutputs> {});
-
-      // Q -> E
-      integrate(output_to_outfd, output_bases, yq, ye);
-   }
-
-   IntegratorContext ctx;
-   qfunc_t qfunc;
-   inputs_t inputs;
-   outputs_t outputs;
-
-   std::array<size_t, ninputs> input_to_infd;
-   std::array<size_t, noutputs> output_to_outfd;
-
-   std::array<FieldBasis, ninputs> input_bases;
-   std::array<FieldBasis, noutputs> output_bases;
-
-   std::array<std::vector<int>, ninputs>  input_qlayouts;
-   std::array<std::vector<int>, noutputs> output_qlayouts;
-
-   int gnqp = 0;
-   Array<int> xq_offsets, yq_offsets;
-   mutable BlockVector xq, yq;
-};
-
-}
-}

fem/dfem/backends/global_qf/derivative_action.hpp

@@ -1,180 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "../util.hpp"
-#include <utility>
-
-namespace mfem::future::GlobalQFImpl
-{
-
-template<
-   int derivative_id,
-   typename qfunc_t,
-   typename inputs_t,
-   typename outputs_t,
-   size_t ninputs = tuple_size<inputs_t>::value,
-   size_t noutputs = tuple_size<outputs_t>::value>
-struct DerivativeAction
-{
-   using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
-
-   DerivativeAction(
-      IntegratorContext ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs):
-      ctx(ctx), qfunc(qfunc),
-      qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
-      inputs(inputs), outputs(outputs)
-   {
-      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(inputs, input_to_infd, ctx.infds);
-      check_consistency(outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(
-         outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
-      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
-
-      const int nqp = ctx.ir.GetNPoints();
-      gnqp = nqp * ctx.nentities;
-
-      xq_offsets.SetSize(ninputs + 1);
-      xq_offsets[0] = 0;
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         const auto input = get<i>(inputs);
-         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
-      });
-      xq_offsets.PartialSum();
-      InitBlockVector(xq, xq_offsets);
-
-      yq_offsets.SetSize(noutputs + 1);
-      yq_offsets[0] = 0;
-      constexpr_for<0, noutputs>([&](auto i)
-      {
-         const auto output = get<i>(outputs);
-         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
-      });
-      yq_offsets.PartialSum();
-      InitBlockVector(yq, yq_offsets);
-
-      // Shadow blocks use the same offsets as xq so tensor_array views
-      shadow_xq_offsets.SetSize(xq_offsets.Size());
-      shadow_xq_offsets = xq_offsets;
-      InitBlockVector(shadow_xq, shadow_xq_offsets);
-
-      dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
-
-      const size_t direction_fd_idx = FindIdx(derivative_id, ctx.infds);
-      MFEM_ASSERT(direction_fd_idx != SIZE_MAX,
-                  "derivative direction field not found in infds");
-      direction_fd = ctx.infds[direction_fd_idx];
-   }
-
-   void operator()(
-      const std::vector<Vector *> &xe,
-      const Vector *de,
-      std::vector<Vector *> &ye)
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      // E -> Q
-      interpolate(input_to_infd, input_bases, xe, xq);
-
-      constexpr auto input_active =
-         detail::make_activity_map<derivative_id>(inputs_t{});
-
-      MFEM_ASSERT(de != nullptr, "derivative direction vector is null");
-      restriction(direction_fd, direction_rcache, *de, direction_e,
-                  dof_ordering);
-
-      shadow_xq = 0.0;
-      shadow_xq.SyncToBlocks();
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         if (!input_active[i]) { return; }
-         input_bases[i].forward(direction_e, shadow_xq.GetBlock(i));
-      });
-
-      static_assert(detail::supports_tensor_array_qfunc<qfunc_t,
-                    inputs_t,
-                    outputs_t>::value,
-                    "qfunc signature not supported by default backend Action");
-
-      // Q -> Q
-      yq = 0.0;
-      yq.SyncToBlocks();
-      if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
-      {
-         detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t, inputs_t,
-                outputs_t>(
-                   qfunc,
-                   qfunc_shadow,
-                   xq,
-                   shadow_xq,
-                   yq,
-                   gnqp,
-                   input_qlayouts,
-                   output_qlayouts,
-                   std::make_index_sequence<ninputs> {},
-                   std::make_index_sequence<noutputs> {});
-      }
-      else
-      {
-         detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
-            qfunc,
-            xq,
-            shadow_xq,
-            yq,
-            gnqp,
-            input_qlayouts,
-            output_qlayouts,
-            std::make_index_sequence<ninputs> {},
-            std::make_index_sequence<noutputs> {});
-      }
-
-      // Q -> E
-      integrate(output_to_outfd, output_bases, yq, ye);
-   }
-
-   IntegratorContext ctx;
-   qfunc_t qfunc;
-   qfunc_shadow_t qfunc_shadow;
-   inputs_t inputs;
-   outputs_t outputs;
-
-   std::array<size_t, ninputs> input_to_infd;
-   std::array<size_t, noutputs> output_to_outfd;
-
-   std::array<FieldBasis, ninputs> input_bases;
-   std::array<FieldBasis, noutputs> output_bases;
-
-   std::array<std::vector<int>, ninputs> input_qlayouts;
-   std::array<std::vector<int>, noutputs> output_qlayouts;
-
-   int gnqp = 0;
-   Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
-   mutable BlockVector xq, shadow_xq, yq;
-
-   FieldDescriptor direction_fd;
-   ElementDofOrdering dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   mutable Vector direction_e;
-   mutable RestrictionCache<Entity::Element> direction_rcache;
-};
-
-} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_apply.hpp

@@ -1,242 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "../util.hpp"
-
-#include <array>
-#include <utility>
-#include <vector>
-
-namespace mfem::future::GlobalQFImpl
-{
-
-// Q-function-shape-agnostic cached forward apply (J·v)
-template<
-   int derivative_id,
-   typename inputs_t,
-   typename outputs_t>
-struct DerivativeApply
-{
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-
-   DerivativeApply(
-      IntegratorContext ctx,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache):
-      ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
-      qp_cache(qp_cache)
-   {
-      create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(this->inputs, input_to_infd, ctx.infds);
-      check_consistency(this->outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(
-         this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(
-         this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      const int nqp = ctx.ir.GetNPoints();
-      const int ne = ctx.nentities;
-      num_qp = nqp;
-      gnqp = nqp * ne;
-
-      // Precompute Q-space BlockVector layouts
-      dir_q_offsets.SetSize(n_inputs + 1);
-      dir_q_offsets[0] = 0;
-      constexpr_for<0, n_inputs>([&](auto i)
-      {
-         dir_q_offsets[i + 1] =
-            dir_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
-      });
-      InitBlockVector(dir_q_local, dir_q_offsets);
-
-      result_q_offsets.SetSize(n_outputs + 1);
-      result_q_offsets[0] = 0;
-      constexpr_for<0, n_outputs>([&](auto i)
-      {
-         result_q_offsets[i + 1] =
-            result_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
-      });
-      InitBlockVector(result_q_local, result_q_offsets);
-
-      // Cache layout metadata (must match DerivativeSetup)
-      residual_size_on_qp = 0;
-      trial_vdim = 0;
-      total_trial_op_dim = 0;
-
-      constexpr auto activity =
-         detail::make_activity_map<derivative_id>(inputs_t{});
-
-      constexpr_for<0, n_inputs>([&](auto i)
-      {
-         if (!activity[i]) { return; }
-         const auto &fop = get<i>(this->inputs);
-         trial_vdim = fop.vdim;
-         total_trial_op_dim += fop.size_on_qp / fop.vdim;
-      });
-
-      constexpr_for<0, n_outputs>([&](auto i)
-      { residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
-      residual_size_on_qp *= trial_vdim * total_trial_op_dim;
-   }
-
-   void operator()(
-      const std::vector<Vector *> & /*xe*/,
-      const Vector *direction_l,
-      std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      MFEM_ASSERT(direction_l != nullptr,
-                  "Global DerivativeApply: direction vector is null");
-
-      // Re-zero pre-allocated Q temporaries
-      dir_q_local = 0.0;
-      dir_q_local.SyncToBlocks();
-      result_q_local = 0.0;
-      result_q_local.SyncToBlocks();
-
-      // Restrict trial direction from the derivative field
-      size_t in_fd = SIZE_MAX;
-      constexpr_for<0, n_inputs>([&](auto i)
-      {
-         if (get<i>(inputs).GetFieldId() == derivative_id)
-         {
-            in_fd = input_to_infd[i.value];
-         }
-      });
-      MFEM_ASSERT(in_fd != SIZE_MAX,
-                  "DerivativeApply: derivative field not found among inputs");
-
-      const auto &fd = ctx.infds[in_fd];
-
-      Vector dir_e;
-      restriction(fd, direction_rcache, *direction_l, dir_e,
-                  ElementDofOrdering::LEXICOGRAPHIC);
-
-      // Forward the trial direction into active input Q block
-      constexpr_for<0, n_inputs>([&](auto s)
-      {
-         if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
-         input_bases[s.value].forward(dir_e, dir_q_local.GetBlock(s.value));
-      });
-
-      const real_t *cache_ptr = qp_cache.Read();
-      const int res_sz = residual_size_on_qp;
-      const int gnqp_local = gnqp;
-      const int num_qp_local = num_qp;
-      const int trial_vdim_local = trial_vdim;
-      const int total_trial_op_dim_local = total_trial_op_dim;
-
-      constexpr_for<0, n_outputs>([&](auto o)
-      {
-         const int tv_o = get<o>(outputs).vdim;
-         const int to_o = get<o>(outputs).size_on_qp / tv_o;
-         const int out_base = [&]
-         {
-            int off = 0;
-            constexpr_for<0, o.value>([&](auto prev)
-            { off += get<prev>(outputs).size_on_qp; });
-            return off;
-         }();
-
-         real_t *res_o = result_q_local.GetBlock(o.value).ReadWrite();
-
-         int m_offset = 0;
-         constexpr_for<0, n_inputs>([&](auto s)
-         {
-            if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
-
-            const int tv = get<s>(inputs).vdim;
-            const int to = get<s>(inputs).size_on_qp / tv;
-            const real_t *dir_s = dir_q_local.GetBlock(s.value).Read();
-
-            mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
-            {
-               // Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
-               // read adjacent addresses for a fixed cache_idx.
-               const int cache_base =
-                  (gq % num_qp_local) +
-                  num_qp_local * res_sz * (gq / num_qp_local);
-
-               for (int j = 0; j < tv; ++j)
-               {
-                  for (int m = 0; m < to; ++m)
-                  {
-                     const real_t v = dir_s[(j * to + m) + (tv * to) * gq];
-                     const int m_global = m + m_offset;
-
-                     for (int i = 0; i < tv_o; ++i)
-                     {
-                        for (int k = 0; k < to_o; ++k)
-                        {
-                           const int out_comp = out_base + i * to_o + k;
-
-                           const int cache_idx =
-                              out_comp * trial_vdim_local * total_trial_op_dim_local +
-                              j * total_trial_op_dim_local + m_global;
-
-                           const real_t c =
-                              cache_ptr[cache_base + num_qp_local * cache_idx];
-                           res_o[(i * to_o + k) + (tv_o * to_o) * gq] += c * v;
-                        }
-                     }
-                  }
-               }
-            });
-            m_offset += to;
-         });
-      });
-
-      result_q_local.SyncToBlocks();
-
-      // Map result Q back to output fields
-      constexpr_for<0, n_outputs>([&](auto o)
-      {
-         const size_t out_fd = output_to_outfd[o.value];
-         output_bases[o.value].transpose(result_q_local.GetBlock(o.value),
-                                         *ye[out_fd]);
-      });
-   }
-
-private:
-   IntegratorContext ctx;
-   inputs_t inputs;
-   outputs_t outputs;
-   const Vector &qp_cache;
-
-   std::array<size_t, n_inputs> input_to_infd;
-   std::array<size_t, n_outputs> output_to_outfd;
-
-   std::array<FieldBasis, n_inputs> input_bases;
-   std::array<FieldBasis, n_outputs> output_bases;
-
-   int gnqp = 0;
-   int num_qp = 0;
-
-   Array<int> dir_q_offsets;
-   Array<int> result_q_offsets;
-   mutable BlockVector dir_q_local;
-   mutable BlockVector result_q_local;
-   mutable RestrictionCache<Entity::Element> direction_rcache;
-
-   int residual_size_on_qp = 0;
-   int trial_vdim = 0;
-   int total_trial_op_dim = 0;
-};
-
-} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_apply_transpose.hpp

@@ -1,260 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "../util.hpp"
-
-#include <array>
-#include <utility>
-#include <vector>
-
-namespace mfem::future::GlobalQFImpl
-{
-
-// Q-function-shape-agnostic cached transpose apply (Jᵀ·w)
-template<
-   int derivative_id,
-   typename qfunc_t,
-   typename inputs_t,
-   typename outputs_t>
-struct DerivativeApplyTranspose
-{
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-
-   DerivativeApplyTranspose(
-      IntegratorContext ctx,
-      qfunc_t /*qfunc*/,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache):
-      ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
-      qp_cache(qp_cache), dir_out_l(n_outputs), dir_out_e(n_outputs)
-   {
-      create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(this->inputs, input_to_infd, ctx.infds);
-      check_consistency(this->outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(
-         this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(
-         this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      const int nqp = ctx.ir.GetNPoints();
-      const int ne = ctx.nentities;
-      num_qp = nqp;
-      gnqp = nqp * ne;
-
-      // Precompute Q-space BlockVector layouts
-      dir_q_offsets.SetSize(n_outputs + 1);
-      dir_q_offsets[0] = 0;
-      constexpr_for<0, n_outputs>([&](auto i)
-      {
-         dir_q_offsets[i + 1] =
-            dir_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
-      });
-      InitBlockVector(dir_q_local, dir_q_offsets);
-
-      result_q_offsets.SetSize(n_inputs + 1);
-      result_q_offsets[0] = 0;
-      constexpr_for<0, n_inputs>([&](auto i)
-      {
-         result_q_offsets[i + 1] =
-            result_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
-      });
-      InitBlockVector(result_q_local, result_q_offsets);
-
-      // Cache layout metadata
-      residual_size_on_qp = 0;
-      trial_vdim = 0;
-      total_trial_op_dim = 0;
-
-      constexpr auto activity =
-         detail::make_activity_map<derivative_id>(inputs_t{});
-
-      constexpr_for<0, n_inputs>([&](auto i)
-      {
-         if (!activity[i]) { return; }
-         const auto &fop = get<i>(this->inputs);
-         trial_vdim = fop.vdim;
-         total_trial_op_dim += fop.size_on_qp / fop.vdim;
-      });
-
-      constexpr_for<0, n_outputs>([&](auto i)
-      { residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
-      residual_size_on_qp *= trial_vdim * total_trial_op_dim;
-   }
-
-   void operator()(
-      const std::vector<Vector *> & /*xe*/,
-      const Vector *direction_l,
-      std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      MFEM_ASSERT(direction_l != nullptr,
-                  "Global DerivativeApplyTranspose: direction vector is null");
-
-      // Re-zero the pre-allocated Q temporaries
-      dir_q_local = 0.0;
-      result_q_local = 0.0;
-      dir_q_local.SyncToBlocks();
-      result_q_local.SyncToBlocks();
-
-      // Bring test cotangent to quadrature points
-      pull_output_cotangents_to_q(direction_l, dir_q_local);
-
-      // Contract qp_cache with test directions at quadrature points
-      const real_t *cache_ptr = qp_cache.Read();
-      const int res_sz = residual_size_on_qp;
-      const int gnqp_local = gnqp;
-      const int num_qp_local = num_qp;
-      const int trial_vdim_local = trial_vdim;
-      const int total_trial_op_dim_local = total_trial_op_dim;
-
-      constexpr_for<0, n_outputs>([&](auto o)
-      {
-         const int tv_o = get<o>(outputs).vdim;
-         const int to_o = get<o>(outputs).size_on_qp / tv_o;
-         const int out_base = [&]
-         {
-            int off = 0;
-            constexpr_for<0, o.value>([&](auto prev)
-            { off += get<prev>(outputs).size_on_qp; });
-            return off;
-         }();
-
-         const int size_o = get<o>(outputs).size_on_qp;
-         const real_t *dir_o = dir_q_local.GetBlock(o.value).Read();
-
-         int m_offset = 0;
-         constexpr_for<0, n_inputs>([&](auto s)
-         {
-            if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
-
-            const int size_s = get<s>(inputs).size_on_qp;
-            const int to_s = size_s / trial_vdim_local;
-            real_t *res_s = result_q_local.GetBlock(s.value).ReadWrite();
-
-            mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
-            {
-               // Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
-               // read adjacent addresses for a fixed cache_idx.
-               const int cache_base =
-                  (gq % num_qp_local) +
-                  num_qp_local * res_sz * (gq / num_qp_local);
-
-               for (int i = 0; i < tv_o; ++i)
-               {
-                  for (int k = 0; k < to_o; ++k)
-                  {
-                     const int out_comp = out_base + i * to_o + k;
-                     const real_t w = dir_o[(i * to_o + k) + size_o * gq];
-
-                     for (int j = 0; j < trial_vdim_local; ++j)
-                     {
-                        for (int m = 0; m < to_s; ++m)
-                        {
-                           const int m_global = m + m_offset;
-                           const int cache_idx =
-                              out_comp * trial_vdim_local * total_trial_op_dim_local +
-                              j * total_trial_op_dim_local + m_global;
-
-                           const real_t c =
-                              cache_ptr[cache_base + num_qp_local * cache_idx];
-                           res_s[(j * to_s + m) + size_s * gq] += c * w;
-                        }
-                     }
-                  }
-               }
-            });
-            m_offset += to_s;
-         });
-      });
-
-      // Map result Q back to the trial (input) fields
-      constexpr_for<0, n_inputs>([&](auto s)
-      {
-         if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
-
-         const size_t in_fd = input_to_infd[s.value];
-         input_bases[s.value].transpose(
-            result_q_local.GetBlock(s.value), *ye[in_fd]);
-      });
-   }
-
-private:
-   IntegratorContext ctx;
-   inputs_t inputs;
-   outputs_t outputs;
-   const Vector &qp_cache;
-
-   std::array<size_t, n_inputs> input_to_infd;
-   std::array<size_t, n_outputs> output_to_outfd;
-
-   std::array<FieldBasis, n_inputs> input_bases;
-   std::array<FieldBasis, n_outputs> output_bases;
-
-   int gnqp = 0;
-   int num_qp = 0;
-
-   // Pre-allocated Q-space temporaries
-   Array<int> dir_q_offsets;
-   Array<int> result_q_offsets;
-   mutable BlockVector dir_q_local;
-   mutable BlockVector result_q_local;
-
-   // Pre-allocated owning storage for output cotangent temporaries
-   mutable std::array<Vector, n_outputs> dir_out_l_owned;
-   mutable std::array<Vector, n_outputs> dir_out_e_owned;
-   mutable std::vector<Vector *> dir_out_l;
-   mutable std::vector<Vector *> dir_out_e;
-   mutable RestrictionCache<Entity::Element> out_rcache;
-
-   int residual_size_on_qp = 0;
-   int trial_vdim = 0;
-   int total_trial_op_dim = 0;
-
-   /// Pull output cotangents from L-space into the pre-allocated Q BlockVector
-   void pull_output_cotangents_to_q(const Vector *direction_l,
-                                    BlockVector &dir_q) const
-   {
-      int l_offset = 0;
-      constexpr_for<0, n_outputs>([&](auto i)
-      {
-         const size_t outfd = output_to_outfd[i];
-         const auto &fd = ctx.outfds[outfd];
-         const int l_size = GetVSize(fd);
-
-         dir_out_l_owned[i] =
-            Vector(*const_cast<Vector *>(direction_l), l_offset, l_size);
-         dir_out_e_owned[i].SetSize(0);
-         dir_out_e_owned[i].UseDevice(true);
-
-         dir_out_l[i] = &dir_out_l_owned[i];
-         dir_out_e[i] = &dir_out_e_owned[i];
-         l_offset += l_size;
-      });
-
-      restriction(ctx.outfds, out_rcache, dir_out_l, dir_out_e);
-
-      constexpr_for<0, n_outputs>([&](auto i)
-      {
-         output_bases[i.value].forward(*dir_out_e[i], dir_q.GetBlock(i.value));
-      });
-      dir_q.SyncToBlocks();
-   }
-};
-
-} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_setup.hpp

@@ -1,244 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "../util.hpp"
-#include <utility>
-
-namespace mfem::future::GlobalQFImpl
-{
-
-template<
-   int derivative_id,
-   typename qfunc_t,
-   typename inputs_t,
-   typename outputs_t,
-   size_t ninputs = tuple_size<inputs_t>::value,
-   size_t noutputs = tuple_size<outputs_t>::value>
-struct DerivativeSetup
-{
-   using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
-
-   DerivativeSetup(
-      IntegratorContext ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      Vector &qp_cache) :
-      ctx(ctx),
-      qfunc(qfunc),
-      qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
-      inputs(inputs),
-      outputs(outputs),
-      qp_cache(qp_cache)
-   {
-      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
-      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
-
-      check_consistency(inputs, input_to_infd, ctx.infds);
-      check_consistency(outputs, output_to_outfd, ctx.outfds);
-
-      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
-      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
-
-      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
-      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
-
-      const int nqp = ctx.ir.GetNPoints();
-      num_qp = nqp;
-      nentities = ctx.nentities;
-      gnqp = nqp * nentities;
-
-      xq_offsets.SetSize(ninputs + 1);
-      xq_offsets[0] = 0;
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         xq_offsets[i + 1] = nqp * get<i>(inputs).size_on_qp * nentities;
-      });
-      xq_offsets.PartialSum();
-      InitBlockVector(xq, xq_offsets);
-
-      shadow_xq_offsets.SetSize(xq_offsets.Size());
-      shadow_xq_offsets = xq_offsets;
-      InitBlockVector(shadow_xq, shadow_xq_offsets);
-
-      yq_offsets.SetSize(noutputs + 1);
-      yq_offsets[0] = 0;
-      constexpr_for<0, noutputs>([&](auto o)
-      {
-         yq_offsets[o + 1] = nqp * get<o>(outputs).size_on_qp * nentities;
-      });
-      yq_offsets.PartialSum();
-      InitBlockVector(yq, yq_offsets);
-
-      total_out_size_on_qp = 0;
-      constexpr_for<0, noutputs>([&](auto o)
-      {
-         total_out_size_on_qp += get<o>(outputs).size_on_qp;
-         out_vdim[o] = get<o>(outputs).vdim;
-         out_op_dim[o] = get<o>(outputs).size_on_qp / get<o>(outputs).vdim;
-      });
-
-      activity_map = detail::make_activity_map<derivative_id>(inputs_t {});
-
-      trial_vdim = 0;
-      total_trial_op_dim = 0;
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         if (!activity_map[i]) { return; }
-         const auto inp = get<i>(inputs);
-         trial_vdim = inp.vdim;
-         total_trial_op_dim += inp.size_on_qp / inp.vdim;
-      });
-
-      constexpr_for<0, ninputs>([&](auto i)
-      {
-         input_size_on_qp_arr[i] = get<i>(inputs).size_on_qp;
-      });
-
-      residual_size_on_qp = total_out_size_on_qp * trial_vdim * total_trial_op_dim;
-      qp_cache.SetSize(residual_size_on_qp * num_qp * nentities);
-      qp_cache.UseDevice(true);
-   }
-
-   void operator()(const std::vector<Vector *> &xe)
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      interpolate(input_to_infd, input_bases, xe, xq);
-
-      const int gnqp_local = gnqp;
-      const int num_qp_local = num_qp;
-      const int trial_vdim_local = trial_vdim;
-      const int total_trial_op_dim_local = total_trial_op_dim;
-      const int residual_size_local = residual_size_on_qp;
-
-      for (int j = 0; j < trial_vdim; j++)
-      {
-         int m_offset = 0;
-         constexpr_for<0, ninputs>([&](auto s)
-         {
-            if (!activity_map[s]) { return; }
-
-            const int input_vdim_s  = get<s>(inputs).vdim;
-            const int input_size_s  = input_size_on_qp_arr[s];
-            const int trial_op_dim_s = input_size_s / input_vdim_s;
-
-            for (int m = 0; m < trial_op_dim_s; m++)
-            {
-               shadow_xq = 0.0;
-               shadow_xq.SyncToBlocks();
-
-               // Set component (j + input_vdim_s * m) to 1 at all QPs
-               const int c_shadow = j + input_vdim_s * m;
-               real_t *shadow_ptr = shadow_xq.GetBlock(s.value).ReadWrite();
-               mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
-               {
-                  shadow_ptr[c_shadow + input_size_s * gq] = 1.0;
-               });
-
-               yq = 0.0;
-               yq.SyncToBlocks();
-
-               if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
-               {
-                  detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t,
-                         inputs_t, outputs_t>(
-                            qfunc, qfunc_shadow, xq, shadow_xq, yq, gnqp,
-                            input_qlayouts, output_qlayouts,
-                            std::make_index_sequence<ninputs> {},
-                            std::make_index_sequence<noutputs> {});
-               }
-               else
-               {
-                  detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
-                     qfunc, xq, shadow_xq, yq, gnqp,
-                     input_qlayouts, output_qlayouts,
-                     std::make_index_sequence<ninputs> {},
-                     std::make_index_sequence<noutputs> {});
-               }
-
-               real_t *cache_d = qp_cache.ReadWrite();
-
-               // Write yq into the cache column
-               const int m_global = m + m_offset;
-               const int j_cur    = j;
-               int out_offset = 0;
-               constexpr_for<0, noutputs>([&](auto o)
-               {
-                  const int test_vdim_o   = out_vdim[o];
-                  const int test_op_dim_o = out_op_dim[o];
-                  const int yq_out_size   = test_vdim_o * test_op_dim_o;
-                  const int out_offset_o  = out_offset;
-                  const real_t *yq_d      = yq.GetBlock(o.value).Read();
-
-                  // The cache is (q, cache_idx, e) with the quadrature index
-                  // fastest, so gq is the fastest-varying thread index to keep
-                  // the stores coalesced.
-                  mfem::forall(gnqp_local * yq_out_size, [=] MFEM_HOST_DEVICE(int idx)
-                  {
-                     const int gq       = idx % gnqp_local;
-                     const int c_out    = idx / gnqp_local;
-                     const int q        = gq % num_qp_local;
-                     const int entity   = gq / num_qp_local;
-                     const int out_comp = out_offset_o + c_out;
-                     const int cache_idx =
-                        out_comp * trial_vdim_local * total_trial_op_dim_local +
-                        j_cur * total_trial_op_dim_local +
-                        m_global;
-                     cache_d[q + num_qp_local *
-                               (cache_idx + residual_size_local * entity)] =
-                                yq_d[c_out + yq_out_size * gq];
-                  });
-                  out_offset += yq_out_size;
-               });
-            }
-            m_offset += trial_op_dim_s;
-         });
-      }
-   }
-
-   IntegratorContext ctx;
-   qfunc_t qfunc;
-   qfunc_shadow_t qfunc_shadow;
-   inputs_t inputs;
-   outputs_t outputs;
-   Vector &qp_cache;
-
-   std::array<size_t, ninputs> input_to_infd;
-   std::array<size_t, noutputs> output_to_outfd;
-
-   std::array<FieldBasis, ninputs> input_bases;
-   std::array<FieldBasis, noutputs> output_bases;
-
-   std::array<std::vector<int>, ninputs>  input_qlayouts;
-   std::array<std::vector<int>, noutputs> output_qlayouts;
-
-   int gnqp = 0;
-   int num_qp = 0;
-   int nentities = 0;
-
-   Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
-   mutable BlockVector xq, shadow_xq, yq;
-
-   int total_out_size_on_qp = 0;
-   int trial_vdim = 0;
-   int total_trial_op_dim = 0;
-   int residual_size_on_qp = 0;
-
-   std::array<int, noutputs> out_vdim {};
-   std::array<int, noutputs> out_op_dim {};
-   std::array<int, ninputs>  input_size_on_qp_arr {};
-   std::array<bool, ninputs> activity_map {};
-};
-
-} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/prelude.hpp

@@ -1,219 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "action.hpp"
-#include "derivative_action.hpp"
-#include "derivative_setup.hpp"
-
-#include "derivative_apply_transpose.hpp"
-
-#include "../local_qf/derivative_apply.hpp"
-#include "../local_qf/derivative_assemble.hpp"
-#include "../local_qf/derivative_assemble_diagonal.hpp"
-
-#include "../scratch_bank.hpp"
-
-namespace mfem::future
-{
-
-namespace detail
-{
-
-template <typename T>
-struct LocalQFShapeArg
-{
-   using type = std::remove_const_t<T>&;
-};
-
-template <typename scalar_t, int ndims, int... tensor_sizes>
-struct LocalQFShapeArg<tensor_ndarray<scalar_t, ndims, tensor_sizes...>>
-{
-   using scalar_type = std::remove_const_t<scalar_t>;
-   using type = std::conditional_t<
-                sizeof...(tensor_sizes) == 0,
-                scalar_type,
-                tensor<scalar_type, tensor_sizes...>>&;
-};
-
-template <typename scalar_t, int... tensor_sizes>
-struct LocalQFShapeArg<tensor<scalar_t, tensor_sizes...>>
-{
-   using scalar_type = std::remove_const_t<scalar_t>;
-   using type = std::conditional_t<
-                sizeof...(tensor_sizes) == 0,
-                scalar_type,
-                tensor<scalar_type, tensor_sizes...>>&;
-};
-
-template <typename qf_param_ts>
-struct LocalQFShapeFunction;
-
-template <typename... qf_param_ts>
-struct LocalQFShapeFunction<tuple<qf_param_ts...>>
-{
-   void operator()(
-      typename LocalQFShapeArg<qf_param_decay_t<qf_param_ts>>::type...) const;
-};
-
-template <typename qfunc_t>
-using LocalQFShapeFunctionFor = LocalQFShapeFunction<
-                                typename get_function_signature<qfunc_t>::type::parameter_ts>;
-
-} // namespace detail
-
-struct GlobalQFBackend
-{
-   /**
-    * @brief Make an action for a global Q-function.
-    *
-    * @param ctx The integrator context.
-    * @param args The arguments to the action.
-    * @return The action.
-    */
-   template<
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeAction(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      return GlobalQFImpl::Action(ctx, qfunc, inputs, outputs);
-   }
-
-   /**
-    * @brief Make a derivative action for a global Q-function.
-    *
-    * @tparam derivative_id The id of the derivative.
-    * @param ctx The integrator context.
-    * @param args The arguments to the derivative action.
-    * @return The derivative action.
-    */
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeAction(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      return GlobalQFImpl::DerivativeAction<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs);
-   }
-
-   /**
-    * @brief Make a derivative setup for a global Q-function.
-    *
-    * @tparam derivative_id The id of the derivative.
-    * @param ctx The integrator context.
-    * @param args The arguments to the derivative setup.
-    * @return The derivative setup.
-    */
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeSetup(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      Vector &qp_cache)
-   {
-      return GlobalQFImpl::DerivativeSetup<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeApply(
-      const IntegratorContext &ctx,
-      const qfunc_t & /*qfunc*/,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeApply<
-             derivative_id,
-             detail::LocalQFShapeFunctionFor<qfunc_t>,
-             inputs_t,
-             outputs_t>(ctx,
-                        detail::LocalQFShapeFunctionFor<qfunc_t> {},
-                        inputs,
-                        outputs,
-                        qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeApplyTranspose(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return GlobalQFImpl::DerivativeApplyTranspose<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeAssemble(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeAssemble<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   auto static MakeDerivativeAssembleDiagonal(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeAssembleDiagonal<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs, qp_cache);
-   }
-};
-
-} // namespace mfem::future

fem/dfem/backends/local_qf/action.hpp

@@ -1,658 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../fieldoperator.hpp"
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include "../../integrator_ctx.hpp"
-#include "../util.hpp"
-
-#include <utility>
-
-namespace mfem::future::LocalQFImpl
-{
-
-template <typename outputs_t, typename index_seq>
-struct action_outputs_direct_impl;
-
-template <typename outputs_t, std::size_t... Is>
-struct action_outputs_direct_impl<outputs_t, std::index_sequence<Is...>>
-{
-   static constexpr bool value =
-      ((is_identity_fop_v<tuple_element_t<Is, outputs_t>> ||
-        is_functionalvalue_fop_v<tuple_element_t<Is, outputs_t>>) && ...);
-};
-
-template <typename outputs_t>
-constexpr bool action_outputs_direct_v = action_outputs_direct_impl<outputs_t,
-               std::make_index_sequence<tuple_size<outputs_t>::value>>::value;
-
-template <typename qfunc_t, typename inputs_t, typename outputs_t,
-          typename index_seq>
-struct action_outputs_direct_value_impl;
-
-template <typename qfunc_t, typename inputs_t, typename outputs_t,
-          std::size_t... Is>
-struct action_outputs_direct_value_impl<qfunc_t, inputs_t, outputs_t,
-          std::index_sequence<Is...>>
-{
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr bool value =
-      ((is_identity_fop_v<tuple_element_t<Is, outputs_t>> ||
-        is_functionalvalue_fop_v<tuple_element_t<Is, outputs_t>>) && ...)
-      && ((!qf_param_uses_dual_v<
-           typename qf_param_slot<qfunc_t, n_inputs + Is>::qf_decay_param_t>)
-          && ...);
-};
-
-template <typename qfunc_t, typename inputs_t, typename outputs_t>
-constexpr bool action_outputs_direct_value_v =
-   action_outputs_direct_value_impl<qfunc_t, inputs_t, outputs_t,
-   std::make_index_sequence<tuple_size<outputs_t>::value>>::value;
-
-template<typename qfunc_t, typename inputs_t, typename outputs_t>
-class Action
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-   using args_tuple_t = decay_tuple<qf_param_ts>;
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
-                 "LocalQF: q-function arity must match inputs + outputs");
-
-   template<typename backend_t, std::size_t I, typename RArgs, typename InXEs>
-   static MFEM_HOST_DEVICE decltype(auto) direct_input_arg(
-      RArgs &rargs,
-      const InXEs &in_XE,
-      const int qx,
-      const int qy,
-      const int qz,
-      const int e)
-   {
-      const auto &XE = in_XE[I];
-      using FOP = tuple_element_t<I, inputs_t>;
-      using ARG = typename qf_param_slot<qfunc_t, I>::qf_reg_param_t;
-      if constexpr (is_identity_fop_v<FOP>)
-      {
-         using DT = typename qf_param_slot<qfunc_t, I>::qf_decay_param_t;
-         if constexpr (qf_param_uses_dual_v<DT>)
-         {
-            return backend_t::template identity_qp_pull_dual<DT>(
-               false, XE, XE, qx, qy, qz, e);
-         }
-         else
-         {
-            return as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-         }
-      }
-      else if constexpr (is_weight_fop_v<FOP>)
-      {
-         return XE(qx, qy, qz, 0, 0);
-      }
-      else if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-      {
-         return backend_t::template qp_pull<ARG>(get<I>(rargs), qx, qy, qz);
-      }
-      else
-      {
-         static_assert(false, "Unsupported");
-      }
-   }
-
-   template<std::size_t I, typename OutYEs>
-   static MFEM_HOST_DEVICE decltype(auto) direct_output_arg(
-      const OutYEs &out_YE,
-      const int qx,
-      const int qy,
-      const int qz,
-      const int e)
-   {
-      constexpr size_t o = n_inputs + I;
-      const auto &YE = out_YE[I];
-      using DT = typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-      using ARG = typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
-      if constexpr (std::is_same_v<DT, real_t>)
-      {
-         return YE(0, qx, qy, qz, e);
-      }
-      else
-      {
-         return as_tensor<ARG>(&YE(0, qx, qy, qz, e));
-      }
-   }
-
-   template<typename backend_t, typename RArgs, typename InXEs, typename OutYEs,
-            std::size_t... InIs, std::size_t... OutIs>
-   static MFEM_HOST_DEVICE void call_qfunc_direct(
-      const qfunc_t &qfunc,
-      RArgs &rargs,
-      const InXEs &in_XE,
-      const OutYEs &out_YE,
-      const int qx,
-      const int qy,
-      const int qz,
-      const int e,
-      std::index_sequence<InIs...>,
-      std::index_sequence<OutIs...>)
-   {
-      qfunc(direct_input_arg<backend_t, InIs>(rargs, in_XE, qx, qy, qz, e)...,
-            direct_output_arg<OutIs>(out_YE, qx, qy, qz, e)...);
-   }
-
-   const qfunc_t qfunc;
-   const inputs_t inputs;
-   const outputs_t outputs;
-   const IntegratorContext ctx;
-   const std::vector<const DofToQuad *> dtqs;
-   // inputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_inputs> input_dtq;
-   const std::array<size_t, n_inputs> input_idx; // input to field
-   const std::array<const real_t *, n_inputs> input_B, input_G;
-   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
-   // outputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_outputs> output_dtq;
-   const std::array<size_t, n_outputs> output_idx; // output to field
-   const std::array<const real_t *, n_outputs> output_B, output_G;
-   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
-   // other constants
-   const int dim, ne, nq, q1d;
-
-public:
-   ////////////////////////////////////////////////////////
-   Action() = delete;
-
-   Action(IntegratorContext ctx,
-          qfunc_t qfunc,
-          inputs_t inputs,
-          outputs_t outputs):
-      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
-      dtqs(make_dtqs(ctx)),
-      // inputs: dtq, idx, B, G, d1d, q1d, vdim
-      input_dtq(create_dtq_maps<Entity::Element>(
-                   inputs,
-                   dtqs,
-                   create_union_field_map_for_dtq(ctx, inputs),
-                   ctx.unionfds,
-                   ctx.ir)),
-      input_idx(create_input_vector_map(ctx, inputs)),
-      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
-      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
-      input_vdim(get_vdim(inputs)),
-      // outputs: dtq, idx, B, G, d1d, q1d, vdim
-      output_dtq(create_dtq_maps<Entity::Element>(
-                    outputs,
-                    dtqs,
-                    create_union_field_map_for_dtq(ctx, outputs),
-                    ctx.unionfds,
-                    ctx.ir)),
-      output_idx(create_output_vector_map(ctx, outputs)),
-      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
-      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
-      output_vdim(get_vdim(outputs)),
-      // other constants
-      dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
-      q1d(tensor_1d_size(nq, dim))
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-   }
-
-   template<typename Backend>
-   void run_kernels(const std::vector<Vector *> &xe,
-                    std::vector<Vector *> &ye) const
-   {
-      Backend::Run(dim,
-                   q1d,
-                   // arguments
-                   ctx,
-                   qfunc,
-                   // inputs
-                   input_idx,
-                   input_B,
-                   input_G,
-                   input_vdim,
-                   input_d1d,
-                   input_q1d,
-                   // outputs
-                   output_idx,
-                   output_B,
-                   output_G,
-                   output_vdim,
-                   output_d1d,
-                   output_q1d,
-                   // input and output vectors
-                   xe,
-                   ye,
-                   // fallback arguments
-                   dim,
-                   q1d);
-   }
-
-   void operator()(const std::vector<Vector *> &xe,
-                   std::vector<Vector *> &ye) const
-   {
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<ActionLO>(xe, ye);
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<ActionHO>(xe, ye);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-   }
-
-   ////////////////////////////////////////////////////////
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void
-   action_callback(const IntegratorContext &ctx,
-                   const qfunc_t &qfunc,
-                   // inputs: idx, B, G, vdim, d1d, q1d
-                   const std::array<size_t, n_inputs> &in_idx,
-                   const std::array<const real_t *, n_inputs> in_B,
-                   const std::array<const real_t *, n_inputs> in_G,
-                   const std::array<int, n_inputs> &in_vdim,
-                   const std::array<int, n_inputs> &in_d1d,
-                   const std::array<int, n_inputs> &in_q1d,
-                   // outputs: idx, B, G, vdim, d1d, q1d
-                   const std::array<size_t, n_outputs> &out_idx,
-                   const std::array<const real_t *, n_outputs> out_B,
-                   const std::array<const real_t *, n_outputs> out_G,
-                   const std::array<int, n_outputs> &out_vdim,
-                   const std::array<int, n_outputs> &out_d1d,
-                   const std::array<int, n_outputs> &out_q1d,
-                   const std::vector<Vector *> &xe,
-                   std::vector<Vector *> &ye,
-                   // fallback arguments
-                   const int dim,
-                   const int q1d)
-   {
-      if (ctx.attr.Size() == 0) { return; }
-      MFEM_CONTRACT_VAR(dim);
-      MFEM_ASSERT(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-
-      static constexpr auto B2D = backend_t::DIM == 2;
-      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::Q1D;
-      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
-
-      const int ne = ctx.nentities;
-
-      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
-
-      // --------------------------------------------------
-      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = in_idx[i];
-         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else if constexpr (is_weight_fop_v<FOP>)
-         {
-            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
-                        "tensor-product IR expected");
-            in_XE[i] = Reshape(
-                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      // --------------------------------------------------
-      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
-      for_constexpr<n_outputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = out_idx[i];
-         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
-         using FOP = tuple_element_t<i, outputs_t>;
-         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
-         {
-            MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP> || is_functionalvalue_fop_v<FOP>)
-         {
-            MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else
-         {
-            static_assert(false, "Unsupported FieldOperator");
-         }
-      });
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      dfem::forall<MTPB>(
-         [=] MFEM_HOST_DEVICE(const int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         // -----------------------------------------------
-         // Inputs and outputs argument registers
-         // -----------------------------------------------
-         action_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
-
-         // -----------------------------------------------
-         // Shared memory
-         // -----------------------------------------------
-         MFEM_SHARED typename backend_t::Shared smem;
-
-         // -----------------------------------------------
-         // Load inputs
-         // -----------------------------------------------
-         for_constexpr<n_inputs>([&](auto ic)
-         {
-            constexpr size_t i = ic.value;
-            const auto &XE = in_XE[i];
-            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
-            ;
-            const real_t *B = in_B[i], *G = in_G[i];
-            auto &rarg = get<i>(rargs);
-            using FOP = tuple_element_t<i, inputs_t>;
-            if constexpr (is_value_fop<FOP>::value)
-            {
-               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               using XE_t = decltype(XE);
-               using rarg_t = decltype(rarg);
-               using qf_param_t =
-                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
-               backend_t::template LoadGradient<RNK, rarg_t, XE_t, qf_param_t>(
-                  smem, e, d, q, q1d, B, G, XE, rarg);
-            }
-            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
-            {
-               // qp values are read directly from in_XE / IR
-            }
-            else
-            {
-               static_assert(false, "Unsupported");
-            }
-         });
-
-         // -----------------------------------------------
-         // Evaluate the quadrature function
-         // Warning: no 'DIRECT' on the 'Z' direction,
-         // as one backend may need to iterate over it.
-         // -----------------------------------------------
-         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-               {
-                  if constexpr (action_outputs_direct_value_v<qfunc_t, inputs_t,
-                                outputs_t>)
-                  {
-                     call_qfunc_direct<backend_t>(
-                        qfunc, rargs, in_XE, out_YE, qx, qy, qz, e,
-                        std::make_index_sequence<n_inputs> {},
-                        std::make_index_sequence<n_outputs> {});
-                  }
-                  else
-                  {
-                     args_tuple_t qargs;
-
-                     // --------------------------------------
-                     // Pulling arguments from registers to qargs tuple
-                     // --------------------------------------
-                     for_constexpr<n_inputs>([&](auto ic)
-                     {
-                        constexpr size_t i = ic.value;
-                        auto &qarg = get<i>(qargs);
-                        const auto &XE = in_XE[i];
-                        using FOP = tuple_element_t<i, inputs_t>;
-                        using ARG =
-                           typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
-                        if constexpr (is_identity_fop_v<FOP>)
-                        {
-                           using DT =
-                              typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-                           if constexpr (qf_param_uses_dual_v<DT>)
-                           {
-                              qarg = backend_t::template identity_qp_pull_dual<DT>(
-                                 false, XE, XE, qx, qy, qz, e);
-                           }
-                           else
-                           {
-                              qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-                           }
-                        }
-                        else if constexpr (is_weight_fop_v<FOP>)
-                        {
-                           qarg = XE(qx, qy, qz, 0, 0);
-                        }
-                        else if constexpr (is_value_fop_v<FOP> ||
-                                           is_gradient_fop_v<FOP>)
-                        {
-                           qarg = backend_t::template qp_pull<ARG>(
-                              get<i>(rargs), qx, qy, qz);
-                        }
-                        else
-                        {
-                           static_assert(false, "Unsupported");
-                        }
-                     });
-
-                     // --------------------------------------
-                     // Call the quadrature function
-                     // --------------------------------------
-                     call_qfunc_no_move(qfunc, qargs);
-
-                     // --------------------------------------
-                     // Pushing arguments from qargs tuple to registers
-                     // --------------------------------------
-                     for_constexpr<n_outputs>([&](auto ic)
-                     {
-                        constexpr size_t i = ic.value, o = n_inputs + i;
-                        const auto &qarg = get<o>(qargs);
-                        const auto &YE = out_YE[i];
-                        using FOP = tuple_element_t<i, outputs_t>;
-                        using ARG =
-                           typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
-                        if constexpr (is_identity_fop_v<FOP> ||
-                                      is_functionalvalue_fop_v<FOP>)
-                        {
-                           using DT =
-                              typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-                           if constexpr (qf_param_uses_dual_v<DT>)
-                           {
-                              backend_t::identity_qp_write_value(
-                                 YE, qx, qy, qz, e, qarg);
-                           }
-                           else
-                           {
-                              as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
-                           }
-                        }
-                        else if constexpr (is_value_fop_v<FOP> ||
-                                           is_gradient_fop_v<FOP>)
-                        {
-                           auto &rarg = get<o>(rargs);
-                           backend_t::template qp_push<ARG>(
-                              rarg, qx, qy, qz, qarg);
-                        }
-                        else
-                        {
-                           static_assert(false, "Unsupported");
-                        }
-                     });
-                  }
-               }
-            }
-         }
-         if constexpr (!action_outputs_direct_v<outputs_t>)
-         {
-            MFEM_SYNC_THREAD;
-         }
-
-         // -----------------------------------------------
-         // Integrate outputs
-         // -----------------------------------------------
-         if constexpr (!action_outputs_direct_v<outputs_t>)
-         {
-            for_constexpr<n_outputs>([&](auto ic)
-            {
-               constexpr size_t i = ic.value, o = n_inputs + i;
-               const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
-               const auto B = out_B[i], G = out_G[i];
-               const auto &YE = out_YE[i];
-               auto &rarg = get<o>(rargs);
-               using FOP = tuple_element_t<i, outputs_t>;
-               if constexpr (is_value_fop_v<FOP>)
-               {
-                  backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
-               }
-               else if constexpr (is_gradient_fop_v<FOP>)
-               {
-                  using YE_t = decltype(YE);
-                  using rarg_t = decltype(rarg);
-                  // Both the rank and the extents have to come from the output
-                  // parameter slot o, not from the output index i.
-                  using qf_param_t =
-                     typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-                  constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
-                  backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
-                     smem, e, d, q, Q1D, B, G, YE, rarg);
-               }
-               else if constexpr (is_identity_fop_v<FOP> ||
-                                  is_functionalvalue_fop_v<FOP>)
-               {
-                  // nothing to do
-               }
-               else
-               {
-                  static_assert(false, "Unsupported");
-               }
-            });
-         }
-      },
-      ne,
-      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
-                                  q1d, in_d1d, out_d1d)),
-      0,
-      nullptr);
-   }
-   using KernelType = decltype(&Action::action_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionLO, KernelType, (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionHO, KernelType, (int, int) );
-};
-
-// Low Order kernels
-template<typename qfunc_t, typename inputs_t, typename outputs_t>
-template<int DIM, int Q1D>
-inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
-Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Kernel()
-{
-   static_assert(Q1D <= LocalQFLOBackend<DIM>::MQ1);
-   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
-   if constexpr (DIM == 3 && Q1D == LocalQFLOBackendMQ1() &&
-                 action_outputs_direct_value_v<qfunc_t, inputs_t, outputs_t>)
-   {
-      return action_t::template action_callback<LocalQFLOBackend<DIM, Q1D, Q1D / 2>>;
-   }
-   else
-   {
-      return action_t::template action_callback<LocalQFLOBackend<DIM, Q1D>>;
-   }
-}
-
-// Low Order fallback
-template<typename qfunc_t, typename inputs_t, typename outputs_t>
-inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
-Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Fallback(int dim, int q1d)
-{
-   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
-   using ActionLO = typename action_t::ActionLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<ActionLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<ActionLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-// High Order kernels
-template<typename qfunc_t, typename inputs_t, typename outputs_t>
-template<int DIM, int Q1D>
-inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
-Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Kernel()
-{
-   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
-   return action_t::template action_callback<LocalQFHOBackend<DIM>, Q1D>;
-}
-
-// High Order fallback
-template<typename qfunc_t, typename inputs_t, typename outputs_t>
-inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
-Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Fallback(int dim, int q1d)
-{
-   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
-   using ActionHO = typename action_t::ActionHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<ActionHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<ActionHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_action.hpp

@@ -1,926 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include "../util.hpp"
-
-namespace mfem::future::LocalQFImpl
-{
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-class DerivativeAction
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-   using args_tuple_t = decay_tuple<qf_param_ts>;
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
-                 "LocalQF: q-function arity must match inputs + outputs");
-
-   /// Which inputs carry a tangent, i.e. are attached to the field being
-   /// differentiated against. This is a property of `inputs_t` and
-   /// `derivative_id` alone, so it is available at compile time: it decides the
-   /// Enzyme activity of every q-function parameter, which loads the tangent
-   /// pass has to do, and how large the shadow register bank has to be. The
-   /// runtime `input_is_dependent` below holds the same values and is kept for
-   /// the host-side sizing checks.
-   static constexpr auto input_activity =
-      mfem::future::detail::make_activity_map <
-   static_cast<std::size_t>(derivative_id) > (inputs_t {});
-   static_assert(input_activity.size() == n_inputs);
-
-   /// Shadow register bank: only the active input slots are materialized.
-   template <typename backend_t, int MQ1, std::size_t... Is>
-   static auto shadow_bank_type(std::index_sequence<Is...>)
-#ifdef MFEM_USE_ENZYME
-   -> masked_input_args_reg_t<backend_t, qfunc_t, MQ1, input_activity[Is]...>;
-#else
-   // The dual-number path pulls through every input slot unconditionally.
-   -> input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1>;
-#endif
-
-   template <typename backend_t, int MQ1>
-   using shadow_bank_t = decltype(shadow_bank_type<backend_t, MQ1>(
-   std::make_index_sequence<n_inputs> {}));
-
-   /// Per-quadrature-point shadow argument tuple. Only the active inputs and
-   /// the outputs are materialized where supported; the rest are `enzyme_const`
-   /// and their shadow slots are never addressed. This is the innermost live
-   /// state of the kernel, and on device it shares a per-thread register budget
-   /// capped by the launch bounds, so the dead slots are worth removing
-   /// explicitly rather than hoping the optimizer splits the tuple.
-#ifdef _MSC_VER
-   using shadow_args_t = args_tuple_t;
-#else
-   template <std::size_t... Is>
-   static auto shadow_tuple_type(std::index_sequence<Is...>)
-   -> masked_args_tuple_t < args_tuple_t,
-   (Is<n_inputs ? input_activity[Is] : true)... >;
-
-   using shadow_args_t = decltype(shadow_tuple_type(
-   std::make_index_sequence<n_inputs + n_outputs> {}));
-#endif
-
-#ifdef MFEM_USE_ENZYME
-   /// Forward-mode call with the activity of every q-function parameter fixed
-   /// at compile time. Outputs are always active; inputs follow
-   /// `input_activity`, so an inactive input (the mesh nodes and the quadrature
-   /// weight, for a derivative w.r.t. the trial field) is marked `enzyme_const`
-   /// rather than dup'd with a zero tangent. Without this Enzyme differentiates
-   /// everything those inputs feed - for a diffusion q-function the whole
-   /// inv(J) / det(J) chain - to produce a tangent that is structurally zero.
-   // `qf_t` is deduced because the kernel captures the q-function by value into
-   // a const lambda, so it arrives here as `const qfunc_t`.
-   template <typename qf_t, std::size_t... Is>
-   MFEM_FUTURE_ALWAYS_INLINE
-   MFEM_HOST_DEVICE static void call_fwddiff(qf_t &qfunc,
-                                             args_tuple_t &primal_args,
-                                             shadow_args_t &shadow_args,
-                                             std::index_sequence<Is...>)
-   {
-      mfem::future::call_enzyme_fwddiff_active <
-      (Is < n_inputs ? input_activity[Is] : true)... > (
-         qfunc, primal_args, shadow_args);
-   }
-#endif
-
-   qfunc_t qfunc;
-   const inputs_t inputs;
-   const outputs_t outputs;
-   const IntegratorContext ctx;
-   const std::vector<const DofToQuad *> dtqs;
-   // inputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_inputs> input_dtq;
-   const std::array<size_t, n_inputs> input_idx;
-   const std::array<const real_t *, n_inputs> input_B, input_G;
-   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
-   // outputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_outputs> output_dtq;
-   const std::array<size_t, n_outputs> output_idx;
-   const std::array<const real_t *, n_outputs> output_B, output_G;
-   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
-   // other constants
-   const int dim, ne, nq, q1d;
-
-   std::array<bool, n_inputs> input_is_dependent;
-   FieldDescriptor direction_fd;
-   mutable Vector direction_e;
-   mutable RestrictionCache<Entity::Element> direction_rcache;
-
-public:
-   //////////////////////////////////////////////////////////////////
-   DerivativeAction() = delete;
-
-   DerivativeAction(IntegratorContext ctx,
-                    qfunc_t qfunc,
-                    inputs_t inputs,
-                    outputs_t outputs):
-      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
-      dtqs(make_dtqs(ctx)),
-      // inputs: dtq, idx, B, G, d1d, q1d, vdim
-      input_dtq(create_dtq_maps<Entity::Element>(
-                   inputs,
-                   dtqs,
-                   create_union_field_map_for_dtq(ctx, inputs),
-                   ctx.unionfds,
-                   ctx.ir)),
-      input_idx(create_input_vector_map(ctx, inputs)),
-      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
-      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
-      input_vdim(get_vdim(inputs)),
-      // outputs: dtq, idx, B, G, d1d, q1d, vdim
-      output_dtq(create_dtq_maps<Entity::Element>(
-                    outputs,
-                    dtqs,
-                    create_union_field_map_for_dtq(ctx, outputs),
-                    ctx.unionfds,
-                    ctx.ir)),
-      output_idx(create_output_vector_map(ctx, outputs)),
-      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
-      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
-      output_vdim(get_vdim(outputs)),
-      // other constants
-      dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
-      q1d(tensor_1d_size(nq, dim))
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-
-      // Determine which inputs are dependent on the derivative direction
-      auto dependency_map = make_dependency_map(inputs);
-      auto it = dependency_map.find(derivative_id);
-      MFEM_ASSERT(it != dependency_map.end(),
-                  "Derivative ID not found in dependency map");
-      input_is_dependent = it->second;
-
-      // Find direction field index
-      int direction_field_idx = -1;
-      for (size_t uf = 0; uf < nfields; uf++)
-      {
-         if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
-         {
-            direction_field_idx = static_cast<int>(uf);
-            break;
-         }
-      }
-      MFEM_ASSERT(
-         direction_field_idx != -1,
-         "LocalQFBackend: derivative direction field not found in unionfds");
-      direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename Backend>
-   void run_kernels(const std::vector<Vector *> &xe,
-                    std::vector<Vector *> &ye)
-   {
-      Backend::Run(dim,
-                   q1d,
-                   // arguments
-                   ctx,
-                   qfunc,
-                   // inputs
-                   input_idx,
-                   input_B,
-                   input_G,
-                   input_vdim,
-                   input_d1d,
-                   input_q1d,
-                   // outputs
-                   output_idx,
-                   output_B,
-                   output_G,
-                   output_vdim,
-                   output_d1d,
-                   output_q1d,
-                   // input and output vectors
-                   xe,
-                   ye,
-                   input_is_dependent,
-                   direction_e,
-                   // fallback arguments
-                   dim,
-                   q1d);
-   }
-
-   //////////////////////////////////////////////////////////////////
-   void operator()(const std::vector<Vector *> &xe,
-                   const Vector *direction_l,
-                   std::vector<Vector *> &ye)
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      MFEM_ASSERT(direction_l != nullptr,
-                  "LocalQF DerivativeAction: direction vector is null");
-
-      restriction(direction_fd,
-                  direction_rcache,
-                  *direction_l,
-                  direction_e,
-                  ElementDofOrdering::LEXICOGRAPHIC);
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<DerivativeActionLO>(xe, ye);
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<DerivativeActionHO>(xe, ye);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-   }
-
-
-   template<typename backend_t, int T_Q1D>
-   struct DerivativeActionKernelData
-   {
-      qfunc_t qfunc;
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
-      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
-      std::array<const real_t *, n_inputs> in_B;
-      std::array<const real_t *, n_inputs> in_G;
-      std::array<int, n_inputs> in_d1d;
-      std::array<int, n_inputs> in_q1d;
-      std::array<const real_t *, n_outputs> out_B;
-      std::array<const real_t *, n_outputs> out_G;
-      std::array<int, n_outputs> out_d1d;
-      std::array<int, n_outputs> out_q1d;
-      std::array<bool, n_inputs> input_dep;
-      const int *d_attr;
-      bool has_attr;
-      const int *d_elem_attr;
-      int q1d;
-   };
-
-   template<typename backend_t, int T_Q1D>
-   MFEM_FUTURE_ALWAYS_INLINE MFEM_HOST_DEVICE static void
-   derivative_action_kernel_body(
-      const DerivativeActionKernelData<backend_t, T_Q1D> &data, const int e)
-   {
-      static constexpr auto B2D = backend_t::DIM == 2;
-      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
-      auto &qfunc = data.qfunc;
-      const auto &in_XE = data.in_XE;
-      const auto &in_XE_dir = data.in_XE_dir;
-      const auto &out_YE = data.out_YE;
-      const auto &in_B = data.in_B;
-      const auto &in_G = data.in_G;
-      const auto &in_d1d = data.in_d1d;
-      const auto &in_q1d = data.in_q1d;
-      const auto &out_B = data.out_B;
-      const auto &out_G = data.out_G;
-      const auto &out_d1d = data.out_d1d;
-      const auto &out_q1d = data.out_q1d;
-      const auto &input_dep = data.input_dep;
-      MFEM_CONTRACT_VAR(input_dep);
-      const auto d_attr = data.d_attr;
-      const bool has_attr = data.has_attr;
-      const auto d_elem_attr = data.d_elem_attr;
-      const int q1d = data.q1d;
-      if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-      // -----------------------------------------------
-      // Inputs and outputs argument registers
-      // -----------------------------------------------
-      args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
-      shadow_bank_t<backend_t, MQ1> sargs; // shadow, active inputs only
-
-      // -----------------------------------------------
-      // Shared memory
-      // -----------------------------------------------
-      MFEM_SHARED typename backend_t::Shared smem;
-
-      // -----------------------------------------------
-      // Load primal inputs (rargs)
-      // -----------------------------------------------
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const auto &XE = in_XE[i];
-         const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
-         const real_t *B = in_B[i], *G = in_G[i];
-         auto &rarg = get<i>(rargs);
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (is_value_fop<FOP>::value)
-         {
-            backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
-         }
-         else if constexpr (is_gradient_fop_v<FOP>)
-         {
-            constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
-            using FieldParamT =
-               typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-            backend_t::template LoadGradient<RNK,
-                                             decltype(rarg),
-                                             decltype(XE),
-                                             FieldParamT>(
-                                                smem, e, d, q, Q1D, B, G, XE, rarg);
-         }
-         else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
-         {
-            // qp values are read directly from in_XE / IR
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      // -----------------------------------------------
-      // Load tangent directions (sargs)
-      // -----------------------------------------------
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (input_activity[i] &&
-                       (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>))
-         {
-            const auto &XE = in_XE_dir[i];
-            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
-            const real_t *B = in_B[i], *G = in_G[i];
-            auto &sarg = get<i>(sargs); // shadow argument register
-            if constexpr (is_value_fop_v<FOP>)
-            {
-               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
-            }
-            else
-            {
-               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
-               using FieldParamT =
-                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-               backend_t::template LoadGradient<RNK,
-                                                decltype(sarg),
-                                                decltype(XE),
-                                                FieldParamT>(
-                                                   smem, e, d, q, Q1D, B, G, XE, sarg);
-            }
-         }
-         else
-         {
-            // Inactive input, or an input read straight from quadrature
-            // point data (weight / identity): nothing to interpolate.
-            static_assert(!input_activity[i] || is_weight_fop_v<FOP> ||
-                          is_identity_fop_v<FOP>, "Unsupported");
-         }
-      });
-
-      // -----------------------------------------------
-      // Evaluate the quadrature function
-      // Warning: no 'DIRECT' on the 'Z' direction,
-      // as one backend may need to iterate over it.
-      // -----------------------------------------------
-      MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-            {
-#ifdef MFEM_USE_ENZYME
-               args_tuple_t primal_args {};
-               shadow_args_t shadow_args {};
-
-               // --------------------------------------
-               // Pulling arguments from registers to primal and shadow
-               // tuples
-               // --------------------------------------
-               for_constexpr<n_inputs>([&](auto ic)
-               {
-                  constexpr size_t i = ic.value;
-                  auto &parg = get<i>(primal_args);
-                  auto &targ = get<i>(shadow_args);
-                  const auto &XE = in_XE[i];
-                  const auto &XEd = in_XE_dir[i];
-                  using FOP = tuple_element_t<i, inputs_t>;
-                  using ARG =
-                     typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
-                  MFEM_CONTRACT_VAR(targ);
-                  MFEM_CONTRACT_VAR(XEd);
-                  if constexpr (is_identity_fop_v<FOP>)
-                  {
-                     parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-                     if constexpr (input_activity[i])
-                     {
-                        targ = as_tensor<ARG>(&XEd(0, qx, qy, qz, e));
-                     }
-                  }
-                  else if constexpr (is_weight_fop_v<FOP>)
-                  {
-                     parg = XE(qx, qy, qz, 0, 0);
-                  }
-                  else if constexpr (is_value_fop_v<FOP> ||
-                                     is_gradient_fop_v<FOP>)
-                  {
-                     parg = backend_t::template qp_pull<ARG>(
-                        get<i>(rargs), qx, qy, qz);
-                     if constexpr (input_activity[i])
-                     {
-                        targ = backend_t::template qp_pull<ARG>(
-                           get<i>(sargs), qx, qy, qz);
-                     }
-                  }
-                  else
-                  {
-                     static_assert(false, "Unsupported");
-                  }
-               });
-
-               // --------------------------------------
-               // Call the quadrature function. Inactive inputs are
-               // enzyme_const, so their shadow slots are never read and are
-               // deliberately left unset above.
-               // --------------------------------------
-               call_fwddiff(qfunc, primal_args, shadow_args,
-                            std::make_index_sequence<n_inputs + n_outputs> {});
-
-               // --------------------------------------
-               // Pushing arguments from enzyme_shadow tuple to registers
-               // --------------------------------------
-               for_constexpr<n_outputs>([&](auto ic)
-               {
-                  constexpr size_t i = ic.value, o = n_inputs + i;
-                  const auto &qout = get<o>(shadow_args);
-                  auto &YE = out_YE[i];
-                  using FOP = tuple_element_t<i, outputs_t>;
-                  using ARG =
-                     typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
-                  if constexpr (is_identity_fop_v<FOP>)
-                  {
-                     as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qout;
-                  }
-                  else if constexpr (is_value_fop_v<FOP> ||
-                                     is_gradient_fop_v<FOP>)
-                  {
-                     auto &rarg = get<o>(rargs);
-                     backend_t::template qp_push_tangent<ARG>(
-                        rarg, qx, qy, qz, qout);
-                  }
-                  else
-                  {
-                     static_assert(false, "Unsupported");
-                  }
-               });
-#else  // MFEM_USE_ENZYME
-               args_tuple_t qargs;
-
-               // --------------------------------------
-               // Pulling arguments from registers to qargs tuple
-               // --------------------------------------
-               for_constexpr<n_inputs>([&](auto ic)
-               {
-                  constexpr size_t i = ic.value;
-                  auto &qarg = get<i>(qargs);
-                  const auto &XE = in_XE[i];
-                  const auto &XEd = in_XE_dir[i];
-                  using FOP = tuple_element_t<i, inputs_t>;
-                  using ARG =
-                     typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
-                  if constexpr (is_identity_fop_v<FOP>)
-                  {
-                     using DT =
-                        typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-                     if constexpr (qf_param_uses_dual_v<DT>)
-                     {
-                        qarg = backend_t::template identity_qp_pull_dual<DT>(
-                           input_dep[i], XE, XEd, qx, qy, qz, e);
-                     }
-                     else
-                     {
-                        qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-                     }
-                  }
-                  else if constexpr (is_weight_fop_v<FOP>)
-                  {
-                     qarg = XE(qx, qy, qz, 0, 0);
-                  }
-                  else if constexpr (is_value_fop_v<FOP> ||
-                                     is_gradient_fop_v<FOP>)
-                  {
-                     qarg = backend_t::template qp_pull_directional<ARG>(
-                        get<i>(rargs),
-                        get<i>(sargs),
-                        qx,
-                        qy,
-                        qz,
-                        input_dep[i]);
-                  }
-                  else
-                  {
-                     static_assert(false, "Unsupported");
-                  }
-               });
-
-               // --------------------------------------
-               // Call the quadrature function
-               // --------------------------------------
-               call_qfunc_no_move(qfunc, qargs);
-
-               // --------------------------------------
-               // Pushing arguments from qargs tuple to registers
-               // --------------------------------------
-               for_constexpr<n_outputs>([&](auto ic)
-               {
-                  constexpr size_t i = ic.value, o = n_inputs + i;
-                  const auto &qarg = get<o>(qargs);
-                  const auto &YE = out_YE[i];
-                  using FOP = tuple_element_t<i, outputs_t>;
-                  using ARG =
-                     typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
-                  if constexpr (is_identity_fop_v<FOP>)
-                  {
-                     using DT =
-                        typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-                     if constexpr (qf_param_uses_dual_v<DT>)
-                     {
-                        backend_t::identity_qp_write_tangent(
-                           YE, qx, qy, qz, e, qarg);
-                     }
-                     else
-                     {
-                        as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
-                     }
-                  }
-                  else if constexpr (is_value_fop_v<FOP> ||
-                                     is_gradient_fop_v<FOP>)
-                  {
-                     auto &rarg = get<o>(rargs);
-                     backend_t::template qp_push_tangent<ARG>(
-                        rarg, qx, qy, qz, qarg);
-                  }
-                  else
-                  {
-                     static_assert(false, "Unsupported");
-                  }
-               });
-#endif // MFEM_USE_ENZYME
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // -----------------------------------------------
-      // Integrate outputs
-      // -----------------------------------------------
-      for_constexpr<n_outputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value, o = n_inputs + i;
-         const int d = out_d1d[i], q = out_q1d[i];
-         const auto B = out_B[i], G = out_G[i];
-         auto &YE = out_YE[i];
-         auto &rarg = get<o>(rargs);
-         using FOP = tuple_element_t<i, outputs_t>;
-         if constexpr (is_value_fop_v<FOP>)
-         {
-            backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
-         }
-         else if constexpr (is_gradient_fop_v<FOP>)
-         {
-            using YE_t = decltype(YE);
-            using rarg_t = decltype(rarg);
-            using qf_param_t =
-               typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-            constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
-            backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
-               smem, e, d, q, q1d, B, G, YE, rarg);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            // nothing to do
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-   }
-
-   template<typename backend_t, int T_Q1D>
-   struct DerivativeActionKernelBody
-   {
-      MFEM_FUTURE_ALWAYS_INLINE MFEM_HOST_DEVICE static void run(
-         const DerivativeActionKernelData<backend_t, T_Q1D> &data, const int e)
-      {
-         derivative_action_kernel_body<backend_t, T_Q1D>(data, e);
-      }
-   };
-
-   //////////////////////////////////////////////////////////////////
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void
-   derivative_action_callback(const IntegratorContext &ctx,
-                              qfunc_t &qfunc,
-                              // inputs: idx, B, G, vdim, d1d, q1d
-                              const std::array<size_t, n_inputs> &in_idx,
-                              const std::array<const real_t *, n_inputs> in_B,
-                              const std::array<const real_t *, n_inputs> in_G,
-                              const std::array<int, n_inputs> &in_vdim,
-                              const std::array<int, n_inputs> &in_d1d,
-                              const std::array<int, n_inputs> &in_q1d,
-                              // outputs: idx, B, G, vdim, d1d, q1d
-                              const std::array<size_t, n_outputs> &out_idx,
-                              const std::array<const real_t *, n_outputs> out_B,
-                              const std::array<const real_t *, n_outputs> out_G,
-                              const std::array<int, n_outputs> &out_vdim,
-                              const std::array<int, n_outputs> &out_d1d,
-                              const std::array<int, n_outputs> &out_q1d,
-                              const std::vector<Vector *> &xe,
-                              std::vector<Vector *> &ye,
-                              const std::array<bool, n_inputs> &input_dep,
-                              const Vector &direction_e,
-                              // fallback arguments
-                              const int dim,
-                              const int q1d)
-   {
-      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-      // Dependency is resolved at compile time through `input_activity`; the
-      // runtime array is only carried for the non-Enzyme dual-number path.
-      MFEM_CONTRACT_VAR(input_dep);
-
-      if (ctx.attr.Size() == 0) { return; }
-
-      static constexpr auto B2D = backend_t::DIM == 2;
-
-      const int ne = ctx.nentities;
-
-      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
-
-      // --------------------------------------------------
-      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = in_idx[i];
-         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else if constexpr (is_weight_fop_v<FOP>)
-         {
-            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
-                        "tensor-product IR expected");
-            in_XE[i] = Reshape(
-                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      const auto d_direction = direction_e.Read();
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = in_idx[i];
-         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            if constexpr (input_activity[i])
-            {
-               MFEM_ASSERT(direction_e.Size() == xe[k]->Size(),
-                           "direction E-vector size mismatch for input " << i);
-               in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
-            }
-            else
-            {
-               in_XE_dir[i] = in_XE[i];
-            }
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            if constexpr (input_activity[i])
-            {
-               MFEM_VERIFY(direction_e.Size() == xe[k]->Size(),
-                           "direction E-vector size mismatch (identity input) "
-                           << i);
-               in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
-            }
-            else
-            {
-               in_XE_dir[i] = in_XE[i];
-            }
-         }
-         else if constexpr (is_weight_fop_v<FOP>) { in_XE_dir[i] = in_XE[i]; }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      // --------------------------------------------------
-      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
-      for_constexpr<n_outputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = out_idx[i];
-         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
-         using FOP = tuple_element_t<i, outputs_t>;
-         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
-         {
-            MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else
-         {
-            static_assert(false, "Unsupported FieldOperator");
-         }
-      });
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      DerivativeActionKernelData<backend_t, T_Q1D> data
-      {
-         qfunc,
-         in_XE,
-         in_XE_dir,
-         out_YE,
-         in_B,
-         in_G,
-         in_d1d,
-         in_q1d,
-         out_B,
-         out_G,
-         out_d1d,
-         out_q1d,
-         input_dep,
-         d_attr,
-         has_attr,
-         d_elem_attr,
-         q1d
-      };
-
-      const auto blocks = backend_t::thread_blocks(
-                             compute_kernel_thread_1d<inputs_t, outputs_t>(q1d, in_d1d, out_d1d));
-
-      if (Device::Allows(Backend::CUDA_MASK) ||
-          Device::Allows(Backend::HIP_MASK))
-      {
-         dfem::forall_data<backend_t::MAX_THREADS_PER_BLOCK(),
-              DerivativeActionKernelBody<backend_t, T_Q1D>>(
-                 data, ne, blocks);
-      }
-      else if (Device::Allows(Backend::CPU_MASK))
-      {
-         for (int e = 0; e < ne; e++)
-         {
-            derivative_action_kernel_body<backend_t, T_Q1D>(data, e);
-         }
-      }
-      else
-      {
-         MFEM_ABORT("no compute backend available");
-      }
-   }
-
-   using DerivativeKernelType =
-      decltype(&DerivativeAction::derivative_action_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionLO,
-                                     DerivativeKernelType,
-                                     (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionHO,
-                                     DerivativeKernelType,
-                                     (int, int) );
-};
-
-// Low Order kernels
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeKernelType
-DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeActionLO::Kernel()
-{
-   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
-   using derivative_action_t =
-      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return derivative_action_t::template derivative_action_callback<
-             LocalQFLOBackend<DIM, Q1D>>;
-}
-
-// Low Order fallback
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeKernelType
-DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeActionLO::Fallback(int dim, int q1d)
-{
-   using derivative_action_t =
-      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeActionLO =
-      typename derivative_action_t::DerivativeActionLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<DerivativeActionLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<DerivativeActionLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-// High Order kernels
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeKernelType
-DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeActionHO::Kernel()
-{
-   using derivative_action_t =
-      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return derivative_action_t::
-          template derivative_action_callback<LocalQFHOBackend<DIM>, Q1D>;
-}
-
-// High Order fallback
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeKernelType
-DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeActionHO::Fallback(int dim, int q1d)
-{
-   using derivative_action_t =
-      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeActionHO = typename derivative_action_t::DerivativeActionHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<DerivativeActionHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<DerivativeActionHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_apply.hpp

@@ -1,738 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include <array>
-
-namespace mfem::future::LocalQFImpl
-{
-
-// Cached Jacobian apply: J·v from qp_cache filled by DerivativeSetup
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-class DerivativeApply
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-   using args_tuple_t = decay_tuple<qf_param_ts>;
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
-                 "LocalQF: q-function arity must match inputs + outputs");
-
-   const inputs_t inputs;
-   const outputs_t outputs;
-   const IntegratorContext ctx;
-   const Vector &qp_cache;
-   const std::vector<const DofToQuad *> dtqs;
-   // inputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_inputs> input_dtq;
-   const std::array<size_t, n_inputs> input_idx;
-   const std::array<const real_t *, n_inputs> input_B, input_G;
-   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
-   // outputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_outputs> output_dtq;
-   const std::array<size_t, n_outputs> output_idx;
-   const std::array<const real_t *, n_outputs> output_B, output_G;
-   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
-   // Jacobian cache metadata
-   const std::array<bool, n_inputs> input_is_dependent;
-   const std::array<int, n_inputs> input_size_on_qp;
-   const std::array<int, n_outputs> out_op_dim;
-   const std::array<int, n_outputs> out_offsets;
-   const int output_size_on_qp;
-   const int trial_vdim;
-   const int total_trial_op_dim;
-   const int residual_size_on_qp;
-   // other constants
-   const int dim, ne, nq, q1d;
-   FieldDescriptor direction_fd;
-   mutable Vector direction_e;
-   mutable RestrictionCache<Entity::Element> direction_rcache;
-
-   template <std::size_t slot>
-   static constexpr int ParamRank()
-   {
-      using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
-      return qf_param_shape<param_t>::rank;
-   }
-
-   template <std::size_t slot, int dim_idx>
-   static constexpr int ParamExtent()
-   {
-      using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
-      return qf_param_shape<param_t>::extents[dim_idx];
-   }
-
-   template <typename fop_t, std::size_t slot>
-   static constexpr int StaticVDim()
-   {
-      constexpr int rank = ParamRank<slot>();
-      if constexpr (is_gradient_fop_v<fop_t>)
-      {
-         if constexpr (rank <= 1) { return 1; }
-         else { return ParamExtent<slot, 0>(); }
-      }
-      else
-      {
-         if constexpr (rank == 0) { return 1; }
-         else { return ParamExtent<slot, 0>(); }
-      }
-   }
-
-   template <typename fop_t, std::size_t slot>
-   static constexpr int StaticOpDim()
-   {
-      constexpr int rank = ParamRank<slot>();
-      if constexpr (is_gradient_fop_v<fop_t>)
-      {
-         if constexpr (rank == 0) { return 1; }
-         else if constexpr (rank == 1) { return ParamExtent<slot, 0>(); }
-         else { return ParamExtent<slot, 1>(); }
-      }
-      else
-      {
-         if constexpr (rank <= 1) { return 1; }
-         else { return ParamExtent<slot, 1>(); }
-      }
-   }
-
-   template <std::size_t input_slot>
-   static constexpr bool StaticInputDep()
-   {
-      using fop_t = tuple_element_t<input_slot, inputs_t>;
-      return fop_t::GetFieldId() == derivative_id;
-   }
-
-   template <std::size_t input_slot>
-   static constexpr int StaticInputVDim()
-   {
-      using fop_t = tuple_element_t<input_slot, inputs_t>;
-      return StaticVDim<fop_t, input_slot>();
-   }
-
-   template <std::size_t input_slot>
-   static constexpr int StaticInputOpDim()
-   {
-      using fop_t = tuple_element_t<input_slot, inputs_t>;
-      return StaticOpDim<fop_t, input_slot>();
-   }
-
-   template <std::size_t output_slot>
-   static constexpr int StaticOutputVDim()
-   {
-      using fop_t = tuple_element_t<output_slot, outputs_t>;
-      return StaticVDim<fop_t, n_inputs + output_slot>();
-   }
-
-   template <std::size_t output_slot>
-   static constexpr int StaticOutputOpDim()
-   {
-      using fop_t = tuple_element_t<output_slot, outputs_t>;
-      return StaticOpDim<fop_t, n_inputs + output_slot>();
-   }
-
-   template <std::size_t output_slot>
-   static constexpr int StaticOutputOffset()
-   {
-      int offset = 0;
-      for_constexpr<output_slot>([&](auto oc)
-      {
-         constexpr size_t o = oc.value;
-         offset += StaticOutputVDim<o>() * StaticOutputOpDim<o>();
-      });
-      return offset;
-   }
-
-   template <std::size_t input_slot>
-   static constexpr int StaticInputOpOffset()
-   {
-      int offset = 0;
-      for_constexpr<input_slot>([&](auto sc)
-      {
-         constexpr size_t s = sc.value;
-         if constexpr (StaticInputDep<s>()) { offset += StaticInputOpDim<s>(); }
-      });
-      return offset;
-   }
-
-   static constexpr int StaticTrialVDim()
-   {
-      int vdim = 1;
-      for_constexpr<n_inputs>([&](auto sc)
-      {
-         constexpr size_t s = sc.value;
-         if constexpr (StaticInputDep<s>()) { vdim = StaticInputVDim<s>(); }
-      });
-      return vdim;
-   }
-
-   static constexpr int StaticTotalTrialOpDim()
-   {
-      int op_dim = 0;
-      for_constexpr<n_inputs>([&](auto sc)
-      {
-         constexpr size_t s = sc.value;
-         if constexpr (StaticInputDep<s>()) { op_dim += StaticInputOpDim<s>(); }
-      });
-      return op_dim;
-   }
-
-public:
-   DerivativeApply() = delete;
-
-   DerivativeApply(IntegratorContext ctx,
-                   qfunc_t /*qfunc*/,
-                   inputs_t inputs,
-                   outputs_t outputs,
-                   const Vector &qp_cache_in):
-      inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache_in),
-      dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
-                                         inputs,
-                                         dtqs,
-                                         create_union_field_map_for_dtq(ctx, inputs),
-                                         ctx.unionfds,
-                                         ctx.ir)),
-      input_idx(create_input_vector_map(ctx, inputs)),
-      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
-      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
-      input_vdim(get_vdim(inputs)),
-      output_dtq(create_dtq_maps<Entity::Element>(
-                    outputs,
-                    dtqs,
-                    create_union_field_map_for_dtq(ctx, outputs),
-                    ctx.unionfds,
-                    ctx.ir)),
-      output_idx(create_output_vector_map(ctx, outputs)),
-      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
-      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
-      output_vdim(get_vdim(outputs)),
-      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
-      input_size_on_qp(
-         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
-                           out_op_dim(compute_out_op_dim(outputs)),
-                           out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
-                           output_size_on_qp(
-                              [&]
-   {
-      int s = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      { s += get<o>(outputs).size_on_qp; });
-      return s;
-   }()),
-   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
-   total_trial_op_dim(compute_total_trial_op_dim(
-                         inputs, input_is_dependent, input_size_on_qp)),
-   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
-   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
-   q1d(tensor_1d_size(nq, dim))
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-
-      int direction_field_idx = -1;
-      for (size_t uf = 0; uf < nfields; uf++)
-      {
-         if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
-         {
-            direction_field_idx = static_cast<int>(uf);
-            break;
-         }
-      }
-      MFEM_ASSERT(direction_field_idx != -1,
-                  "DerivativeApply: derivative direction field not found");
-
-      direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename Backend>
-   void run_kernels(std::vector<Vector *> &ye) const
-   {
-      Backend::Run(dim,
-                   q1d,
-                   ctx,
-                   qp_cache,
-                   // inputs
-                   input_idx,
-                   input_B,
-                   input_G,
-                   input_vdim,
-                   input_d1d,
-                   input_q1d,
-                   input_size_on_qp,
-                   input_is_dependent,
-                   // outputs
-                   output_idx,
-                   output_B,
-                   output_G,
-                   output_vdim,
-                   output_d1d,
-                   output_q1d,
-                   out_op_dim,
-                   out_offsets,
-                   trial_vdim,
-                   total_trial_op_dim,
-                   residual_size_on_qp,
-                   output_size_on_qp,
-                   direction_e,
-                   ye,
-                   // fallback arguments
-                   dim,
-                   q1d);
-   }
-
-   //////////////////////////////////////////////////////////////////
-   void operator()(const std::vector<Vector *> &,
-                   const Vector *direction_l,
-                   std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      MFEM_ASSERT(direction_l != nullptr,
-                  "LocalQF DerivativeApply: direction vector is null");
-
-      restriction(direction_fd,
-                  direction_rcache,
-                  *direction_l,
-                  direction_e,
-                  ElementDofOrdering::LEXICOGRAPHIC);
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<DerivativeApplyLO>(ye);
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<DerivativeApplyHO>(ye);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void
-   derivative_apply_callback(const IntegratorContext &ctx,
-                             const Vector &qp_cache,
-                             // inputs: idx, B, G, vdim, d1d, q1d
-                             const std::array<size_t, n_inputs> & /*in_idx*/,
-                             const std::array<const real_t *, n_inputs> in_B,
-                             const std::array<const real_t *, n_inputs> in_G,
-                             const std::array<int, n_inputs> &in_vdim,
-                             const std::array<int, n_inputs> &in_d1d,
-                             const std::array<int, n_inputs> &in_q1d,
-                             const std::array<int, n_inputs> &in_size_on_qp,
-                             const std::array<bool, n_inputs> &input_dep,
-                             // outputs: idx, B, G, vdim, d1d, q1d
-                             const std::array<size_t, n_outputs> &out_idx,
-                             const std::array<const real_t *, n_outputs> out_B,
-                             const std::array<const real_t *, n_outputs> out_G,
-                             const std::array<int, n_outputs> &out_vdim,
-                             const std::array<int, n_outputs> &out_d1d,
-                             const std::array<int, n_outputs> &out_q1d,
-                             const std::array<int, n_outputs> &out_op_dim,
-                             const std::array<int, n_outputs> &out_offsets,
-                             const int trial_vdim,
-                             const int total_trial_op_dim,
-                             const int residual_size_on_qp,
-                             const int output_size_on_qp,
-                             const Vector &direction_e,
-                             std::vector<Vector *> &ye,
-                             // fallback arguments
-                             const int dim,
-                             const int q1d)
-   {
-      MFEM_CONTRACT_VAR(input_dep);
-      MFEM_CONTRACT_VAR(in_size_on_qp);
-      MFEM_CONTRACT_VAR(out_vdim);
-      MFEM_CONTRACT_VAR(out_op_dim);
-      MFEM_CONTRACT_VAR(out_offsets);
-      MFEM_CONTRACT_VAR(trial_vdim);
-      MFEM_CONTRACT_VAR(total_trial_op_dim);
-      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-      if (ctx.attr.Size() == 0) { return; }
-
-      static constexpr auto B2D = backend_t::DIM == 2;
-      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
-      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
-
-      const int ne = ctx.nentities;
-      const int nq = ctx.ir.GetNPoints();
-      MFEM_CONTRACT_VAR(output_size_on_qp);
-
-      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
-
-      // --------------------------------------------------
-      // DIRECTION (trial): XE_dir for the dependent inputs
-      // --------------------------------------------------
-      const auto d_direction = direction_e.Read();
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
-         using FOP = tuple_element_t<i, inputs_t>;
-         if (!input_dep[i]) { return; }
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
-         }
-         else if constexpr (is_weight_fop_v<FOP>) { /* never a direction */ }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      // --------------------------------------------------
-      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
-      for_constexpr<n_outputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = out_idx[i];
-         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
-         using FOP = tuple_element_t<i, outputs_t>;
-         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
-         {
-            MFEM_VERIFY(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            MFEM_VERIFY(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else
-         {
-            static_assert(false, "Unsupported FieldOperator");
-         }
-      });
-
-      auto cache_tensor = DeviceTensor<3, const real_t>(
-                             qp_cache.Read(), nq, residual_size_on_qp, ne);
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      dfem::forall<MTPB>(
-         [=] MFEM_HOST_DEVICE(const int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         // -----------------------------------------------
-         // Output integration registers, trial direction (shadow) registers
-         // and shared memory. `rargs` only ever holds test-function data, so
-         // it is an output-only bank: slot `o` is q-function parameter
-         // `n_inputs + o`.
-         // -----------------------------------------------
-         output_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
-         input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> sargs;
-         MFEM_SHARED typename backend_t::Shared smem;
-
-         // -----------------------------------------------
-         // Load trial direction (sargs) for the dependent inputs
-         // -----------------------------------------------
-         for_constexpr<n_inputs>([&](auto ic)
-         {
-            constexpr size_t i = ic.value;
-            if constexpr (!StaticInputDep<i>()) { return; }
-            const auto &XE = in_XE_dir[i];
-            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
-            const real_t *B = in_B[i], *G = in_G[i];
-            auto &sarg = get<i>(sargs);
-            using FOP = tuple_element_t<i, inputs_t>;
-            if constexpr (is_value_fop<FOP>::value)
-            {
-               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
-               using FieldParamT =
-                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-               backend_t::template LoadGradient<RNK,
-                                                decltype(sarg),
-                                                decltype(XE),
-                                                FieldParamT>(
-                                                   smem, e, d, q, Q1D, B, G, XE, sarg);
-            }
-            else if constexpr (is_identity_fop_v<FOP> || is_weight_fop_v<FOP>)
-            {
-               // identity read at qp; weight is never a trial direction
-            }
-            else
-            {
-               static_assert(false, "Unsupported");
-            }
-         });
-         MFEM_SYNC_THREAD;
-
-         // -----------------------------------------------
-         // Contract the cached Jacobian with the trial direction at each
-         // quadrature point and push the result into the test registers.
-         // -----------------------------------------------
-         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-
-                  // The trial direction at this quadrature point is the same
-                  // for every test row (i, k), so pull each dependent input
-                  // slot out of the register bank once, here, instead of once
-                  // per row inside the contraction below.
-                  args_tuple_t dvecs {};
-                  for_constexpr<n_inputs>([&](auto sc)
-                  {
-                     constexpr size_t s = sc.value;
-                     if constexpr (StaticInputDep<s>())
-                     {
-                        using SARG =
-                           typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
-                        get<s>(dvecs) = backend_t::template qp_pull<SARG>(
-                           get<s>(sargs), qx, qy, qz);
-                     }
-                  });
-
-                  for_constexpr<n_outputs>([&](auto oc)
-                  {
-                     constexpr size_t o = oc.value, ao = n_inputs + o;
-                     using FOP = tuple_element_t<o, outputs_t>;
-                     using ARG =
-                        typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
-                     constexpr int tv = StaticOutputVDim<o>();
-                     constexpr int to = StaticOutputOpDim<o>();
-                     constexpr int offset_o = StaticOutputOffset<o>();
-                     constexpr int trial_vdim_ct = StaticTrialVDim();
-                     constexpr int total_trial_op_dim_ct = StaticTotalTrialOpDim();
-
-                     ARG fhat{};
-                     MFEM_UNROLL(tv)
-                     for (int i = 0; i < tv; i++)
-                     {
-                        MFEM_UNROLL(to)
-                        for (int k = 0; k < to; k++)
-                        {
-                           const int row = offset_o + i * to + k;
-                           const int cache_row =
-                              row * trial_vdim_ct * total_trial_op_dim_ct;
-                           real_t sum = 0.0;
-                           for_constexpr<n_inputs>([&](auto sc)
-                           {
-                              constexpr size_t s = sc.value;
-                              if constexpr (StaticInputDep<s>())
-                              {
-                                 constexpr int op_dim_s = StaticInputOpDim<s>();
-                                 constexpr int m_offset = StaticInputOpOffset<s>();
-                                 const auto &dvec = get<s>(dvecs);
-                                 MFEM_UNROLL(trial_vdim_ct)
-                                 for (int j = 0; j < trial_vdim_ct; j++)
-                                 {
-                                    MFEM_UNROLL(op_dim_s)
-                                    for (int m = 0; m < op_dim_s; m++)
-                                    {
-                                       const int cache_idx =
-                                          cache_row + j * total_trial_op_dim_ct +
-                                          (m + m_offset);
-                                       sum += cache_tensor(q, cache_idx, e) *
-                                              qf_value_at(dvec, j, m);
-                                    }
-                                 }
-                              }
-                           });
-                           qf_set_value_at(fhat, i, k, sum);
-                        }
-                     }
-
-                     auto &YE = out_YE[o];
-                     if constexpr (is_identity_fop_v<FOP>)
-                     {
-                        MFEM_UNROLL(tv)
-                        for (int i = 0; i < tv; i++)
-                        {
-                           MFEM_UNROLL(to)
-                           for (int k = 0; k < to; k++)
-                           {
-                              YE(i + tv * k, qx, qy, qz, e) =
-                                 qf_value_at(fhat, i, k);
-                           }
-                        }
-                     }
-                     else
-                     {
-                        backend_t::template qp_push<ARG>(
-                           get<o>(rargs), qx, qy, qz, fhat);
-                     }
-                  });
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // -----------------------------------------------
-         // Integrate value / gradient outputs to the test dofs
-         // -----------------------------------------------
-         for_constexpr<n_outputs>([&](auto ic)
-         {
-            constexpr size_t i = ic.value, o = n_inputs + i;
-            const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
-            const auto B = out_B[i], G = out_G[i];
-            auto &YE = out_YE[i];
-            auto &rarg = get<i>(rargs);
-            using FOP = tuple_element_t<i, outputs_t>;
-            if constexpr (is_value_fop_v<FOP>)
-            {
-               backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, rarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               using YE_t = decltype(YE);
-               using rarg_t = decltype(rarg);
-               using qf_param_t =
-                  typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
-               constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
-               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
-                  smem, e, d, q, Q1D, B, G, YE, rarg);
-            }
-            else if constexpr (is_identity_fop_v<FOP>) { /* written at qp */ }
-            else
-            {
-               static_assert(false, "Unsupported");
-            }
-         });
-      },
-      ne,
-      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
-                                  q1d, in_d1d, out_d1d)),
-      0,
-      nullptr);
-   }
-
-   using ApplyKernelType =
-      decltype(&DerivativeApply::derivative_apply_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyLO,
-                                     ApplyKernelType,
-                                     (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyHO,
-                                     ApplyKernelType,
-                                     (int, int) );
-};
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-ApplyKernelType
-DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeApplyLO::Kernel()
-{
-   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
-   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return apply_t::template derivative_apply_callback<
-             LocalQFLOBackend<DIM, Q1D>>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-ApplyKernelType
-DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeApplyLO::Fallback(int dim, int q1d)
-{
-   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeApplyLO = typename apply_t::DerivativeApplyLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<DerivativeApplyLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<DerivativeApplyLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-ApplyKernelType
-DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeApplyHO::Kernel()
-{
-   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return apply_t::template derivative_apply_callback<LocalQFHOBackend<DIM>,
-                                                      Q1D>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-ApplyKernelType
-DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeApplyHO::Fallback(int dim, int q1d)
-{
-   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeApplyHO = typename apply_t::DerivativeApplyHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<DerivativeApplyHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<DerivativeApplyHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_apply_transpose.hpp

@@ -1,637 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include <array>
-
-namespace mfem::future::LocalQFImpl
-{
-
-// Cached transposed Jacobian apply: Jᵀ·w from the qp_cache
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-class DerivativeApplyTranspose
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-   using args_tuple_t = decay_tuple<qf_param_ts>;
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
-                 "LocalQF: q-function arity must match inputs + outputs");
-
-   // Input tuple slot referencing the derivative field (compile-time)
-   static constexpr size_t deriv_input_idx_ct = []() constexpr
-   {
-      size_t idx = SIZE_MAX;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         using FOP = tuple_element_t<i, inputs_t>;
-         if (FOP::GetFieldId() == derivative_id) { idx = i; }
-      });
-      return idx;
-   }();
-   static_assert(deriv_input_idx_ct < n_inputs,
-                 "DerivativeApplyTranspose: derivative input slot not found");
-
-   const inputs_t inputs;
-   const outputs_t outputs;
-   const IntegratorContext ctx;
-   const Vector &qp_cache; // Jacobian cache from DerivativeSetup
-   const std::vector<const DofToQuad *> dtqs;
-   // inputs: dtq, B, G, d1d, q1d, vdim (trial / derivative fields)
-   const std::array<DofToQuadMap, n_inputs> input_dtq;
-   const std::array<const real_t *, n_inputs> input_B, input_G;
-   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
-   // outputs: dtq, idx, B, G, d1d, q1d, vdim (test / cotangent fields)
-   const std::array<DofToQuadMap, n_outputs> output_dtq;
-   const std::array<size_t, n_outputs> output_idx;
-   const std::array<const real_t *, n_outputs> output_B, output_G;
-   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
-   // Jacobian cache metadata
-   const std::array<bool, n_inputs> input_is_dependent;
-   const std::array<int, n_inputs> input_size_on_qp;
-   const std::array<int, n_outputs> out_op_dim;
-   const std::array<int, n_outputs> out_offsets;
-   const int output_size_on_qp;
-   const int trial_vdim;
-   const int total_trial_op_dim;
-   const int residual_size_on_qp;
-   // other constants
-   const int dim, ne, nq, q1d;
-   const size_t deriv_infd_idx; // index of the derivative field in ye
-   // output cotangent restriction workspace (blocked by element)
-   std::array<int, n_outputs> out_elem_dof_size;
-   mutable Vector dir_out_e;
-   /// One restriction cache per output field, resolved on first use.
-   mutable std::array<RestrictionCache<Entity::Element>, n_outputs>
-   out_rcaches;
-
-public:
-   //////////////////////////////////////////////////////////////////
-   DerivativeApplyTranspose() = delete;
-
-   DerivativeApplyTranspose(IntegratorContext ctx,
-                            qfunc_t /*qfunc*/,
-                            inputs_t inputs,
-                            outputs_t outputs,
-                            const Vector &qp_cache):
-      inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache),
-      dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
-                                         inputs,
-                                         dtqs,
-                                         create_union_field_map_for_dtq(ctx, inputs),
-                                         ctx.unionfds,
-                                         ctx.ir)),
-      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
-      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
-      input_vdim(get_vdim(inputs)),
-      output_dtq(create_dtq_maps<Entity::Element>(
-                    outputs,
-                    dtqs,
-                    create_union_field_map_for_dtq(ctx, outputs),
-                    ctx.unionfds,
-                    ctx.ir)),
-      output_idx(create_output_vector_map(ctx, outputs)),
-      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
-      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
-      output_vdim(get_vdim(outputs)),
-      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
-      input_size_on_qp(
-         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
-                           out_op_dim(compute_out_op_dim(outputs)),
-                           out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
-                           output_size_on_qp(
-                              [&]
-   {
-      int s = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      { s += get<o>(outputs).size_on_qp; });
-      return s;
-   }()),
-   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
-   total_trial_op_dim(compute_total_trial_op_dim(
-                         inputs, input_is_dependent, input_size_on_qp)),
-   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
-   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
-   q1d(tensor_1d_size(nq, dim)),
-   deriv_infd_idx(find_infd_index(ctx, derivative_id)), out_elem_dof_size{}
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-      MFEM_ASSERT(
-         deriv_infd_idx != SIZE_MAX,
-         "DerivativeApplyTranspose: derivative field not found in infds");
-
-      // Size the workspace that holds the output cotangent(s) in element
-      // layout.
-      int total_dir_e_size = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         const int elem_sz = compute_element_dof_sz(
-            ctx.outfds[output_idx[o]], ne, ElementDofOrdering::LEXICOGRAPHIC);
-         out_elem_dof_size[o] = elem_sz;
-         total_dir_e_size += elem_sz;
-      });
-      dir_out_e.SetSize(total_dir_e_size * ne);
-      dir_out_e.UseDevice(true);
-      dir_out_e.Read();
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename Backend>
-   void run_kernels(std::vector<Vector *> &ye) const
-   {
-      Backend::Run(dim,
-                   q1d,
-                   ctx,
-                   qp_cache,
-                   dir_out_e,
-                   // inputs (integration target metadata)
-                   input_B,
-                   input_G,
-                   input_vdim,
-                   input_d1d,
-                   input_q1d,
-                   input_size_on_qp,
-                   input_is_dependent,
-                   // outputs (direction interpolation metadata)
-                   output_B,
-                   output_G,
-                   output_vdim,
-                   output_d1d,
-                   output_q1d,
-                   out_op_dim,
-                   out_offsets,
-                   trial_vdim,
-                   total_trial_op_dim,
-                   residual_size_on_qp,
-                   output_size_on_qp,
-                   deriv_infd_idx,
-                   ye,
-                   // fallback arguments
-                   dim,
-                   q1d);
-   }
-
-   //////////////////////////////////////////////////////////////////
-   void operator()(const std::vector<Vector *> & /*xe*/,
-                   const Vector *direction_l,
-                   std::vector<Vector *> &ye) const
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      MFEM_ASSERT(direction_l != nullptr,
-                  "LocalQF DerivativeApplyTranspose: direction vector is null");
-
-      // Restrict output cotangent from L-vectors into element layout
-      // (dir_out_e).
-      int l_offset = 0;
-      int e_offset = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         const size_t outfd = output_idx[o];
-         const auto &fd = ctx.outfds[outfd];
-         const int l_size = GetVSize(fd);
-         Vector dir_o_l(*const_cast<Vector *>(direction_l), l_offset, l_size);
-         dir_o_l.UseDevice(true);
-         const int elem_sz = out_elem_dof_size[o];
-         Vector dir_o_e(dir_out_e, e_offset, elem_sz * ne);
-         dir_o_e.UseDevice(true);
-         restriction(fd, out_rcaches[o], dir_o_l, dir_o_e,
-                     ElementDofOrdering::LEXICOGRAPHIC);
-         l_offset += l_size;
-         e_offset += elem_sz * ne;
-      });
-
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<DerivativeApplyTransposeLO>(ye);
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<DerivativeApplyTransposeHO>(ye);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void derivative_apply_transpose_callback(
-      const IntegratorContext &ctx,
-      const Vector &qp_cache,
-      const Vector &dir_e, // restricted, concatenated output cotangents
-      // inputs (integration target metadata)
-      const std::array<const real_t *, n_inputs> in_B,
-      const std::array<const real_t *, n_inputs> in_G,
-      const std::array<int, n_inputs> &in_vdim,
-      const std::array<int, n_inputs> &in_d1d,
-      const std::array<int, n_inputs> &in_q1d,
-      const std::array<int, n_inputs> &in_size_on_qp,
-      const std::array<bool, n_inputs> &input_dep,
-      // outputs (direction interpolation metadata)
-      const std::array<const real_t *, n_outputs> out_B,
-      const std::array<const real_t *, n_outputs> out_G,
-      const std::array<int, n_outputs> &out_vdim,
-      const std::array<int, n_outputs> &out_d1d,
-      const std::array<int, n_outputs> &out_q1d,
-      const std::array<int, n_outputs> &out_op_dim,
-      const std::array<int, n_outputs> &out_offsets,
-      const int trial_vdim,
-      const int total_trial_op_dim,
-      const int residual_size_on_qp,
-      const int output_size_on_qp,
-      const size_t deriv_infd_idx,
-      std::vector<Vector *> &ye,
-      // fallback arguments
-      const int dim,
-      const int q1d)
-   {
-      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-      if (ctx.attr.Size() == 0) { return; }
-
-      static constexpr auto B2D = backend_t::DIM == 2;
-      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
-      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
-
-      const int ne = ctx.nentities;
-      const int nq = ctx.ir.GetNPoints();
-      MFEM_CONTRACT_VAR(output_size_on_qp);
-      MFEM_CONTRACT_VAR(in_q1d);
-
-      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
-
-      // --------------------------------------------------
-      // DIRECTION (test cotangent): out_XE_dir, concatenated per output
-      // --------------------------------------------------
-      const auto d_dir = dir_e.Read();
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_outputs> out_XE_dir;
-      int e_offset = 0;
-      for_constexpr<n_outputs>([&](auto oc)
-      {
-         constexpr size_t o = oc.value;
-         const int d = out_d1d[o], q = out_q1d[o], v = out_vdim[o];
-         using FOP = tuple_element_t<o, outputs_t>;
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            out_XE_dir[o] = Reshape(d_dir + e_offset, d, d, B2D ? 1 : d, v, ne);
-            e_offset += k_dim(d) * v * ne;
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            out_XE_dir[o] = Reshape(d_dir + e_offset, v, q, q, B2D ? 1 : q, ne);
-            e_offset += k_dim(q) * v * ne;
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      // --------------------------------------------------
-      // DERIVATIVE TRIAL FIELD: ye_XE (accumulates Jᵀ w)
-      // --------------------------------------------------
-      const int d_in = in_d1d[deriv_input_idx_ct];
-      const int v_in = in_vdim[deriv_input_idx_ct];
-      auto ye_XE = Reshape(
-                      ye[deriv_infd_idx]->ReadWrite(), d_in, d_in, B2D ? 1 : d_in, v_in, ne);
-
-      auto cache_tensor = DeviceTensor<3, const real_t>(
-                             qp_cache.Read(), nq, residual_size_on_qp, ne);
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      dfem::forall<MTPB>(
-         [=] MFEM_HOST_DEVICE(const int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         // -----------------------------------------------
-         // Output cotangent (direction) registers live in the output slots;
-         // the trial integration data is pushed into the input slots.
-         // -----------------------------------------------
-         args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
-         MFEM_SHARED typename backend_t::Shared smem;
-
-         // -----------------------------------------------
-         // Interpolate the test cotangent to quadrature points (output slots)
-         // -----------------------------------------------
-         for_constexpr<n_outputs>([&](auto oc)
-         {
-            constexpr size_t o = oc.value, ao = n_inputs + o;
-            using FOP = tuple_element_t<o, outputs_t>;
-            const auto &XE = out_XE_dir[o];
-            const int d = out_d1d[o], q = out_q1d[o], Q1D = q1d;
-            const real_t *B = out_B[o], *G = out_G[o];
-            auto &oarg = get<ao>(rargs);
-            if constexpr (is_value_fop_v<FOP>)
-            {
-               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, oarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               constexpr auto RNK = qf_param_slot<qfunc_t, ao>::extents.size();
-               using FieldParamT =
-                  typename qf_param_slot<qfunc_t, ao>::qf_decay_param_t;
-               backend_t::template LoadGradient<RNK,
-                                                decltype(oarg),
-                                                decltype(XE),
-                                                FieldParamT>(
-                                                   smem, e, d, q, Q1D, B, G, XE, oarg);
-            }
-            else if constexpr (is_identity_fop_v<FOP>)
-            {
-               // identity cotangent is read directly at qp from out_XE_dir
-            }
-            else
-            {
-               static_assert(false, "Unsupported");
-            }
-         });
-         MFEM_SYNC_THREAD;
-
-         // -----------------------------------------------
-         // Contract the transposed cached Jacobian with the test cotangent at
-         // each quadrature point and push the trial result into the dependent
-         // input registers.
-         // -----------------------------------------------
-         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-
-                  // The test cotangent at this quadrature point is the same
-                  // for every trial column (j, m), so pull each interpolated
-                  // output slot out of the register bank once, here, instead
-                  // of once per column inside the contraction below. Identity
-                  // outputs have no register bank and are read from
-                  // out_XE_dir at the point of use.
-                  args_tuple_t wvecs {};
-                  for_constexpr<n_outputs>([&](auto oc)
-                  {
-                     constexpr size_t o = oc.value, ao = n_inputs + o;
-                     using OFOP = tuple_element_t<o, outputs_t>;
-                     if constexpr (is_value_fop_v<OFOP> ||
-                                   is_gradient_fop_v<OFOP>)
-                     {
-                        using OARG =
-                           typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
-                        get<ao>(wvecs) = backend_t::template qp_pull<OARG>(
-                           get<ao>(rargs), qx, qy, qz);
-                     }
-                  });
-
-                  int m_offset = 0;
-                  for_constexpr<n_inputs>([&](auto sc)
-                  {
-                     constexpr size_t s = sc.value;
-                     if (!input_dep[s]) { return; }
-                     using SARG =
-                        typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
-                     const int vdim_s = in_vdim[s];
-                     const int op_dim_s = in_size_on_qp[s] / vdim_s;
-
-                     SARG fhat{};
-                     for (int j = 0; j < trial_vdim; j++)
-                     {
-                        for (int m = 0; m < op_dim_s; m++)
-                        {
-                           const int col =
-                              j * total_trial_op_dim + (m + m_offset);
-                           real_t sum = 0.0;
-                           for_constexpr<n_outputs>([&](auto oc)
-                           {
-                              constexpr size_t o = oc.value, ao = n_inputs + o;
-                              using OFOP = tuple_element_t<o, outputs_t>;
-                              const int tv = out_vdim[o], to = out_op_dim[o];
-                              const auto offset_o = out_offsets[o];
-                              const auto &cache = cache_tensor;
-                              if constexpr (is_value_fop_v<OFOP> ||
-                                            is_gradient_fop_v<OFOP>)
-                              {
-                                 const auto &wvec = get<ao>(wvecs);
-                                 for (int i = 0; i < tv; i++)
-                                 {
-                                    for (int k = 0; k < to; k++)
-                                    {
-                                       const int row = offset_o + i * to + k;
-                                       const int cache_idx =
-                                          row * trial_vdim *
-                                          total_trial_op_dim +
-                                          col;
-                                       sum += cache(q, cache_idx, e) *
-                                              qf_value_at(wvec, i, k);
-                                    }
-                                 }
-                              }
-                              else if constexpr (is_identity_fop_v<OFOP>)
-                              {
-                                 const auto &XEo = out_XE_dir[o];
-                                 for (int i = 0; i < tv; i++)
-                                 {
-                                    for (int k = 0; k < to; k++)
-                                    {
-                                       const int row = offset_o + i * to + k;
-                                       const int cache_idx =
-                                          row * trial_vdim *
-                                          total_trial_op_dim +
-                                          col;
-                                       sum += cache(q, cache_idx, e) *
-                                              XEo(i + tv * k, qx, qy, qz, e);
-                                    }
-                                 }
-                              }
-                           });
-                           qf_set_value_at(fhat, j, m, sum);
-                        }
-                     }
-                     backend_t::template qp_push<SARG>(
-                        get<s>(rargs), qx, qy, qz, fhat);
-                     m_offset += op_dim_s;
-                  });
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // -----------------------------------------------
-         // Integrate the trial result into the derivative field dofs. Multiple
-         // dependent input slots (e.g. value and gradient of the same field)
-         // accumulate into ye_XE via the writers' '+=' semantics.
-         // -----------------------------------------------
-         for_constexpr<n_inputs>([&](auto sc)
-         {
-            constexpr size_t s = sc.value;
-            if (!input_dep[s]) { return; }
-            using FOP = tuple_element_t<s, inputs_t>;
-            const int d = in_d1d[s], q = in_q1d[s], Q1D = q1d;
-            const real_t *B = in_B[s], *G = in_G[s];
-            auto &sarg = get<s>(rargs);
-            auto &YE = ye_XE;
-            if constexpr (is_value_fop_v<FOP>)
-            {
-               backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, sarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               using YE_t = decltype(YE);
-               using rarg_t = decltype(sarg);
-               using qf_param_t =
-                  typename qf_param_slot<qfunc_t, s>::qf_decay_param_t;
-               constexpr auto RNK = qf_param_slot<qfunc_t, s>::extents.size();
-               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
-                  smem, e, d, q, Q1D, B, G, YE, sarg);
-            }
-            else
-            {
-               // identity / weight derivative targets are not produced here
-            }
-         });
-      },
-      ne,
-      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
-                                  q1d, in_d1d, out_d1d)),
-      0,
-      nullptr);
-   }
-
-   using TransposeKernelType =
-      decltype(&DerivativeApplyTranspose::
-               derivative_apply_transpose_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeLO,
-                                     TransposeKernelType,
-                                     (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeHO,
-                                     TransposeKernelType,
-                                     (int, int) );
-};
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeApplyTranspose<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::TransposeKernelType
-       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeApplyTransposeLO::Kernel()
-{
-   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
-   using transpose_t =
-      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return transpose_t::template derivative_apply_transpose_callback<
-             LocalQFLOBackend<DIM, Q1D>>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeApplyTranspose<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::TransposeKernelType
-       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeApplyTransposeLO::Fallback(int dim, int q1d)
-{
-   using transpose_t =
-      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeApplyTransposeLO =
-      typename transpose_t::DerivativeApplyTransposeLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeApplyTranspose<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::TransposeKernelType
-       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeApplyTransposeHO::Kernel()
-{
-   using transpose_t =
-      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return transpose_t::
-          template derivative_apply_transpose_callback<LocalQFHOBackend<DIM>, Q1D>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeApplyTranspose<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::TransposeKernelType
-       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeApplyTransposeHO::Fallback(int dim, int q1d)
-{
-   using transpose_t =
-      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeApplyTransposeHO =
-      typename transpose_t::DerivativeApplyTransposeHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_assemble_diagonal.hpp

@@ -1,664 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include <algorithm>
-#include <array>
-#include <vector>
-
-namespace mfem::future::LocalQFImpl
-{
-
-// Assemble the diagonal of one row block of a cached Jacobian (tensor 2D/3D).
-//
-// The derivative is a block column, one row block per output field. Only a
-// block whose test space is the trial space is square, and only a square block
-// has a diagonal at all, so the row block is chosen per call and checked.
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-class DerivativeAssembleDiagonal
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-
-   const IntegratorContext ctx;
-   const Vector &qp_cache;
-   inputs_t inputs;
-   outputs_t outputs;
-   const bool use_sum_factorization;
-   const std::vector<const DofToQuad *> dtqs;
-   const std::array<DofToQuadMap, n_inputs> input_dtq_maps;
-   const std::array<DofToQuadMap, n_outputs> output_dtq_maps;
-   const std::array<bool, n_inputs> input_is_dependent;
-   const size_t trial_field_uf;
-   /// Column space of every row block; null when the differentiated field is
-   /// not an FE space, in which case no block has a diagonal.
-   const ParFiniteElementSpace *trial_fes;
-   const std::array<int, n_outputs> out_vdim;
-   const std::array<int, n_outputs> out_op_dim;
-   const std::array<int, n_outputs> out_offsets;
-   const int output_size_on_qp;
-   /// Row blocks: the distinct output field ids, in order of first appearance.
-   /// Outputs sharing a field id are summed into one diagonal, which is how
-   /// Value<U> + Gradient<U> becomes mass plus diffusion.
-   const std::vector<int> group_field_ids;
-   const std::array<int, n_outputs> out_group;
-   const std::vector<const ParFiniteElementSpace *> group_fes;
-   const std::vector<int> group_test_vdim;
-   const std::vector<int> group_num_test_dof;
-   const std::vector<int> group_num_test_dof_1d;
-   /// Whether a row block has a diagonal: its test space has to be an FE space
-   /// and has to *be* the trial space, and no output on it may be an Identity.
-   const std::vector<bool> group_has_diagonal;
-   const int trial_vdim;
-   const int total_trial_op_dim;
-   const int num_trial_dof_1d;
-   const int residual_size_on_qp;
-   const int dim, ne, nq, q1d;
-   const std::array<int, n_inputs> inputs_trial_op_dim;
-   mutable std::vector<Vector> group_Ye_mem;
-
-   /// Distinct output field ids, in order of first appearance.
-   static std::vector<int> compute_group_field_ids(const outputs_t &outs)
-   {
-      std::vector<int> ids;
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         const int fid = get<o>(outs).GetFieldId();
-         if (std::find(ids.begin(), ids.end(), fid) == ids.end())
-         {
-            ids.push_back(fid);
-         }
-      });
-      return ids;
-   }
-
-public:
-   DerivativeAssembleDiagonal() = delete;
-
-   DerivativeAssembleDiagonal(IntegratorContext ctx_in,
-                              qfunc_t /*qfunc*/,
-                              inputs_t inputs_in,
-                              outputs_t outputs_in,
-                              const Vector &qp_cache_in):
-      ctx(ctx_in), qp_cache(qp_cache_in), inputs(inputs_in),
-      outputs(outputs_in), use_sum_factorization(
-         [&]
-   {
-      const Element::Type etype =
-         Element::TypeFromGeometry(ctx_in.mesh.GetTypicalElementGeometry());
-      return (etype == Element::QUADRILATERAL || etype == Element::HEXAHEDRON);
-   }()),
-   dtqs(
-      [&]
-   {
-      const DofToQuad::Mode dtq_mode = use_sum_factorization
-      ? DofToQuad::Mode::TENSOR
-      : DofToQuad::Mode::FULL;
-      std::vector<const DofToQuad *> maps;
-      maps.reserve(ctx_in.unionfds.size());
-      for (const auto &field : ctx_in.unionfds)
-      {
-         maps.emplace_back(
-            GetDofToQuad<Entity::Element>(field, ctx_in.ir, dtq_mode));
-      }
-      return maps;
-   }()),
-   input_dtq_maps(create_dtq_maps<Entity::Element>(
-                     inputs,
-                     dtqs,
-                     create_union_field_map_for_dtq(ctx_in, inputs),
-                     ctx_in.unionfds,
-                     ctx_in.ir)),
-   output_dtq_maps(create_dtq_maps<Entity::Element>(
-                      outputs,
-                      dtqs,
-                      create_union_field_map_for_dtq(ctx_in, outputs),
-                      ctx_in.unionfds,
-                      ctx_in.ir)),
-   input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
-   trial_field_uf(find_union_field_index(ctx_in, derivative_id)),
-   trial_fes(
-      [&]() -> const ParFiniteElementSpace *
-   {
-      if (trial_field_uf >= ctx_in.unionfds.size()) { return nullptr; }
-      const auto *fes = std::get_if<const ParFiniteElementSpace *>(
-         &ctx_in.unionfds[trial_field_uf].data);
-      return fes ? *fes : nullptr;
-   }()),
-   out_vdim(get_vdim(outputs_in)),
-   out_op_dim(compute_out_op_dim(outputs_in)),
-   out_offsets(compute_out_offsets(out_vdim, out_op_dim)),
-   output_size_on_qp(
-      [&]
-   {
-      int s = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      { s += get<o>(outputs_in).size_on_qp; });
-      return s;
-   }()),
-   group_field_ids(compute_group_field_ids(outputs_in)),
-   out_group(
-      [&]
-   {
-      std::array<int, n_outputs> g {};
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         const int fid = get<o>(outputs_in).GetFieldId();
-         const auto &ids = group_field_ids;
-         g[o] = static_cast<int>(std::find(ids.begin(), ids.end(), fid)
-                                 - ids.begin());
-      });
-      return g;
-   }()),
-   group_fes(
-      [&]
-   {
-      std::vector<const ParFiniteElementSpace *> v(group_field_ids.size(),
-                                                   nullptr);
-      for (size_t g = 0; g < v.size(); g++)
-      {
-         const size_t uf = find_union_field_index(ctx_in, group_field_ids[g]);
-         if (uf >= ctx_in.unionfds.size()) { continue; }
-         const auto *fes = std::get_if<const ParFiniteElementSpace *>(
-                              &ctx_in.unionfds[uf].data);
-         v[g] = fes ? *fes : nullptr;
-      }
-      return v;
-   }()),
-   group_test_vdim(
-      [&]
-   {
-      std::vector<int> v(group_field_ids.size(), 0);
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         v[out_group[o]] = get<o>(outputs_in).vdim;
-      });
-      return v;
-   }()),
-   group_num_test_dof(
-      [&]
-   {
-      std::vector<int> v(group_field_ids.size(), 0);
-      for (size_t g = 0; g < v.size(); g++)
-      {
-         if (group_fes[g] == nullptr) { continue; }
-         v[g] = group_fes[g]->GetFE(0)->GetDof();
-      }
-      return v;
-   }()),
-   group_num_test_dof_1d(
-      [&]
-   {
-      std::vector<int> v(group_field_ids.size(), 0);
-      for (size_t g = 0; g < v.size(); g++)
-      {
-         if (group_num_test_dof[g] > 0)
-         {
-            v[g] = tensor_1d_size(group_num_test_dof[g],
-                                  ctx_in.mesh.Dimension());
-         }
-      }
-      return v;
-   }()),
-   group_has_diagonal(
-      [&]
-   {
-      // A diagonal needs row space == column space, so only a row block on the
-      // trial space qualifies. Squareness alone cannot pick a block when
-      // several output fields share that space, which is why the caller names
-      // the row. Identity outputs are quadrature point data and are excluded
-      // for the same reason as in DerivativeAssemble: they cannot be
-      // contracted, and every output on a field lands in the same block.
-      std::vector<bool> v(group_field_ids.size(), false);
-      if (trial_fes == nullptr) { return v; }
-      for (size_t g = 0; g < v.size(); g++)
-      {
-         v[g] = (group_fes[g] != nullptr) && (group_fes[g] == trial_fes);
-      }
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         using output_fop_t = std::decay_t<decltype(get<o>(outputs_in))>;
-         if constexpr (is_identity_fop_v<output_fop_t>)
-         {
-            v[out_group[o]] = false;
-         }
-      });
-      return v;
-   }()),
-   trial_vdim(compute_trial_vdim(inputs, derivative_id)), total_trial_op_dim(
-      [&]
-   {
-      const auto input_size_on_qp =
-      get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs>{});
-      return compute_total_trial_op_dim(
-         inputs, input_is_dependent, input_size_on_qp);
-   }()),
-   num_trial_dof_1d(
-      trial_fes ? tensor_1d_size(trial_fes->GetFE(0)->GetDof(),
-                                 ctx_in.mesh.Dimension())
-      : 0),
-   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
-   dim(ctx_in.mesh.Dimension()), ne(ctx_in.nentities),
-   nq(ctx_in.ir.GetNPoints()), q1d(tensor_1d_size(nq, dim)),
-   inputs_trial_op_dim(
-      [&]
-   {
-      std::array<int, n_inputs> itod{};
-      for_constexpr<n_inputs>([&](auto i)
-      {
-         itod[i] = input_is_dependent[i]
-                   ? get<i>(inputs).size_on_qp / get<i>(inputs).vdim
-                   : 0;
-      });
-      return itod;
-   }()),
-   group_Ye_mem()
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-      MFEM_ASSERT(
-         trial_field_uf != SIZE_MAX,
-         "DerivativeAssembleDiagonal: trial field not found in unionfds");
-      MFEM_ASSERT(trial_vdim > 0,
-                  "LocalQFBackend: could not determine trial vdim");
-      MFEM_ASSERT(total_trial_op_dim > 0,
-                  "LocalQFBackend: no dependent inputs found");
-      for_constexpr<n_outputs>([&](auto o)
-      {
-         MFEM_ASSERT(out_vdim[o] == group_test_vdim[out_group[o]],
-                     "DerivativeAssembleDiagonal: outputs on one field must "
-                     "share its vdim");
-      });
-
-      group_Ye_mem.resize(group_field_ids.size());
-      for (size_t g = 0; g < group_Ye_mem.size(); g++)
-      {
-         if (!group_has_diagonal[g]) { continue; }
-         group_Ye_mem[g].SetSize(group_num_test_dof[g] * group_test_vdim[g] *
-                                 ne);
-         group_Ye_mem[g].UseDevice(true);
-      }
-   }
-
-   /// Index of the row block for output field @a field_id, or -1.
-   int FindGroup(int field_id) const
-   {
-      const auto &ids = group_field_ids;
-      const auto it = std::find(ids.begin(), ids.end(), field_id);
-      return (it == ids.end()) ? -1 : static_cast<int>(it - ids.begin());
-   }
-
-   template<typename Backend>
-   void run_kernels(const int g) const
-   {
-      Backend::Run(dim,
-                   q1d,
-                   ctx,
-                   qp_cache,
-                   group_Ye_mem[g],
-                   inputs,
-                   outputs,
-                   output_dtq_maps,
-                   input_dtq_maps,
-                   out_group,
-                   g,
-                   group_test_vdim[g],
-                   out_op_dim,
-                   out_offsets,
-                   output_size_on_qp,
-                   group_num_test_dof[g],
-                   group_num_test_dof_1d[g],
-                   trial_vdim,
-                   total_trial_op_dim,
-                   residual_size_on_qp,
-                   inputs_trial_op_dim,
-                   nq,
-                   ne,
-                   q1d,
-                   dim);
-   }
-
-   /// Add this integrator's contribution to the diagonal of the row block of
-   /// output field @a out_field_id. Adds nothing if the integrator writes no
-   /// square, basis-backed block for that field; the caller is responsible for
-   /// rejecting a row that no integrator can serve.
-   void operator()(const int out_field_id, Vector &diag_e) const
-   {
-      const int g = FindGroup(out_field_id);
-      if (g < 0 || !group_has_diagonal[g]) { return; }
-      if (ctx.attr.Size() == 0) { return; }
-
-      if (!(use_sum_factorization && (dim == 2 || dim == 3)))
-      {
-         MFEM_ABORT("DerivativeAssembleDiagonal optimized path is implemented "
-                    "for tensor-product 2D/3D elements only");
-      }
-      MFEM_VERIFY(group_num_test_dof_1d[g] == num_trial_dof_1d,
-                  "DerivativeAssembleDiagonal requires matching tensor dofs");
-      const auto &limits = DeviceDofQuadLimits::Get();
-      MFEM_VERIFY(group_num_test_dof_1d[g] <= limits.MAX_D1D, "");
-      MFEM_VERIFY(q1d <= limits.MAX_Q1D, "");
-
-      group_Ye_mem[g] = 0.0;
-
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<DerivativeAssembleDiagonalLO>(g);
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<DerivativeAssembleDiagonalHO>(g);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-
-      diag_e += group_Ye_mem[g];
-   }
-
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void derivative_assemble_diagonal_callback(
-      const IntegratorContext &ctx,
-      const Vector &qp_cache,
-      Vector &Ye_mem,
-      const inputs_t &inputs,
-      const outputs_t &outputs,
-      const std::array<DofToQuadMap, n_outputs> &output_dtq_maps,
-      const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
-      const std::array<int, n_outputs> &out_group,
-      const int row_group,
-      const int test_vdim,
-      const std::array<int, n_outputs> &out_op_dim,
-      const std::array<int, n_outputs> &out_offsets,
-      const int output_size_on_qp,
-      const int num_test_dof,
-      const int num_test_dof_1d,
-      const int trial_vdim,
-      const int total_trial_op_dim,
-      const int residual_size_on_qp,
-      const std::array<int, n_inputs> &inputs_trial_op_dim,
-      const int nq,
-      const int ne,
-      const int q1d,
-      const int dim)
-   {
-      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-      if (ctx.attr.Size() == 0) { return; }
-
-      static constexpr bool B2D = backend_t::DIM == 2;
-      static constexpr int MTPB = backend_t::MAX_THREADS_PER_BLOCK();
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      auto cache_tensor = DeviceTensor<3, const real_t>(
-                             qp_cache.Read(), nq, residual_size_on_qp, ne);
-      const int num_dofs_per_elem = num_test_dof * test_vdim;
-      auto Ye = Reshape(Ye_mem.ReadWrite(), num_dofs_per_elem, ne);
-
-      dfem::forall<MTPB>(
-         [=] MFEM_HOST_DEVICE(const int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         // The cache is written with the quadrature index fastest, then the
-         // trial op index, then the (test vdim, test op) rows of all outputs
-         // stacked via out_offsets.
-         auto qpdc = Reshape(&cache_tensor(0, 0, e),
-                             nq,
-                             total_trial_op_dim,
-                             trial_vdim,
-                             output_size_on_qp);
-
-         // Backend-owned shared scratch for the sum-factorized contraction.
-         MFEM_SHARED typename backend_t::Shared s_diag;
-         const int nz_dof = B2D ? 1 : num_test_dof_1d;
-
-         for (int vd = 0; vd < test_vdim; vd++)
-         {
-            auto Y = Reshape(&Ye(vd * num_test_dof, e),
-                             num_test_dof_1d,
-                             num_test_dof_1d,
-                             nz_dof);
-
-            MFEM_FOREACH_THREAD(dz_t, z, nz_dof)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(dy_t, y, num_test_dof_1d)
-               {
-                  MFEM_FOREACH_THREAD_DIRECT(dx_t, x, num_test_dof_1d)
-                  { Y(dx_t, dy_t, dz_t) = 0.0; }
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            // Accumulate every output belonging to the requested row block.
-            // This sums multiple contributions, such as Value<U> +
-            // Gradient<U>, while skipping outputs on the other row blocks.
-            // The row is a run time choice, so unlike the field id it cannot
-            // gate the instantiation; is_identity_fop_v still does, since
-            // eval_test has no meaning for quadrature point data.
-            for_constexpr<n_outputs>([&](auto o)
-            {
-               using test_fop_t = std::decay_t<decltype(get<o>(outputs))>;
-               if constexpr (!is_identity_fop_v<test_fop_t>)
-               {
-                  if (out_group[static_cast<int>(o)] != row_group) { return; }
-                  const auto &out_dtq = output_dtq_maps[o];
-                  const int test_op_dim = out_op_dim[static_cast<int>(o)];
-
-                  // Test-basis factor along a spatial axis
-                  const auto eval_test =
-                     [&](const int k, const int axis, const int q, const int d)
-                  {
-                     const auto &B = out_dtq.B;
-                     const auto &G = out_dtq.G;
-                     if constexpr (is_value_fop<test_fop_t>::value)
-                     {
-                        return (k == 0) ? B(q, 0, d) : 0.0;
-                     }
-                     else if constexpr (is_gradient_fop<test_fop_t>::value)
-                     {
-                        return (k == axis) ? G(q, 0, d) : B(q, 0, d);
-                     }
-                     else
-                     {
-                        return 0.0;
-                     }
-                  };
-
-                  for (int k = 0; k < test_op_dim; k++)
-                  {
-                     const int row = out_offsets[static_cast<int>(o)] +
-                                     vd * test_op_dim + k;
-                     int m_offset = 0;
-                     for_constexpr<n_inputs>([&](auto s)
-                     {
-                        using fop_t = std::decay_t<decltype(get<s>(inputs))>;
-                        const int trial_op_dim =
-                           inputs_trial_op_dim[static_cast<int>(s)];
-                        if (trial_op_dim == 0) { return; }
-
-                        const auto &in_dtq = input_dtq_maps[s];
-                        const auto eval_input =
-                           [&](const int m, const int axis, const int q,
-                               const int d)
-                        {
-                           if constexpr (is_value_fop<fop_t>::value)
-                           {
-                              return (m == 0) ? in_dtq.B(q, 0, d) : 0.0;
-                           }
-                           else if constexpr (is_gradient_fop<fop_t>::value)
-                           {
-                              return (m == axis) ? in_dtq.G(q, 0, d)
-                                     : in_dtq.B(q, 0, d);
-                           }
-                           else
-                           {
-                              return 0.0;
-                           }
-                        };
-
-                        for (int m = 0; m < trial_op_dim; m++)
-                        {
-                           const int col = m_offset + m;
-                           backend_t::DiagContract(
-                              s_diag,
-                              num_test_dof_1d,
-                              q1d,
-                              nz_dof,
-                              [&](int axis, int q, int d)
-                           { return eval_test(k, axis, q, d); },
-                           [&](int axis, int q, int d)
-                           { return eval_input(m, axis, q, d); },
-                           [&](int q) { return qpdc(q, col, vd, row); },
-                           [&](int dx, int dy, int dz, real_t u)
-                           { Y(dx, dy, dz) += u; });
-                        }
-                        m_offset += trial_op_dim;
-                     });
-                  }
-               }
-            });
-         }
-      },
-      ne,
-      backend_t::thread_blocks(std::max(q1d, num_test_dof_1d)),
-      0,
-      nullptr);
-   }
-
-   using DiagonalKernelType =
-      decltype(&DerivativeAssembleDiagonal::
-               derivative_assemble_diagonal_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalLO,
-                                     DiagonalKernelType,
-                                     (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalHO,
-                                     DiagonalKernelType,
-                                     (int, int) );
-};
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeAssembleDiagonal<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::DiagonalKernelType
-       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeAssembleDiagonalLO::Kernel()
-{
-   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
-   using diag_t =
-      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return diag_t::template derivative_assemble_diagonal_callback<
-             LocalQFLOBackend<DIM, Q1D>>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeAssembleDiagonal<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::DiagonalKernelType
-       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeAssembleDiagonalLO::Fallback(int dim, int q1d)
-{
-   using diag_t =
-      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeAssembleDiagonalLO =
-      typename diag_t::DerivativeAssembleDiagonalLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeAssembleDiagonal<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::DiagonalKernelType
-       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeAssembleDiagonalHO::Kernel()
-{
-   using diag_t =
-      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return diag_t::template derivative_assemble_diagonal_callback<
-             LocalQFHOBackend<DIM>,
-             Q1D>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeAssembleDiagonal<derivative_id,
-       qfunc_t,
-       inputs_t,
-       outputs_t>::DiagonalKernelType
-       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
-       DerivativeAssembleDiagonalHO::Fallback(int dim, int q1d)
-{
-   using diag_t =
-      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeAssembleDiagonalHO =
-      typename diag_t::DerivativeAssembleDiagonalHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_setup.hpp

@@ -1,620 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-
-#include "kernels.hpp"
-#include "util.hpp"
-
-#include <array>
-
-namespace mfem::future::LocalQFImpl
-{
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-class DerivativeSetup
-{
-   static constexpr auto inout_tuple =
-   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
-   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
-   static constexpr size_t nfields =
-      count_unique_field_ids(filtered_inout_tuple);
-
-   using qf_signature = typename get_function_signature<qfunc_t>::type;
-   using qf_param_ts = typename qf_signature::parameter_ts;
-   using args_tuple_t = decay_tuple<qf_param_ts>;
-
-   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
-   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
-   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
-                 "LocalQF: q-function arity must match inputs + outputs");
-
-   qfunc_t qfunc;
-   const inputs_t inputs;
-   const outputs_t outputs;
-   const IntegratorContext ctx;
-   Vector &qp_cache;
-   const std::vector<const DofToQuad *> dtqs;
-   // inputs: dtq, idx, B, G, d1d, q1d, vdim
-   const std::array<DofToQuadMap, n_inputs> input_dtq;
-   const std::array<size_t, n_inputs> input_idx;
-   const std::array<const real_t *, n_inputs> input_B, input_G;
-   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
-   // Jacobian cache metadata
-   const std::array<bool, n_inputs> input_is_dependent;
-   const std::array<int, n_inputs> input_size_on_qp;
-   const std::array<int, n_outputs> out_vdim;
-   const std::array<int, n_outputs> out_op_dim;
-   const std::array<int, n_outputs> out_offsets;
-   const int output_size_on_qp;
-   const int trial_vdim;
-   const int total_trial_op_dim;
-   const int residual_size_on_qp;
-   // other constants
-   const int dim, ne, nq, q1d;
-
-public:
-   //////////////////////////////////////////////////////////////////
-   DerivativeSetup() = delete;
-
-   DerivativeSetup(IntegratorContext ctx,
-                   qfunc_t qfunc,
-                   inputs_t inputs,
-                   outputs_t outputs,
-                   Vector &qp_cache):
-      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
-      qp_cache(qp_cache), dtqs(make_dtqs(ctx)),
-      input_dtq(create_dtq_maps<Entity::Element>(
-                   inputs,
-                   dtqs,
-                   create_union_field_map_for_dtq(ctx, inputs),
-                   ctx.unionfds,
-                   ctx.ir)),
-      input_idx(create_input_vector_map(ctx, inputs)),
-      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
-      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
-      input_vdim(get_vdim(inputs)),
-      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
-      input_size_on_qp(
-         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
-                           out_vdim(get_vdim(outputs)), out_op_dim(compute_out_op_dim(outputs)),
-                           out_offsets(compute_out_offsets(out_vdim, out_op_dim)), output_size_on_qp(
-                              [&]
-   {
-      int s = 0;
-      for_constexpr<n_outputs>([&](auto o)
-      { s += get<o>(outputs).size_on_qp; });
-      return s;
-   }()),
-   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
-   total_trial_op_dim(compute_total_trial_op_dim(
-                         inputs, input_is_dependent, input_size_on_qp)),
-   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
-   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
-   q1d(tensor_1d_size(nq, dim))
-   {
-      MFEM_ASSERT(ctx.unionfds.size() == nfields,
-                  "LocalQFBackend: unionfds size mismatch");
-
-      qp_cache.SetSize(ne * nq * residual_size_on_qp);
-      qp_cache.UseDevice(true);
-   }
-
-   //////////////////////////////////////////////////////////////////
-   void operator()(const std::vector<Vector *> &xe)
-   {
-      if (ctx.attr.Size() == 0) { return; }
-
-      // Quadrature index is fastest-varying so that adjacent threads (one per
-      // quadrature point) touch adjacent addresses.
-      auto cache_tensor = DeviceTensor<3, real_t>(
-                             qp_cache.ReadWrite(), nq, residual_size_on_qp, ne);
-
-      if (q1d <= LocalQFLOBackendMQ1())
-      {
-         run_kernels<DerivativeSetupLO>(xe, cache_tensor);
-      }
-      else if (q1d <= LocalQFHOBackendMQ1())
-      {
-         run_kernels<DerivativeSetupHO>(xe, cache_tensor);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
-      }
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename Backend>
-   void run_kernels(const std::vector<Vector *> &xe,
-                    DeviceTensor<3, real_t> &cache_tensor)
-   {
-      Backend::Run(dim,
-                   q1d,
-                   ctx,
-                   qfunc,
-                   // inputs
-                   input_idx,
-                   input_B,
-                   input_G,
-                   input_vdim,
-                   input_d1d,
-                   input_q1d,
-                   input_size_on_qp,
-                   input_is_dependent,
-                   // outputs / cache metadata
-                   out_vdim,
-                   out_op_dim,
-                   out_offsets,
-                   trial_vdim,
-                   total_trial_op_dim,
-                   residual_size_on_qp,
-                   // vectors
-                   xe,
-                   cache_tensor,
-                   // fallback arguments
-                   dim,
-                   q1d);
-   }
-
-   //////////////////////////////////////////////////////////////////
-   /// Zeroes the q-function *output* slots of an argument tuple.
-   ///
-   /// The argument tuple is built once per quadrature point and reused for
-   /// every trial seed. The q-function writes through its output parameters,
-   /// so those slots have to be restored before each call to give every seed
-   /// the same starting state a freshly value-initialized tuple would.
-   static MFEM_HOST_DEVICE inline void reset_output_args(args_tuple_t &args)
-   {
-      for_constexpr<n_outputs>([&](auto oc)
-      {
-         constexpr size_t ao = n_inputs + oc.value;
-         auto &oarg = get<ao>(args);
-         oarg = std::remove_reference_t<decltype(oarg)> {};
-      });
-   }
-
-   //////////////////////////////////////////////////////////////////
-   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
-   static void
-   derivative_setup_callback(const IntegratorContext &ctx,
-                             qfunc_t &qfunc,
-                             // inputs: idx, B, G, vdim, d1d, q1d
-                             const std::array<size_t, n_inputs> &in_idx,
-                             const std::array<const real_t *, n_inputs> in_B,
-                             const std::array<const real_t *, n_inputs> in_G,
-                             const std::array<int, n_inputs> &in_vdim,
-                             const std::array<int, n_inputs> &in_d1d,
-                             const std::array<int, n_inputs> &in_q1d,
-                             const std::array<int, n_inputs> &in_size_on_qp,
-                             const std::array<bool, n_inputs> &input_dep,
-                             // outputs / cache metadata
-                             const std::array<int, n_outputs> &out_vdim,
-                             const std::array<int, n_outputs> &out_op_dim,
-                             const std::array<int, n_outputs> &out_offsets,
-                             const int trial_vdim,
-                             const int total_trial_op_dim,
-                             const int residual_size_on_qp,
-                             const std::vector<Vector *> &xe,
-                             DeviceTensor<3, real_t> &cache_tensor,
-                             // fallback arguments
-                             const int dim,
-                             const int q1d)
-   {
-      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
-      if (ctx.attr.Size() == 0) { return; }
-
-      static constexpr auto B2D = backend_t::DIM == 2;
-      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
-      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
-
-      const int ne = ctx.nentities;
-      MFEM_CONTRACT_VAR(residual_size_on_qp);
-
-      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
-
-      // --------------------------------------------------
-      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
-      // --------------------------------------------------
-      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
-      for_constexpr<n_inputs>([&](auto ic)
-      {
-         constexpr size_t i = ic.value;
-         const size_t k = in_idx[i];
-         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
-         using FOP = tuple_element_t<i, inputs_t>;
-         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
-         }
-         else if constexpr (is_identity_fop_v<FOP>)
-         {
-            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
-            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
-         }
-         else if constexpr (is_weight_fop_v<FOP>)
-         {
-            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
-                        "tensor-product IR expected");
-            in_XE[i] = Reshape(
-                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      });
-
-      const auto d_attr = ctx.attr.Read();
-      const bool has_attr = ctx.attr.Size() > 0;
-      const auto d_elem_attr = ctx.elem_attr->Read();
-
-      dfem::forall<MTPB>(
-         [=] MFEM_HOST_DEVICE(const int e, void *)
-      {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-         // -----------------------------------------------
-         // Inputs argument registers + shared memory
-         // -----------------------------------------------
-         input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
-         MFEM_SHARED typename backend_t::Shared smem;
-
-         // -----------------------------------------------
-         // Load primal inputs (rargs) once for this element
-         // -----------------------------------------------
-         for_constexpr<n_inputs>([&](auto ic)
-         {
-            constexpr size_t i = ic.value;
-            const auto &XE = in_XE[i];
-            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
-            const real_t *B = in_B[i], *G = in_G[i];
-            auto &rarg = get<i>(rargs);
-            using XE_t = decltype(XE);
-            using rarg_t = decltype(rarg);
-            using FOP = tuple_element_t<i, inputs_t>;
-            if constexpr (is_value_fop<FOP>::value)
-            {
-               backend_t::template LoadValue<rarg_t, XE_t>(
-                  smem, e, d, q, Q1D, B, XE, rarg);
-            }
-            else if constexpr (is_gradient_fop_v<FOP>)
-            {
-               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
-               using FieldParamT =
-                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-               backend_t::template LoadGradient<RNK, rarg_t, XE_t, FieldParamT>(
-                  smem, e, d, q, q1d, B, G, XE, rarg);
-            }
-            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
-            {
-               // qp values are read directly from in_XE / IR
-            }
-            else
-            {
-               static_assert(false, "Unsupported");
-            }
-         });
-         MFEM_SYNC_THREAD;
-
-         // -----------------------------------------------
-         // Build the primal arguments once per quadrature point, then, for
-         // each trial seed (j, dependent input s, m), differentiate the
-         // q-function with a unit tangent and store the result row in the
-         // cache. Nothing in the primal pull depends on the seed, so the
-         // thread loop is the outermost one here. The seed loops only touch
-         // per-thread state, hence no barrier inside them.
-         // Warning: no 'DIRECT' on the 'Z' direction,
-         // as one backend may need to iterate over it.
-         // -----------------------------------------------
-         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-#ifdef MFEM_USE_ENZYME
-                  args_tuple_t primal_args {};
-                  for_constexpr<n_inputs>([&](auto ic)
-                  {
-                     constexpr size_t i = ic.value;
-                     auto &parg = get<i>(primal_args);
-                     const auto &XE = in_XE[i];
-                     using FOP = tuple_element_t<i, inputs_t>;
-                     using ARG =
-                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
-                     if constexpr (is_identity_fop_v<FOP>)
-                     {
-                        parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-                     }
-                     else if constexpr (is_weight_fop_v<FOP>)
-                     {
-                        parg = XE(qx, qy, qz, 0, 0);
-                     }
-                     else if constexpr (is_value_fop_v<FOP> ||
-                                        is_gradient_fop_v<FOP>)
-                     {
-                        parg = backend_t::template qp_pull<ARG>(
-                           get<i>(rargs), qx, qy, qz);
-                     }
-                     else
-                     {
-                        static_assert(false, "Unsupported");
-                     }
-                  });
-
-                  for (int j = 0; j < trial_vdim; j++)
-                  {
-                     int m_offset = 0;
-                     for_constexpr<n_inputs>([&](auto sc)
-                     {
-                        constexpr size_t s = sc.value;
-                        if (!input_dep[s]) { return; }
-
-                        const int vdim_s = in_vdim[s];
-                        const int op_dim_s = in_size_on_qp[s] / vdim_s;
-
-                        for (int m = 0; m < op_dim_s; m++)
-                        {
-                           const int col_m = m + m_offset;
-
-                           // Enzyme writes through the output slots of the
-                           // primal tuple, so they are reset per seed.
-                           reset_output_args(primal_args);
-
-                           args_tuple_t shadow_args {};
-                           qf_set_value_at(get<s>(shadow_args), j, m, 1.0);
-
-                           call_enzyme_fwddiff(qfunc, primal_args, shadow_args);
-
-                           for_constexpr<n_outputs>([&](auto oc)
-                           {
-                              constexpr size_t o = oc.value, ao = n_inputs + o;
-                              using out_fop_t =
-                                 std::decay_t<tuple_element_t<o, outputs_t>>;
-                              const auto &tangent = get<ao>(shadow_args);
-                              const int tv = out_vdim[o], to = out_op_dim[o];
-                              for (int i = 0; i < tv; i++)
-                              {
-                                 for (int k = 0; k < to; k++)
-                                 {
-                                    const int row = out_offsets[o] + i * to + k;
-                                    const int cache_idx =
-                                       row * trial_vdim * total_trial_op_dim +
-                                       j * total_trial_op_dim + col_m;
-                                    // An Identity output is flat quadrature
-                                    // point data: its FieldOperator vdim counts
-                                    // components, not rows of the q-function
-                                    // argument's shape. The two index form
-                                    // assumes vdim == extents[0] and would run
-                                    // off the end of, say, a
-                                    // tensor<real_t, DIM, DIM> bound to a vdim
-                                    // DIM*DIM space, so read it flat with the
-                                    // same column major packing that
-                                    // identity_qp_write_value writes.
-                                    if constexpr (is_identity_fop_v<out_fop_t>)
-                                    {
-                                       cache_tensor(q, cache_idx, e) =
-                                          qf_flat_value(tangent, i * to + k);
-                                    }
-                                    else
-                                    {
-                                       cache_tensor(q, cache_idx, e) =
-                                          qf_value_at(tangent, i, k);
-                                    }
-                                 }
-                              }
-                           });
-                        }
-                        m_offset += op_dim_s;
-                     });
-                  }
-#else  // MFEM_USE_ENZYME
-                  args_tuple_t qargs {};
-                  for_constexpr<n_inputs>([&](auto ic)
-                  {
-                     constexpr size_t i = ic.value;
-                     auto &qarg = get<i>(qargs);
-                     const auto &XE = in_XE[i];
-                     using FOP = tuple_element_t<i, inputs_t>;
-                     using ARG =
-                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
-                     if constexpr (is_identity_fop_v<FOP>)
-                     {
-                        using DT =
-                           typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
-                        if constexpr (qf_param_uses_dual_v<DT>)
-                        {
-                           qarg = backend_t::template identity_qp_pull_dual<DT>(
-                              false, XE, XE, qx, qy, qz, e);
-                        }
-                        else
-                        {
-                           qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
-                        }
-                     }
-                     else if constexpr (is_weight_fop_v<FOP>)
-                     {
-                        qarg = XE(qx, qy, qz, 0, 0);
-                     }
-                     else if constexpr (is_value_fop_v<FOP> ||
-                                        is_gradient_fop_v<FOP>)
-                     {
-                        qarg = backend_t::template qp_pull<ARG>(
-                           get<i>(rargs), qx, qy, qz);
-                     }
-                     else
-                     {
-                        static_assert(false, "Unsupported");
-                     }
-                  });
-
-                  for (int j = 0; j < trial_vdim; j++)
-                  {
-                     int m_offset = 0;
-                     for_constexpr<n_inputs>([&](auto sc)
-                     {
-                        constexpr size_t s = sc.value;
-                        if (!input_dep[s]) { return; }
-
-                        const int vdim_s = in_vdim[s];
-                        const int op_dim_s = in_size_on_qp[s] / vdim_s;
-
-                        for (int m = 0; m < op_dim_s; m++)
-                        {
-                           const int col_m = m + m_offset;
-
-                           // The q-function writes through the output slots,
-                           // so they are reset per seed.
-                           reset_output_args(qargs);
-
-                           qf_set_gradient_at(get<s>(qargs), j, m, 1.0);
-
-                           call_qfunc_no_move(qfunc, qargs);
-
-                           for_constexpr<n_outputs>([&](auto oc)
-                           {
-                              constexpr size_t o = oc.value, ao = n_inputs + o;
-                              const auto &tangent = get<ao>(qargs);
-                              const int tv = out_vdim[o], to = out_op_dim[o];
-                              for (int i = 0; i < tv; i++)
-                              {
-                                 for (int k = 0; k < to; k++)
-                                 {
-                                    const int row = out_offsets[o] + i * to + k;
-                                    const int cache_idx =
-                                       row * trial_vdim * total_trial_op_dim +
-                                       j * total_trial_op_dim + col_m;
-                                    cache_tensor(q, cache_idx, e) =
-                                       qf_gradient_at(tangent, i, k);
-                                 }
-                              }
-                           });
-
-                           // Clear the seed so the next direction starts from
-                           // the pristine (zero-tangent) primal state.
-                           qf_set_gradient_at(get<s>(qargs), j, m, 0.0);
-                        }
-                        m_offset += op_dim_s;
-                     });
-                  }
-#endif // MFEM_USE_ENZYME
-               }
-            }
-         }
-      },
-      ne,
-      backend_t::thread_blocks(
-         compute_kernel_thread_1d<inputs_t>(q1d, in_d1d)),
-      0,
-      nullptr);
-   }
-
-   using SetupKernelType =
-      decltype(&DerivativeSetup::derivative_setup_callback<>);
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupLO,
-                                     SetupKernelType,
-                                     (int, int) );
-   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupHO,
-                                     SetupKernelType,
-                                     (int, int) );
-};
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-SetupKernelType
-DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeSetupLO::Kernel()
-{
-   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
-   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return setup_t::template derivative_setup_callback<
-             LocalQFLOBackend<DIM, Q1D>>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-SetupKernelType
-DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeSetupLO::Fallback(int dim, int q1d)
-{
-   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeSetupLO = typename setup_t::DerivativeSetupLO;
-   if (dim == 2)
-   {
-      return DispatchLOKernelByQ1D<DerivativeSetupLO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchLOKernelByQ1D<DerivativeSetupLO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-template<int DIM, int Q1D>
-inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-SetupKernelType
-DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeSetupHO::Kernel()
-{
-   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   return setup_t::template derivative_setup_callback<LocalQFHOBackend<DIM>,
-                                                      Q1D>;
-}
-
-template<int derivative_id,
-         typename qfunc_t,
-         typename inputs_t,
-         typename outputs_t>
-inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-SetupKernelType
-DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
-DerivativeSetupHO::Fallback(int dim, int q1d)
-{
-   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
-   using DerivativeSetupHO = typename setup_t::DerivativeSetupHO;
-   if (dim == 2)
-   {
-      return DispatchHOKernelByQ1D<DerivativeSetupHO, 2>(q1d);
-   }
-   else if (dim == 3)
-   {
-      return DispatchHOKernelByQ1D<DerivativeSetupHO, 3>(q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-      return nullptr;
-   }
-}
-
-} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/fwddiff_transformer.hpp

@@ -1,277 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../util.hpp"
-
-#ifdef MFEM_USE_ENZYME
-
-namespace mfem::future
-{
-
-template <typename T> struct function_traits;
-
-template <typename R, typename C, typename... Args>
-struct function_traits<R (C::*)(Args...) const>
-{
-   using primal_return_type = R;
-   using args_tuple = std::tuple<Args...>;
-   static constexpr size_t arity = sizeof...(Args);
-};
-
-// Component count and writable counterpart of a per-point argument, which
-// is either a tensor or a plain scalar.
-template <typename Arg> struct qp_traits
-{
-   static_assert(std::is_arithmetic_v<Arg>,
-                 "per-point arguments must be tensors or scalars");
-   using view_type = Arg;
-   static constexpr int components = 1;
-};
-
-template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
-{
-   using view_type = tensor<std::remove_const_t<T>, Sizes...>;
-   static constexpr int components = (Sizes * ... * 1);
-};
-
-// Generic FwdDiff: computes the full gradient of a pointwise qfunction at a
-// single quadrature point.
-//
-// active_input is the index of the argument to differentiate with respect
-// to; active_output is the index of the (scalar) output argument whose
-// derivative is taken.
-//
-// operator()(args...) takes the qfunction's arguments, except that the
-// active output position receives the *gradient*, shaped like the active
-// input (d(output)/d(input component)).
-//
-// Per input component d, one enzyme fwddiff call with the one-hot seed e_d
-// in the input tangent yields gradient entry d, i.e. grad_components enzyme
-// calls per point. The output tangent is not pre-zeroed, so the qfunction
-// must fully write its output (tangent stores overwrite).
-//
-// operator() is MFEM_HOST_DEVICE, stateless and allocation-free, so it is
-// callable inside a GPU kernel: seed, primal scratch and tangent all live
-// on the stack.
-template <typename Func, size_t active_input, size_t active_output>
-struct FwdDiff
-{
-   using traits = function_traits<decltype(&Func::operator())>;
-   using args_tuple = typename traits::args_tuple;
-   static constexpr size_t arity = traits::arity;
-
-   static_assert(std::is_void_v<typename traits::primal_return_type>,
-                 "FwdDiff only supports primal functions with void return type");
-   static_assert(active_input < arity && active_output < arity,
-                 "active argument indices must be within the function arity");
-   static_assert(active_input != active_output,
-                 "active input and output must be different arguments");
-
-   using input_type =
-      std::decay_t<std::tuple_element_t<active_input, args_tuple>>;
-   using output_type =
-      std::decay_t<std::tuple_element_t<active_output, args_tuple>>;
-
-   using grad_type = typename qp_traits<input_type>::view_type;
-   using output_view = typename qp_traits<output_type>::view_type;
-
-   static constexpr int grad_components = qp_traits<input_type>::components;
-
-   static_assert(qp_traits<output_type>::components == 1,
-                 "gradient output requires a scalar output");
-
-   // Signature of the differentiated qfunction: the primal arguments, with
-   // the Active output slot receiving the (writable) gradient instead.
-   // Exposed through create_function_signature below so that
-   // DifferentiableOperator can deduce the parameter types, which it cannot
-   // do from the variadic operator().
-   template <size_t I>
-   using qf_arg_t = std::conditional_t<I == active_output, grad_type &,
-         std::tuple_element_t<I, args_tuple>>;
-
-   template <size_t... Is>
-   static FunctionSignature<void(qf_arg_t<Is>...)>
-   signature_impl(std::index_sequence<Is...>);
-
-   using signature =
-      decltype(signature_impl(std::make_index_sequence<arity> {}));
-
-   // d-th scalar of a per-point argument in flat row-major order, regardless
-   // of rank, built on the native operator[] (tensor has no flat-index
-   // accessor; flatten() returns a copy, so it cannot be written through).
-   MFEM_HOST_DEVICE static double &component(double &t, int) { return t; }
-
-   template <typename T, int n0, int... n>
-   MFEM_HOST_DEVICE static T &component(tensor<T, n0, n...> &t, int d)
-   {
-      if constexpr (sizeof...(n) == 0)
-      {
-         return t[d];
-      }
-      else
-      {
-         constexpr int stride = (n * ... * 1);
-         return component(t[d / stride], d % stride);
-      }
-   }
-
-   // Plain function with the qfunction's exact (reference) signature, so it
-   // can be handed to Enzyme as a function pointer; references are pointers
-   // to Enzyme, so primal arguments and shadows are passed by address below.
-   template <size_t... Is>
-   MFEM_HOST_DEVICE static void
-   static_call(std::tuple_element_t<Is, args_tuple>... args)
-   {
-      Func{}(args...);
-   }
-
-   template <size_t... Is>
-   static constexpr auto fn_ptr(std::index_sequence<Is...>)
-   {
-      return &static_call<Is...>;
-   }
-   static constexpr auto fn()
-   {
-      return fn_ptr(std::make_index_sequence<arity> {});
-   }
-
-   // Writable, zero-initialized scratch with the shape of argument I, used
-   // as its enzyme shadow.
-   template <size_t I>
-   using shadow_t = typename qp_traits<
-                    std::decay_t<std::tuple_element_t<I, args_tuple>>>::view_type;
-
-   template <size_t... Is>
-   MFEM_HOST_DEVICE static auto make_shadows(std::index_sequence<Is...>)
-   {
-      return mfem::future::make_tuple(shadow_t<Is> {}...);
-   }
-
-   template <typename Shadows, size_t... Is>
-   MFEM_HOST_DEVICE static auto make_shadow_ptrs(Shadows &shadows,
-                                                 std::index_sequence<Is...>)
-   {
-      return mfem::future::make_tuple(&mfem::future::get<int(Is)>(shadows)...);
-   }
-
-   // The caller's argument pointers, except the active output slot, which
-   // points to scalar scratch: the caller's slot holds the gradient, while
-   // the primal function writes its scalar output there.
-   template <size_t I, typename Ptrs>
-   MFEM_HOST_DEVICE static auto primal_ptr(Ptrs &ptrs, output_view &primal)
-   {
-      if constexpr (I == active_output) { return &primal; }
-      else { return mfem::future::get<int(I)>(ptrs); }
-   }
-
-   template <typename Ptrs, size_t... Is>
-   MFEM_HOST_DEVICE static auto make_primal_ptrs(Ptrs &ptrs,
-                                                 output_view &primal,
-                                                 std::index_sequence<Is...>)
-   {
-      return mfem::future::make_tuple(primal_ptr<Is>(ptrs, primal)...);
-   }
-
-   // Single flat enzyme call. The activity markers must appear directly in
-   // the __enzyme_fwddiff argument list — Enzyme cannot trace markers that
-   // were forwarded through function parameters (e.g. at -O0, where nothing
-   // is inlined). Every argument is therefore enzyme_dup'd in one sticky
-   // group; Const arguments simply carry a zero tangent, which is equivalent
-   // to marking them enzyme_const.
-   //
-   // always_inline is load-bearing: when FwdDiff is itself differentiated
-   // (second derivatives, forward-over-forward), Enzyme only recognizes this
-   // nested __enzyme_fwddiff call if it sits at most one call level below
-   // the function handed to the outer __enzyme_fwddiff. Without inlining
-   // (-O0) it sits two levels down (wrapper -> operator() -> call_enzyme)
-   // and the outer pass treats it as a regular call: the activity marker
-   // ints then receive undef shadows, which misaligns the argument pairing
-   // (observed as "cannot compute with global variable that doesn't have
-   // marked shadow global" at compile time or null-shadow segfaults at
-   // runtime). The always-inliner runs even at -O0, hoisting this call into
-   // operator() where the nested handling applies.
-   template <typename PrimalPtrs, typename ShadowPtrs, size_t... Is>
-   MFEM_FUTURE_ALWAYS_INLINE
-   MFEM_HOST_DEVICE static void call_enzyme(PrimalPtrs &primal_ptrs,
-                                            ShadowPtrs &shadow_ptrs,
-                                            std::index_sequence<Is...>)
-   {
-      __enzyme_fwddiff<void>(fn(), enzyme_dup,
-                             mfem::future::get<int(Is)>(primal_ptrs)...,
-                             enzyme_interleave,
-                             mfem::future::get<int(Is)>(shadow_ptrs)...,
-                             enzyme_runtime_activity);
-   }
-
-   template <typename... Args>
-   MFEM_HOST_DEVICE void operator()(Args &&...args) const
-   {
-      static_assert(sizeof...(Args) == arity, "Wrong number of arguments");
-      auto ptrs = mfem::future::make_tuple(&args...);
-
-      auto &grad = *mfem::future::get<int(active_output)>(ptrs);
-      static_assert(std::is_same_v<std::decay_t<decltype(grad)>, grad_type>,
-                    "gradient argument must be shaped like the Active input "
-                    "(with writable scalars)");
-
-      constexpr auto seq = std::make_index_sequence<arity> {};
-
-      output_view primal{};
-      auto primal_ptrs = make_primal_ptrs(ptrs, primal, seq);
-
-      auto shadows = make_shadows(seq);
-      auto shadow_ptrs = make_shadow_ptrs(shadows, seq);
-      auto &seed = mfem::future::get<int(active_input)>(shadows);
-      auto &tangent = mfem::future::get<int(active_output)>(shadows);
-
-      // One enzyme call per input component d: seed e_d in the input tangent
-      // and read gradient entry d off the output tangent.
-      for (int d = 0; d < grad_components; d++)
-      {
-         component(seed, d) = 1.0;
-         call_enzyme(primal_ptrs, shadow_ptrs, seq);
-         component(grad, d) = component(tangent, 0);
-         component(seed, d) = 0.0;
-      }
-   }
-
-   static void print() { print_impl(std::make_index_sequence<arity> {}); }
-
-   template <size_t... Is> static void print_impl(std::index_sequence<Is...>)
-   {
-      mfem::out << "for d in [0, " << grad_components
-                << "): __enzyme_fwddiff<void>(fptr, enzyme_dup";
-      ((mfem::out << ", "
-        << get_type_name<std::tuple_element_t<Is, args_tuple>>()),
-       ...);
-      mfem::out << ", enzyme_interleave";
-      (([&]
-      {
-         if constexpr (Is == active_input) { mfem::out << ", e_d seed"; }
-         else if constexpr (Is == active_output) { mfem::out << ", tangent out"; }
-         else { mfem::out << ", zero tangent"; }
-      }()),
-      ...);
-      mfem::out << ")\n";
-   }
-};
-
-template <typename Func, size_t active_input, size_t active_output>
-struct create_function_signature<FwdDiff<Func, active_input, active_output>>
-{
-   using type =
-      typename FwdDiff<Func, active_input, active_output>::signature;
-};
-
-} // namespace mfem::future
-
-#endif // MFEM_USE_ENZYME

fem/dfem/backends/local_qf/kernels.cpp

@@ -1,83 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-// Explicit instantiation of the local Q-function backend device code
-
-#include "../../../../config/config.hpp"
-
-#ifdef MFEM_USE_MPI
-
-#include "kernels.hpp"
-
-namespace mfem::future
-{
-
-// ────────────────────────────────────────────────────────────────────────────
-// Low-order backends instantiations for (DIM, Q1D)
-// ────────────────────────────────────────────────────────────────────────────
-template struct lo_ker_backend<2, 2>;
-template struct lo_ker_backend<2, 3>;
-template struct lo_ker_backend<2, 4>;
-template struct lo_ker_backend<2, 5>;
-template struct lo_ker_backend<2, 6>;
-template struct lo_ker_backend<2, 7>;
-template struct lo_ker_backend<2, 8>;
-
-template struct lo_ker_backend<3, 2>;
-template struct lo_ker_backend<3, 3>;
-template struct lo_ker_backend<3, 4>;
-template struct lo_ker_backend<3, 5>;
-template struct lo_ker_backend<3, 6>;
-template struct lo_ker_backend<3, 7>;
-template struct lo_ker_backend<3, 8>;
-
-template struct LocalQFLOBackend<2, 2>;
-template struct LocalQFLOBackend<2, 3>;
-template struct LocalQFLOBackend<2, 4>;
-template struct LocalQFLOBackend<2, 5>;
-template struct LocalQFLOBackend<2, 6>;
-template struct LocalQFLOBackend<2, 7>;
-template struct LocalQFLOBackend<2, 8>;
-
-template struct LocalQFLOBackend<3, 2>;
-template struct LocalQFLOBackend<3, 3>;
-template struct LocalQFLOBackend<3, 4>;
-template struct LocalQFLOBackend<3, 5>;
-template struct LocalQFLOBackend<3, 6>;
-template struct LocalQFLOBackend<3, 7>;
-template struct LocalQFLOBackend<3, 8>;
-
-// ────────────────────────────────────────────────────────────────────────────
-// High-order backends instantiations for (DIM, Q1D)
-// ────────────────────────────────────────────────────────────────────────────
-template struct ho_ker_backend<2, 8>;
-template struct ho_ker_backend<2, 10>;
-template struct ho_ker_backend<2, 12>;
-template struct ho_ker_backend<2, 16>;
-
-template struct ho_ker_backend<3, 8>;
-template struct ho_ker_backend<3, 10>;
-template struct ho_ker_backend<3, 12>;
-template struct ho_ker_backend<3, 16>;
-
-template struct LocalQFHOBackend<2, 8>;
-template struct LocalQFHOBackend<2, 10>;
-template struct LocalQFHOBackend<2, 12>;
-template struct LocalQFHOBackend<2, 16>;
-
-template struct LocalQFHOBackend<3, 8>;
-template struct LocalQFHOBackend<3, 10>;
-template struct LocalQFHOBackend<3, 12>;
-template struct LocalQFHOBackend<3, 16>;
-
-} // namespace mfem::future
-
-#endif // MFEM_USE_MPI

fem/dfem/backends/local_qf/kernels.hpp

@@ -1,77 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "kernels_lo.hpp" // IWYU pragma: export
-#include "kernels_ho.hpp" // IWYU pragma: export
-
-namespace mfem::future
-{
-
-// ────────────────────────────────────────────────────────────────────────────
-// Low-order backends instantiations for (DIM, Q1D)
-// ────────────────────────────────────────────────────────────────────────────
-extern template struct lo_ker_backend<2, 2>;
-extern template struct lo_ker_backend<2, 3>;
-extern template struct lo_ker_backend<2, 4>;
-extern template struct lo_ker_backend<2, 5>;
-extern template struct lo_ker_backend<2, 6>;
-extern template struct lo_ker_backend<2, 7>;
-extern template struct lo_ker_backend<2, 8>;
-
-extern template struct lo_ker_backend<3, 2>;
-extern template struct lo_ker_backend<3, 3>;
-extern template struct lo_ker_backend<3, 4>;
-extern template struct lo_ker_backend<3, 5>;
-extern template struct lo_ker_backend<3, 6>;
-extern template struct lo_ker_backend<3, 7>;
-extern template struct lo_ker_backend<3, 8>;
-
-extern template struct LocalQFLOBackend<2, 2>;
-extern template struct LocalQFLOBackend<2, 3>;
-extern template struct LocalQFLOBackend<2, 4>;
-extern template struct LocalQFLOBackend<2, 5>;
-extern template struct LocalQFLOBackend<2, 6>;
-extern template struct LocalQFLOBackend<2, 7>;
-extern template struct LocalQFLOBackend<2, 8>;
-
-extern template struct LocalQFLOBackend<3, 2>;
-extern template struct LocalQFLOBackend<3, 3>;
-extern template struct LocalQFLOBackend<3, 4>;
-extern template struct LocalQFLOBackend<3, 5>;
-extern template struct LocalQFLOBackend<3, 6>;
-extern template struct LocalQFLOBackend<3, 7>;
-extern template struct LocalQFLOBackend<3, 8>;
-
-// ────────────────────────────────────────────────────────────────────────────
-// High-order backends instantiations for (DIM, Q1D)
-// ────────────────────────────────────────────────────────────────────────────
-extern template struct ho_ker_backend<2, 8>;
-extern template struct ho_ker_backend<2, 10>;
-extern template struct ho_ker_backend<2, 12>;
-extern template struct ho_ker_backend<2, 16>;
-
-extern template struct ho_ker_backend<3, 8>;
-extern template struct ho_ker_backend<3, 10>;
-extern template struct ho_ker_backend<3, 12>;
-extern template struct ho_ker_backend<3, 16>;
-
-extern template struct LocalQFHOBackend<2, 8>;
-extern template struct LocalQFHOBackend<2, 10>;
-extern template struct LocalQFHOBackend<2, 12>;
-extern template struct LocalQFHOBackend<2, 16>;
-
-extern template struct LocalQFHOBackend<3, 8>;
-extern template struct LocalQFHOBackend<3, 10>;
-extern template struct LocalQFHOBackend<3, 12>;
-extern template struct LocalQFHOBackend<3, 16>;
-
-} // namespace mfem::future

fem/dfem/backends/local_qf/kernels_ho.hpp

@@ -1,803 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../../kernels.hpp"
-namespace ker = mfem::kernels::internal;
-
-#include "../../util.hpp" // for ThreadBlocks
-#include "util.hpp"
-
-namespace mfem::future
-{
-
-// ────────────────────────────────────────────────────────────────────────────
-inline constexpr int LocalQFHOBackendMQ1() { return 16; }
-
-// ────────────────────────────────────────────────────────────────────────────
-/// Register type for one HO q-function parameter
-template<typename KerOps, typename T, int rank = qf_param_shape<T>::rank>
-struct ho_qreg;
-
-template<typename KerOps, typename T>
-struct ho_qreg<KerOps, T, 0>
-{
-   using type = typename KerOps::template val_reg_t<1>;
-};
-
-template<typename KerOps, typename T>
-struct ho_qreg<KerOps, T, 1>
-{
-   static constexpr int e0 = qf_param_shape<T>::extents[0];
-   using type = typename KerOps::template val_reg_t<e0>;
-};
-
-template<typename KerOps, typename T>
-struct ho_qreg<KerOps, T, 2>
-{
-   static constexpr int VDIM = qf_param_shape<T>::extents[0];
-   static constexpr int SDIM = qf_param_shape<T>::extents[1];
-   using type = typename KerOps::template del_reg_t<VDIM, SDIM>;
-};
-
-template<typename KerOps, typename T>
-using ho_qreg_t = typename ho_qreg<KerOps, T>::type;
-
-// ────────────────────────────────────────────────────────────────────────────
-namespace hok
-{
-
-/// Load one quadrature-point value
-template<int DIM, typename T, typename Reg>
-MFEM_HOST_DEVICE inline auto load_at(Reg &reg, int qx, int qy, int qz)
-{
-   static_assert(DIM == 2 || DIM == 3);
-   constexpr int RNK = qf_param_shape<T>::rank;
-   if constexpr (DIM == 2)
-   {
-      MFEM_CONTRACT_VAR(qz);
-      if constexpr (RNK == 0) { return T{ reg(0, qy, qx) }; }
-      else if constexpr (RNK == 1)
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         T t{};
-         MFEM_UNROLL(e0)
-         for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qy, qx); }
-         return t;
-      }
-      else
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         constexpr int e1 = qf_param_shape<T>::extents[1];
-         T t;
-         MFEM_UNROLL(e0)
-         for (int i = 0; i < e0; ++i)
-         {
-            MFEM_UNROLL(e1)
-            for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qy, qx); }
-         }
-         return t;
-      }
-   }
-   else
-   {
-      if constexpr (RNK == 0) { return T{ reg(0, qz, qy, qx) }; }
-      else if constexpr (RNK == 1)
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         T t{};
-         MFEM_UNROLL(e0)
-         for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qz, qy, qx); }
-         return t;
-      }
-      else
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         constexpr int e1 = qf_param_shape<T>::extents[1];
-         T t;
-         MFEM_UNROLL(e0)
-         for (int i = 0; i < e0; ++i)
-         {
-            MFEM_UNROLL(e1)
-            for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qz, qy, qx); }
-         }
-         return t;
-      }
-   }
-}
-
-template<bool tangent, typename U>
-MFEM_HOST_DEVICE inline auto qp_store(const U &v)
-{
-   if constexpr (tangent) { return qf_store_gradient(v); }
-   else
-   {
-      return qf_store_value(v);
-   }
-}
-
-// Store primal value or dual tangent at one quadrature point
-template<int DIM, typename T, typename Reg, bool tangent>
-MFEM_HOST_DEVICE inline void
-store_at(Reg &reg, int qx, int qy, int qz, const T &out)
-{
-   static_assert(DIM == 2 || DIM == 3);
-   constexpr int RNK = qf_param_shape<T>::rank;
-   if constexpr (DIM == 2)
-   {
-      MFEM_CONTRACT_VAR(qz);
-      if constexpr (RNK == 0) { reg(0, qy, qx) = qp_store<tangent>(out); }
-      else if constexpr (RNK == 1)
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         MFEM_UNROLL(e0)
-         for (int dd = 0; dd < e0; ++dd)
-         {
-            reg(dd, qy, qx) = qp_store<tangent>(out(dd));
-         }
-      }
-      else
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         constexpr int e1 = qf_param_shape<T>::extents[1];
-         MFEM_UNROLL(e0)
-         for (int i = 0; i < e0; ++i)
-         {
-            MFEM_UNROLL(e1)
-            for (int j = 0; j < e1; ++j)
-            {
-               reg(i, j, qy, qx) = qp_store<tangent>(out(i, j));
-            }
-         }
-      }
-   }
-   else
-   {
-      if constexpr (RNK == 0) { reg(0, qz, qy, qx) = qp_store<tangent>(out); }
-      else if constexpr (RNK == 1)
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         MFEM_UNROLL(e0)
-         for (int dd = 0; dd < e0; ++dd)
-         {
-            reg(dd, qz, qy, qx) = qp_store<tangent>(out(dd));
-         }
-      }
-      else
-      {
-         constexpr int e0 = qf_param_shape<T>::extents[0];
-         constexpr int e1 = qf_param_shape<T>::extents[1];
-         MFEM_UNROLL(e0)
-         for (int i = 0; i < e0; ++i)
-         {
-            MFEM_UNROLL(e1)
-            for (int j = 0; j < e1; ++j)
-            {
-               reg(i, j, qz, qy, qx) = qp_store<tangent>(out(i, j));
-            }
-         }
-      }
-   }
-}
-
-// Pull primal/tangent pair into a dual q-function argument
-template<int DIM, typename T, typename Reg>
-MFEM_HOST_DEVICE inline auto
-pull_directional(Reg &preg, Reg &sreg, int qx, int qy, int qz, bool dependent)
-{
-   if constexpr (!qf_param_uses_dual_v<T>)
-   {
-      return load_at<DIM, T>(preg, qx, qy, qz);
-   }
-   else
-   {
-      if (!dependent) { return load_at<DIM, T>(preg, qx, qy, qz); }
-      constexpr int RNK = qf_param_shape<T>::rank;
-      if constexpr (DIM == 2)
-      {
-         MFEM_CONTRACT_VAR(qz);
-         if constexpr (RNK == 0)
-         {
-            return T{ preg(0, qy, qx), sreg(0, qy, qx) };
-         }
-         else if constexpr (RNK == 1)
-         {
-            constexpr int e0 = qf_param_shape<T>::extents[0];
-            T t{};
-            MFEM_UNROLL(e0)
-            for (int dd = 0; dd < e0; ++dd)
-            {
-               t(dd) = { preg(dd, qy, qx), sreg(dd, qy, qx) };
-            }
-            return t;
-         }
-         else
-         {
-            constexpr int e0 = qf_param_shape<T>::extents[0];
-            constexpr int e1 = qf_param_shape<T>::extents[1];
-            T t;
-            MFEM_UNROLL(e0)
-            for (int i = 0; i < e0; ++i)
-            {
-               MFEM_UNROLL(e1)
-               for (int j = 0; j < e1; ++j)
-               {
-                  t(i, j) = { preg(i, j, qy, qx), sreg(i, j, qy, qx) };
-               }
-            }
-            return t;
-         }
-      }
-      else
-      {
-         if constexpr (RNK == 0)
-         {
-            return T{ preg(0, qz, qy, qx), sreg(0, qz, qy, qx) };
-         }
-         else if constexpr (RNK == 1)
-         {
-            constexpr int e0 = qf_param_shape<T>::extents[0];
-            T t{};
-            MFEM_UNROLL(e0)
-            for (int dd = 0; dd < e0; ++dd)
-            {
-               t(dd) = { preg(dd, qz, qy, qx), sreg(dd, qz, qy, qx) };
-            }
-            return t;
-         }
-         else
-         {
-            constexpr int e0 = qf_param_shape<T>::extents[0];
-            constexpr int e1 = qf_param_shape<T>::extents[1];
-            T t;
-            MFEM_UNROLL(e0)
-            for (int i = 0; i < e0; ++i)
-            {
-               MFEM_UNROLL(e1)
-               for (int j = 0; j < e1; ++j)
-               {
-                  t(i, j) = { preg(i, j, qz, qy, qx), sreg(i, j, qz, qy, qx) };
-               }
-            }
-            return t;
-         }
-      }
-   }
-}
-
-} // namespace hok
-
-// ────────────────────────────────────────────────────────────────────────────
-/// HO tensor-product kernels
-template<int T_DIM, int MQ1>
-struct ho_ker_backend
-{
-   static constexpr int DIM = T_DIM;
-   static_assert(DIM == 2 || DIM == 3);
-
-   template<int VDIM>
-   using val_reg_t = std::conditional_t<(DIM == 2),
-         ker::v_regs2d_t<VDIM, MQ1>,
-         ker::v_regs3d_t<VDIM, MQ1>>;
-
-   template<int VDIM, int SDIM>
-   using del_reg_t = std::conditional_t<(DIM == 2),
-         ker::vd_regs2d_t<VDIM, SDIM, MQ1>,
-         ker::vd_regs3d_t<VDIM, SDIM, MQ1>>;
-
-   struct Shared
-   {
-      real_t M[MQ1][MQ1], B[MQ1][MQ1], G[MQ1][MQ1];
-   };
-
-   template<typename XE_t, typename Dofs>
-   static MFEM_HOST_DEVICE void
-   load_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
-   {
-      if constexpr (DIM == 2) { ker::LoadDofs2d(e, d, XE, dofs); }
-      else
-      {
-         ker::LoadDofs3d(e, d, XE, dofs);
-      }
-   }
-
-   template<int VDIM, int SDIM, typename XE_t, typename Dofs>
-   static MFEM_HOST_DEVICE void
-   load_grad_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
-   {
-      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
-      load_dofs(e, d, XE, dofs);
-   }
-
-   template<typename Smem, typename Dofs, typename ArgReg>
-   static MFEM_HOST_DEVICE void
-   eval_value(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
-   {
-      if constexpr (DIM == 2) { ker::Eval2d(d, q, s.M, s.B, dofs, rarg); }
-      else
-      {
-         ker::Eval3d(d, q, s.M, s.B, dofs, rarg);
-      }
-   }
-
-   template<int VDIM, int SDIM, typename Smem, typename Dofs, typename ArgReg>
-   static MFEM_HOST_DEVICE void
-   grad(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
-   {
-      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
-      if constexpr (DIM == 2) { ker::Grad2d(d, q, s.M, s.B, s.G, dofs, rarg); }
-      else
-      {
-         ker::Grad3d(d, q, s.M, s.B, s.G, dofs, rarg);
-      }
-   }
-
-   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
-   static MFEM_HOST_DEVICE void write_value(const int d,
-                                            const int q,
-                                            const int e,
-                                            Smem &s,
-                                            ArgReg &rarg,
-                                            Dofs &dofs,
-                                            YE_t &YE)
-   {
-      if constexpr (DIM == 2)
-      {
-         ker::EvalTranspose2d(d, q, s.M, s.B, rarg, dofs);
-         ker::WriteDofs2d(e, d, dofs, YE);
-      }
-      else
-      {
-         ker::EvalTranspose3d(d, q, s.M, s.B, rarg, dofs);
-         ker::WriteDofs3d(e, d, dofs, YE);
-      }
-   }
-
-   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
-   static MFEM_HOST_DEVICE void write_gradient_2d(const int d,
-                                                  const int q,
-                                                  const int e,
-                                                  Smem &s,
-                                                  ArgReg &rarg,
-                                                  Dofs &dofs,
-                                                  YE_t &YE)
-   {
-      ker::GradTranspose2d(d, q, s.M, s.B, s.G, rarg, dofs);
-      ker::WriteDofs2d(e, d, dofs, YE);
-   }
-
-   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
-   static MFEM_HOST_DEVICE void write_gradient_3d(const int d,
-                                                  const int q,
-                                                  const int e,
-                                                  Smem &s,
-                                                  ArgReg &rarg,
-                                                  Dofs &dofs,
-                                                  YE_t &YE)
-   {
-      ker::GradTranspose3d(d, q, s.M, s.B, s.G, rarg, dofs);
-      ker::WriteDofs3d(e, d, dofs, YE);
-   }
-
-   template<int VDIM,
-            int SDIM,
-            typename Smem,
-            typename Dofs,
-            typename ArgReg,
-            typename YE_t>
-   static MFEM_HOST_DEVICE void write_gradient(const int d,
-                                               const int q,
-                                               const int e,
-                                               Smem &s,
-                                               ArgReg &rarg,
-                                               Dofs &dofs,
-                                               YE_t &YE)
-   {
-      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
-      if constexpr (DIM == 2) { write_gradient_2d(d, q, e, s, rarg, dofs, YE); }
-      else
-      {
-         write_gradient_3d(d, q, e, s, rarg, dofs, YE);
-      }
-   }
-};
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int T_DIM, int T_Q1D = LocalQFHOBackendMQ1()>
-struct LocalQFHOBackend
-{
-   // ─────────────────────────────────────────────────────
-   static constexpr int DIM = T_DIM, MQ1 = T_Q1D, Q1D = T_Q1D;
-   static_assert(DIM == 2 || DIM == 3);
-
-   // ─────────────────────────────────────────────────────
-   static inline ThreadBlocks thread_blocks(const int q1d)
-   {
-      MFEM_ASSERT(q1d <= Q1D, "q1d must be <= " << Q1D);
-      return { q1d, q1d, 1 };
-   }
-
-   // ─────────────────────────────────────────────────────
-   static inline constexpr int MAX_THREADS_PER_BLOCK() { return Q1D * Q1D; }
-
-   // ─────────────────────────────────────────────────────
-   using backend_t = ho_ker_backend<DIM, Q1D>;
-
-   // ─────────────────────────────────────────────────────
-   using Shared = typename backend_t::Shared;
-
-   // ─────────────────────────────────────────────────────
-   template<typename WT, typename WI, typename Cache, typename AddY>
-   static MFEM_HOST_DEVICE inline void DiagContract(Shared &s,
-                                                    const int num_dof_1d,
-                                                    const int q1d,
-                                                    const int nz_dof,
-                                                    WT wt,
-                                                    WI wi,
-                                                    Cache cache,
-                                                    AddY add_y)
-   {
-      MFEM_CONTRACT_VAR(nz_dof);
-      const int nqz = (DIM == 3) ? q1d : 1;
-      const int ndz = (DIM == 3) ? num_dof_1d : 1;
-
-      ker::s_regs3d_t<MQ1> rz, ry;
-      auto &smem = s.M;
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            for (int dz = 0; dz < ndz; dz++)
-            {
-               real_t u = 0.0;
-               for (int qz = 0; qz < nqz; qz++)
-               {
-                  const int q = qx + (qy + qz * q1d) * q1d;
-                  const real_t wz =
-                     (DIM == 3) ? (wt(2, qz, dz) * wi(2, qz, dz)) : real_t(1);
-                  u += wz * cache(q);
-               }
-               rz[dz][qy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      for (int dz = 0; dz < ndz; dz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-            { smem[qy][qx] = rz[dz][qy][qx]; }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < q1d; qy++)
-               {
-                  u += wt(1, qy, dy) * wi(1, qy, dy) * smem[qy][qx];
-               }
-               ry[dz][dy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-
-      for (int dz = 0; dz < ndz; dz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-            { smem[dy][qx] = ry[dz][dy][qx]; }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, num_dof_1d)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < q1d; qx++)
-               {
-                  u += wt(0, qx, dx) * wi(0, qx, dx) * smem[dy][qx];
-               }
-               add_y(dx, dy, dz, u);
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename T>
-   using QReg = ho_qreg_t<backend_t, T>;
-
-   // ─────────────────────────────────────────────────────
-   template<typename ArgRegT, typename XE_T>
-   static inline MFEM_HOST_DEVICE void LoadValue(Shared &s,
-                                                 const int e,
-                                                 const int d,
-                                                 const int q,
-                                                 const int,
-                                                 const real_t *B,
-                                                 const XE_T &XE,
-                                                 ArgRegT &rarg)
-   {
-      ker::LoadMatrix(d, q, B, s.B);
-      std::remove_reference_t<ArgRegT> dofs;
-      backend_t::load_dofs(e, d, XE, dofs);
-      backend_t::eval_value(d, q, s, dofs, rarg);
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<int RNK,
-            typename ArgRegT,
-            typename XE_T,
-            typename FieldParamT = ArgRegT>
-   static inline MFEM_HOST_DEVICE void LoadGradient(Shared &s,
-                                                    const int e,
-                                                    const int d,
-                                                    const int q,
-                                                    const int,
-                                                    const real_t *B,
-                                                    const real_t *G,
-                                                    const XE_T &XE,
-                                                    ArgRegT &rarg)
-   {
-      ker::LoadMatrix(d, q, B, s.B);
-      ker::LoadMatrix(d, q, G, s.G);
-      static_assert(RNK == 1 || RNK == 2);
-      static constexpr int VDIM =
-         (RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
-      static constexpr int SDIM = (RNK == 1)
-                                  ? qf_param_shape<FieldParamT>::extents[0]
-                                  : qf_param_shape<FieldParamT>::extents[1];
-      if constexpr (SDIM == DIM)
-      {
-         typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
-         if constexpr (RNK == 1) { backend_t::load_dofs(e, d, XE, dofs); }
-         else
-         {
-            backend_t::template load_grad_dofs<VDIM, SDIM>(e, d, XE, dofs);
-         }
-         backend_t::template grad<VDIM, SDIM>(d, q, s, dofs, rarg);
-      }
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename T>
-   static MFEM_HOST_DEVICE inline auto
-   qp_pull(QReg<T> &reg, int qx, int qy, int qz)
-   { return hok::load_at<DIM, T>(reg, qx, qy, qz); }
-
-   // ─────────────────────────────────────────────────────
-   template<typename T>
-   static MFEM_HOST_DEVICE inline auto qp_pull_directional(
-      QReg<T> &preg, QReg<T> &sreg, int qx, int qy, int qz, bool dependent)
-   { return hok::pull_directional<DIM, T>(preg, sreg, qx, qy, qz, dependent); }
-
-   // ─────────────────────────────────────────────────────
-   template<typename DT, typename XE_T>
-   static MFEM_HOST_DEVICE inline DT identity_qp_pull_dual(bool dependent,
-                                                           const XE_T &XP,
-                                                           const XE_T &XD,
-                                                           int qx,
-                                                           int qy,
-                                                           int qz,
-                                                           int e)
-   {
-      constexpr int RNK = qf_param_shape<DT>::rank;
-      if constexpr (RNK == 0)
-      {
-         DT t{};
-         t.value = XP(0, qx, qy, qz, e);
-         t.gradient = dependent ? XD(0, qx, qy, qz, e) : 0.0;
-         return t;
-      }
-      else if constexpr (RNK == 1)
-      {
-         constexpr int e0 = qf_param_shape<DT>::extents[0];
-         DT t{};
-         MFEM_UNROLL(e0)
-         for (int dd = 0; dd < e0; ++dd)
-         {
-            t(dd).value = XP(dd, qx, qy, qz, e);
-            t(dd).gradient = dependent ? XD(dd, qx, qy, qz, e) : 0.0;
-         }
-         return t;
-      }
-      else if constexpr (RNK == 2)
-      {
-         constexpr int e0 = qf_param_shape<DT>::extents[0];
-         constexpr int e1 = qf_param_shape<DT>::extents[1];
-         DT t{};
-         MFEM_UNROLL(e0)
-         for (int i = 0; i < e0; ++i)
-         {
-            MFEM_UNROLL(e1)
-            for (int j = 0; j < e1; ++j)
-            {
-               t(i, j).value = XP(i + e0 * j, qx, qy, qz, e);
-               t(i, j).gradient =
-                  dependent ? XD(i + e0 * j, qx, qy, qz, e) : 0.0;
-            }
-         }
-         return t;
-      }
-      else
-      {
-         static_assert(false, "Unsupported");
-      }
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename T>
-   static MFEM_HOST_DEVICE inline void
-   qp_push(QReg<T> &reg, int qx, int qy, int qz, const T &out)
-   { hok::store_at<DIM, T, decltype(reg), false>(reg, qx, qy, qz, out); }
-
-   // ─────────────────────────────────────────────────────
-   template<typename T>
-   static MFEM_HOST_DEVICE inline void
-   qp_push_tangent(QReg<T> &reg, int qx, int qy, int qz, const T &out)
-   {
-      hok::store_at<DIM, T, decltype(reg), qf_param_uses_dual_v<T>>(
-                                                                    reg, qx, qy, qz, out);
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename DT, typename YE_T>
-   static MFEM_HOST_DEVICE inline void identity_qp_write_value(
-      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
-   {
-      constexpr int RNK = qf_param_shape<DT>::rank;
-      if constexpr (qf_param_uses_dual_v<DT>)
-      {
-         if constexpr (RNK == 0)
-         {
-            YE(0, qx, qy, qz, e) = qf_store_value(qout);
-         }
-         else if constexpr (RNK == 1)
-         {
-            constexpr int e0 = qf_param_shape<DT>::extents[0];
-            MFEM_UNROLL(e0)
-            for (int dd = 0; dd < e0; ++dd)
-            {
-               YE(dd, qx, qy, qz, e) = qf_store_value(qout(dd));
-            }
-         }
-         else if constexpr (RNK == 2)
-         {
-            constexpr int e0 = qf_param_shape<DT>::extents[0];
-            constexpr int e1 = qf_param_shape<DT>::extents[1];
-            MFEM_UNROLL(e0)
-            for (int i = 0; i < e0; ++i)
-            {
-               MFEM_UNROLL(e1)
-               for (int j = 0; j < e1; ++j)
-               {
-                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_value(qout(i, j));
-               }
-            }
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      }
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename DT, typename YE_T>
-   static MFEM_HOST_DEVICE inline void identity_qp_write_tangent(
-      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
-   {
-      constexpr int RNK = qf_param_shape<DT>::rank;
-      if constexpr (qf_param_uses_dual_v<DT>)
-      {
-         if constexpr (RNK == 0)
-         {
-            YE(0, qx, qy, qz, e) = qf_store_gradient(qout);
-         }
-         else if constexpr (RNK == 1)
-         {
-            constexpr int e0 = qf_param_shape<DT>::extents[0];
-            MFEM_UNROLL(e0)
-            for (int dd = 0; dd < e0; ++dd)
-            {
-               YE(dd, qx, qy, qz, e) = qf_store_gradient(qout(dd));
-            }
-         }
-         else if constexpr (RNK == 2)
-         {
-            constexpr int e0 = qf_param_shape<DT>::extents[0];
-            constexpr int e1 = qf_param_shape<DT>::extents[1];
-            MFEM_UNROLL(e0)
-            for (int i = 0; i < e0; ++i)
-            {
-               MFEM_UNROLL(e1)
-               for (int j = 0; j < e1; ++j)
-               {
-                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_gradient(qout(i, j));
-               }
-            }
-         }
-         else
-         {
-            static_assert(false, "Unsupported");
-         }
-      }
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<typename ArgRegT, typename YE_T>
-   static inline MFEM_HOST_DEVICE void WriteValue(Shared &s,
-                                                  const int e,
-                                                  const int d,
-                                                  const int q,
-                                                  const int,
-                                                  const real_t *B,
-                                                  YE_T &YE,
-                                                  ArgRegT &rarg)
-   {
-      ker::LoadMatrix(d, q, B, s.B);
-      std::remove_reference_t<ArgRegT> dofs;
-      backend_t::write_value(d, q, e, s, rarg, dofs, YE);
-   }
-
-   // ─────────────────────────────────────────────────────
-   template<int RNK,
-            typename ArgRegT,
-            typename YE_T,
-            typename FieldParamT = ArgRegT>
-   static inline MFEM_HOST_DEVICE void WriteGradient(Shared &s,
-                                                     const int e,
-                                                     const int d,
-                                                     const int q,
-                                                     const int,
-                                                     const real_t *B,
-                                                     const real_t *G,
-                                                     YE_T &YE,
-                                                     ArgRegT &rarg)
-   {
-      ker::LoadMatrix(d, q, B, s.B);
-      ker::LoadMatrix(d, q, G, s.G);
-      static_assert(RNK == 1 || RNK == 2);
-      static constexpr int VDIM =
-         (RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
-      static constexpr int SDIM = (RNK == 1)
-                                  ? qf_param_shape<FieldParamT>::extents[0]
-                                  : qf_param_shape<FieldParamT>::extents[1];
-      if constexpr (SDIM == DIM)
-      {
-         typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
-         backend_t::template write_gradient<VDIM, SDIM>(
-            d, q, e, s, rarg, dofs, YE);
-      }
-   }
-};
-
-/// @brief Dispatch to a compile-time HO kernel with MQ1 >= runtime @a q1d.
-template <typename HOKernelTable, int DIM, int MQ1 = LocalQFHOBackendMQ1()>
-inline typename HOKernelTable::KernelSignature
-DispatchHOKernelByQ1D(int q1d)
-{
-   MFEM_VERIFY(q1d >= 2 && q1d <= MQ1,
-               "Unsupported HO quadrature order: " << q1d);
-   return HOKernelTable::template Kernel<DIM, MQ1>();
-}
-
-} // namespace mfem::future

fem/dfem/backends/local_qf/prelude.hpp

@@ -1,242 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../integrator_ctx.hpp"
-#include "action.hpp"
-#include "derivative_action.hpp"
-#include "derivative_setup.hpp"
-#include "derivative_apply.hpp"
-#include "derivative_assemble.hpp"
-#include "derivative_assemble_diagonal.hpp"
-#include "derivative_apply_transpose.hpp"
-
-namespace mfem::future
-{
-
-struct LocalQFBackend
-{
-   /**
-    * @brief Make an action for a local Q-function backend.
-    *
-    * @param ctx The integrator context.
-    * @param args The arguments to the action.
-    * @return The action.
-    */
-   template<typename qfunc_t, typename inputs_t, typename outputs_t>
-   static auto MakeAction(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      return LocalQFImpl::Action<qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs);
-   }
-
-   /**
-    * @brief Make a derivative action for a local Q-function backend.
-    *
-    * @tparam derivative_id The id of the derivative.
-    * @param ctx The integrator context.
-    * @param args The arguments to the derivative action.
-    * @return The derivative action.
-    */
-   template<int id, typename qfunc_t, typename inputs_t, typename outputs_t>
-   static auto MakeDerivativeAction(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs)
-   {
-      return LocalQFImpl::DerivativeAction<id, qfunc_t, inputs_t, outputs_t>(
-                ctx, qfunc, inputs, outputs);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   static auto MakeDerivativeSetup(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeSetup<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
-                                                          outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   static auto MakeDerivativeApply(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeApply<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
-                                                          outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   static auto MakeDerivativeApplyTranspose(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeApplyTranspose<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
-                                                          outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   static auto MakeDerivativeAssemble(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeAssemble<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
-                                                          outputs, qp_cache);
-   }
-
-   template<
-      int derivative_id,
-      typename qfunc_t,
-      typename inputs_t,
-      typename outputs_t>
-   static auto MakeDerivativeAssembleDiagonal(
-      const IntegratorContext &ctx,
-      const qfunc_t &qfunc,
-      inputs_t inputs,
-      outputs_t outputs,
-      const Vector &qp_cache)
-   {
-      return LocalQFImpl::DerivativeAssembleDiagonal<
-             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
-                                                          outputs, qp_cache);
-   }
-};
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, typename QT, typename IT, typename OT>
-inline void AddAction()
-{
-   using ker = LocalQFImpl::Action<QT, IT, OT>;
-   if constexpr (Q1D <= 8)
-   {
-      ker::ActionLO::template Specialization<DIM, Q1D>::Add();
-   }
-   else
-   {
-      ker::ActionHO::template Specialization<DIM, Q1D>::Add();
-   }
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
-inline void AddDerivativeAction()
-{
-   using ker = LocalQFImpl::DerivativeAction<DID, QT, IT, OT>;
-   if constexpr (Q1D <= 8)
-   {
-      ker::DerivativeActionLO::template Specialization<DIM, Q1D>::Add();
-   }
-   else
-   {
-      ker::DerivativeActionHO::template Specialization<DIM, Q1D>::Add();
-   }
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
-inline void AddDerivativeSetup()
-{
-   using ker = LocalQFImpl::DerivativeSetup<DID, QT, IT, OT>;
-   if constexpr (Q1D <= 8)
-   {
-      ker::DerivativeSetupLO::template Specialization<DIM, Q1D>::Add();
-   }
-   else
-   {
-      ker::DerivativeSetupHO::template Specialization<DIM, Q1D>::Add();
-   }
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
-inline void AddDerivativeApply()
-{
-   using ker = LocalQFImpl::DerivativeApply<DID, QT, IT, OT>;
-   if constexpr (Q1D <= 8)
-   {
-      ker::DerivativeApplyLO::template Specialization<DIM, Q1D>::Add();
-   }
-   else
-   {
-      ker::DerivativeApplyHO::template Specialization<DIM, Q1D>::Add();
-   }
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
-inline void AddDerivativeApplyTranspose()
-{
-   using ker = LocalQFImpl::DerivativeApplyTranspose<DID, QT, IT, OT>;
-   if constexpr (Q1D <= 8)
-   {
-      ker::DerivativeApplyTransposeLO::template Specialization<DIM, Q1D>::Add();
-   }
-   else
-   {
-      ker::DerivativeApplyTransposeHO::template Specialization<DIM, Q1D>::Add();
-   }
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-template<int DIM, int Q1D, typename QT, typename IT, typename OT,
-         typename derivative_ids_t = std::index_sequence<>>
-inline void AddLocalSpecializations()
-{
-   AddAction<DIM, Q1D, QT, IT, OT>();
-
-   for_constexpr([&](auto i)
-   {
-      using derivative_id = decltype(i);
-      AddDerivativeAction<DIM, Q1D, derivative_id::value, QT, IT, OT>();
-      AddDerivativeSetup<DIM, Q1D, derivative_id::value, QT, IT, OT>();
-      AddDerivativeApply<DIM, Q1D, derivative_id::value, QT, IT, OT>();
-      AddDerivativeApplyTranspose<DIM, Q1D, derivative_id::value, QT, IT, OT>();
-   }, derivative_ids_t{});
-}
-
-}

fem/dfem/backends/local_qf/revdiff_transformer.hpp

@@ -1,606 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../util.hpp"
-#include "util.hpp"
-#include "../../util.hpp"
-
-namespace mfem::future
-{
-
-template <typename T> struct function_traits;
-
-template <typename R, typename C, typename... Args>
-struct function_traits<R (C::*)(Args...) const>
-{
-   using primal_return_type = R;
-   using args_tuple = tuple<Args...>;
-   static constexpr size_t arity = sizeof...(Args);
-};
-
-/// Scalar-level view of a per-point argument: the plain value type it stores
-/// and the dual type used to carry a first-order tangent alongside it.
-template <typename T>
-struct qp_scalar_traits
-{
-   using view_type = T;
-   using dual_type = dual<T, T>;
-};
-
-template <typename V, typename G>
-struct qp_scalar_traits<dual<V, G>>
-{
-   using view_type = V;
-   using dual_type = dual<V, G>;
-};
-
-// Component count and writable counterpart of a per-point argument, which
-// is either a tensor or a plain scalar. `view_type` keeps the argument's own
-// scalar type; `dual_type` is the same shape with a dual scalar, used for the
-// gradient blocks of the native dual-number backend.
-template <typename Arg> struct qp_traits
-{
-   static_assert(std::is_arithmetic_v<Arg> || is_dual_number<Arg>::value,
-                 "per-point arguments must be tensors or scalars");
-   using scalar_type = std::remove_const_t<Arg>;
-   using view_type = scalar_type;
-   using dual_type = typename qp_scalar_traits<scalar_type>::dual_type;
-   static constexpr int components = 1;
-};
-
-template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
-{
-   using scalar_type = std::remove_const_t<T>;
-   using view_type = tensor<scalar_type, Sizes...>;
-   using dual_type =
-      tensor<typename qp_scalar_traits<scalar_type>::dual_type, Sizes...>;
-   static constexpr int components = (Sizes * ... * 1);
-};
-
-template <typename... T1s, typename... T2s>
-constexpr tuple<T1s..., T2s...> concat_tuples(tuple<T1s...>, tuple<T2s...>);
-
-///////////////////////////////////////////////////////////////////////////////
-/// Nested ("hyper") dual utilities, used for second derivatives on the native
-/// dual-number backend.
-///
-/// A second derivative taken with plain duals would have to reuse the single
-/// gradient slot that already carries the incoming direction. Lifting the
-/// scalar to `dual<dual<V,G>, dual<V,G>>` adds a second, independent slot:
-///
-///   dual(a, b) -> ((a, c), (b, d))
-///
-/// `a`/`b` stay the incoming primal/direction, `c` is seeded per component and
-/// `d` returns the second-order result.
-template <typename T>
-struct make_nested_qp_type
-{
-   using type = T;
-};
-
-template <typename V, typename G>
-struct make_nested_qp_type<dual<V, G>>
-{
-   using type = dual<dual<V, G>, dual<V, G>>;
-};
-
-template <typename S, int... Sizes>
-struct make_nested_qp_type<tensor<S, Sizes...>>
-{
-   using type = tensor<typename make_nested_qp_type<S>::type, Sizes...>;
-};
-
-template <typename T>
-using make_nested_qp_type_t = typename make_nested_qp_type<T>::type;
-
-using native_dual_t = typename qp_scalar_traits<real_t>::dual_type;
-using nested_native_dual_t = make_nested_qp_type_t<native_dual_t>;
-
-/// Rebinds a q-function's scalar template parameter so its arguments are
-/// nested duals. Only the leading scalar parameter is rebound; any remaining
-/// non-type parameters (e.g. `dim`) are carried through unchanged. This
-/// requires q-functions of the form `QFunc<scalar_t>` or `QFunc<scalar_t,
-/// Params...>`; `supported` reports whether that shape was matched, so callers
-/// can fail with a readable static_assert.
-template <typename qfunc_t, typename nested_scalar_t, typename = void>
-struct rebind_qfunc_scalar
-{
-   static constexpr bool supported = false;
-};
-
-template <template <typename> class qfunc_template_t,
-          typename old_scalar_t,
-          typename nested_scalar_t>
-struct rebind_qfunc_scalar<qfunc_template_t<old_scalar_t>, nested_scalar_t>
-{
-   static constexpr bool supported = true;
-   using type = qfunc_template_t<nested_scalar_t>;
-};
-
-template <template <typename, auto...> class qfunc_template_t,
-          typename old_scalar_t,
-          auto... Params,
-          typename nested_scalar_t>
-struct rebind_qfunc_scalar<qfunc_template_t<old_scalar_t, Params...>,
-          nested_scalar_t,
-       std::enable_if_t<(sizeof...(Params) > 0)>>
-       {
-          static constexpr bool supported = true;
-          using type = qfunc_template_t<nested_scalar_t, Params...>;
-       };
-
-template <typename qfunc_t, typename nested_scalar_t>
-using rebind_qfunc_scalar_t =
-   typename rebind_qfunc_scalar<qfunc_t, nested_scalar_t>::type;
-
-/// Copies a q-function argument into its nested-dual counterpart, mapping
-/// dual(a, b) -> ((a, 0), (b, 0)). The inner gradients stay zero; the caller
-/// seeds one of them per component.
-template <typename Dst, typename Src>
-MFEM_HOST_DEVICE void lift_to_nested_arg(const Src &src, Dst &dst)
-{
-   using dst_t = std::decay_t<Dst>;
-   constexpr bool dst_uses_dual = is_dual_number<dst_t>::value ||
-                                  qf_param_uses_dual_v<dst_t> ||
-                                  is_nested_dual_number<dst_t>::value ||
-                                  qf_param_uses_nested_dual_v<dst_t>;
-   if constexpr (dst_uses_dual)
-   {
-      constexpr int ncomp = qp_traits<dst_t>::components;
-      for (int component = 0; component < ncomp; component++)
-      {
-         qf_set_flat_value(dst, component, qf_flat_value(src, component));
-         qf_set_flat_gradient(dst, component, qf_flat_gradient(src, component));
-      }
-   }
-   else
-   {
-      // Destination carries no derivative slots: a plain copy is the lift.
-      dst = src;
-   }
-}
-
-// RevDiff: computes the full gradient of a pointwise qfunction at a single
-// quadrature point using one Enzyme reverse-mode (autodiff) call.
-//
-// operator()(args...) takes all qfunction input primals followed by one
-// writable gradient output per Active input (same shape as that input).
-// The qfunction's own output is not passed; Enzyme writes it to stack
-// scratch (enzyme_dupnoneed).
-//
-// A single __enzyme_autodiff call with the output adjoint seeded to 1
-// yields all gradient blocks simultaneously — O(1) calls per point
-// regardless of input size, vs O(ncomp) for forward mode.
-//
-// operator() is MFEM_HOST_DEVICE and allocation-free.
-//
-// Without Enzyme the same interface is served by a forward-mode dual-number
-// fallback (`call_dual_rev`), which seeds one component at a time. `mode`
-// selects what that fallback is being asked for: `Eval` is the plain gradient,
-// `Derivative` is a gradient taken inside an outer derivative, which lifts the
-// q-function to nested duals so seeding does not clobber the outer direction.
-// With Enzyme both modes use the single reverse-mode call and `mode` is inert.
-enum class RevDiffDualMode
-{
-   Eval,
-   Derivative
-};
-
-// Number of Active inputs and their argument indices, in ascending order.
-// A qfunction may have several Active inputs at once: e.g. a field's value
-// u and its gradient dudx both feed the output and both must be
-// differentiated (the chain-rule contraction with the value/gradient shape
-// functions then happens at the FE-operator level). We produce one gradient
-// block, d(output)/d(input), per Active input — each computed with the other
-// Active inputs frozen, so they come out as isolated partials.
-template <typename activity_t, size_t num_inputs, size_t... Is>
-constexpr size_t count_active_inputs(std::index_sequence<Is...>)
-{
-   return ((Is < num_inputs && qf_param_is_active_v<activity_t, Is>
-            ? size_t{1} : size_t{0}) + ...);
-}
-
-template <typename activity_t, size_t num_inputs, size_t num_active,
-          size_t... Is>
-constexpr std::array<size_t, num_active>
-collect_active_inputs(std::index_sequence<Is...>)
-{
-   std::array<size_t, num_active> idx{};
-   size_t j = 0;
-   (((Is < num_inputs && qf_param_is_active_v<activity_t, Is>)
-     ? (idx[j++] = Is) : size_t{0}), ...);
-   return idx;
-}
-
-template <typename Func, typename InputActivityTuple,
-          typename OutputActivityTuple,
-          RevDiffDualMode mode = RevDiffDualMode::Eval>
-struct RevDiff
-{
-   using traits = function_traits<decltype(&Func::operator())>;
-   using args_tuple = typename traits::args_tuple;
-   using activity =
-      decltype(concat_tuples(InputActivityTuple{}, OutputActivityTuple{}));
-   static constexpr size_t arity = traits::arity;
-   static constexpr size_t num_inputs = tuple_size<InputActivityTuple>::value;
-
-   static_assert(std::is_void_v<typename traits::primal_return_type>,
-                 "RevDiff only supports primal functions with void return type");
-   static_assert(tuple_size<activity>::value == arity,
-                 "Number of input and output activity tags must match function "
-                 "arity");
-
-   static constexpr size_t num_active_inputs =
-      count_active_inputs<activity, num_inputs>(
-         std::make_index_sequence<arity> {});
-
-   static constexpr auto active_inputs =
-      collect_active_inputs<activity, num_inputs, num_active_inputs>(
-   std::make_index_sequence<arity> {});
-
-   // Slot index of argument I in the active_inputs array (compile-time).
-   template <size_t I>
-   static constexpr size_t slot_of()
-   {
-      for (size_t s = 0; s < num_active_inputs; s++)
-         if (active_inputs[s] == I) { return s; }
-      return num_active_inputs;
-   }
-
-   static constexpr size_t active_output =
-      find_single_active_qparam<activity, num_inputs, arity>();
-   static_assert(active_output < arity,
-                 "gradient mode requires exactly one Active output");
-   static_assert(num_active_inputs >= 1,
-                 "gradient mode requires at least one Active input");
-   static_assert(tuple_size<OutputActivityTuple>::value == 1,
-                 "gradient mode requires exactly one (scalar) output");
-
-   using output_type =
-      std::decay_t<tuple_element_t<active_output, args_tuple>>;
-
-   using output_view = typename qp_traits<output_type>::view_type;
-
-   static_assert(qp_traits<output_type>::components == 1,
-                 "gradient output requires a scalar output");
-
-   template <size_t I>
-   using primal_arg_t = tuple_element_t<I, args_tuple>;
-
-   // True when reverse mode is served by the dual-number fallback rather than
-   // Enzyme. Everything below that widens a type to a dual is gated on this, so
-   // an Enzyme build sees exactly the types it saw before nested duals existed.
-#ifdef MFEM_USE_ENZYME
-   static constexpr bool native_dual_backend = false;
-#else
-   static constexpr bool native_dual_backend = true;
-#endif
-
-   static constexpr bool use_native_dual_derivative =
-      native_dual_backend && (mode == RevDiffDualMode::Derivative);
-
-   // Under the native-dual second-derivative path the active primals arrive
-   // carrying the outer direction, so they must be dual-typed.
-   template <size_t I>
-   using derivative_arg_t =
-      std::conditional_t<use_native_dual_derivative &&
-      qf_param_is_active_v<activity, I>,
-      typename qp_traits<std::decay_t<tuple_element_t<I, args_tuple>>>::dual_type,
-      primal_arg_t<I>>;
-
-   template <size_t S>
-   using active_arg_decay_t =
-      std::decay_t<tuple_element_t<active_inputs[S], args_tuple>>;
-
-   template <size_t S>
-   static constexpr bool active_arg_uses_dual()
-   {
-      return native_dual_backend &&
-             (is_dual_number<active_arg_decay_t<S>>::value ||
-              qf_param_uses_dual_v<active_arg_decay_t<S>>);
-   }
-
-   // A gradient block mirrors its active input's shape. It needs a dual scalar
-   // whenever the fallback has to return a value and a tangent through it.
-   template <size_t S>
-   using grad_arg_t =
-      std::conditional_t<use_native_dual_derivative || active_arg_uses_dual<S>(),
-      typename qp_traits<active_arg_decay_t<S>>::dual_type,
-      typename qp_traits<active_arg_decay_t<S>>::view_type>
-      &;
-
-   template <typename qfunc_type>
-   using qfunc_args_tuple_t =
-      decay_tuple<typename function_traits<decltype(&qfunc_type::operator())>::args_tuple>;
-
-   template <size_t... Is, size_t... Ss>
-   static FunctionSignature<void(derivative_arg_t<Is>..., grad_arg_t<Ss>...)>
-   signature_impl(std::index_sequence<Is...>, std::index_sequence<Ss...>);
-
-   using signature = decltype(signature_impl(std::make_index_sequence<num_inputs> {},
-                                             std::make_index_sequence<num_active_inputs> {}));
-
-   Func func {};
-
-   RevDiff() = default;
-   MFEM_HOST_DEVICE explicit RevDiff(const Func &func_) : func(func_) { }
-
-   // Plain function with the qfunction's exact (reference) signature, plus the
-   // configured qfunction instance, so it can be handed to Enzyme as a function
-   // pointer without default-constructing away runtime qfunction state.
-   template <size_t... Is>
-   MFEM_HOST_DEVICE static MFEM_FUTURE_ALWAYS_INLINE void
-   static_call(Func *func, tuple_element_t<Is, args_tuple>... args)
-   {
-      (*func)(args...);
-   }
-
-   template <size_t... Is>
-   static constexpr auto fn_ptr(std::index_sequence<Is...>)
-   {
-      return &static_call<Is...>;
-   }
-   static constexpr auto fn()
-   {
-      return fn_ptr(std::make_index_sequence<arity> {});
-   }
-
-   // Load primal inputs from the pointer tuple into a local qargs copy. Dual
-   // gradient parts are implicitly zero because qargs is value-initialized.
-   template <typename QArgs, typename AllPtrs, size_t... Is>
-   MFEM_HOST_DEVICE static MFEM_FUTURE_ALWAYS_INLINE void load_qargs(
-      QArgs &qargs, AllPtrs &ptrs, std::index_sequence<Is...>)
-   {
-      ((mfem::future::get<int(Is)>(qargs) =
-           *mfem::future::get<int(Is)>(ptrs)), ...);
-   }
-
-   template <typename QArgs, typename AllPtrs, size_t... Is>
-   MFEM_HOST_DEVICE static MFEM_FUTURE_ALWAYS_INLINE void
-   lift_qargs_to_nested_dual(QArgs &qargs, AllPtrs &ptrs,
-                             std::index_sequence<Is...>)
-   {
-      (lift_to_nested_arg(*mfem::future::get<int(Is)>(ptrs),
-                          mfem::future::get<Is>(qargs)), ...);
-   }
-
-   // The nested-dual q-function is a *different* type — its scalar template
-   // parameter is rebound — so a configured instance cannot simply be copied
-   // over. Runtime q-function state must still survive, or the second
-   // derivative would silently be taken of a differently-parameterised energy.
-   //
-   // Three cases, in order:
-   //   * the rebound type converts from this one: use that conversion;
-   //   * no state at all: nothing to carry;
-   //   * same size and trivially copyable: none of the members depend on the
-   //     rebound scalar, so the two are layout-identical and the state copies
-   //     over bytewise. A member that *did* depend on the scalar would change
-   //     the size and land in the static_assert below instead.
-   template <typename nested_func_t>
-   MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE nested_func_t
-   make_nested_func() const
-   {
-      if constexpr (std::is_constructible_v<nested_func_t, const Func &>)
-      {
-         return nested_func_t(func);
-      }
-      else if constexpr (std::is_empty_v<Func>)
-      {
-         return nested_func_t {};
-      }
-      else
-      {
-         static_assert(std::is_trivially_copyable_v<Func> &&
-                       std::is_trivially_copyable_v<nested_func_t> &&
-                       sizeof(Func) == sizeof(nested_func_t),
-                       "second derivatives on the native dual backend rebind "
-                       "the q-function's scalar type; a q-function whose state "
-                       "depends on that scalar must be constructible from its "
-                       "rebound form");
-         nested_func_t nested {};
-         const auto *src = reinterpret_cast<const unsigned char *>(&func);
-         auto *dst = reinterpret_cast<unsigned char *>(&nested);
-         for (size_t b = 0; b < sizeof(Func); b++) { dst[b] = src[b]; }
-         return nested;
-      }
-   }
-
-   // Seed the s-th Active input one component at a time and read the resulting
-   // gradient block back out. This is the forward-mode dual-number stand-in for
-   // one reverse-mode call: O(ncomp) evaluations instead of O(1).
-   template <size_t S, typename AllPtrs>
-   MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void seed_active_input(
-      AllPtrs &ptrs) const
-   {
-      constexpr size_t input_idx = active_inputs[S];
-      using active_arg_t = std::decay_t<tuple_element_t<input_idx, args_tuple>>;
-      constexpr int ncomp = qp_traits<active_arg_t>::components;
-
-      for (int component = 0; component < ncomp; component++)
-      {
-         if constexpr (mode == RevDiffDualMode::Eval)
-         {
-            // Fresh value-initialized qargs: primals loaded below, all dual
-            // gradient parts start at zero, so no explicit clear is needed.
-            qfunc_args_tuple_t<Func> qargs {};
-            load_qargs(qargs, ptrs, std::make_index_sequence<num_inputs> {});
-
-            auto &grad = *mfem::future::get<num_inputs + S>(ptrs);
-
-            qf_set_flat_gradient(mfem::future::get<input_idx>(qargs), component,
-                                 1.0);
-
-            call_qfunc_no_move(func, qargs);
-
-            auto &out = mfem::future::get<active_output>(qargs);
-
-            qf_set_flat_value(grad, component, qf_flat_gradient(out, 0));
-         }
-         else
-         {
-            // Lift the incoming dual (a, b) to ((a, c), (b, d)): b is the outer
-            // Hessian-action direction, c is this loop's component seed. After
-            // evaluating E the nested output holds ((E, dE/dx_i), (E'[b],
-            // H_i[b])), and we hand dfem back (dE/dx_i, H_i[b]).
-            static_assert(rebind_qfunc_scalar<Func, nested_native_dual_t>::supported,
-                          "RevDiff native-dual derivative mode requires "
-                          "q-function types of the form QFunc<scalar_t> so they "
-                          "can be rebound to nested dual scalars");
-            using nested_func_t = rebind_qfunc_scalar_t<Func, nested_native_dual_t>;
-            qfunc_args_tuple_t<nested_func_t> nested_qargs {};
-
-            lift_qargs_to_nested_dual(nested_qargs, ptrs,
-                                      std::make_index_sequence<num_inputs> {});
-
-            qf_set_flat_value_gradient(
-               mfem::future::get<input_idx>(nested_qargs), component, 1.0);
-
-            call_qfunc_no_move(make_nested_func<nested_func_t>(), nested_qargs);
-
-            auto &out = mfem::future::get<active_output>(nested_qargs);
-            auto &grad = *mfem::future::get<num_inputs + S>(ptrs);
-
-            qf_set_flat_value(grad, component, qf_flat_value_gradient(out, 0));
-            qf_set_flat_gradient(grad, component,
-                                 qf_flat_gradient_gradient(out, 0));
-         }
-      }
-   }
-
-   // Dual-number fallback for the whole reverse-mode call: one seeded sweep per
-   // Active input.
-   template <typename AllPtrs>
-   MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void call_dual_rev(
-      AllPtrs &ptrs) const
-   {
-      for_constexpr<num_active_inputs>([&](auto s)
-      {
-         seed_active_input<decltype(s)::value>(ptrs);
-      });
-   }
-
-#ifdef MFEM_USE_ENZYME
-   // Recursive builder of the per-argument reverse-mode enzyme call.
-   template <size_t I = 0, typename AllPtrs, typename... Built>
-   MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void
-   call_enzyme_rev(AllPtrs &ptrs, output_view &scratch, output_view &adjoint,
-                   Built... built) const
-   {
-      if constexpr (I == arity)
-      {
-         __enzyme_autodiff<void>(fn(), enzyme_const, const_cast<Func *>(&func),
-                                 built...);
-      }
-      else if constexpr (I == active_output)
-      {
-         // Output: primal written to scratch (unused), adjoint seeded to 1.
-         call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
-                                enzyme_dupnoneed, &scratch, &adjoint);
-      }
-      else if constexpr (qf_param_is_active_v<activity, I>)
-      {
-         // Active input: gradient accumulates into its grad-output slot.
-         call_enzyme_rev<I + 1>(
-            ptrs, scratch, adjoint, built..., enzyme_dup,
-            mfem::future::get<int(I)>(ptrs),
-            mfem::future::get<int(num_inputs + slot_of<I>())>(ptrs));
-      }
-      else
-      {
-         // Const input: primal only, no shadow.
-         call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
-                                enzyme_const, mfem::future::get<int(I)>(ptrs));
-      }
-   }
-#endif // MFEM_USE_ENZYME
-
-   // Zero all gradient outputs before the enzyme call (Enzyme accumulates).
-   template <typename AllPtrs, size_t... Ss>
-   MFEM_HOST_DEVICE static
-   MFEM_FUTURE_ALWAYS_INLINE void zero_grads(
-      AllPtrs &ptrs,
-      std::index_sequence<Ss...>)
-   {
-      ((*mfem::future::get<int(num_inputs + Ss)>(ptrs) =
-           std::decay_t<decltype(*mfem::future::get<int(num_inputs + Ss)>(ptrs))> {}),
-       ...);
-   }
-
-   // Called once per quadrature point. Arguments are, in order:
-   //   * the primal value of every qfunction input (active and const), then
-   //   * one gradient output per Active input (ascending index order), each
-   //     shaped like its Active input.
-   // The qfunction's own output slot is not passed; Enzyme writes it to stack
-   // scratch (enzyme_dupnoneed). A single __enzyme_autodiff call yields all
-   // gradient blocks simultaneously.
-   template <typename... Args>
-   MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void operator()(
-      Args &&...args) const
-   {
-      static_assert(sizeof...(Args) == num_inputs + num_active_inputs,
-                    "expected one primal per input plus one gradient output per "
-                    "Active input");
-      auto ptrs = mfem::future::make_tuple(&args...);
-      zero_grads(ptrs, std::make_index_sequence<num_active_inputs> {});
-#ifdef MFEM_USE_ENZYME
-      output_view out_scratch {};
-      output_view out_adjoint{1.0}; // seed: d(output)/d(output) = 1
-      call_enzyme_rev(ptrs, out_scratch, out_adjoint);
-#else
-      call_dual_rev(ptrs);
-#endif
-   }
-
-   static MFEM_FUTURE_ALWAYS_INLINE void print() { print_impl(std::make_index_sequence<arity> {}); }
-
-   template <size_t... Is> static MFEM_FUTURE_ALWAYS_INLINE void print_impl(
-      std::index_sequence<Is...>)
-   {
-      mfem::out << "__enzyme_autodiff<void>(fptr";
-      (([&]
-      {
-         auto name = get_type_name<tuple_element_t<Is, args_tuple>>();
-         if constexpr (Is == active_output)
-            mfem::out << ", enzyme_dupnoneed, " << name << ", adjoint=1";
-         else if constexpr (qf_param_is_active_v<activity, Is>)
-            mfem::out << ", enzyme_dup, " << name << ", grad out";
-         else
-         {
-            mfem::out << ", enzyme_const, " << name;
-         }
-      }()),
-      ...);
-      mfem::out << ")\n";
-   }
-};
-
-template <typename Func, typename InputActivityTuple,
-          typename OutputActivityTuple, RevDiffDualMode mode>
-struct create_function_signature<RevDiff<Func, InputActivityTuple,
-          OutputActivityTuple, mode>>
-{
-   using type = typename
-                RevDiff<Func, InputActivityTuple, OutputActivityTuple, mode>::signature;
-};
-
-/// Builds the reverse-mode transform of @a f, differentiating the inputs marked
-/// Active in @a activity_t.
-///
-/// A factory rather than a plain declaration of a RevDiff variable for compatibility with MSVC.
-template <typename activity_t, RevDiffDualMode mode = RevDiffDualMode::Eval,
-          typename func_t>
-auto make_revdiff(const func_t &f)
-{
-   return RevDiff<func_t, activity_t, tuple<Active>, mode>(f);
-}
-
-} // namespace mfem::future

fem/dfem/backends/scratch_bank.hpp

@@ -1,327 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-#include "../../../general/error.hpp"
-#include "../../../linalg/vector.hpp"
-#include "../tuple.hpp"
-#include <initializer_list>
-#include <memory>
-#include <type_traits>
-#include <utility>
-#include <vector>
-
-namespace mfem::future
-{
-// Scratch storage and q-function shadow helpers for dFEM backends. The bank
-// supports two scratch kinds:
-// - quadrature-point scratch: real_t buffers sized as NQ * components_per_qp,
-// - global scratch: one tuple of qfunction-local temporaries, independent of
-//   NQ, used for values such as flags, scalars, or small Vector workspaces.
-//
-// @a scalar_t is the scalar the owning q-function uses at a quadrature point.
-// With Enzyme this is real_t and the tangent lives in a separate shadow bank.
-// Without Enzyme the q-function is evaluated on native duals, which carry the
-// tangent inside the value itself; the bank then widens its backing storage
-// accordingly so that a scratch entry can round-trip a dual without dropping
-// the gradient. Backing storage stays a real_t Vector in both cases, so the
-// device and shadow plumbing is unchanged.
-template <typename scalar_t, typename... GlobalScratchTypes>
-struct ScratchBank
-{
-   static_assert(sizeof(scalar_t) % sizeof(real_t) == 0,
-                 "scratch scalar must be a whole number of real_t");
-
-   /// Number of real_t needed to back one scalar_t scratch entry.
-   static constexpr int scalar_size = sizeof(scalar_t) / sizeof(real_t);
-
-
-   //=================================
-   ///<--- Global scratch utilities.
-   //=================================
-
-   using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
-
-   template <typename T>
-   static T MakeGlobalScratchShadow(const T &)
-   {
-      return T {};
-   }
-
-   static Vector MakeGlobalScratchShadow(const Vector &primal)
-   {
-      Vector shadow(primal.Size());
-      shadow.UseDevice(true);
-      shadow = 0.0;
-      return shadow;
-   }
-
-   template <typename Tuple, size_t... Is>
-   static auto MakeGlobalScratchShadowTuple(const Tuple &primal,
-                                            std::index_sequence<Is...>)
-   {
-      return make_tuple(MakeGlobalScratchShadow(get<Is>(primal))...);
-   }
-
-   template <typename Tuple>
-   static auto MakeGlobalScratchShadowTuple(const Tuple &primal)
-   {
-      return MakeGlobalScratchShadowTuple(
-                primal, std::make_index_sequence<tuple_size<Tuple>::value> {});
-   }
-
-
-   //===========================
-   ///<--- Scratch objects
-   //===========================
-
-   mutable GlobalScratchTuple global;
-
-   int nq = 0;
-   std::vector<int> components;
-   std::vector<int> sizes;
-   std::vector<std::shared_ptr<Vector>> owned;
-   std::vector<real_t *> ptrs;
-
-
-   //===========================
-   ///<--- Setter methods
-   //===========================
-
-   void SetScratch(const int nq_,
-                   std::initializer_list<int> components_per_qp = {1})
-   {
-      SetScratch(nq_, std::vector<int>(components_per_qp));
-   }
-
-   void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
-   {
-      nq = nq_;
-      components.clear();
-      sizes.clear();
-      owned.clear();
-      ptrs.clear();
-      for (int component_count : components_per_qp)
-      {
-         AddScratch(component_count);
-      }
-   }
-
-   void AddScratch(const int components_per_qp = 1)
-   {
-      MFEM_VERIFY(nq > 0, "SetScratch must be called before AddScratch");
-      MFEM_VERIFY(components_per_qp > 0,
-                  "scratch components per quadrature point must be positive");
-      owned.push_back(std::make_shared<Vector>());
-      Vector &scratch = *owned.back();
-      const int size = components_per_qp * nq * scalar_size;
-      scratch.SetSize(size);
-      scratch.UseDevice(true);
-      scratch = 0.0;
-      components.push_back(components_per_qp);
-      sizes.push_back(scratch.Size());
-      ptrs.push_back(scratch.ReadWrite());
-   }
-
-   void SetGlobalScratch(const GlobalScratchTuple &global_)
-   {
-      global = global_;
-   }
-
-
-   //===========================
-   ///<--- Getter methods
-   //===========================
-
-   /// Scratch buffer @a i viewed as the q-function's scalar type.
-   scalar_t *GetScratchPointer(const int i) const
-   {
-      return reinterpret_cast<scalar_t *>(ptrs[i]);
-   }
-
-   scalar_t *operator[](const int i) const { return GetScratchPointer(i); }
-
-   /// Raw real_t backing storage of scratch buffer @a i. Its size is
-   /// scalar_size times the number of scalar_t entries.
-   Vector &GetScratchVector(const int i) const { return *owned[i]; }
-
-   template <int I>
-   auto &GetGlobalScratch() const
-   {
-      return get<I>(global);
-   }
-
-
-   //===========================
-   ///<--- Utils methods
-   //===========================
-
-   void CloneScratchLayoutTo(ScratchBank &shadow) const
-   {
-      shadow.SetScratch(nq, components);
-      shadow.SetGlobalScratch(MakeGlobalScratchShadowTuple(global));
-   }
-
-   int Size() const { return static_cast<int>(ptrs.size()); }
-};
-
-// Shared base for Q-functions that use ScratchBank. Under Enzyme a matching
-// scratch shadow is created for forward differentiation; with native duals the
-// tangent rides along in the scratch entry and no shadow is created.
-template <typename scalar_t, typename... GlobalScratchTypes>
-struct QFWithScratch
-{
-   using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
-   using ScratchScalar = scalar_t;
-
-   /// Number of real_t backing one scratch entry; see ScratchBank.
-   static constexpr int scalar_size =
-      ScratchBank<scalar_t, GlobalScratchTypes...>::scalar_size;
-
-   int nq = 0;
-   ScratchBank<scalar_t, GlobalScratchTypes...> scratch;
-
-   void SetScratch(const int nq_,
-                   std::initializer_list<int> components_per_qp = {1})
-   {
-      nq = nq_;
-      scratch.SetScratch(nq, components_per_qp);
-   }
-
-   void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
-   {
-      nq = nq_;
-      scratch.SetScratch(nq, components_per_qp);
-   }
-
-   void SetScratch(const int nq_, const int num_scratch_elem,
-                   const int components_per_qp = 1)
-   {
-      nq = nq_;
-      scratch.SetScratch(nq,
-                         std::vector<int>(num_scratch_elem, components_per_qp));
-   }
-
-   void SetGlobalScratch(const GlobalScratchTuple &global_scratch_)
-   {
-      scratch.SetGlobalScratch(global_scratch_);
-   }
-
-   Vector &GetScratchVector(const int i) const
-   {
-      return scratch.GetScratchVector(i);
-   }
-
-   scalar_t *GetScratchPointer(const int i) const
-   {
-      return scratch.GetScratchPointer(i);
-   }
-
-   template <int I>
-   auto &GetGlobalScratch() const
-   {
-      return scratch.template GetGlobalScratch<I>();
-   }
-
-   void CloneScratchLayoutTo(QFWithScratch &shadow) const
-   {
-      shadow.nq = nq;
-      scratch.CloneScratchLayoutTo(shadow.scratch);
-   }
-
-   QFWithScratch CreateShadow() const
-   {
-      QFWithScratch shadow;
-      CloneScratchLayoutTo(shadow);
-      return shadow;
-   }
-};
-
-/// Q-function base with quadrature-point scratch only. @a scalar_t is the
-/// scalar the q-function signature uses (real_t under Enzyme, dual otherwise).
-template <typename scalar_t = real_t>
-using QFWithScratchType = QFWithScratch<scalar_t>;
-
-/// Q-function base with quadrature-point scratch and a global scratch tuple.
-template <typename scalar_t = real_t>
-using QFWithGlobalScratchType =
-   QFWithScratch<scalar_t, bool, real_t, Vector>;
-
-namespace detail
-{
-
-template <typename T>
-struct qfunc_uses_scratch
-{
-private:
-   template <typename scalar_t, typename... GlobalScratchTypes>
-   static std::true_type Test(
-      const QFWithScratch<scalar_t, GlobalScratchTypes...> *);
-
-   static std::false_type Test(...);
-
-public:
-   static constexpr bool value = decltype(Test(
-                                             static_cast<std::remove_cv_t<std::remove_reference_t<T>> *>(nullptr)))::value;
-};
-
-template <typename T>
-inline constexpr bool qfunc_uses_scratch_v =
-   qfunc_uses_scratch<T>::value;
-
-struct unused_qfunc_shadow { };
-
-// A separate shadow scratch bank only exists for Enzyme, which writes tangents
-// into shadow memory. The native dual fallback carries the tangent inside the
-// scratch entry itself (see ScratchBank::scalar_size), so a shadow bank would
-// be allocated and never read; it is dropped entirely there.
-template <typename T>
-inline constexpr bool qfunc_needs_shadow_v =
-#ifdef MFEM_USE_ENZYME
-   qfunc_uses_scratch_v<T>;
-#else
-   false;
-#endif
-
-template <typename qfunc_t, bool needs_shadow>
-struct qfunc_shadow_type
-{
-   using type = unused_qfunc_shadow;
-};
-
-template <typename qfunc_t>
-struct qfunc_shadow_type<qfunc_t, true>
-{
-   using type = decltype(std::declval<const qfunc_t &>().CreateShadow());
-};
-
-template <typename qfunc_t>
-using qfunc_shadow_t = typename qfunc_shadow_type<qfunc_t,
-      qfunc_needs_shadow_v<qfunc_t>>::type;
-
-// Create a persistent q-function shadow if one is needed, otherwise return an empty struct.
-template <typename qfunc_t>
-inline qfunc_shadow_t<qfunc_t> MakeQFunctionShadowStorage(
-   const qfunc_t &qfunc)
-{
-   if constexpr (qfunc_needs_shadow_v<qfunc_t>)
-   {
-      return qfunc.CreateShadow();
-   }
-   else
-   {
-      MFEM_CONTRACT_VAR(qfunc);
-      return {};
-   }
-}
-
-} // namespace detail
-
-}

fem/dfem/doperator.cpp

@@ -11,366 +11,92 @@
 
 #include "doperator.hpp"
 
-#include <algorithm>
-
 #ifdef MFEM_USE_MPI
 
 using namespace mfem;
 using namespace mfem::future;
 
-namespace
+void DifferentiableOperator::SetParameters(std::vector<Vector *> p) const
 {
-int GetTotalTrueVSize(const std::vector<FieldDescriptor> &fds)
-{
-   int size = 0;
-   for (const auto &fd : fds)
+   MFEM_ASSERT(parameters.size() == p.size(),
+               "number of parameters doesn't match descriptors");
+   for (size_t i = 0; i < parameters.size(); i++)
    {
-      size += mfem::future::GetTrueVSize(fd);
+      p[i]->Read();
+      parameters_l[i] = *p[i];
    }
-   return size;
-}
-
-template <typename map_t>
-const typename map_t::mapped_type &FindOrDefault(
-   const map_t &map, const typename map_t::key_type &id,
-   const typename map_t::mapped_type &fallback)
-{
-   const auto it = map.find(id);
-   return it == map.end() ? fallback : it->second;
-}
-
-template <typename map_t>
-typename map_t::mapped_type FindOrEmpty(
-   const map_t &map, const typename map_t::key_type &id)
-{
-   const auto it = map.find(id);
-   return it == map.end() ? typename map_t::mapped_type{} : it->second;
-}
-
-const std::vector<derivative_action_t> &SelectActionCallbacks(
-   const std::vector<derivative_action_t> &direct_actions,
-   const DerivativeActionMap &cached_actions,
-   size_t derivative_id,
-   bool use_cached_setup)
-{
-   if (use_cached_setup)
-   {
-      const auto it_apply = cached_actions.find(derivative_id);
-      if (it_apply != cached_actions.end() && !it_apply->second.empty())
-      {
-         return it_apply->second;
-      }
-   }
-
-   return direct_actions;
-}
-
-struct DerivativeCallbackSet
-{
-   const DerivativeActionMap &actions;
-   const DerivativeActionMap &cached_actions;
-   const DerivativeActionMap &transpose_actions;
-   const DerivativeFieldMap &outfds;
-   const SparseAssemblyMap &assemble_sparse;
-   const HypreAssemblyMap &assemble_hypre;
-   const DiagonalAssemblyMap &assemble_diagonal;
-   const DerivativeSetupMap &setup;
-   const char *missing_action_message;
-};
-
-struct SecondDerivativeCallbackSet
-{
-   const SecondDerivativeActionMap &actions;
-   const SecondDerivativeActionMap &cached_actions;
-   const SecondDerivativeActionMap &transpose_actions;
-   const SecondDerivativeFieldMap &outfds;
-   const SecondSparseAssemblyMap &assemble_sparse;
-   const SecondHypreAssemblyMap &assemble_hypre;
-   const SecondDiagonalAssemblyMap &assemble_diagonal;
-   const SecondDerivativeSetupMap &setup;
-   const char *missing_action_message;
-};
-
-template <typename vector_t>
-std::shared_ptr<DerivativeOperator> MakeStatefulDerivativeOperator(
-   size_t derivative_id,
-   const vector_t &x,
-   const std::vector<FieldDescriptor> &infds,
-   const std::vector<FieldDescriptor> &default_outfds,
-   const DerivativeCallbackSet &callbacks,
-   bool use_cached_setup,
-   bool lvector_mode,
-   bool functional_gradient = false)
-{
-   const auto it_action = callbacks.actions.find(derivative_id);
-   MFEM_ASSERT(it_action != callbacks.actions.end(),
-               callbacks.missing_action_message << derivative_id);
-
-   const size_t dfidx = FindIdx(derivative_id, infds);
-   const auto &doutfds =
-      FindOrDefault(callbacks.outfds, derivative_id, default_outfds);
-   const auto &mult_callbacks =
-      SelectActionCallbacks(it_action->second, callbacks.cached_actions,
-                            derivative_id, use_cached_setup);
-
-   return std::make_shared<DerivativeOperator>(
-             GetTotalTrueVSize(doutfds),
-             GetTrueVSize(infds[dfidx]),
-             mult_callbacks,
-             FindOrEmpty(callbacks.transpose_actions, derivative_id),
-             infds[dfidx],
-             x,
-             infds,
-             doutfds,
-             FindOrEmpty(callbacks.assemble_sparse, derivative_id),
-             FindOrEmpty(callbacks.assemble_hypre, derivative_id),
-             FindOrEmpty(callbacks.assemble_diagonal, derivative_id),
-             FindOrEmpty(callbacks.setup, derivative_id),
-             lvector_mode,
-             functional_gradient);
-}
-
-const std::vector<derivative_action_t> &SelectSecondDerivativeActionCallbacks(
-   const std::vector<derivative_action_t> &direct_actions,
-   const SecondDerivativeActionMap &cached_actions,
-   second_derivative_key_t derivative_key,
-   bool use_cached_setup)
-{
-   if (use_cached_setup)
-   {
-      const auto it_apply = cached_actions.find(derivative_key);
-      if (it_apply != cached_actions.end() && !it_apply->second.empty())
-      {
-         return it_apply->second;
-      }
-   }
-
-   return direct_actions;
-}
-
-template <typename vector_t>
-std::shared_ptr<DerivativeOperator> MakeStatefulSecondDerivativeOperator(
-   size_t gradient_id,
-   size_t direction_id,
-   const vector_t &x,
-   const std::vector<FieldDescriptor> &infds,
-   const std::vector<FieldDescriptor> &default_outfds,
-   const SecondDerivativeCallbackSet &callbacks,
-   bool use_cached_setup,
-   bool lvector_mode)
-{
-   const second_derivative_key_t derivative_key{gradient_id, direction_id};
-   const auto it_action = callbacks.actions.find(derivative_key);
-   MFEM_ASSERT(it_action != callbacks.actions.end(),
-               callbacks.missing_action_message << "(" << gradient_id << ", "
-               << direction_id << ")");
-
-   const size_t dfidx = FindIdx(direction_id, infds);
-   const auto &doutfds =
-      FindOrDefault(callbacks.outfds, derivative_key, default_outfds);
-   const auto &mult_callbacks =
-      SelectSecondDerivativeActionCallbacks(
-         it_action->second, callbacks.cached_actions, derivative_key,
-         use_cached_setup);
-
-   return std::make_shared<DerivativeOperator>(
-             GetTotalTrueVSize(doutfds),
-             GetTrueVSize(infds[dfidx]),
-             mult_callbacks,
-             FindOrEmpty(callbacks.transpose_actions, derivative_key),
-             infds[dfidx],
-             x,
-             infds,
-             doutfds,
-             FindOrEmpty(callbacks.assemble_sparse, derivative_key),
-             FindOrEmpty(callbacks.assemble_hypre, derivative_key),
-             FindOrEmpty(callbacks.assemble_diagonal, derivative_key),
-             FindOrEmpty(callbacks.setup, derivative_key),
-             lvector_mode);
-}
 }
 
 DifferentiableOperator::DifferentiableOperator(
-   const std::vector<FieldDescriptor> &infds,
-   const std::vector<FieldDescriptor> &outfds,
+   const std::vector<FieldDescriptor> &solutions,
+   const std::vector<FieldDescriptor> &parameters,
    const ParMesh &mesh) :
-   Operator(),
    mesh(mesh),
-   infds(infds),
-   outfds(outfds)
+   solutions(solutions),
+   parameters(parameters)
 {
-   unionfds.clear();
-   unionfds.insert(unionfds.end(), infds.begin(), infds.end());
-   unionfds.insert(unionfds.end(), outfds.begin(), outfds.end());
-   std::sort(unionfds.begin(), unionfds.end());
-   auto last = std::unique(unionfds.begin(), unionfds.end());
-   unionfds.erase(last, unionfds.end());
+   fields.resize(solutions.size() + parameters.size());
+   fields_e.resize(fields.size());
+   solutions_l.resize(solutions.size());
+   parameters_l.resize(parameters.size());
 
-   infields_l.resize(infds.size());
-   for (size_t i = 0; i < infds.size(); i++)
+   for (size_t i = 0; i < solutions.size(); i++)
    {
-      infields_l[i] = new Vector(GetVSize(infds[i]));
+      fields[i] = solutions[i];
    }
 
-   infields_e.resize(infds.size());
-}
-
-void DifferentiableOperator::SetMultLevel(MultLevel level)
-{
-   mult_level = level;
-}
-
-void DifferentiableOperator::Mult(const Vector &x, Vector &y) const
-{
-   MFEM_ASSERT(!action_callbacks.empty(),
-               "no integrators have been set");
-
-   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&x),
-               "x needs to be a BlockVector");
-
-   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&y),
-               "y needs to be a BlockVector");
-
-   const auto &bx = static_cast<const BlockVector &>(x);
-   auto &by = static_cast<BlockVector &>(y);
-
-   Mult(bx, by);
-}
-
-void DifferentiableOperator::DisableTensorProductStructure(bool disable)
-{
-   use_tensor_product_structure = !disable;
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
-   size_t derivative_id, const Vector &x)
-{
-   return MakeStatefulDerivativeOperator(
-             derivative_id, x, infds, outfds,
+   for (size_t i = 0; i < parameters.size(); i++)
    {
-      derivative_action_callbacks,
-      derivative_apply_callbacks,
-      daction_transpose_callbacks,
-      derivative_outfds,
-      assemble_derivative_sparsematrix_callbacks,
-      assemble_derivative_hypreparmatrix_callbacks,
-      assemble_diagonal_callbacks,
-      derivative_setup_callbacks,
-      "no derivative action has been found for ID "
-   },
-   true,
-   mult_level == MultLevel::LVECTOR,
-   IsFunctionalDerivative(derivative_id));
+      fields[i + solutions.size()] = parameters[i];
+   }
 }
 
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
-   size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
+
+void FDJacobian::Mult(const Vector &v, Vector &y) const
 {
-   return MakeStatefulDerivativeOperator(
-             derivative_id, x, infds, outfds,
+   // See [1] for choice of eps.
+   //
+   // [1] Woodward, C.S., Gardner, D.J. and Evans, K.J., 2015. On the use of
+   // finite difference matrix-vector products in Newton-Krylov solvers for
+   // implicit climate dynamics with spectral elements. Procedia Computer
+   // Science, 51, pp.2036-2045.
+   real_t eps;
+   if (fixed_eps > 0.0)
    {
-      derivative_action_callbacks,
-      derivative_apply_callbacks,
-      daction_transpose_callbacks,
-      derivative_outfds,
-      assemble_derivative_sparsematrix_callbacks,
-      assemble_derivative_hypreparmatrix_callbacks,
-      assemble_diagonal_callbacks,
-      derivative_setup_callbacks,
-      "no derivative action has been found for ID "
-   },
-   use_cached_setup,
-   mult_level == MultLevel::LVECTOR,
-   IsFunctionalDerivative(derivative_id));
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
-   size_t derivative_id)
-{
-   MFEM_ASSERT(HasFunctionalIntegrator(),
-               "stateless GetDerivative is available only for functionals");
-
-   const auto it_action = derivative_action_callbacks.find(derivative_id);
-   MFEM_ASSERT(it_action != derivative_action_callbacks.end(),
-               "no derivative action has been found for ID " << derivative_id);
-
-   const size_t dfidx = FindIdx(derivative_id, infds);
-   const auto &doutfds =
-      FindOrDefault(derivative_outfds, derivative_id, outfds);
-
-   return std::make_shared<DerivativeOperator>(
-             GetTotalTrueVSize(doutfds),
-             GetTrueVSize(infds[dfidx]),
-             it_action->second,
-             infds,
-             doutfds);
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
-   size_t derivative_id, const Vector &x)
-{
-   return GetSecondDerivative(derivative_id, derivative_id, x);
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
-   size_t gradient_id, size_t direction_id, const Vector &x)
-{
-   MFEM_ASSERT(HasFunctionalIntegrator(),
-               "second derivatives are available only for functionals");
-
-   return MakeStatefulSecondDerivativeOperator(
-             gradient_id, direction_id, x, infds, outfds,
+      eps = fixed_eps;
+   }
+   else
    {
-      second_derivative_action_callbacks,
-      second_derivative_apply_callbacks,
-      second_daction_transpose_callbacks,
-      second_derivative_outfds,
-      assemble_second_derivative_sparsematrix_callbacks,
-      assemble_second_derivative_hypreparmatrix_callbacks,
-      assemble_second_derivative_diagonal_callbacks,
-      second_derivative_setup_callbacks,
-      "no second derivative action has been found for ID "
-   },
-   false,
-   mult_level == MultLevel::LVECTOR);
-}
+      const real_t vnorm_local = v.Norml2();
+      real_t vnorm;
+      MPI_Allreduce(&vnorm_local, &vnorm, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                    MPI_COMM_WORLD);
+      eps = lambda * (lambda + xnorm / vnorm);
+   }
 
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
-   size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
-{
-   return GetSecondDerivative(derivative_id, derivative_id, x, use_cached_setup);
-}
-
-std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
-   size_t gradient_id, size_t direction_id, const MultiVector &x,
-   const bool use_cached_setup)
-{
-   MFEM_ASSERT(HasFunctionalIntegrator(),
-               "second derivatives are available only for functionals");
-
-   return MakeStatefulSecondDerivativeOperator(
-             gradient_id, direction_id, x, infds, outfds,
+   // x + eps * v
    {
-      second_derivative_action_callbacks,
-      second_derivative_apply_callbacks,
-      second_daction_transpose_callbacks,
-      second_derivative_outfds,
-      assemble_second_derivative_sparsematrix_callbacks,
-      assemble_second_derivative_hypreparmatrix_callbacks,
-      assemble_second_derivative_diagonal_callbacks,
-      second_derivative_setup_callbacks,
-      "no second derivative action has been found for ID "
-   },
-   use_cached_setup,
-   mult_level == MultLevel::LVECTOR);
-}
+      const auto d_v = v.Read();
+      const auto d_x = x.Read();
+      auto d_xpev = xpev.Write();
+      mfem::forall(x.Size(), [=] MFEM_HOST_DEVICE (int i)
+      {
+         d_xpev[i] = d_x[i] + eps * d_v[i];
+      });
+   }
 
-bool DifferentiableOperator::HasSecondDerivative(size_t gradient_id,
-                                                 size_t direction_id) const
-{
-   const second_derivative_key_t derivative_key{gradient_id, direction_id};
-   const auto it = second_derivative_action_callbacks.find(derivative_key);
-   return it != second_derivative_action_callbacks.end() && !it->second.empty();
+   // y = f(x + eps * v)
+   op.Mult(xpev, y);
+
+   // y = (f(x + eps * v) - f(x)) / eps
+   {
+      const auto d_f = f.Read();
+      auto d_y = y.ReadWrite();
+      mfem::forall(f.Size(), [=] MFEM_HOST_DEVICE (int i)
+      {
+         d_y[i] = (d_y[i] - d_f[i]) / eps;
+      });
+   }
 }
 
 #endif // MFEM_USE_MPI

fem/dfem/fielddescriptor.hpp

@@ -1,64 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../fespace.hpp"
-#include "../qspace.hpp"
-#include "parameterspace.hpp"
-
-namespace mfem::future
-{
-
-/// @brief FieldDescriptor struct
-///
-/// This struct is used to store information about a field.
-struct FieldDescriptor
-{
-   using data_variant_t =
-      std::variant<const FiniteElementSpace *,
-      const ParFiniteElementSpace *,
-      const VectorQuadratureSpace *,
-      const ParameterSpace *>;
-
-   /// Field ID
-   std::size_t id;
-
-   /// Field variant
-   data_variant_t data;
-
-   /// Default constructor
-   FieldDescriptor() :
-      id(SIZE_MAX), data(data_variant_t{}) {}
-
-   /// Constructor
-   template <typename T>
-   FieldDescriptor(std::size_t field_id, const T* v) :
-      id(field_id), data(v) {}
-
-   bool operator==(const FieldDescriptor& other) const
-   {
-      return id == other.id;
-   }
-
-   bool operator<(const FieldDescriptor& other) const
-   {
-      return id < other.id;
-   }
-
-   friend void swap(FieldDescriptor& a, FieldDescriptor& b)
-   {
-      using std::swap;
-      swap(a.id, b.id);
-      swap(a.data, b.data);
-   }
-};
-
-}

fem/dfem/fieldoperator.hpp

@@ -10,7 +10,6 @@
 // CONTRIBUTING.md for details.
 #pragma once
 
-#include <ostream>
 #include <type_traits>
 
 namespace mfem::future
@@ -75,15 +74,6 @@ struct is_identity_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_identity_fop<Identity<FIELD_ID>> : std::true_type {};
 
-template <typename T>
-constexpr bool is_identity_fop_v = is_identity_fop<T>::value;
-
-template <int FIELD_ID>
-inline std::ostream& operator<<(std::ostream& out, Identity<FIELD_ID>)
-{
-   return out << "Identity<" << FIELD_ID << ">";
-}
-
 /// @brief Weight FieldOperator.
 ///
 /// This FieldOperator is used to signal that this field contains the quadrature
@@ -100,14 +90,6 @@ struct is_weight_fop : std::false_type {};
 template <>
 struct is_weight_fop<Weight> : std::true_type {};
 
-template <typename T>
-constexpr bool is_weight_fop_v = is_weight_fop<T>::value;
-
-inline std::ostream& operator<<(std::ostream& out, Weight)
-{
-   return out << "Weight";
-}
-
 /// @brief Value FieldOperator.
 ///
 /// This FieldOperator is used to signal that the field contains the
@@ -119,20 +101,11 @@ public:
    constexpr Value() : FieldOperator<FIELD_ID>() {};
 };
 
-template <typename T>
+template< typename T >
 struct is_value_fop : std::false_type {};
 
-template <int T>
-struct is_value_fop<Value<T>> : std::true_type {};
-
-template <typename T>
-constexpr bool is_value_fop_v = is_value_fop<T>::value;
-
 template <int FIELD_ID>
-inline std::ostream& operator<<(std::ostream& out, Value<FIELD_ID>)
-{
-   return out << "Value<" << FIELD_ID << ">";
-}
+struct is_value_fop<Value<FIELD_ID>> : std::true_type {};
 
 /// @brief Gradient FieldOperator.
 ///
@@ -151,15 +124,6 @@ struct is_gradient_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_gradient_fop<Gradient<FIELD_ID>> : std::true_type {};
 
-template <typename T>
-constexpr bool is_gradient_fop_v = is_gradient_fop<T>::value;
-
-template <int FIELD_ID>
-inline std::ostream& operator<<(std::ostream& out, Gradient<FIELD_ID>)
-{
-   return out << "Gradient<" << FIELD_ID << ">";
-}
-
 /// @brief Sum FieldOperator.
 ///
 /// This FieldOperator is commonly used to signal that an output of a quadrature
@@ -177,39 +141,4 @@ struct is_sum_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_sum_fop<Sum<FIELD_ID>> : std::true_type {};
 
-template <typename T>
-constexpr bool is_sum_fop_v = is_sum_fop<T>::value;
-
-template <int FIELD_ID>
-inline std::ostream& operator<<(std::ostream& out, Sum<FIELD_ID>)
-{
-   return out << "Sum<" << FIELD_ID << ">";
-}
-
-/// @brief FunctionalValue FieldOperator.
-///
-/// This FieldOperator is commonly used to signal that an output of a quadrature
-/// function should be summed.
-template <int FIELD_ID = -1>
-class FunctionalValue : public FieldOperator<FIELD_ID>
-{
-public:
-   constexpr FunctionalValue() : FieldOperator<FIELD_ID>() {};
-};
-
-template< typename T >
-struct is_functionalvalue_fop : std::false_type {};
-
-template <int FIELD_ID>
-struct is_functionalvalue_fop<FunctionalValue<FIELD_ID>> : std::true_type {};
-
-template <typename T>
-constexpr bool is_functionalvalue_fop_v = is_functionalvalue_fop<T>::value;
-
-template <int FIELD_ID>
-inline std::ostream& operator<<(std::ostream& out, FunctionalValue<FIELD_ID>)
-{
-   return out << "FunctionalValue<" << FIELD_ID << ">";
-}
-
 } // namespace mfem::future

fem/dfem/integrate.hpp

@@ -0,0 +1,536 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "util.hpp"
+
+namespace mfem::future
+{
+
+template <typename output_t>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields_impl(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const output_t &output,
+   const DofToQuadMap &dtq)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   // assuming the quadrature point residual has to "play nice with
+   // the test function"
+   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [num_qp, cdim, num_dof] = B.GetShape();
+      const int vdim = output.vdim > 0 ? output.vdim : cdim ;
+      for (int dof = 0; dof < num_dof; dof++)
+      {
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            real_t acc = 0.0;
+            for (int qp = 0; qp < num_qp; qp++)
+            {
+               acc += B(qp, 0, dof) * f(vd, 0, qp);
+            }
+            y(dof, vd) += acc;
+         }
+      }
+   }
+   else if constexpr (
+      is_gradient_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [num_qp, dim, num_dof] = G.GetShape();
+      const int vdim = output.vdim;
+      for (int dof = 0; dof < num_dof; dof++)
+      {
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            real_t acc = 0.0;
+            for (int d = 0; d < dim; d++)
+            {
+               for (int qp = 0; qp < num_qp; qp++)
+               {
+                  acc += G(qp, d, dof) * f(vd, d, qp);
+               }
+            }
+            y(dof, vd) += acc;
+         }
+      }
+   }
+   else if constexpr (is_sum_fop<std::decay_t<output_t>>::value)
+   {
+      // This is the "integral over all quadrature points type" applying
+      // B = 1 s.t. B^T * C \in R^1.
+      const auto [num_qp, unused, unused1] = B.GetShape();
+      auto cc = Reshape(&f(0, 0, 0), num_qp);
+      for (int i = 0; i < num_qp; i++)
+      {
+         y(0, 0) += cc(i);
+      }
+   }
+   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [num_qp, unused, num_dof] = B.GetShape();
+      const auto vdim = output.vdim;
+      auto cc = Reshape(&f(0, 0, 0), num_qp * vdim);
+      auto yy = Reshape(&y(0, 0), num_qp * vdim);
+      for (int i = 0; i < num_qp * vdim; i++)
+      {
+         yy(i) = cc(i);
+      }
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
+                        " for this field descriptor");
+   }
+}
+
+template <typename output_t>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields_tensor_impl_1d(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const output_t &output,
+   const DofToQuadMap &dtq,
+   std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, vdim);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dx, x, d1d)
+         {
+            real_t acc = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               acc += fqp(vd, 0, qx) * B(qx, 0, dx);
+            }
+            yd(dx, vd) = acc;
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = G.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, vdim);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dx, x, d1d)
+         {
+            real_t acc = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               acc += fqp(vd, 0, qx) * G(qx, 0, dx);
+            }
+            yd(dx, vd) = acc;
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
+      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
+
+      for (int sq = 0; sq < output.size_on_qp; sq++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            yqp(sq, qx) = fqp(sq, qx);
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
+                        "for this field descriptor with sum factorization on"
+                        " tensor product elements");
+   }
+}
+
+template <typename output_t>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields_tensor_impl_2d(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const output_t &output,
+   const DofToQuadMap &dtq,
+   std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
+
+      auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               real_t acc = 0.0;
+               for (int qx = 0; qx < q1d; qx++)
+               {
+                  acc += fqp(vd, 0, qx, qy) * B(qx, 0, dx);
+               }
+               s0(qy, dx) = acc;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               real_t acc = 0.0;
+               for (int qy = 0; qy < q1d; qy++)
+               {
+                  acc += s0(qy, dx) * B(qy, 0, dy);
+               }
+               yd(dx, dy, vd) += acc;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = G.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
+
+      auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
+      auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               real_t uv[2] = {0.0, 0.0};
+               for (int qx = 0; qx < q1d; qx++)
+               {
+                  uv[0] += fqp(vd, 0, qx, qy) * G(qx, 0, dx);
+                  uv[1] += fqp(vd, 1, qx, qy) * B(qx, 0, dx);
+               }
+               s0(qy, dx) = uv[0];
+               s1(qy, dx) = uv[1];
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               real_t uv[2] = {0.0, 0.0};
+               for (int qy = 0; qy < q1d; qy++)
+               {
+                  uv[0] += s0(qy, dx) * B(qy, 0, dy);
+                  uv[1] += s1(qy, dx) * G(qy, 0, dy);
+               }
+               yd(dx, dy, vd) += uv[0] + uv[1];
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+
+      // // TODO: Check if this is the right fix for all cases
+      // auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
+      // auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
+      // for (int sq = 0; sq < output.size_on_qp; sq++)
+      // {
+      //    MFEM_FOREACH_THREAD(qx, x, q1d)
+      //    {
+      //       yqp(sq, qx) = fqp(sq, qx);
+      //    }
+      //    MFEM_SYNC_THREAD;
+      // }
+
+      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d);
+      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d);
+
+      for (int sq = 0; sq < output.size_on_qp; sq++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               yqp(sq, qx, qy) = fqp(sq, qx, qy);
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
+                        " for this field descriptor with sum factorization on"
+                        " tensor product elements");
+   }
+}
+
+template <typename output_t>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields_tensor_impl_3d(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const output_t &output,
+   const DofToQuadMap &dtq,
+   std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
+
+      auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
+      auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d, d1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               MFEM_FOREACH_THREAD(qz, z, q1d)
+               {
+                  real_t acc = 0.0;
+                  for (int qx = 0; qx < q1d; qx++)
+                  {
+                     acc += fqp(vd, 0, qx, qy, qz) * B(qx, 0, dx);
+                  }
+                  s0(qz, qy, dx) = acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               MFEM_FOREACH_THREAD(qz, z, q1d)
+               {
+                  real_t acc = 0.0;
+                  for (int qy = 0; qy < q1d; qy++)
+                  {
+                     acc += s0(qz, qy, dx) * B(qy, 0, dy);
+                  }
+                  s1(qz, dy, dx) = acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               MFEM_FOREACH_THREAD(dz, z, d1d)
+               {
+                  real_t acc = 0.0;
+                  for (int qz = 0; qz < q1d; qz++)
+                  {
+                     acc += s1(qz, dy, dx) * B(qz, 0, dz);
+                  }
+                  yd(dx, dy, dz, vd) += acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = G.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
+
+      auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
+      auto s1 = Reshape(&scratch_mem[1](0), q1d, q1d, d1d);
+      auto s2 = Reshape(&scratch_mem[2](0), q1d, q1d, d1d);
+      auto s3 = Reshape(&scratch_mem[3](0), q1d, d1d, d1d);
+      auto s4 = Reshape(&scratch_mem[4](0), q1d, d1d, d1d);
+      auto s5 = Reshape(&scratch_mem[5](0), q1d, d1d, d1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qz, z, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD(dx, x, d1d)
+               {
+                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  for (int qx = 0; qx < q1d; qx++)
+                  {
+                     uvw[0] += fqp(vd, 0, qx, qy, qz) * G(qx, 0, dx);
+                     uvw[1] += fqp(vd, 1, qx, qy, qz) * B(qx, 0, dx);
+                     uvw[2] += fqp(vd, 2, qx, qy, qz) * B(qx, 0, dx);
+                  }
+                  s0(qz, qy, dx) = uvw[0];
+                  s1(qz, qy, dx) = uvw[1];
+                  s2(qz, qy, dx) = uvw[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(qz, z, q1d)
+         {
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            {
+               MFEM_FOREACH_THREAD(dx, x, d1d)
+               {
+                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  for (int qy = 0; qy < q1d; qy++)
+                  {
+                     uvw[0] += s0(qz, qy, dx) * B(qy, 0, dy);
+                     uvw[1] += s1(qz, qy, dx) * G(qy, 0, dy);
+                     uvw[2] += s2(qz, qy, dx) * B(qy, 0, dy);
+                  }
+                  s3(qz, dy, dx) = uvw[0];
+                  s4(qz, dy, dx) = uvw[1];
+                  s5(qz, dy, dx) = uvw[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dz, z, d1d)
+         {
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            {
+               MFEM_FOREACH_THREAD(dx, x, d1d)
+               {
+                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  for (int qz = 0; qz < q1d; qz++)
+                  {
+                     uvw[0] += s3(qz, dy, dx) * B(qz, 0, dz);
+                     uvw[1] += s4(qz, dy, dx) * B(qz, 0, dz);
+                     uvw[2] += s5(qz, dy, dx) * G(qz, 0, dz);
+                  }
+                  yd(dx, dy, dz, vd) += uvw[0] + uvw[1] + uvw[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d, q1d);
+      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
+
+      for (int sq = 0; sq < output.size_on_qp; sq++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD(qz, z, q1d)
+               {
+                  yqp(sq, qx, qy, qz) = fqp(sq, qx, qy, qz);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
+                        " for this field descriptor with sum factorization on"
+                        " tensor product elements");
+   }
+}
+
+template <typename output_t>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const output_t &output,
+   const DofToQuadMap &dtq,
+   std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 1)
+      {
+         map_quadrature_data_to_fields_tensor_impl_1d(y, f, output, dtq, scratch_mem);
+      }
+      else if (dimension == 2)
+      {
+         map_quadrature_data_to_fields_tensor_impl_2d(y, f, output, dtq, scratch_mem);
+      }
+      else if (dimension == 3)
+      {
+         map_quadrature_data_to_fields_tensor_impl_3d(y, f, output, dtq, scratch_mem);
+      }
+      else { MFEM_ABORT_KERNEL("dimension not supported"); }
+   }
+   else
+   {
+      map_quadrature_data_to_fields_impl(y, f, output, dtq);
+   }
+}
+
+} // namespace mfem::future

fem/dfem/integrator_ctx.hpp

@@ -1,37 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include <typeindex>
-#include <unordered_map>
-#include <vector>
-
-#include "../../general/array.hpp"
-#include "fielddescriptor.hpp"
-
-namespace mfem::future
-{
-
-struct IntegratorContext
-{
-   const ParMesh &mesh;
-   const Array<int> *elem_attr;
-   Array<int> attr;
-   const int nentities;
-   const std::vector<FieldDescriptor> &infds;
-   const std::vector<FieldDescriptor> &outfds;
-   const std::vector<FieldDescriptor> &unionfds;
-   const IntegrationRule &ir;
-   std::unordered_map<std::type_index, std::vector<int>> &in_qlayouts;
-   std::unordered_map<std::type_index, std::vector<int>> &out_qlayouts;
-};
-
-}

fem/dfem/interpolate.hpp

@@ -0,0 +1,678 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "util.hpp"
+
+namespace mfem::future
+{
+
+template <typename field_operator_t>
+MFEM_HOST_DEVICE inline
+void map_field_to_quadrature_data_tensor_product_3d(
+   DeviceTensor<2> &field_qp,
+   const DofToQuadMap &dtq,
+   const DeviceTensor<1> &field_e,
+   const field_operator_t &input,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
+   {
+      auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d, q1d);
+      auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
+      auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dz, z, d1d)
+         {
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            {
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               {
+                  real_t acc = 0.0;
+                  for (int dx = 0; dx < d1d; dx++)
+                  {
+                     acc += B(qx, 0, dx) * field(dx, dy, dz, vd);
+                  }
+                  s0(dz, dy, qx) = acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dz, z, d1d)
+         {
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               MFEM_FOREACH_THREAD(qy, y, q1d)
+               {
+                  real_t acc = 0.0;
+                  for (int dy = 0; dy < d1d; dy++)
+                  {
+                     acc += s0(dz, dy, qx) * B(qy, 0, dy);
+                  }
+                  s1(dz, qy, qx) = acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(qz, z, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               {
+                  real_t acc = 0.0;
+                  for (int dz = 0; dz < d1d; dz++)
+                  {
+                     acc += s1(dz, qy, qx) * B(qz, 0, dz);
+                  }
+                  fqp(vd, qx, qy, qz) = acc;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (
+      is_gradient_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const int dim = input.dim;
+      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d, q1d);
+
+      auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
+      auto s1 = Reshape(&scratch_mem[1](0), d1d, d1d, q1d);
+      auto s2 = Reshape(&scratch_mem[2](0), d1d, q1d, q1d);
+      auto s3 = Reshape(&scratch_mem[3](0), d1d, q1d, q1d);
+      auto s4 = Reshape(&scratch_mem[4](0), d1d, q1d, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dz, z, d1d)
+         {
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            {
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               {
+                  real_t uv[2] = {0.0, 0.0};
+                  for (int dx = 0; dx < d1d; dx++)
+                  {
+                     const real_t f = field(dx, dy, dz, vd);
+                     uv[0] += f * B(qx, 0, dx);
+                     uv[1] += f * G(qx, 0, dx);
+                  }
+                  s0(dz, dy, qx) = uv[0];
+                  s1(dz, dy, qx) = uv[1];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(dz, z, d1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               {
+                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  for (int dy = 0; dy < d1d; dy++)
+                  {
+                     const real_t s0i = s0(dz, dy, qx);
+                     uvw[0] += s1(dz, dy, qx) * B(qy, 0, dy);
+                     uvw[1] += s0i * G(qy, 0, dy);
+                     uvw[2] += s0i * B(qy, 0, dy);
+                  }
+                  s2(dz, qy, qx) = uvw[0];
+                  s3(dz, qy, qx) = uvw[1];
+                  s4(dz, qy, qx) = uvw[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(qz, z, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               {
+                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  for (int dz = 0; dz < d1d; dz++)
+                  {
+                     uvw[0] += s2(dz, qy, qx) * B(qz, 0, dz);
+                     uvw[1] += s3(dz, qy, qx) * B(qz, 0, dz);
+                     uvw[2] += s4(dz, qy, qx) * G(qz, 0, dz);
+                  }
+                  fqp(vd, 0, qx, qy, qz) = uvw[0];
+                  fqp(vd, 1, qx, qy, qz) = uvw[1];
+                  fqp(vd, 2, qx, qy, qz) = uvw[2];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   // TODO: Create separate function for clarity
+   else if constexpr (
+      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
+   {
+      const int num_qp = integration_weights.GetShape()[0];
+      // TODO: eeek
+      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
+      auto w = Reshape(&integration_weights[0], q1d, q1d, q1d);
+      auto f = Reshape(&field_qp[0], q1d, q1d, q1d);
+      MFEM_FOREACH_THREAD(qx, x, q1d)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(qz, z, q1d)
+            {
+               f(qx, qy, qz) = w(qx, qy, qz);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const int q1d = B.GetShape()[0];
+      auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d * q1d);
+      field_qp = field;
+   }
+   else
+   {
+      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
+                    "can't map field to quadrature data");
+   }
+}
+
+template <typename field_operator_t>
+MFEM_HOST_DEVICE inline
+void map_field_to_quadrature_data_tensor_product_2d(
+   DeviceTensor<2> &field_qp,
+   const DofToQuadMap &dtq,
+   const DeviceTensor<1> &field_e,
+   const field_operator_t &input,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
+   {
+      auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d);
+      auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               real_t acc = 0.0;
+               for (int dx = 0; dx < d1d; dx++)
+               {
+                  acc += B(qx, 0, dx) * field(dx, dy, vd);
+               }
+               s0(dy, qx) = acc;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            {
+               real_t acc = 0.0;
+               for (int dy = 0; dy < d1d; dy++)
+               {
+                  acc += s0(dy, qx) * B(qy, 0, dy);
+               }
+               fqp(vd, qx, qy) = acc;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else if constexpr (
+      is_gradient_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const int dim = input.dim;
+      const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d);
+
+      auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
+      auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dy, y, d1d)
+         {
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               real_t uv[2] = {0.0, 0.0};
+               for (int dx = 0; dx < d1d; dx++)
+               {
+                  const real_t f = field(dx, dy, vd);
+                  uv[0] += f * B(qx, 0, dx);
+                  uv[1] += f * G(qx, 0, dx);
+               }
+               s0(dy, qx) = uv[0];
+               s1(dy, qx) = uv[1];
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               real_t uv[2] = {0.0, 0.0};
+               for (int dy = 0; dy < d1d; dy++)
+               {
+                  const real_t s0i = s0(dy, qx);
+                  uv[0] += s1(dy, qx) * B(qy, 0, dy);
+                  uv[1] += s0i * G(qy, 0, dy);
+               }
+               fqp(vd, 0, qx, qy) = uv[0];
+               fqp(vd, 1, qx, qy) = uv[1];
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   // TODO: Create separate function for clarity
+   else if constexpr (
+      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
+   {
+      const int num_qp = integration_weights.GetShape()[0];
+      // TODO: eeek
+      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
+      auto w = Reshape(&integration_weights[0], q1d, q1d);
+      auto f = Reshape(&field_qp[0], q1d, q1d);
+      MFEM_FOREACH_THREAD(qx, x, q1d)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            f(qx, qy) = w(qx, qy);
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const int q1d = B.GetShape()[0];
+      auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d);
+      field_qp = field;
+   }
+   else
+   {
+      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
+                    "can't map field to quadrature data");
+   }
+}
+
+
+template <typename field_operator_t>
+MFEM_HOST_DEVICE inline
+void map_field_to_quadrature_data_tensor_product_1d(
+   DeviceTensor<2> &field_qp,
+   const DofToQuadMap &dtq,
+   const DeviceTensor<1> &field_e,
+   const field_operator_t &input,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
+   {
+      auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const auto field = Reshape(&field_e[0], d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t acc = 0.0;
+            for (int dx = 0; dx < d1d; dx++)
+            {
+               acc += B(qx, 0, dx) * field(dx, vd);
+            }
+            fqp(vd, qx) = acc;
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (
+      is_gradient_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const int dim = input.dim;
+      const auto field = Reshape(&field_e[0], d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t acc = 0.0;
+            for (int dx = 0; dx < d1d; dx++)
+            {
+               acc += G(qx, 0, dx) * field(dx, vd);
+            }
+            fqp(vd, 0, qx) = acc;
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   // TODO: Create separate function for clarity
+   else if constexpr (
+      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
+   {
+      const int num_qp = integration_weights.GetShape()[0];
+      // TODO: eeek
+      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
+      auto w = Reshape(&integration_weights[0], q1d);
+      auto f = Reshape(&field_qp[0], q1d);
+      MFEM_FOREACH_THREAD(qx, x, q1d)
+      {
+         f(qx) = w(qx);
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const int q1d = B.GetShape()[0];
+      auto field = Reshape(&field_e[0], input.size_on_qp, q1d);
+      field_qp = field;
+   }
+   else
+   {
+      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
+                    "can't map field to quadrature data");
+   }
+}
+
+template <typename field_operator_t>
+MFEM_HOST_DEVICE
+void map_field_to_quadrature_data(
+   DeviceTensor<2> field_qp,
+   const DofToQuadMap &dtq,
+   const DeviceTensor<1> &field_e,
+   const field_operator_t &input,
+   const DeviceTensor<1, const real_t> &integration_weights)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+   if constexpr (is_value_fop<field_operator_t>::value)
+   {
+      auto [num_qp, dim, num_dof] = B.GetShape();
+      const int vdim = input.vdim;
+      const auto field = Reshape(&field_e(0), num_dof, vdim);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         for (int qp = 0; qp < num_qp; qp++)
+         {
+            real_t acc = 0.0;
+            for (int dof = 0; dof < num_dof; dof++)
+            {
+               acc += B(qp, 0, dof) * field(dof, vd);
+            }
+            field_qp(vd, qp) = acc;
+         }
+      }
+   }
+   else if constexpr (is_gradient_fop<field_operator_t>::value)
+   {
+      const auto [num_qp, dim, num_dof] = G.GetShape();
+      const int vdim = input.vdim;
+      const auto field = Reshape(&field_e(0), num_dof, vdim);
+
+      auto f = Reshape(&field_qp[0], vdim, dim, num_qp);
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         for (int qp = 0; qp < num_qp; qp++)
+         {
+            for (int d = 0; d < dim; d++)
+            {
+               real_t acc = 0.0;
+               for (int dof = 0; dof < num_dof; dof++)
+               {
+                  acc += G(qp, d, dof) * field(dof, vd);
+               }
+               f(vd, d, qp) = acc;
+            }
+         }
+      }
+   }
+   else if constexpr (std::is_same_v<field_operator_t, Weight>)
+   {
+      const int num_qp = integration_weights.GetShape()[0];
+      auto f = Reshape(&field_qp[0], num_qp);
+      for (int qp = 0; qp < num_qp; qp++)
+      {
+         f(qp) = integration_weights(qp);
+      }
+   }
+   else if constexpr (is_identity_fop<field_operator_t>::value)
+   {
+      auto [num_qp, unused, num_dof] = B.GetShape();
+      const int size_on_qp = input.size_on_qp;
+      const auto field = Reshape(&field_e[0], size_on_qp * num_qp);
+      auto f = Reshape(&field_qp[0], size_on_qp * num_qp);
+      for (int i = 0; i < size_on_qp * num_qp; i++)
+      {
+         f(i) = field(i);
+      }
+   }
+   else
+   {
+      static_assert(dfem::always_false<field_operator_t>,
+                    "can't map field to quadrature data");
+
+   }
+}
+
+template <typename field_operator_ts, size_t num_inputs, size_t num_fields>
+MFEM_HOST_DEVICE inline
+void map_fields_to_quadrature_data(
+   std::array<DeviceTensor<2>, num_inputs> &fields_qp,
+   const std::array<DeviceTensor<1>, num_fields> &fields_e,
+   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
+   const std::array<size_t, num_inputs> &input_to_field,
+   const field_operator_ts &fops,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const int &dimension,
+   const bool &use_sum_factorization = false)
+{
+   // When the input_to_field map returns -1, this means the requested input
+   // is the integration weight. Weights don't have a user defined field
+   // attached to them and we create a dummy field which is not accessed
+   // inside the functions it is passed to.
+   const auto dummy_field_weight = DeviceTensor<1>(nullptr, 0);
+   for_constexpr<num_inputs>([&](auto i)
+   {
+      const DeviceTensor<1> &field_e =
+         (input_to_field[i] == SIZE_MAX) ? dummy_field_weight :
+         fields_e[input_to_field[i]];
+
+      if (use_sum_factorization)
+      {
+         if (dimension == 1)
+         {
+            map_field_to_quadrature_data_tensor_product_1d(
+               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
+               integration_weights, scratch_mem);
+         }
+         else if (dimension == 2)
+         {
+            map_field_to_quadrature_data_tensor_product_2d(
+               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
+               integration_weights, scratch_mem);
+         }
+         else if (dimension == 3)
+         {
+            map_field_to_quadrature_data_tensor_product_3d(
+               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
+               integration_weights, scratch_mem);
+         }
+         else
+         {
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+            MFEM_ABORT("unsupported dimension");
+#endif
+         }
+      }
+      else
+      {
+         map_field_to_quadrature_data(
+            fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
+            integration_weights);
+      }
+   });
+}
+
+template <typename field_operator_t>
+MFEM_HOST_DEVICE
+void map_field_to_quadrature_data_conditional(
+   DeviceTensor<2> &field_qp,
+   const DeviceTensor<1> &field_e,
+   const DofToQuadMap &dtqmap,
+   field_operator_t &fop,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const bool &condition,
+   const int &dimension,
+   const bool &use_sum_factorization = false)
+{
+   if (condition)
+   {
+      if (use_sum_factorization)
+      {
+         if (dimension == 1)
+         {
+            map_field_to_quadrature_data_tensor_product_1d(
+               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
+
+         }
+         else if (dimension == 2)
+         {
+            map_field_to_quadrature_data_tensor_product_2d(
+               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
+         }
+         else if (dimension == 3)
+         {
+            map_field_to_quadrature_data_tensor_product_3d(
+               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
+         }
+      }
+      else
+      {
+         map_field_to_quadrature_data(
+            field_qp, dtqmap, field_e, fop, integration_weights);
+      }
+   }
+}
+
+template <size_t num_fields, size_t num_inputs, typename field_operator_ts>
+MFEM_HOST_DEVICE
+void map_fields_to_quadrature_data_conditional(
+   std::array<DeviceTensor<2>, num_inputs> &fields_qp,
+   const std::array<DeviceTensor<1, const real_t>, num_fields> &fields_e,
+   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
+   field_operator_ts fops,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const std::array<bool, num_inputs> &conditions,
+   const bool &use_sum_factorization = false)
+{
+   for_constexpr<num_inputs>([&](auto i)
+   {
+      map_field_to_quadrature_data_conditional(
+         fields_qp[i], fields_e[i], dtqmaps[i], get<i>(fops), integration_weights,
+         scratch_mem, conditions[i], use_sum_factorization);
+   });
+}
+
+template <size_t num_inputs, typename field_operator_ts>
+MFEM_HOST_DEVICE
+void map_direction_to_quadrature_data_conditional(
+   std::array<DeviceTensor<2>, num_inputs> &directions_qp,
+   const DeviceTensor<1> &direction_e,
+   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
+   field_operator_ts fops,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const std::array<bool, num_inputs> &conditions,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   for_constexpr<num_inputs>([&](auto i)
+   {
+      if (conditions[i])
+      {
+         if (use_sum_factorization)
+         {
+            if (dimension == 1)
+            {
+               map_field_to_quadrature_data_tensor_product_1d(
+                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
+                  integration_weights, scratch_mem);
+            }
+            else if (dimension == 2)
+            {
+               map_field_to_quadrature_data_tensor_product_2d(
+                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
+                  integration_weights, scratch_mem);
+            }
+            else if (dimension == 3)
+            {
+               map_field_to_quadrature_data_tensor_product_3d(
+                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
+                  integration_weights, scratch_mem);
+            }
+         }
+         else
+         {
+            map_field_to_quadrature_data(
+               directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
+               integration_weights);
+         }
+      }
+   });
+}
+
+}

fem/dfem/parameterspace.hpp

@@ -20,7 +20,7 @@ namespace mfem::future
 class ParameterSpace
 {
 public:
-   ParameterSpace(int vdim = 1) : vdim(vdim) { dtq.FE = nullptr; dtq.IntRule = nullptr; }
+   ParameterSpace(int vdim = 1) : vdim(vdim) {}
 
    /// @brief Get vector dimension at each point
    ///
@@ -43,7 +43,7 @@ public:
    /// Get spatial dimension
    ///
    /// returns always 1.
-   constexpr int Dimension() const
+   int Dimension() const
    {
       return 1;
    }
@@ -65,7 +65,7 @@ public:
    /// It should not be used by a user.
    ///
    /// returns identity by default that is lazy evaluated.
-   virtual const Operator* GetElementRestriction(ElementDofOrdering) const
+   virtual const Operator* GetElementRestriction(ElementDofOrdering o) const
    {
       if (!elem_restr)
       {
@@ -74,14 +74,11 @@ public:
       return elem_restr.get();
    }
 
-   virtual const Operator* GetB() const = 0;
-
-   virtual const Operator* GetBt() const = 0;
-
 protected:
    int vdim;
    DofToQuad dtq;
-   mutable std::unique_ptr<Operator> prolongation, elem_restr, B, Bt;
+   mutable std::unique_ptr<Operator> prolongation;
+   mutable std::unique_ptr<Operator> elem_restr;
 };
 
 /// @brief Uniform parameter space

fem/dfem/qfunction_apply.hpp

@@ -0,0 +1,619 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "util.hpp"
+#include "qfunction_transform.hpp"
+
+namespace mfem::future
+{
+
+/// @brief Call a qfunction with the given parameters.
+///
+/// @param qfunc the qfunction to call.
+/// @param input_shmem the input shared memory.
+/// @param residual_shmem the residual shared memory.
+/// @param rs_qp the size of the residual.
+/// @param num_qp the number of quadrature points.
+/// @param q1d the number of quadrature points in 1D.
+/// @param dimension the spatial dimension.
+/// @param use_sum_factorization whether to use sum factorization.
+/// @tparam qf_param_ts the tuple type of the qfunction parameters.
+template <
+   typename qf_param_ts,
+   typename qfunc_t,
+   std::size_t num_fields>
+MFEM_HOST_DEVICE inline
+void call_qfunction(
+   qfunc_t &qfunc,
+   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
+   DeviceTensor<2> &residual_shmem,
+   const int &rs_qp,
+   const int &num_qp,
+   const int &q1d,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 1)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
+         {
+            auto qf_args = decay_tuple<qf_param_ts> {};
+            auto r = Reshape(&residual_shmem(0, q), rs_qp);
+            apply_kernel(r, qfunc, qf_args, input_shmem, q);
+         }
+      }
+      else if (dimension == 2)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               const int q = qx + q1d * qy;
+               auto qf_args = decay_tuple<qf_param_ts> {};
+               auto r = Reshape(&residual_shmem(0, q), rs_qp);
+               apply_kernel(r, qfunc, qf_args, input_shmem, q);
+            }
+         }
+      }
+      else if (dimension == 3)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+                  auto qf_args = decay_tuple<qf_param_ts> {};
+                  auto r = Reshape(&residual_shmem(0, q), rs_qp);
+                  apply_kernel(r, qfunc, qf_args, input_shmem, q);
+               }
+            }
+         }
+      }
+      else
+      {
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+         MFEM_ABORT("unsupported dimension for sum factorization");
+#endif
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else
+   {
+      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
+      {
+         auto qf_args = decay_tuple<qf_param_ts> {};
+         auto r = Reshape(&residual_shmem(0, q), rs_qp);
+         apply_kernel(r, qfunc, qf_args, input_shmem, q);
+      }
+   }
+}
+
+/// @brief Call a qfunction with the given parameters and
+/// compute it's derivative action.
+///
+/// @param qfunc the qfunction to call.
+/// @param input_shmem the input shared memory.
+/// @param shadow_shmem the shadow shared memory.
+/// @param residual_shmem the residual shared memory.
+/// @param das_qp the size of the derivative action.
+/// @param num_qp the number of quadrature points.
+/// @param q1d the number of quadrature points in 1D.
+/// @param dimension the spatial dimension.
+/// @param use_sum_factorization whether to use sum factorization.
+/// @tparam qf_param_ts the tuple type of the qfunction parameters.
+template <
+   typename qf_param_ts,
+   typename qfunc_t,
+   std::size_t num_fields>
+MFEM_HOST_DEVICE inline
+void call_qfunction_derivative_action(
+   qfunc_t &qfunc,
+   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
+   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   DeviceTensor<2> &residual_shmem,
+   const int &das_qp,
+   const int &num_qp,
+   const int &q1d,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 1)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
+         {
+            auto r = Reshape(&residual_shmem(0, q), das_qp);
+            auto qf_args = decay_tuple<qf_param_ts> {};
+#ifdef MFEM_USE_ENZYME
+            auto qf_shadow_args = decay_tuple<qf_param_ts> {};
+            apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
+                                        shadow_shmem, q);
+#else
+            apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
+#endif
+         }
+      }
+      else if (dimension == 2)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               const int q = qx + q1d * qy;
+               auto r = Reshape(&residual_shmem(0, q), das_qp);
+               auto qf_args = decay_tuple<qf_param_ts> {};
+#ifdef MFEM_USE_ENZYME
+               auto qf_shadow_args = decay_tuple<qf_param_ts> {};
+               apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
+                                           shadow_shmem, q);
+#else
+               apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
+#endif
+            }
+         }
+      }
+      else if (dimension == 3)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+                  auto r = Reshape(&residual_shmem(0, q), das_qp);
+                  auto qf_args = decay_tuple<qf_param_ts> {};
+#ifdef MFEM_USE_ENZYME
+                  auto qf_shadow_args = decay_tuple<qf_param_ts> {};
+                  apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
+                                              shadow_shmem, q);
+#else
+                  apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
+#endif
+               }
+            }
+         }
+      }
+      else
+      {
+         MFEM_ABORT_KERNEL("unsupported dimension");
+      }
+   }
+   else
+   {
+      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
+      {
+         auto r = Reshape(&residual_shmem(0, q), das_qp);
+         auto qf_args = decay_tuple<qf_param_ts> {};
+#ifdef MFEM_USE_ENZYME
+         auto qf_shadow_args = decay_tuple<qf_param_ts> {};
+         apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
+                                     shadow_shmem, q);
+#else
+         apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
+#endif
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+namespace detail
+{
+template <
+   typename qf_param_ts,
+   typename qfunc_t,
+   std::size_t num_fields>
+MFEM_HOST_DEVICE inline
+void call_qfunction_derivative(
+   qfunc_t &qfunc,
+   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
+   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   DeviceTensor<2> &residual_shmem,
+   DeviceTensor<5> &qpdc,
+   const DeviceTensor<1, const real_t> &itod,
+   const int &das_qp,
+   const int &q)
+{
+   const int test_vdim = qpdc.GetShape()[0];
+   const int test_op_dim = qpdc.GetShape()[1];
+   const int trial_vdim = qpdc.GetShape()[2];
+   const int num_qp = qpdc.GetShape()[4];
+   const size_t num_inputs = itod.GetShape()[0];
+
+   for (int j = 0; j < trial_vdim; j++)
+   {
+      int m_offset = 0;
+      for (size_t s = 0; s < num_inputs; s++)
+      {
+         const int trial_op_dim = static_cast<int>(itod(s));
+         if (trial_op_dim == 0)
+         {
+            continue;
+         }
+
+         auto d_qp = Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
+         for (int m = 0; m < trial_op_dim; m++)
+         {
+            d_qp(j, m, q) = 1.0;
+
+            auto r = Reshape(&residual_shmem(0, q), das_qp);
+            auto qf_args = decay_tuple<qf_param_ts> {};
+#ifdef MFEM_USE_ENZYME
+            auto qf_shadow_args = decay_tuple<qf_param_ts> {};
+            apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
+                                        shadow_shmem, q);
+#else
+            apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
+#endif
+            d_qp(j, m, q) = 0.0;
+
+            auto f = Reshape(&r(0), test_vdim, test_op_dim);
+            for (int i = 0; i < test_vdim; i++)
+            {
+               for (int k = 0; k < test_op_dim; k++)
+               {
+                  qpdc(i, k, j, m + m_offset, q) = f(i, k);
+               }
+            }
+         }
+         m_offset += trial_op_dim;
+      }
+   }
+}
+}
+
+/// @brief Call a qfunction with the given parameters and
+/// compute it's derivative represented by the Jacobian on
+/// each quadrature point.
+///
+/// @param qfunc the qfunction to call.
+/// @param input_shmem the input shared memory.
+/// @param shadow_shmem the shadow shared memory.
+/// @param residual_shmem the residual shared memory.
+/// @param qpdc the quadrature point data cache holding the resulting
+/// Jacobians on each quadrature point.
+/// @param itod inputs trial operator dimension.
+/// If input is dependent the value corresponds to the spatial dimension, otherwise
+/// a zero indicates non-dependence on the variable.
+/// @param das_qp the size of the derivative action.
+/// @param q1d the number of quadrature points in 1D.
+/// @param dimension the spatial dimension.
+/// @param use_sum_factorization whether to use sum factorization.
+/// @tparam qf_param_ts the tuple type of the qfunction parameters.
+template <
+   typename qf_param_ts,
+   typename qfunc_t,
+   std::size_t num_fields>
+MFEM_HOST_DEVICE inline
+void call_qfunction_derivative(
+   qfunc_t &qfunc,
+   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
+   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   DeviceTensor<2> &residual_shmem,
+   DeviceTensor<5> &qpdc,
+   const DeviceTensor<1, const real_t> &itod,
+   const int &das_qp,
+   const int &q1d,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 1)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
+         {
+            detail::call_qfunction_derivative<qf_param_ts>(
+               qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
+         }
+      }
+      else if (dimension == 2)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               const int q = qx + q1d * qy;
+               detail::call_qfunction_derivative<qf_param_ts>(
+                  qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
+            }
+         }
+      }
+      else if (dimension == 3)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+                  detail::call_qfunction_derivative<qf_param_ts>(
+                     qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
+               }
+            }
+         }
+      }
+      else
+      {
+         MFEM_ABORT_KERNEL("unsupported dimension");
+      }
+   }
+   else
+   {
+      const int num_qp = qpdc.GetShape()[4];
+      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
+      {
+         detail::call_qfunction_derivative<qf_param_ts>(
+            qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+namespace detail
+{
+
+/// @brief Apply the quadrature point data cache (qpdc) to a vector
+/// (usually a direction) on quadrature point q.
+///
+/// The qpdc consists of compatible data to be used for integration with a test
+/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
+/// function including integration weights and necessesary transformations.
+///
+/// @param fhat the qpdc applied to a vector in shadow_memory.
+/// @param shadow_shmem the shadow shared memory.
+/// @param qpdc the quadrature point data cache holding the resulting
+/// Jacobians on each quadrature point.
+/// @param itod inputs trial operator dimension.
+/// If input is dependent the value corresponds to the spatial dimension, otherwise
+/// a zero indicates non-dependence on the variable.
+/// @param q the current quadrature point index.
+template <size_t num_fields>
+MFEM_HOST_DEVICE inline
+void apply_qpdc(
+   DeviceTensor<3> &fhat,
+   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   const DeviceTensor<5, const real_t> &qpdc,
+   const DeviceTensor<1, const real_t> &itod,
+   const int &q)
+{
+   const int test_vdim = qpdc.GetShape()[0];
+   const int test_op_dim = qpdc.GetShape()[1];
+   const int trial_vdim = qpdc.GetShape()[2];
+   const int num_qp = qpdc.GetShape()[4];
+   const size_t num_inputs = itod.GetShape()[0];
+
+   for (int i = 0; i < test_vdim; i++)
+   {
+      for (int k = 0; k < test_op_dim; k++)
+      {
+         real_t sum = 0.0;
+         int m_offset = 0;
+         for (size_t s = 0; s < num_inputs; s++)
+         {
+            const int trial_op_dim = static_cast<int>(itod(s));
+            if (trial_op_dim == 0)
+            {
+               continue;
+            }
+            const auto d_qp =
+               Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
+            for (int j = 0; j < trial_vdim; j++)
+            {
+               for (int m = 0; m < trial_op_dim; m++)
+               {
+                  sum += qpdc(i, k, j, m + m_offset, q) * d_qp(j, m, q);
+               }
+            }
+            m_offset += trial_op_dim;
+         }
+         fhat(i, k, q) = sum;
+      }
+   }
+}
+}
+
+/// @brief Apply the quadrature point data cache (qpdc) to a vector
+/// (usually a direction).
+///
+/// The qpdc consists of compatible data to be used for integration with a test
+/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
+/// function including integration weights and necessesary transformations.
+///
+/// @param fhat the qpdc applied to a vector in shadow_memory.
+/// @param shadow_shmem the shadow shared memory.
+/// @param qpdc the quadrature point data cache holding the resulting
+/// Jacobians on each quadrature point.
+/// @param itod inputs trial operator dimension.
+/// If input is dependent the value corresponds to the spatial dimension, otherwise
+/// a zero indicates non-dependence on the variable.
+/// @param q1d number of quadrature points in 1D.
+/// @param dimension spatial dimension.
+/// @param use_sum_factorization whether to use sum factorization.
+template <size_t num_fields>
+MFEM_HOST_DEVICE inline
+void apply_qpdc(
+   DeviceTensor<3> &fhat,
+   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
+   const DeviceTensor<5, const real_t> &qpdc,
+   const DeviceTensor<1, const real_t> &itod,
+   const int &q1d,
+   const int &dimension,
+   const bool &use_sum_factorization)
+{
+   if (use_sum_factorization)
+   {
+      if (dimension == 1)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
+         {
+            detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+         }
+      }
+      else if (dimension == 2)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               const int q = qx + q1d * qy;
+               detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+            }
+         }
+      }
+      else if (dimension == 3)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+                  detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+               }
+            }
+         }
+      }
+      else
+      {
+         MFEM_ABORT_KERNEL("unsupported dimension");
+      }
+   }
+   else
+   {
+      const int num_qp = qpdc.GetShape()[4];
+      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
+      {
+         detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
+      }
+   }
+}
+
+template <typename qfunc_t, typename args_ts, size_t num_args>
+MFEM_HOST_DEVICE inline
+void apply_kernel(
+   DeviceTensor<1, real_t> &f_qp,
+   const qfunc_t &qfunc,
+   args_ts &args,
+   const std::array<DeviceTensor<2>, num_args> &u,
+   int qp)
+{
+   process_qf_args(u, args, qp);
+   process_qf_result(f_qp, get<0>(apply(qfunc, args)));
+}
+
+template <typename qfunc_t, typename arg_ts, size_t num_args>
+MFEM_HOST_DEVICE inline
+void apply_kernel_native_dual(
+   DeviceTensor<1, real_t> &f_qp,
+   const qfunc_t &qfunc,
+   arg_ts &args,
+   const std::array<DeviceTensor<2>, num_args> &u,
+   const std::array<DeviceTensor<2>, num_args> &v,
+   const int &qp_idx)
+{
+   process_qf_args(u, v, args, qp_idx);
+   auto r = get<0>(apply(qfunc, args));
+   process_derivative_from_native_dual(f_qp, r);
+}
+
+#ifdef MFEM_USE_ENZYME
+
+template <typename func_t, typename... arg_ts>
+MFEM_HOST_DEVICE inline
+auto qfunction_wrapper(const func_t &f, arg_ts &&...args)
+{
+   return f(args...);
+}
+
+// Version for active function arguments only
+//
+// This is an Enzyme regression and can be removed in later versions.
+template <typename qfunc_t, typename arg_ts, std::size_t... Is,
+          typename inactive_arg_ts>
+MFEM_HOST_DEVICE inline
+auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
+                                  arg_ts &&shadow_args,
+                                  std::index_sequence<Is...>,
+                                  inactive_arg_ts &&inactive_args,
+                                  std::index_sequence<>)
+{
+   using qf_return_t = typename create_function_signature<
+                       decltype(&qfunc_t::operator())>::type::return_t;
+   return __enzyme_fwddiff<qf_return_t>(
+             qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...>, enzyme_const,
+             (void *)&qfunc, enzyme_dup, &get<Is>(args)..., enzyme_interleave,
+             &get<Is>(shadow_args)...);
+}
+
+// Interleave function arguments for enzyme
+template <typename qfunc_t, typename arg_ts, std::size_t... Is,
+          typename inactive_arg_ts, std::size_t... Js>
+MFEM_HOST_DEVICE inline
+auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
+                                  arg_ts &&shadow_args,
+                                  std::index_sequence<Is...>,
+                                  inactive_arg_ts &&inactive_args,
+                                  std::index_sequence<Js...>)
+{
+   using qf_return_t = typename create_function_signature<
+                       decltype(&qfunc_t::operator())>::type::return_t;
+   return __enzyme_fwddiff<qf_return_t>(
+             qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...,
+             decltype(get<Js>(inactive_args))...>,
+             enzyme_const, (void *)&qfunc, enzyme_dup, &get<Is>(args)...,
+             enzyme_const, &get<Js>(inactive_args)..., enzyme_interleave,
+             &get<Is>(shadow_args)...);
+}
+
+template <typename qfunc_t, typename arg_ts, typename inactive_arg_ts>
+MFEM_HOST_DEVICE inline
+auto fwddiff_apply_enzyme(qfunc_t &qfunc, arg_ts &&args,
+                          arg_ts &&shadow_args,
+                          inactive_arg_ts &&inactive_args)
+{
+   auto arg_indices = std::make_index_sequence<
+                      tuple_size<std::remove_reference_t<arg_ts>>::value> {};
+
+   auto inactive_arg_indices = std::make_index_sequence<
+                               tuple_size<std::remove_reference_t<inactive_arg_ts>>::value> {};
+
+   return fwddiff_apply_enzyme_indexed(qfunc, args, shadow_args, arg_indices,
+                                       inactive_args, inactive_arg_indices);
+}
+
+template <typename qfunc_t, typename arg_ts, size_t num_args>
+MFEM_HOST_DEVICE inline
+void apply_kernel_fwddiff_enzyme(
+   DeviceTensor<1, real_t> &f_qp,
+   qfunc_t &qfunc,
+   arg_ts &args,
+   arg_ts &shadow_args,
+   const std::array<DeviceTensor<2>, num_args> &u,
+   const std::array<DeviceTensor<2>, num_args> &v,
+   int qp_idx)
+{
+   process_qf_args(u, args, qp_idx);
+   process_qf_args(v, shadow_args, qp_idx);
+   process_qf_result(f_qp,
+                     get<0>(fwddiff_apply_enzyme(qfunc, args, shadow_args, tuple<> {})));
+}
+#endif // MFEM_USE_ENZYME
+
+} // namespace mfem::future

fem/dfem/qfunction_transform.hpp

@@ -0,0 +1,346 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "util.hpp"
+#include "../../linalg/tensor.hpp"
+
+namespace mfem::future
+{
+
+template <typename T0, typename T1, typename T2>
+MFEM_HOST_DEVICE
+void process_qf_arg(const T0 &, const T1 &, T2 &)
+{
+   static_assert(dfem::always_false<T0, T1, T2>,
+                 "process_qf_arg not implemented for arg type");
+}
+
+template <typename T>
+MFEM_HOST_DEVICE
+void process_qf_arg(
+   const DeviceTensor<1, T> &u,
+   const DeviceTensor<1, T> &v,
+   T &arg)
+{
+   arg = u(0);
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   tensor<dual<T, T>, n, m> &arg)
+{
+   for (int i = 0; i < m; i++)
+   {
+      for (int j = 0; j < n; j++)
+      {
+         arg(j, i).value = u((i * n) + j);
+      }
+   }
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   dual<T, T> &arg)
+{
+   arg.value = u(0);
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   const DeviceTensor<1> &v,
+   dual<T, T> &arg)
+{
+   arg.value = u(0);
+   arg.gradient = v(0);
+}
+
+template <typename T, int n>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   const DeviceTensor<1> &v,
+   tensor<dual<T, T>, n> &arg)
+{
+   for (int i = 0; i < n; i++)
+   {
+      arg(i).value = u(i);
+      arg(i).gradient = v(i);
+   }
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   const DeviceTensor<1> &v,
+   tensor<dual<T, T>, n, m> &arg)
+{
+   for (int i = 0; i < m; i++)
+   {
+      for (int j = 0; j < n; j++)
+      {
+         arg(j, i).value = u((i * n) + j);
+         arg(j, i).gradient = v((i * n) + j);
+      }
+   }
+}
+
+template <typename T, int n>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const tensor<dual<T, T>, n> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      r(i) = x(i).value;
+   }
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const tensor<dual<T, T>, n, m> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      for (size_t j = 0; j < m; j++)
+      {
+         r(i + n * j) = x(i, j).value;
+      }
+   }
+}
+
+template <typename arg_type>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<2> &u,
+   const DeviceTensor<2> &v,
+   arg_type &arg,
+   const int &qp)
+{
+   const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
+   const auto v_qp = Reshape(&v(0, qp), v.GetShape()[0]);
+   process_qf_arg(u_qp, v_qp, arg);
+}
+
+template <size_t num_fields, typename qf_args>
+MFEM_HOST_DEVICE inline
+void process_qf_args(
+   const std::array<DeviceTensor<2>, num_fields> &u,
+   const std::array<DeviceTensor<2>, num_fields> &v,
+   qf_args &args,
+   const int &qp)
+{
+   for_constexpr<tuple_size<qf_args>::value>([&](auto i)
+   {
+      process_qf_arg(u[i], v[i], get<i>(args), qp);
+   });
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_derivative_from_native_dual(
+   DeviceTensor<1, T> &r,
+   const tensor<dual<T, T>, n, m> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      for (size_t j = 0; j < m; j++)
+      {
+         r(i + n * j) = x(i, j).gradient;
+      }
+   }
+}
+
+template <typename T, int n>
+MFEM_HOST_DEVICE inline
+void process_derivative_from_native_dual(
+   DeviceTensor<1, T> &r,
+   const tensor<dual<T, T>, n> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      r(i) = x(i).gradient;
+   }
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_derivative_from_native_dual(
+   DeviceTensor<1, T> &r,
+   const dual<T, T> &x)
+{
+   r(0) = x.gradient;
+}
+
+template <typename T0, typename T1>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(const T0 &, T1 &)
+{
+   static_assert(dfem::always_false<T0, T1>,
+                 "process_qf_arg not implemented for arg type");
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1, T> &u,
+   T &arg)
+{
+   arg = u(0);
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1, T> &u,
+   tensor<T> &arg)
+{
+   arg(0) = u(0);
+}
+
+template <typename T, int n>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   tensor<T, n> &arg)
+{
+   for (int i = 0; i < n; i++)
+   {
+      arg(i) = u(i);
+   }
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1> &u,
+   tensor<T, n, m> &arg)
+{
+   for (int i = 0; i < m; i++)
+   {
+      for (int j = 0; j < n; j++)
+      {
+         arg(j, i) = u((i * n) + j);
+      }
+   }
+}
+
+template <typename arg_type>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(const DeviceTensor<2> &u, arg_type &arg, int qp)
+{
+   const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
+   process_qf_arg(u_qp, arg);
+}
+
+template <size_t num_fields, typename qf_args>
+MFEM_HOST_DEVICE inline
+void process_qf_args(
+   const std::array<DeviceTensor<2>, num_fields> &u,
+   qf_args &args,
+   const int &qp)
+{
+   for_constexpr<tuple_size<qf_args>::value>([&](auto i)
+   {
+      process_qf_arg(u[i], get<i>(args), qp);
+   });
+}
+
+template <typename T0, typename T1>
+MFEM_HOST_DEVICE inline
+Vector process_qf_result(T0, T1)
+{
+   static_assert(dfem::always_false<T0, T1>,
+                 "process_qf_result not implemented for result type");
+   return Vector{};
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const T &x)
+{
+   r(0) = x;
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1> &r,
+   const dual<T, T> &x)
+{
+   r(0) = x.value;
+}
+
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const tensor<T> &x)
+{
+   r(0) = x(0);
+}
+
+template <typename T, int n>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const tensor<T, n> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      r(i) = x(i);
+   }
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_result(
+   DeviceTensor<1, T> &r,
+   const tensor<T, n, m> &x)
+{
+   for (size_t i = 0; i < n; i++)
+   {
+      for (size_t j = 0; j < m; j++)
+      {
+         r(i + n * j) = x(i, j);
+      }
+   }
+}
+
+template <typename T, int n, int m>
+MFEM_HOST_DEVICE inline
+void process_qf_arg(
+   const DeviceTensor<1, T> &u,
+   const DeviceTensor<1, T> &v,
+   tensor<T, n, m> &arg)
+{
+   for (int i = 0; i < m; i++)
+   {
+      for (int j = 0; j < n; j++)
+      {
+         arg(j, i) = u((i * n) + j);
+      }
+   }
+}
+
+} // namespace mfem::future

fem/dfem/tensor_functions.hpp

@@ -1,264 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-/**
- * @file tensor_functions.hpp
- *
- * @brief Differentiable functions of tensors
- */
-
-#pragma once
-
-#include <cmath>
-
-#include "../../linalg/dual.hpp"
-#include "../../linalg/tensor.hpp"
-#include "tuple.hpp"
-#include "util.hpp"
-
-// Force-inline every tensor operation under clang
-#if defined(__clang__)
-#pragma clang attribute push (__attribute__((always_inline)), apply_to = function)
-#endif
-
-namespace mfem
-{
-namespace future
-{
-
-/**
- * @brief Differentiable approximation of maximum eigenvale of a symmetric tensor
- *
- * Estimates the maximum eigenvalue using
- * $$
- *     smooth_max_eigenvalue(A) = \frac{1}{\beta} \log\Big( \mathrm{tr}\big(\exp(\beta A) \big) \Big)
- * $$
- * which is equivalent to using the log-sum-exp function on the eigenvalues of A.
- *
- * @param A The input tensor
- * @param beta Sharpness parameter. Must be > 0. Larger values makes the approximation sharper.
- * @return Approximate maximum eigenvalue of A
- */
-template <int n> MFEM_HOST_DEVICE
-real_t smooth_max_eigenvalue_symm(const tensor<real_t, n, n>& A, real_t beta)
-{
-   auto [lambda, V] = eig_symm(A);
-   real_t lambda_max = lambda[n - 1];
-   real_t sum = 0;
-   for (int i = 0; i < n - 1; i++)
-   {
-      sum += std::exp(beta*(lambda[i] - lambda_max));
-   }
-   return lambda_max + std::log1p(sum)/beta;
-}
-
-/**
- * @brief Differentiable approximation of minimum eigenvale of a symmetric tensor
- *
- * Estimates the minimum eigenvalue using
- * $$
- *     smooth_min_eigenvalue(A) = -\frac{1}{\beta} \log\Big( \mathrm{tr}\big(\exp(-\beta A) \big) \Big)
- * $$
- * which is equivalent to using the negated log-sum-exp function on the eigenvalues of -A.
- *
- * @param A The input tensor
- * @param beta Sharpness parameter. Must be > 0. Larger values makes the approximation sharper.
- * @return Approximate minimum eigenvalue of A
- */
-template <int n> MFEM_HOST_DEVICE
-real_t smooth_min_eigenvalue_symm(const tensor<real_t, n, n>& A, real_t beta)
-{
-   return -smooth_max_eigenvalue_symm<n>(-A, beta);
-}
-
-#ifdef MFEM_USE_ENZYME
-
-namespace detail
-{
-
-// Custom forward-mode derivative rule for Enzyme
-template<int n> MFEM_HOST_DEVICE
-dual<real_t, real_t> smooth_max_eigenvalue_symm_fwddiff(
-   const tensor<real_t, n, n>& A, const tensor<real_t, n, n>& A_dot, real_t beta,
-   real_t beta_dot)
-{
-   auto [lambda, V] = eig_symm(A);
-   real_t lambda_max = lambda[n - 1];
-   real_t sum = 0;
-   tensor<real_t, n> eg;
-   tensor<real_t, n> lambda_shifted;
-   for (int i = 0; i < n; i++)
-   {
-      lambda_shifted[i] = lambda[i] - lambda_max;
-      eg[i] = std::exp(beta*lambda_shifted[i]);
-      if (i != n - 1) { sum += eg[i]; }
-   }
-   real_t value = lambda_max + std::log1p(sum)/beta;
-
-   real_t Z = sum + 1.0;
-   real_t derivative{};
-   for (int mu = 0; mu < n; mu++)
-   {
-      real_t w_mu = eg[mu]/Z;
-      for (int i = 0; i < n; i++)
-      {
-         for (int j = 0; j < n; j++)
-         {
-            derivative += w_mu*V[i][mu]*V[j][mu]*A_dot[i][j];
-         }
-      }
-   }
-   derivative += (lambda_max - value + dot(eg, lambda_shifted)/Z)/beta * beta_dot;
-   return {value, derivative};
-}
-
-// Types and functions for Enzyme custom reverse mode derivative
-template <int n>
-struct SmoothMaxEigenvalueSymmTape
-{
-   tensor<real_t, n> lambda;
-   tensor<real_t, n, n> V;
-   tensor<real_t, n> eg;
-   real_t sum;
-   real_t logZ;
-};
-
-template <int n>
-struct SmoothMaxEigenvalueSymmAugmentedReturn
-{
-   void* tape;
-   real_t value;
-};
-
-template <int n> MFEM_HOST_DEVICE
-SmoothMaxEigenvalueSymmAugmentedReturn<n>
-smooth_max_eigenvalue_symm_aug(const tensor<real_t, n, n>* A,
-                               tensor<real_t, n, n>* A_bar,
-                               real_t beta)
-{
-   (void)A_bar; // accumulated in reverse pass
-
-   auto [lambda, V] = eig_symm(*A);
-   const real_t lambda_max = lambda[n - 1];
-
-   tensor<real_t, n> eg;
-   real_t sum = 0;
-   for (int i = 0; i < n; i++)
-   {
-      eg[i] = std::exp(beta*(lambda[i] - lambda_max));
-      if (i != n - 1) { sum += eg[i]; }
-   }
-
-   const real_t logZ = std::log1p(sum);
-   const real_t value = lambda_max + logZ/beta;
-
-   auto* tape = static_cast<SmoothMaxEigenvalueSymmTape<n>*>(
-                   std::malloc(sizeof(SmoothMaxEigenvalueSymmTape<n>)));
-   if (tape)
-   {
-      tape->lambda = lambda;
-      tape->V = V;
-      tape->eg = eg;
-      tape->sum = sum;
-      tape->logZ = logZ;
-   }
-
-   return {static_cast<void*>(tape), value};
-}
-
-template <int n> MFEM_HOST_DEVICE
-real_t smooth_max_eigenvalue_symm_rev(const tensor<real_t, n, n>* A,
-                                      tensor<real_t, n, n>* A_bar,
-                                      real_t beta,
-                                      real_t out_bar,
-                                      void* tape_ptr)
-{
-   (void)A; // all needed info is on the tape
-
-   const auto* tape = static_cast<const SmoothMaxEigenvalueSymmTape<n>*>(tape_ptr);
-   if (!tape)
-   {
-      return 0.0;
-   }
-
-   const real_t Z = tape->sum + 1.0;
-
-   // d/dA = Σ_mu w_mu v_mu v_mu^T, where w_mu = eg[mu]/Z
-   for (int mu = 0; mu < n; mu++)
-   {
-      const real_t w_mu = tape->eg[mu] / Z;
-      for (int i = 0; i < n; i++)
-      {
-         for (int j = 0; j < n; j++)
-         {
-            (*A_bar)[i][j] += out_bar * w_mu * tape->V[i][mu] * tape->V[j][mu];
-         }
-      }
-   }
-
-   // d/dβ = -(log Z)/β^2 + (1/(β Z)) Σ_{i<n-1} exp(β(λ_i-λ_max)) (λ_i-λ_max)
-   real_t dZ_dBeta = 0.0;
-   const real_t& lambda_max = tape->lambda[n - 1];
-   for (int i = 0; i < n - 1; i++)
-   {
-      dZ_dBeta += tape->eg[i] * (tape->lambda[i] - lambda_max);
-   }
-
-   const real_t beta2 = beta * beta;
-   const real_t d_value_dBeta = -(tape->logZ)/beta2 + dZ_dBeta/(beta * Z);
-
-   std::free(const_cast<SmoothMaxEigenvalueSymmTape<n>*>(tape));
-
-   return out_bar * d_value_dBeta;
-}
-
-} // namespace detail
-
-// Register custom derivatives (forward mode) with Enzyme
-__attribute__((used))
-void* __enzyme_register_derivative_smooth_max_eigenvalue_symm_2d[] =
-{
-   reinterpret_cast<void*>(smooth_max_eigenvalue_symm<2>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_fwddiff<2>)
-};
-
-__attribute__((used))
-void* __enzyme_register_derivative_smooth_max_eigenvalue_symm_3d[] =
-{
-   reinterpret_cast<void*>(smooth_max_eigenvalue_symm<3>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_fwddiff<3>)
-};
-
-// Register custom gradients (combined reverse mode) with Enzyme
-__attribute__((used))
-void* __enzyme_register_gradient_smooth_max_eigenvalue_symm_2d[] =
-{
-   reinterpret_cast<void*>(smooth_max_eigenvalue_symm<2>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_aug<2>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_rev<2>)
-};
-
-__attribute__((used))
-void* __enzyme_register_gradient_smooth_max_eigenvalue_symm_3d[] =
-{
-   reinterpret_cast<void*>(smooth_max_eigenvalue_symm<3>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_aug<3>),
-   reinterpret_cast<void*>(detail::smooth_max_eigenvalue_symm_rev<3>)
-};
-
-#endif // MFEM_USE_ENZYME
-
-} // namespace future
-} // namespace mfem
-
-#if defined(__clang__)
-#pragma clang attribute pop
-#endif

fem/dfem/tuple.hpp

@@ -16,24 +16,6 @@
 #include <type_traits>
 #include <tuple>
 
-// Define a portable unreachable macro
-#if defined(__GNUC__) || defined(__clang__)
-#if defined(__CUDACC_VER_MAJOR__)
-#if __CUDACC_VER_MAJOR__ <= 11 && __CUDACC_VER_MINOR__ < 3
-// nvcc didn't add __builtin_unreachable() until cuda 11.3
-#define MFEM_UNREACHABLE()
-#else
-// nvcc >= 11.3
-#define MFEM_UNREACHABLE() __builtin_unreachable()
-#endif
-#else
-// host-only version
-#define MFEM_UNREACHABLE() __builtin_unreachable()
-#endif
-#elif defined(_MSC_VER)
-#define MFEM_UNREACHABLE() __assume(0)
-#endif
-
 namespace mfem::future
 {
 
@@ -41,10 +23,31 @@ namespace mfem::future
 template <typename... T>
 struct tuple;
 
-// Implementation detail: storage using multiple inheritance from tuple_leaf
-// to support structured bindings
+// Implementation detail: storage using multiple inheritance from tuple_leaf,
+// which lets the tuple be defined for an arbitrary number of elements.
+// Structured bindings come from the std::tuple_size / std::tuple_element / get
+// specializations at the bottom of this file, not from the layout.
 namespace detail
 {
+/**
+ * @brief Trait that is true when @a U is a single argument that is (a reference
+ * to) @a Self
+ *
+ * A variadic constructor taking @c "U&&..." is a better match than the copy
+ * constructor for a non-const lvalue of its own type; this is used to constrain
+ * it out of those overload sets.
+ */
+template <typename Self, typename... U>
+struct is_self_arg : std::false_type {};
+
+/// @overload
+template <typename Self, typename U>
+struct is_self_arg<Self, U> : std::is_same<Self, std::decay_t<U>> {};
+
+/// SFINAE guard enabling a constructor for every @a U except @a Self itself
+template <typename Self, typename... U>
+using disable_if_self_t = std::enable_if_t<!is_self_arg<Self, U...>::value>;
+
 /**
  * @brief A single tuple element storage
  * @tparam I The index of this element in the tuple
@@ -59,7 +62,7 @@ struct tuple_leaf
    MFEM_HOST_DEVICE constexpr tuple_leaf() = default;
 
    /// Construct from value
-   template <typename U>
+   template <typename U, typename = disable_if_self_t<tuple_leaf, U>>
    MFEM_HOST_DEVICE constexpr explicit tuple_leaf(U&& v) :
       value(std::forward<U>(v)) {}
 };
@@ -69,8 +72,9 @@ struct tuple_leaf
  * @tparam Indices Index sequence for tuple elements
  * @tparam T The types stored in the tuple
  *
- * This uses multiple inheritance from tuple_leaf base classes to enable
- * structured bindings while maintaining efficient storage.
+ * This uses multiple inheritance from tuple_leaf base classes so that a single
+ * definition covers any number of elements, while keeping the storage layout
+ * (and the trivial copyability that device kernels rely on) of a plain struct.
  */
 template <typename Indices, typename... T>
 struct tuple_impl;
@@ -85,19 +89,61 @@ struct tuple_impl<std::index_sequence<I...>, T...> : tuple_leaf<I, T>...
    /**
     * @brief Construct from values
     * @param args The values to store in the tuple
+    *
+    * @note the arguments are perfectly forwarded, so that constructing a tuple
+    * from lvalues costs exactly one copy per element (taking them by value
+    * would add a copy plus a move).
     */
+   template <typename... U, typename = disable_if_self_t<tuple_impl, U...>>
    MFEM_HOST_DEVICE
-   constexpr explicit tuple_impl(T... args)
-      : tuple_leaf<I, T>(std::forward<T>(args))... {}
+   constexpr explicit tuple_impl(U&&... args)
+      : tuple_leaf<I, T>(std::forward<U>(args))... {}
    };
-}
+
+/**
+ * @brief Element-wise constructibility check, only instantiated once the
+ * argument list is known to have the right length
+ * @tparam Viable whether the arity and self-argument checks have passed
+ * @tparam Tuple the @p tuple being constructed
+ * @tparam U the constructor argument types
+ */
+template <bool Viable, typename Tuple, typename... U>
+struct is_constructible_from : std::false_type {};
+
+/// @overload
+template <typename... T, typename... U>
+struct is_constructible_from<true, tuple<T...>, U...>
+: std::bool_constant<(std::is_constructible_v<T, U&&> && ...)> {};
+
+/**
+ * @brief Trait that is true when @a Tuple can be constructed element-wise from
+ * the argument list @a U
+ * @tparam Tuple the @p tuple being constructed
+ * @tparam U the constructor argument types
+ */
+template <typename Tuple, typename... U>
+struct is_elementwise_constructible : std::false_type {};
+
+/// @overload
+template <typename... T, typename... U>
+struct is_elementwise_constructible<tuple<T...>, U...>
+   : is_constructible_from<sizeof...(U) == sizeof...(T) && sizeof...(U) != 0 &&
+  !is_self_arg<tuple<T...>, U...>::value, tuple<T...>, U...> {};
+
+/// SFINAE guard for the element-wise constructor of @p tuple
+template <typename Tuple, typename... U>
+using enable_elementwise_t =
+   std::enable_if_t<is_elementwise_constructible<Tuple, U...>::value>;
+}  // namespace detail
 
 /**
  * @tparam T the types stored in the tuple
  * @brief This is a class that mimics most of std::tuple's interface,
- * except that it is usable in CUDA kernels and admits some arithmetic operator overloads.
+ * except that it is usable in CUDA kernels and admits some arithmetic operator
+ * overloads.
  *
- * see https://en.cppreference.com/w/cpp/utility/tuple for more information about std::tuple
+ * See https://en.cppreference.com/w/cpp/utility/tuple for more information
+ * about std::tuple.
  */
 template <typename... T>
 struct tuple : detail::tuple_impl<std::index_sequence_for<T...>, T...>
@@ -111,9 +157,16 @@ struct tuple : detail::tuple_impl<std::index_sequence_for<T...>, T...>
    /**
     * @brief Construct tuple from values
     * @param args The values to store
+    *
+    * @note this constructor is deliberately *not* explicit, so that the
+    * copy-list-initialization forms that worked when @p tuple was an aggregate
+    * (@c "tuple<A,B> t = {a,b};", @c "return {a,b};", passing @c "{a,b}" to a
+    * function) keep working.
     */
+   template <typename... U,
+             typename = detail::enable_elementwise_t<tuple, U...>>
    MFEM_HOST_DEVICE
-   constexpr explicit tuple(T... args) : base_type(std::forward<T>(args)...) {}
+   constexpr tuple(U&&... args) : base_type(std::forward<U>(args)...) {}
 
    /// Copy constructor
    MFEM_HOST_DEVICE
@@ -169,10 +222,8 @@ template <class... Types>
 struct tuple_size;
 
 template <class... Types>
-struct tuple_size<tuple<Types...>> :
-                                std::integral_constant<std::size_t, sizeof...(Types)>
-{
-};
+struct tuple_size<tuple<Types...> >
+: std::integral_constant<std::size_t, sizeof...(Types)> {};
 
 /**
  * @brief a struct used to determine the type at index I of a tuple
@@ -188,10 +239,8 @@ struct tuple_element;
 // recursive case
 /// @overload
 template <size_t I, class Head, class... Tail>
-struct tuple_element<I, tuple<Head, Tail...>> : tuple_element<I - 1,
-                                              tuple<Tail...>>
-{
-};
+struct tuple_element<I, tuple<Head, Tail...> >
+   : tuple_element<I - 1, tuple<Tail...>> {};
 
 // base case
 /// @overload
@@ -201,19 +250,20 @@ struct tuple_element<0, tuple<Head, Tail...>>
    using type = Head;  ///< the type at the specified index
 };
 
-/// @brief Type alias mirroring std::tuple_element_t for mfem::future::tuple
-template <size_t I, class T>
-using tuple_element_t = typename tuple_element<I, T>::type;
-
+namespace detail
+{
 /**
  * @brief Get implementation for tuple_leaf - non-const lvalue reference
  * @tparam I the index of the tuple element
  * @tparam T the type of the tuple element
  * @param leaf the tuple_leaf containing the value
  * @return reference to the value
+ *
+ * @note @a T is deduced from the (unique) @p tuple_leaf base class of the
+ * argument, so callers only have to supply the index @a I.
  */
 template <size_t I, typename T>
-MFEM_HOST_DEVICE constexpr T& get_impl(detail::tuple_leaf<I, T>& leaf)
+MFEM_HOST_DEVICE constexpr T& get_impl(tuple_leaf<I, T>& leaf)
 {
    return leaf.value;
 }
@@ -226,8 +276,7 @@ MFEM_HOST_DEVICE constexpr T& get_impl(detail::tuple_leaf<I, T>& leaf)
  * @return const reference to the value
  */
 template <size_t I, typename T>
-MFEM_HOST_DEVICE constexpr const T& get_impl(const detail::tuple_leaf<I, T>&
-                                             leaf)
+MFEM_HOST_DEVICE constexpr const T& get_impl(const tuple_leaf<I, T>& leaf)
 {
    return leaf.value;
 }
@@ -240,7 +289,7 @@ MFEM_HOST_DEVICE constexpr const T& get_impl(const detail::tuple_leaf<I, T>&
  * @return rvalue reference to the value
  */
 template <size_t I, typename T>
-MFEM_HOST_DEVICE constexpr T&& get_impl(detail::tuple_leaf<I, T>&& leaf)
+MFEM_HOST_DEVICE constexpr T&& get_impl(tuple_leaf<I, T>&& leaf)
 {
    return static_cast<T&&>(leaf.value);
 }
@@ -253,11 +302,11 @@ MFEM_HOST_DEVICE constexpr T&& get_impl(detail::tuple_leaf<I, T>&& leaf)
  * @return const rvalue reference to the value
  */
 template <size_t I, typename T>
-MFEM_HOST_DEVICE constexpr const T&& get_impl(const detail::tuple_leaf<I, T>&&
-                                              leaf)
+MFEM_HOST_DEVICE constexpr const T&& get_impl(const tuple_leaf<I, T>&& leaf)
 {
    return static_cast<const T&&>(leaf.value);
 }
+}  // namespace detail
 
 /**
  * @tparam I the tuple index to access
@@ -269,9 +318,7 @@ template <size_t I, typename... T>
 MFEM_HOST_DEVICE constexpr auto& get(tuple<T...>& t)
 {
    static_assert(I < sizeof...(T), "Tuple index out of bounds");
-   using elem_type = typename tuple_element<I, tuple<T...>>::type;
-   using leaf_type = detail::tuple_leaf<I, elem_type>;
-   return get_impl<I>(static_cast<leaf_type&>(t));
+   return detail::get_impl<I>(t);
 }
 
 /**
@@ -284,9 +331,7 @@ template <size_t I, typename... T>
 MFEM_HOST_DEVICE constexpr const auto& get(const tuple<T...>& t)
 {
    static_assert(I < sizeof...(T), "Tuple index out of bounds");
-   using elem_type = typename tuple_element<I, tuple<T...>>::type;
-   using leaf_type = detail::tuple_leaf<I, elem_type>;
-   return get_impl<I>(static_cast<const leaf_type&>(t));
+   return detail::get_impl<I>(t);
 }
 
 /**
@@ -299,9 +344,7 @@ template <size_t I, typename... T>
 MFEM_HOST_DEVICE constexpr auto&& get(tuple<T...>&& t)
 {
    static_assert(I < sizeof...(T), "Tuple index out of bounds");
-   using elem_type = typename tuple_element<I, tuple<T...>>::type;
-   using leaf_type = detail::tuple_leaf<I, elem_type>;
-   return get_impl<I>(static_cast<leaf_type&&>(t));
+   return detail::get_impl<I>(std::move(t));
 }
 
 /**
@@ -314,15 +357,15 @@ template <size_t I, typename... T>
 MFEM_HOST_DEVICE constexpr const auto&& get(const tuple<T...>&& t)
 {
    static_assert(I < sizeof...(T), "Tuple index out of bounds");
-   using elem_type = typename tuple_element<I, tuple<T...>>::type;
-   using leaf_type = detail::tuple_leaf<I, elem_type>;
-   return get_impl<I>(static_cast<const leaf_type&&>(t));
+   return detail::get_impl<I>(std::move(t));
 }
 
 /**
- * @brief a function intended to be used for extracting the ith type from a tuple.
+ * @brief a function intended to be used for extracting the ith type from a
+ * tuple.
  *
- * @note type<i>(my_tuple) returns a value, whereas get<i>(my_tuple) returns a reference
+ * @note type<i>(my_tuple) returns a value, whereas get<i>(my_tuple) returns a
+ * reference
  *
  * @tparam I the index of the tuple to query
  * @tparam T the types stored in the tuple
@@ -336,6 +379,8 @@ MFEM_HOST_DEVICE constexpr auto type(const tuple<T...>& t)
    return get<I>(t);
 }
 
+namespace detail
+{
 /**
  * @brief Helper for applying binary operations element-wise
  *
@@ -357,6 +402,7 @@ MFEM_HOST_DEVICE constexpr auto apply_op_helper(
 {
    return tuple{op(get<I>(x), get<I>(y))...};
 }
+} // namespace detail
 
 /**
  * @tparam S the types stored in the tuple x
@@ -370,7 +416,8 @@ MFEM_HOST_DEVICE constexpr auto operator+(const tuple<S...>& x,
                                           const tuple<T...>& y)
 {
    static_assert(sizeof...(S) == sizeof...(T), "tuples must have same size");
-   return apply_op_helper(x, y, [](auto a, auto b) { return a + b; },
+   return detail::apply_op_helper(x, y,
+   [](const auto& a, const auto& b) { return a + b; },
    std::make_index_sequence<sizeof...(S)> {});
 }
 
@@ -386,7 +433,8 @@ MFEM_HOST_DEVICE constexpr auto operator-(const tuple<S...>& x,
                                           const tuple<T...>& y)
 {
    static_assert(sizeof...(S) == sizeof...(T), "tuples must have same size");
-   return apply_op_helper(x, y, [](auto a, auto b) { return a - b; },
+   return detail::apply_op_helper(x, y,
+   [](const auto& a, const auto& b) { return a - b; },
    std::make_index_sequence<sizeof...(S)> {});
 }
 
@@ -395,14 +443,16 @@ MFEM_HOST_DEVICE constexpr auto operator-(const tuple<S...>& x,
  * @tparam T the types stored in the tuple y
  * @param x a tuple of values
  * @param y a tuple of values
- * @brief return a tuple of values defined by elementwise multiplication of x and y
+ * @brief return a tuple of values defined by elementwise multiplication of x
+ * and y
  */
 template <typename... S, typename... T>
 MFEM_HOST_DEVICE constexpr auto operator*(const tuple<S...>& x,
                                           const tuple<T...>& y)
 {
    static_assert(sizeof...(S) == sizeof...(T), "tuples must have same size");
-   return apply_op_helper(x, y, [](auto a, auto b) { return a * b; },
+   return detail::apply_op_helper(x, y,
+   [](const auto& a, const auto& b) { return a * b; },
    std::make_index_sequence<sizeof...(S)> {});
 }
 
@@ -418,10 +468,13 @@ MFEM_HOST_DEVICE constexpr auto operator/(const tuple<S...>& x,
                                           const tuple<T...>& y)
 {
    static_assert(sizeof...(S) == sizeof...(T), "tuples must have same size");
-   return apply_op_helper(x, y, [](auto a, auto b) { return a / b; },
+   return detail::apply_op_helper(x, y,
+   [](const auto& a, const auto& b) { return a / b; },
    std::make_index_sequence<sizeof...(S)> {});
 }
 
+namespace detail
+{
 /**
  * @brief A helper function for the += operator of tuples
  *
@@ -438,6 +491,7 @@ MFEM_HOST_DEVICE constexpr void inplace_add_helper(
 {
    ((get<I>(x) += get<I>(y)), ...);
 }
+} // namespace detail
 
 /**
  * @tparam T the types stored in the tuples x and y
@@ -446,12 +500,15 @@ MFEM_HOST_DEVICE constexpr void inplace_add_helper(
  * @brief add values contained in y, to the tuple x
  */
 template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator+=(tuple<T...>& x, const tuple<T...>& y)
+MFEM_HOST_DEVICE constexpr tuple<T...>& operator+=(tuple<T...>& x,
+                                                   const tuple<T...>& y)
 {
-   inplace_add_helper(x, y, std::make_index_sequence<sizeof...(T)> {});
+   detail::inplace_add_helper(x, y, std::make_index_sequence<sizeof...(T)> {});
    return x;
 }
 
+namespace detail
+{
 /**
  * @brief A helper function for the -= operator of tuples
  *
@@ -468,6 +525,7 @@ MFEM_HOST_DEVICE constexpr void inplace_sub_helper(
 {
    ((get<I>(x) -= get<I>(y)), ...);
 }
+} // namespace detail
 
 /**
  * @tparam T the types stored in the tuples x and y
@@ -476,12 +534,15 @@ MFEM_HOST_DEVICE constexpr void inplace_sub_helper(
  * @brief subtract values contained in y from the tuple x
  */
 template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator-=(tuple<T...>& x, const tuple<T...>& y)
+MFEM_HOST_DEVICE constexpr tuple<T...>& operator-=(tuple<T...>& x,
+                                                   const tuple<T...>& y)
 {
-   inplace_sub_helper(x, y, std::make_index_sequence<sizeof...(T)> {});
+   detail::inplace_sub_helper(x, y, std::make_index_sequence<sizeof...(T)> {});
    return x;
 }
 
+namespace detail
+{
 /**
  * @brief A helper function for the unary - operator of tuples
  *
@@ -497,18 +558,23 @@ MFEM_HOST_DEVICE constexpr auto unary_minus_helper(
 {
    return tuple{-get<I>(x)...};
 }
+} // namespace detail
 
 /**
  * @tparam T the types stored in the tuple x
  * @param x a tuple of values
- * @brief return a tuple of values defined by applying the unary minus operator to each element of x
+ * @brief return a tuple of values defined by applying the unary minus operator
+ * to each element of x
  */
 template <typename... T>
 MFEM_HOST_DEVICE constexpr auto operator-(const tuple<T...>& x)
 {
-   return unary_minus_helper(x, std::make_index_sequence<sizeof...(T)> {});
+   return detail::unary_minus_helper(
+             x, std::make_index_sequence<sizeof...(T)> {});
 }
 
+namespace detail
+{
 /**
  * @brief A helper function for the * operator of tuples with scalar
  *
@@ -518,14 +584,15 @@ MFEM_HOST_DEVICE constexpr auto operator-(const tuple<T...>& x)
  * @param x tuple of values
  * @return the returned tuple product
  */
-template <typename... T, size_t... I>
+template <typename scalar_t, typename... T, size_t... I>
 MFEM_HOST_DEVICE constexpr auto scalar_mult_helper(
-   real_t a,
+   scalar_t a,
    const tuple<T...>& x,
    std::index_sequence<I...>)
 {
    return tuple{a * get<I>(x)...};
 }
+} // namespace detail
 
 /**
  * @tparam T the types stored in the tuple
@@ -533,10 +600,11 @@ MFEM_HOST_DEVICE constexpr auto scalar_mult_helper(
  * @param x the tuple object
  * @brief multiply each component of x by the value a on the left
  */
-template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator*(real_t a, const tuple<T...>& x)
+template <typename scalar_t, typename... T>
+MFEM_HOST_DEVICE constexpr auto operator*(scalar_t a, const tuple<T...>& x)
 {
-   return scalar_mult_helper(a, x, std::make_index_sequence<sizeof...(T)> {});
+   return detail::scalar_mult_helper(
+             a, x, std::make_index_sequence<sizeof...(T)> {});
 }
 
 /**
@@ -545,12 +613,14 @@ MFEM_HOST_DEVICE constexpr auto operator*(real_t a, const tuple<T...>& x)
  * @param a a scaling factor
  * @brief multiply each component of x by the value a on the right
  */
-template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator*(const tuple<T...>& x, real_t a)
+template <typename scalar_t, typename... T>
+MFEM_HOST_DEVICE constexpr auto operator*(const tuple<T...>& x, scalar_t a)
 {
    return a * x;
 }
 
+namespace detail
+{
 /**
  * @brief A helper function for the / operator of tuples with scalar denominator
  *
@@ -560,14 +630,15 @@ MFEM_HOST_DEVICE constexpr auto operator*(const tuple<T...>& x, real_t a)
  * @param a the constant denominator
  * @return the returned tuple ratio
  */
-template <typename... T, size_t... I>
+template <typename scalar_t, typename... T, size_t... I>
 MFEM_HOST_DEVICE constexpr auto scalar_div_helper(
    const tuple<T...>& x,
-   real_t a,
+   scalar_t a,
    std::index_sequence<I...>)
 {
    return tuple{get<I>(x) / a...};
 }
+} // namespace detail
 
 /**
  * @tparam T the types stored in the tuple x
@@ -575,12 +646,15 @@ MFEM_HOST_DEVICE constexpr auto scalar_div_helper(
  * @param a a denominator
  * @brief return a tuple of values defined by elementwise division of x by a
  */
-template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator/(const tuple<T...>& x, real_t a)
+template <typename scalar_t, typename... T>
+MFEM_HOST_DEVICE constexpr auto operator/(const tuple<T...>& x, scalar_t a)
 {
-   return scalar_div_helper(x, a, std::make_index_sequence<sizeof...(T)> {});
+   return detail::scalar_div_helper(
+             x, a, std::make_index_sequence<sizeof...(T)> {});
 }
 
+namespace detail
+{
 /**
  * @brief A helper function for the / operator with scalar numerator
  *
@@ -590,14 +664,15 @@ MFEM_HOST_DEVICE constexpr auto operator/(const tuple<T...>& x, real_t a)
  * @param x tuple of values
  * @return the returned tuple ratio
  */
-template <typename... T, size_t... I>
+template <typename scalar_t, typename... T, size_t... I>
 MFEM_HOST_DEVICE constexpr auto scalar_div_inv_helper(
-   real_t a,
+   scalar_t a,
    const tuple<T...>& x,
    std::index_sequence<I...>)
 {
    return tuple{a / get<I>(x)...};
 }
+} // namespace detail
 
 /**
  * @tparam T the types stored in the tuple x
@@ -605,12 +680,15 @@ MFEM_HOST_DEVICE constexpr auto scalar_div_inv_helper(
  * @param x a tuple of denominator values
  * @brief return a tuple of values defined by division of a by the elements of x
  */
-template <typename... T>
-MFEM_HOST_DEVICE constexpr auto operator/(real_t a, const tuple<T...>& x)
+template <typename scalar_t, typename... T>
+MFEM_HOST_DEVICE constexpr auto operator/(scalar_t a, const tuple<T...>& x)
 {
-   return scalar_div_inv_helper(a, x, std::make_index_sequence<sizeof...(T)> {});
+   return detail::scalar_div_inv_helper(
+             a, x, std::make_index_sequence<sizeof...(T)> {});
 }
 
+namespace detail
+{
 /**
  * @tparam T the types stored in the tuple
  * @tparam I a list of indices used to access each element of the tuple
@@ -627,6 +705,7 @@ auto& print_helper(std::ostream& out, const tuple<T...>& t,
    out << "}";
    return out;
 }
+} // namespace detail
 
 /**
  * @tparam T the types stored in the tuple
@@ -637,9 +716,12 @@ auto& print_helper(std::ostream& out, const tuple<T...>& t,
 template <typename... T>
 auto& operator<<(std::ostream& out, const tuple<T...>& t)
 {
-   return print_helper(out, t, std::make_index_sequence<sizeof...(T)> {});
+   return detail::print_helper(
+             out, t, std::make_index_sequence<sizeof...(T)> {});
 }
 
+namespace detail
+{
 /**
  * @brief A helper to apply a lambda to a tuple
  *
@@ -656,23 +738,27 @@ MFEM_HOST_DEVICE auto apply_helper(F&& f, tuple<T...>& args,
 {
    return std::forward<F>(f)(get<I>(args)...);
 }
+} // namespace detail
 
 /**
  * @tparam F a callable type
  * @tparam T the types of arguments to be passed in to f
  * @param f the callable object
  * @param args a tuple of arguments
- * @brief a way of passing an n-tuple to a function that expects n separate arguments
+ * @brief a way of passing an n-tuple to a function that expects n separate
+ * arguments
  *
- *   e.g. foo(bar, baz) is equivalent to apply(foo, mfem::tuple(bar,baz));
+ * For example, foo(bar, baz) is equivalent to apply(foo, mfem::tuple(bar,baz)).
  */
 template <typename F, typename... T>
 MFEM_HOST_DEVICE auto apply(F&& f, tuple<T...>& args)
 {
-   return apply_helper(std::forward<F>(f), args,
-                       std::make_index_sequence<sizeof...(T)> {});
+   return detail::apply_helper(std::forward<F>(f), args,
+                               std::make_index_sequence<sizeof...(T)> {});
 }
 
+namespace detail
+{
 /**
  * @overload
  */
@@ -682,21 +768,23 @@ MFEM_HOST_DEVICE auto apply_helper(F&& f, const tuple<T...>& args,
 {
    return std::forward<F>(f)(get<I>(args)...);
 }
+} // namespace detail
 
 /**
  * @tparam F a callable type
  * @tparam T the types of arguments to be passed in to f
  * @param f the callable object
  * @param args a tuple of arguments
- * @brief a way of passing an n-tuple to a function that expects n separate arguments
+ * @brief a way of passing an n-tuple to a function that expects n separate
+ * arguments
  *
- *   e.g. foo(bar, baz) is equivalent to apply(foo, mfem::tuple(bar,baz));
+ * For example, foo(bar, baz) is equivalent to apply(foo, mfem::tuple(bar,baz)).
  */
 template <typename F, typename... T>
 MFEM_HOST_DEVICE auto apply(F&& f, const tuple<T...>& args)
 {
-   return apply_helper(std::forward<F>(f), args,
-                       std::make_index_sequence<sizeof...(T)> {});
+   return detail::apply_helper(std::forward<F>(f), args,
+                               std::make_index_sequence<sizeof...(T)> {});
 }
 
 /**
@@ -714,7 +802,8 @@ struct is_tuple<tuple<T...>> : std::true_type
 };
 
 /**
- * @brief Trait for checking if a type if a @p mfem::tuple containing only @p mfem::tuple
+ * @brief Trait for checking if a type if a @p mfem::tuple containing only
+ * @p mfem::tuple
  */
 template <typename T>
 struct is_tuple_of_tuples : std::false_type
@@ -722,7 +811,8 @@ struct is_tuple_of_tuples : std::false_type
 };
 
 /**
- * @brief Trait for checking if a type if a @p mfem::tuple containing only @p mfem::tuple
+ * @brief Trait for checking if a type if a @p mfem::tuple containing only
+ * @p mfem::tuple
  */
 template <typename... T>
 struct is_tuple_of_tuples<tuple<T...>>
@@ -752,8 +842,8 @@ namespace std
  * @tparam T The types in the mfem::future::tuple
  */
 template <typename... T>
-struct tuple_size<mfem::future::tuple<T...>>
-                                          : integral_constant<size_t, sizeof...(T)> {};
+struct tuple_size<mfem::future::tuple<T...> >
+: integral_constant<size_t, sizeof...(T)> {};
 
 /**
  * @brief Specialization of std::tuple_element for mfem::future::tuple
@@ -766,20 +856,4 @@ struct tuple_element<I, mfem::future::tuple<T...>>
    using type = typename
                 mfem::future::tuple_element<I, mfem::future::tuple<T...>>::type;
 };
-
-template <size_t I, typename... T>
-constexpr decltype(auto) get(mfem::future::tuple<T...>& t) noexcept
-{ return mfem::future::get<I>(t); }
-
-template <size_t I, typename... T>
-constexpr decltype(auto) get(const mfem::future::tuple<T...>& t) noexcept
-{ return mfem::future::get<I>(t); }
-
-template <size_t I, typename... T>
-constexpr decltype(auto) get(mfem::future::tuple<T...>&& t) noexcept
-{ return mfem::future::get<I>(std::move(t)); }
-
-template <size_t I, typename... T>
-constexpr decltype(auto) get(const mfem::future::tuple<T...>&& t) noexcept
-{ return mfem::future::get<I>(std::move(t)); }
-}
+} // namespace std

fem/fe/fe_base.cpp

@@ -1316,13 +1316,14 @@ void VectorFiniteElement::Project_RT(
    }
 }
 
-void VectorFiniteElement::ProjectGrad_RT(
+void VectorFiniteElement::ProjectCurl2D_RT(
    const real_t *nk, const Array<int> &d2n, const FiniteElement &fe,
    ElementTransformation &Trans, DenseMatrix &grad) const
 {
+   // 2D "ProjectCurl_RT"
    if (dim != 2)
    {
-      mfem_error("VectorFiniteElement::ProjectGrad_RT works only in 2D!");
+      mfem_error("VectorFiniteElement::ProjectCurl2D_RT works only in 2D!");
    }
 
    DenseMatrix dshape(fe.GetDof(), fe.GetDim());
@@ -1333,8 +1334,8 @@ void VectorFiniteElement::ProjectGrad_RT(
    for (int k = 0; k < dof; k++)
    {
       fe.CalcDShape(Nodes.IntPoint(k), dshape);
-      tk[0] = nk[d2n[k]*dim+1];
-      tk[1] = -nk[d2n[k]*dim];
+      tk[0] = -nk[d2n[k]*dim+1];
+      tk[1] = nk[d2n[k]*dim];
       dshape.Mult(tk, grad_k);
       for (int j = 0; j < grad_k.Size(); j++)
       {
@@ -1381,7 +1382,7 @@ void VectorFiniteElement::ProjectCurl_ND(
    }
 }
 
-void VectorFiniteElement::ProjectCurl_RT(
+void VectorFiniteElement::ProjectCurl3D_RT(
    const real_t *nk, const Array<int> &d2n, const FiniteElement &fe,
    ElementTransformation &Trans, DenseMatrix &curl) const
 {

fem/fe/fe_base.hpp

@@ -957,10 +957,11 @@ protected:
                    const FiniteElement &fe, ElementTransformation &Trans,
                    DenseMatrix &I) const;
 
-   // rotated gradient in 2D
-   void ProjectGrad_RT(const real_t *nk, const Array<int> &d2n,
-                       const FiniteElement &fe, ElementTransformation &Trans,
-                       DenseMatrix &grad) const;
+   // Input is a scalar representing the Z (out of plane) component, Output is
+   // the X-Y (in-plane) RT curl
+   void ProjectCurl2D_RT(const real_t *nk, const Array<int> &d2n,
+                         const FiniteElement &fe, ElementTransformation &Trans,
+                         DenseMatrix &grad) const;
 
    // Compute the curl as a discrete operator from ND FE (fe) to ND FE (this).
    // The natural FE for the range is RT, so this is an approximation.
@@ -968,9 +969,9 @@ protected:
                        const FiniteElement &fe, ElementTransformation &Trans,
                        DenseMatrix &curl) const;
 
-   void ProjectCurl_RT(const real_t *nk, const Array<int> &d2n,
-                       const FiniteElement &fe, ElementTransformation &Trans,
-                       DenseMatrix &curl) const;
+   void ProjectCurl3D_RT(const real_t *nk, const Array<int> &d2n,
+                         const FiniteElement &fe, ElementTransformation &Trans,
+                         DenseMatrix &curl) const;
 
    /** @brief Project a vector coefficient onto the ND basis functions
        @param tk    Edge tangent vectors for this element type
@@ -1446,6 +1447,8 @@ public:
                                 dof2quad_array_open);
    }
 
+   const Poly_1D::Basis &GetOpenBasis1D() const { return obasis1d; }
+
    virtual ~VectorTensorFiniteElement();
 };
 

fem/fe/fe_rt.hpp

@@ -73,16 +73,11 @@ public:
    void Project(const FiniteElement &fe, ElementTransformation &Trans,
                 DenseMatrix &I) const override
    { Project_RT(nk, dof2nk, fe, Trans, I); }
-   // Gradient + rotation = Curl: H1 -> H(div)
-   void ProjectGrad(const FiniteElement &fe,
-                    ElementTransformation &Trans,
-                    DenseMatrix &grad) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
    // Curl = Gradient + rotation: H1 -> H(div)
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl2D_RT(nk, dof2nk, fe, Trans, curl); }
 
    void GetFaceMap(const int face_id, Array<int> &face_map) const override;
 
@@ -148,7 +143,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 
    /// @brief Return the mapping from lexicographically ordered face DOFs to
    /// lexicographically ordered element DOFs corresponding to local face
@@ -210,16 +205,11 @@ public:
    void Project(const FiniteElement &fe, ElementTransformation &Trans,
                 DenseMatrix &I) const override
    { Project_RT(nk, dof2nk, fe, Trans, I); }
-   // Gradient + rotation = Curl: H1 -> H(div)
-   void ProjectGrad(const FiniteElement &fe,
-                    ElementTransformation &Trans,
-                    DenseMatrix &grad) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
    // Curl = Gradient + rotation: H1 -> H(div)
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl2D_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 
@@ -274,7 +264,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 class RT_WedgeElement : public VectorFiniteElement
@@ -332,7 +322,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 /** Arbitrary order H(Div) basis functions defined on pyramid-shaped elements
@@ -428,7 +418,7 @@ public:
    virtual void ProjectCurl(const FiniteElement &fe,
                             ElementTransformation &Trans,
                             DenseMatrix &curl) const
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 
    void CalcRawVShape(const IntegrationPoint &ip,
                       DenseMatrix &shape) const;

fem/fem.hpp

@@ -52,7 +52,7 @@
 #include "bounds.hpp"
 #include "particleset.hpp"
 
-// #include "dfem/doperator.hpp"
+#include "dfem/doperator.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"

fem/fespace.cpp

@@ -22,6 +22,8 @@
 #include <algorithm>
 #include <cmath>
 #include <cstdarg>
+#include <unordered_map>
+#include <unordered_set>
 
 using namespace std;
 
@@ -4527,6 +4529,210 @@ void FiniteElementSpace
    }
 }
 
+void FiniteElementSpace::GetBoundaryLoopEdgeDofs(
+   const Array<int> &boundary_element_indices,
+   Array<int> &boundary_edge_dofs,
+   Array<int> *dof_edges,
+   Array<int> *dof_boundary_elements) const
+{
+   MFEM_VERIFY(mesh->Dimension() >= 2,
+               "GetBoundaryLoopEdgeDofs requires 2D or 3D meshes to find edge objects");
+
+   boundary_edge_dofs.SetSize(0);
+   if (dof_edges) { dof_edges->SetSize(0); }
+   if (dof_boundary_elements) { dof_boundary_elements->SetSize(0); }
+
+   // A DOF that appears in exactly one selected boundary element lies on the
+   // bounding loop; one appearing in two or more is interior to the boundary
+   // region and is dropped. Count occurrences of each DOF (using scratch maps,
+   // exposed only as parallel-indexed Array<int> below) and record, on first
+   // sight, the local edge and boundary element carrying it.
+   //
+   // The count is over GetEdgeDofs, which returns endpoint vertex DOFs as well
+   // as edge-interior DOFs (relevant for collections such as ND_R2D that carry
+   // vertex DOFs). Edge-interior DOFs occur once per edge, so the count mainly
+   // resolves vertex DOFs: a vertex shared by several elements is interior and
+   // dropped, while a genuine loop-corner (open-curve endpoint) vertex is kept.
+   // This is why we count GetEdgeDofs rather than collecting GetEdgeInteriorDofs,
+   // which would omit the endpoint vertex DOFs the method is documented to keep.
+   // The 3D removal criterion (any edge in two or more faces) matches the
+   // parallel version rather than a parity toggle.
+   std::unordered_map<int, int> dof_count, dof_edge, dof_belem;
+   Array<int> edge_dofs, edges, edge_orientations;
+
+   const int dim = mesh->Dimension();
+   for (int i = 0; i < boundary_element_indices.Size(); ++i)
+   {
+      const int boundary_element_idx = boundary_element_indices[i];
+      std::unordered_set<int> boundary_element_dofs;
+
+      if (dim == 3)
+      {
+         // Boundary elements are 2D faces; extract their 1D edges.
+         int face_index, face_orientation;
+         mesh->GetBdrElementFace(boundary_element_idx, &face_index,
+                                 &face_orientation);
+         mesh->GetFaceEdges(face_index, edges, edge_orientations);
+      }
+      else
+      {
+         // Boundary elements are 1D segments, each being a single edge.
+         mesh->GetBdrElementEdges(boundary_element_idx, edges, edge_orientations);
+         MFEM_VERIFY(edges.Size() == 1,
+                     "2D boundary element should have exactly one edge");
+      }
+
+      for (int j = 0; j < edges.Size(); ++j)
+      {
+         GetEdgeDofs(edges[j], edge_dofs);
+         for (int k = 0; k < edge_dofs.Size(); ++k)
+         {
+            const int dof = edge_dofs[k];
+            // Count each DOF once per boundary element and record metadata the
+            // first time it is seen, so H1 DOFs shared by multiple edges of the
+            // same element are not double counted.
+            if (boundary_element_dofs.insert(dof).second &&
+                dof_count[dof]++ == 0)
+            {
+               dof_edge[dof] = edges[j];
+               dof_belem[dof] = boundary_element_idx;
+            }
+         }
+      }
+   }
+
+   // Emit the DOFs seen in exactly one selected boundary element, in a
+   // deterministic (increasing DOF index) order shared by all output arrays.
+   std::vector<int> kept;
+   kept.reserve(dof_count.size());
+   for (const auto &[dof, count] : dof_count)
+   {
+      if (count == 1) { kept.push_back(dof); }
+   }
+   std::sort(kept.begin(), kept.end());
+
+   boundary_edge_dofs.Reserve(static_cast<int>(kept.size()));
+   if (dof_edges) { dof_edges->Reserve(static_cast<int>(kept.size())); }
+   if (dof_boundary_elements)
+   {
+      dof_boundary_elements->Reserve(static_cast<int>(kept.size()));
+   }
+   for (int dof : kept)
+   {
+      boundary_edge_dofs.Append(dof);
+      if (dof_edges) { dof_edges->Append(dof_edge[dof]); }
+      if (dof_boundary_elements) { dof_boundary_elements->Append(dof_belem[dof]); }
+   }
+}
+
+void FiniteElementSpace::GetBoundaryElementsByAttribute(
+   const Array<int> &bdr_attrs,
+   std::vector<Array<int>> &attr_to_elements)
+{
+   // One (initially empty) list of boundary elements per requested attribute,
+   // indexed to match bdr_attrs.
+   attr_to_elements.assign(bdr_attrs.Size(), Array<int>());
+
+   // Map attribute value -> position in bdr_attrs for quick lookup.
+   std::unordered_map<int, int> attr_to_index;
+   for (int i = 0; i < bdr_attrs.Size(); ++i)
+   {
+      attr_to_index[bdr_attrs[i]] = i;
+   }
+
+   // Bucket boundary elements by their attribute.
+   for (int i = 0; i < mesh->GetNBE(); ++i)
+   {
+      int attr = mesh->GetBdrElement(i)->GetAttribute();
+      auto it = attr_to_index.find(attr);
+      if (it != attr_to_index.end())
+      {
+         attr_to_elements[it->second].Append(i);
+      }
+   }
+}
+
+void FiniteElementSpace::GetBoundaryElementsByAttribute(int bdr_attr,
+                                                        Array<int> &boundary_elements)
+{
+   boundary_elements.SetSize(0);
+
+   for (int i = 0; i < mesh->GetNBE(); ++i)
+   {
+      if (mesh->GetBdrElement(i)->GetAttribute() == bdr_attr)
+      {
+         boundary_elements.Append(i);
+      }
+   }
+}
+
+void FiniteElementSpace::ComputeLoopEdgeOrientations(
+   const Array<int> &dof_edges,
+   const Array<int> &dof_boundary_elements,
+   const Vector &loop_normal,
+   Array<int> &dof_orientations) const
+{
+   MFEM_VERIFY(dof_edges.Size() == dof_boundary_elements.Size(),
+               "dof_edges and dof_boundary_elements must be parallel-indexed");
+
+   const int ndof = dof_edges.Size();
+   dof_orientations.SetSize(ndof);
+
+   Array<int> edge_verts, bdr_elem_verts;
+   Vector edge_vec(3), to_edge_vec(3), cross_product(3);
+   for (int i = 0; i < ndof; i++)
+   {
+      const int edge_id = dof_edges[i];
+      const int bdr_elem_idx = dof_boundary_elements[i];
+
+      // Get edge vertices
+      mesh->GetEdgeVertices(edge_id, edge_verts);
+
+      const real_t *v0 = mesh->GetVertex(edge_verts[0]);
+      const real_t *v1 = mesh->GetVertex(edge_verts[1]);
+
+      // Get boundary element vertices
+      mesh->GetBdrElement(bdr_elem_idx)->GetVertices(bdr_elem_verts);
+
+      // Find the third vertex (not part of the edge)
+      int third_vertex = -1;
+      for (int j = 0; j < bdr_elem_verts.Size(); j++)
+      {
+         int v = bdr_elem_verts[j];
+         if (v != edge_verts[0] && v != edge_verts[1])
+         {
+            third_vertex = v;
+            break;
+         }
+      }
+
+      if (third_vertex == -1)
+      {
+         MFEM_ABORT("Boundary element " << bdr_elem_idx << " has only 2 vertices, "
+                    "but 3D boundary elements must have at least 3 vertices");
+      }
+
+      const real_t *v2 = mesh->GetVertex(third_vertex);
+
+      // Edge vector
+      for (int j = 0; j < 3; j++) { edge_vec[j] = v1[j] - v0[j]; }
+
+      // Vector from third vertex to edge (use edge midpoint)
+      for (int j = 0; j < 3; j++)
+      {
+         real_t edge_midpoint = (v0[j] + v1[j]) * 0.5;
+         to_edge_vec[j] = edge_midpoint - v2[j];
+      }
+
+      // Cross product: to_edge × edge
+      to_edge_vec.cross3D(edge_vec, cross_product);
+
+      // Check alignment with loop normal
+      real_t dot_product = cross_product * loop_normal;
+      dof_orientations[i] = (dot_product > 0) ? 1 : -1;
+   }
+}
+
 FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)
 {
    string buff;

fem/fespace.hpp

@@ -22,6 +22,7 @@
 #include "restriction.hpp"
 #include <iostream>
 #include <unordered_map>
+#include <vector>
 
 namespace mfem
 {
@@ -1389,6 +1390,80 @@ public:
    virtual void GetExteriorTrueDofs(Array<int> &exterior_dofs,
                                     int component = -1) const;
 
+   /** @brief Extract the edge degrees of freedom of a boundary "loop".
+
+       Here a "loop" is the set of boundary edges bounding the region covered by
+       @a boundary_element_indices: in 3D the outer edges of a patch of boundary
+       faces, in 2D the boundary segments themselves. An edge that is shared by
+       two (or more) of the selected boundary elements is interior to that region
+       rather than on its bounding loop, so its DOFs are excluded from the result.
+       This exclusion of interior DOFs is the defining feature of the method.
+
+       The three output arrays share a single indexing: for each valid index @a i,
+       @a dof_edges[i] and @a dof_boundary_elements[i] describe the DOF
+       @a boundary_edge_dofs[i].
+
+       @param[in]  boundary_element_indices Boundary element indices spanning a
+                   boundary surface (3D) or curve (2D).
+       @param[out] boundary_edge_dofs Local DOF indices on the boundary loop.
+       @param[out] dof_edges Optional; local edge index carrying each DOF.
+       @param[out] dof_boundary_elements Optional; a boundary element containing
+                   each DOF.
+
+       @note In 3D the edge DOFs are extracted from the 1D edges of the 2D
+       boundary faces; in 2D they come directly from the 1D boundary segments, so
+       @a dof_edges then holds the boundary element (segment) edge indices.
+       @note This method uses GetEdgeDofs internally, which returns both vertex and
+       edge DOFs. Standard Nédélec elements (ND_FECollection) have no vertex DOFs,
+       so only genuine edge DOFs appear. Collections that carry vertex DOFs (e.g.
+       ND_R2D_FECollection) additionally contribute the vertex DOFs at loop
+       endpoints.
+       @note This is the serial version. For parallel meshes, use the parallel
+       version in ParFiniteElementSpace which handles processor boundaries
+       correctly.
+       @note Requires a 2D or 3D mesh to identify edge objects. The method will
+       assert if called on 1D meshes.
+       @note Only supports conforming meshes; non-conforming meshes are not
+       supported. */
+   void GetBoundaryLoopEdgeDofs(const Array<int> &boundary_element_indices,
+                                Array<int> &boundary_edge_dofs,
+                                Array<int> *dof_edges = nullptr,
+                                Array<int> *dof_boundary_elements = nullptr) const;
+
+   /** @brief Get boundary elements grouped by attribute.
+
+       For each attribute in @a bdr_attrs, collect the indices of all boundary
+       elements carrying that attribute. The result is indexed to match
+       @a bdr_attrs: @a attr_to_elements[i] holds the boundary elements with
+       attribute @a bdr_attrs[i]. */
+   void GetBoundaryElementsByAttribute(
+      const Array<int> &bdr_attrs,
+      std::vector<Array<int>> &attr_to_elements);
+
+   /** @brief Get all boundary elements with a specific attribute. */
+   void GetBoundaryElementsByAttribute(int bdr_attr,
+                                       Array<int> &boundary_elements);
+
+   /** @brief Compute edge orientations relative to a boundary loop direction.
+
+       For each boundary-loop DOF described by @a dof_edges and
+       @a dof_boundary_elements (see GetBoundaryLoopEdgeDofs), determine whether
+       the carrying edge is
+       traversed in the direction consistent with @a loop_normal, following the
+       right-hand rule. Intended for 3D meshes.
+
+       @param[in]  dof_edges Local edge index of each DOF (parallel-indexed with
+                   the boundary_edge_dofs output of GetBoundaryLoopEdgeDofs).
+       @param[in]  dof_boundary_elements A boundary element containing each DOF,
+                   using the same indexing as @a dof_edges.
+       @param[in]  loop_normal Normal vector defining the loop orientation.
+       @param[out] dof_orientations Orientation (+1 or -1) for each DOF, using the
+                   same indexing as @a dof_edges. */
+   void ComputeLoopEdgeOrientations(const Array<int> &dof_edges,
+                                    const Array<int> &dof_boundary_elements,
+                                    const Vector &loop_normal,
+                                    Array<int> &dof_orientations) const;
+
    /// Convert a Boolean marker array to a list containing all marked indices.
    static void MarkerToList(const Array<int> &marker, Array<int> &list);
 

fem/gslib.cpp

@@ -556,7 +556,7 @@ void obboxsurf_calc_3(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, n);
    lag(I0, work, n, 1, 0);
 
-   for (int ie = 0; ie < nel; ie++,x+=n2,y+=n2,z+=n2)
+   for (int ie = 0; (unsigned)ie < nel; ie++,x+=n2,y+=n2,z+=n2)
    {
       struct gslib::dbl_range ab[3];
       struct gslib::dbl_range tb[3];
@@ -780,7 +780,7 @@ void obboxedge_calc_2(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, nr);
    lag(I0r, work, nr,1, 0);
 
-   for (int ie = 0; ie < nel; ie++,x+=nr,y+=nr)
+   for (int ie = 0; (unsigned)ie < nel; ie++,x+=nr,y+=nr)
    {
       double x0[2], A[4];
       struct gslib::dbl_range ab[2], tb[2];
@@ -892,7 +892,7 @@ void obboxedge_calc_3(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, nr);
    lag(I0r, work, nr, 1, 0);
 
-   for (int ie = 0; ie < nel; ie++,x+=nr,y+=nr,z+=nr)
+   for (int ie = 0; (unsigned)ie < nel; ie++,x+=nr,y+=nr,z+=nr)
    {
       double x0[3], A[9], Ai[9];
       struct gslib::dbl_range ab[3], tb[3];
@@ -4518,7 +4518,7 @@ Mesh* FindPointsGSLIB::GetBoundingBoxMesh(int type)
    int eidx = 0;
    if (myid == save_rank)
    {
-      for (int p = 0; p < gsl_comm->np; p++)
+      for (int p = 0; (unsigned)p < gsl_comm->np; p++)
       {
          if (static_cast<unsigned int>(p) != save_rank)
          {

fem/integ/bilininteg_convection_pa.cpp

@@ -178,6 +178,8 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    // Assumes tensor-product elements
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
+   MFEM_VERIFY(el.GetMapType() == FiniteElement::VALUE,
+               "Only value map type currently supported");
    ElementTransformation &Trans = *mesh->GetTypicalElementTransformation();
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Trans);
    if (DeviceCanUseCeed())

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -785,6 +785,23 @@ void PAHcurlL2Setup2D(const int Q1D,
    });
 }
 
+void PAHcurlL2IntSetup2D(const int Q1D, const int NE, const Array<real_t> &w,
+                         Vector &coeff, const Vector &detJ, Vector &op)
+{
+   const int NQ = Q1D*Q1D;
+   auto W = w.Read();
+   auto C = Reshape(coeff.Read(), NQ, NE);
+   auto J = Reshape(detJ.Read(), NQ, NE);
+   auto y = Reshape(op.Write(), NQ, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int q = 0; q < NQ; ++q)
+      {
+         y(q,e) = W[q] * C(q,e) / J(q,e);
+      }
+   });
+}
+
 void PAHcurlL2Setup3D(const int NQ,
                       const int coeffDim,
                       const int NE,

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -1889,13 +1889,17 @@ inline void SmemPACurlCurlApply3D(const int d1d,
    ForallWrap<3>(true, NE, device_kernel, host_kernel, Q1D, Q1D, Q1D);
 }
 
-// PA H(curl)-L2 Assemble 2D kernel
+// PA H(curl)-L2 value Assemble 2D kernel
 void PAHcurlL2Setup2D(const int Q1D,
                       const int NE,
                       const Array<real_t> &w,
                       Vector &coeff,
                       Vector &op);
 
+// PA H(curl)-L2 integral Assemble 2D kernel
+void PAHcurlL2IntSetup2D(const int Q1D, const int NE, const Array<real_t> &w,
+                         Vector &coeff, const Vector &detJ, Vector &op);
+
 // PA H(curl)-L2 Assemble 3D kernel
 void PAHcurlL2Setup3D(const int NQ,
                       const int coeffDim,

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -864,8 +864,656 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
    }); // end of element loop
 }
 
+namespace curlinterp
+{
+constexpr int NBZ3D(int ndof_o, int nquad_o, int mdq)
+{
+   if (ndof_o <= 0 || nquad_o <= 0)
+   {
+      return 1;
+   }
+   int ndof_c = ndof_o + 1;
+   int nquad_c = nquad_o + 1;
+   // z dimension is capped at 64 on nvidia and amd gpus
+   int tmp =
+      std::min((128 + mdq * mdq * (mdq - 1) - 1) / (mdq * mdq * (mdq - 1)), 64);
+   int smem_req =
+      sizeof(mfem::real_t) *
+      ((3 * ndof_c * ndof_c * ndof_o + 2 * 2 * mdq * mdq * mdq) * tmp +
+       ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
+   // assume GPU has at least 48k shared memory
+   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
+}
+}
+
+template <int T_NDOF_O, int T_NQUAD_O>
+void CurlInterpolatorApply3DSmem(const int ne, const int ndof_o,
+                                 const int nquad_o, const Vector &pa,
+                                 const Vector &x_, Vector &y_)
+{
+   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+   constexpr int mnq_o =
+      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
+   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
+   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
+   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
+   int mnq = std::max(ndof_o + 1, nquad_o + 1);
+   auto pa_data = pa.Read();
+   auto x_d = x_.Read();
+   auto y_d = y_.ReadWrite();
+   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
+      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MND_O =
+         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+      constexpr int MNQ_O =
+         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
+      int tidz = MFEM_THREAD_ID(z);
+      // Make mnq a local variable since capturing would result in different
+      // captures between host/device versions, and spuriously fails
+      int mnq = std::max(ndof_o + 1, nquad_o + 1);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
+      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
+      const int NDOF_C = NDOF_O + 1;
+      const int NQUAD_C = NQUAD_O + 1;
+      MFEM_SHARED real_t
+      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
+      auto X_ = Reshape(x_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
+      auto Y = Reshape(y_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
+      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
+      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
+      auto Boo =
+         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
+      MFEM_SHARED real_t X[3][nbz][MND_O * (MND_O + 1) * (MND_O + 1)];
+      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
+      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
+
+      // shapes of buffers always use MNDQ to mitigate shared memory bank
+      // conflicts
+      real_t(*DDQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      real_t(*DQQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
+      real_t(*QQQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      const int offset = NDOF_O * NDOF_C * NDOF_C;
+      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
+      {
+         for (int dim = 0; dim < 3; ++dim)
+         {
+            X[dim][tidz][ix] = X_(ix + dim * offset, e);
+         }
+      }
+      // load basis functions data
+      if (tidz == 0)
+      {
+         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
+         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
+      }
+      MFEM_SYNC_THREAD;
+
+      // x: Vz Bcc Gco Boo - Vy Bcc Boo Gco
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Bcc(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Bcc(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Gco(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_O; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Boo(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_O; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Boo(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Gco(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_O) * NQUAD_C, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // y: Vx Boo Bcc Gco - Vz Gco Bcc Boo
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_O; ++dx)
+         {
+            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Gco(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Bcc(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
+                                           NDOF_O, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Bcc(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Gco(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_O; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Boo(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_C) * NQUAD_O + offsetq, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // z: Vy Gco Boo Bcc - Vx Boo Gco Bcc
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Gco(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_O; ++dx)
+         {
+            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_O; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Boo(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Gco(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Bcc(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Bcc(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_O) * NQUAD_O + 2 * offsetq, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+template <int T_NDOF_O, int T_NQUAD_O>
+void CurlInterpolatorTApply3DSmem(const int ne, const int ndof_o,
+                                  const int nquad_o, const Vector &pa,
+                                  const Vector &x_, Vector &y_)
+{
+   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+   constexpr int mnq_o =
+      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
+   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
+   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
+   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
+   int mnq = std::max(ndof_o + 1, nquad_o + 1);
+   auto pa_data = pa.Read();
+   auto x_d = x_.Read();
+   auto y_d = y_.ReadWrite();
+   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
+      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MND_O =
+         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+      constexpr int MNQ_O =
+         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
+      int tidz = MFEM_THREAD_ID(z);
+      // Make mnq a local variable since capturing would result in different
+      // captures between host/device versions, and spuriously fails
+      int mnq = std::max(ndof_o + 1, nquad_o + 1);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
+      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
+      const int NDOF_C = NDOF_O + 1;
+      const int NQUAD_C = NQUAD_O + 1;
+      MFEM_SHARED real_t
+      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
+      auto X_ = Reshape(x_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
+      auto Y = Reshape(y_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
+      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
+      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
+      auto Boo =
+         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
+      MFEM_SHARED real_t X[3][nbz][MNQ_O * MNQ_O * (MNQ_O + 1)];
+      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
+      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
+
+      // shapes of buffers always use MNDQ to mitigate shared memory bank
+      // conflicts
+      real_t(*QQD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      real_t(*QDD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
+      real_t(*DDD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      const int offset = NDOF_O * NDOF_C * NDOF_C;
+      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offsetq)
+      {
+         for (int dim = 0; dim < 3; ++dim)
+         {
+            X[dim][tidz][ix] = X_(ix + dim * offsetq, e);
+         }
+      }
+      // load basis functions data
+      if (tidz == 0)
+      {
+         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
+         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
+      }
+      MFEM_SYNC_THREAD;
+
+      // x: Vy Boo Bcc Gco - Vz Boo Gco Bcc
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Gco(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_C; ++qz)
+         {
+            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_C; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Bcc(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Gco(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Boo(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Boo(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_C) * NDOF_O, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // y: Vz Gco Boo Bcc - Vx Bcc Boo Gco
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_C; ++qz)
+         {
+            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Gco(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Boo(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
+                                           NQUAD_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Boo(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Gco(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_C; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Bcc(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_O) * NDOF_C + offset, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // z: Vx Bcc Gco Boo - Vy Gco Bcc Boo
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Boo(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Boo(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Gco(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_C; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Bcc(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_C; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Bcc(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Gco(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_C) * NDOF_C + 2 * offset, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
 } // namespace internal
 
+template <int DIM, int NDOF_O, int NQUAD_O>
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyPAKernels::Kernel()
+{
+   if constexpr (DIM == 3)
+   {
+      return internal::CurlInterpolatorApply3DSmem<NDOF_O, NQUAD_O>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+template <int DIM, int NDOF_O, int NQUAD_O>
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyTPAKernels::Kernel()
+{
+   if constexpr (DIM == 3)
+   {
+      return internal::CurlInterpolatorTApply3DSmem<NDOF_O, NQUAD_O>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
 } // namespace mfem
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_interp_pa.cpp

@@ -14,9 +14,218 @@
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 
+#include "bilininteg_hcurlhdiv_kernels.hpp"
+
 namespace mfem
 {
 
+namespace
+{
+
+void PAHcurlApplyCurl2D(const int c_dofs1D,
+                        const int o_dofs1D,
+                        const int NE,
+                        const Array<real_t> &Bo_,
+                        const Array<real_t> &Gc_,
+                        const Vector &x_,
+                        Vector &y_)
+{
+   auto Bo = Reshape(Bo_.Read(), o_dofs1D, o_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), o_dofs1D, o_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int iy = 0; iy < c_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < o_dofs1D; ++ix)
+         {
+            const real_t xv = X(ix + iy * o_dofs1D, e);
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t gy = Gc(oy, iy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  Y(ox, oy, e) -= Bo(ox, ix) * gy * xv;
+               }
+            }
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int iy = 0; iy < o_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < c_dofs1D; ++ix)
+         {
+            const real_t xv = X(y_nd + ix + iy * c_dofs1D, e);
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t by = Bo(oy, iy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  Y(ox, oy, e) += Gc(ox, ix) * by * xv;
+               }
+            }
+         }
+      }
+   });
+}
+
+void PAHcurlApplyCurlTranspose2D(const int c_dofs1D,
+                                 const int o_dofs1D,
+                                 const int NE,
+                                 const Array<real_t> &Bo_,
+                                 const Array<real_t> &Gc_,
+                                 const Vector &x_,
+                                 Vector &y_)
+{
+   auto Bo = Reshape(Bo_.Read(), o_dofs1D, o_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), o_dofs1D, o_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int dy = 0; dy < c_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < o_dofs1D; ++dx)
+         {
+            real_t sum = 0.0;
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t gy = Gc(oy, dy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  sum -= Bo(ox, dx) * gy * X(ox, oy, e);
+               }
+            }
+            Y(dx + dy * o_dofs1D, e) += sum;
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int dy = 0; dy < o_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < c_dofs1D; ++dx)
+         {
+            real_t sum = 0.0;
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t by = Bo(oy, dy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  sum += Gc(ox, dx) * by * X(ox, oy, e);
+               }
+            }
+            Y(y_nd + dx + dy * c_dofs1D, e) += sum;
+         }
+      }
+   });
+}
+
+void PAHdivApplyCurl2D(const int c_dofs1D,
+                       const int o_dofs1D,
+                       const int NE,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Gc_,
+                       const Vector &x_,
+                       Vector &y_)
+{
+   auto Bc = Reshape(Bc_.Read(), c_dofs1D, c_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), c_dofs1D, c_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int iy = 0; iy < c_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < c_dofs1D; ++ix)
+         {
+            const real_t xv = X(ix, iy, e);
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t gy = Gc(oy, iy);
+               for (int ox = 0; ox < c_dofs1D; ++ox)
+               {
+                  Y(ox + oy * c_dofs1D, e) += Bc(ox, ix) * gy * xv;
+               }
+            }
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int iy = 0; iy < c_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < c_dofs1D; ++ix)
+         {
+            const real_t xv = X(ix, iy, e);
+            for (int oy = 0; oy < c_dofs1D; ++oy)
+            {
+               const real_t by = Bc(oy, iy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  Y(y_nd + ox + oy * o_dofs1D, e) -= Gc(ox, ix) * by * xv;
+               }
+            }
+         }
+      }
+   });
+}
+
+void PAHdivApplyCurlTranspose2D(const int c_dofs1D,
+                                const int o_dofs1D,
+                                const int NE,
+                                const Array<real_t> &Bc_,
+                                const Array<real_t> &Gc_,
+                                const Vector &x_,
+                                Vector &y_)
+{
+   auto Bc = Reshape(Bc_.Read(), c_dofs1D, c_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), c_dofs1D, c_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int dy = 0; dy < o_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < c_dofs1D; ++dx)
+         {
+            const real_t xv = X(dx + dy * c_dofs1D, e);
+            for (int iy = 0; iy < c_dofs1D; ++iy)
+            {
+               const real_t gy = Gc(dy, iy);
+               for (int ix = 0; ix < c_dofs1D; ++ix)
+               {
+                  Y(ix, iy, e) += Bc(dx, ix) * gy * xv;
+               }
+            }
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int dy = 0; dy < c_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < o_dofs1D; ++dx)
+         {
+            const real_t xv = X(y_nd + dx + dy * o_dofs1D, e);
+            for (int iy = 0; iy < c_dofs1D; ++iy)
+            {
+               const real_t by = Bc(dy, iy);
+               for (int ix = 0; ix < c_dofs1D; ++ix)
+               {
+                  Y(ix, iy, e) -= Gc(dx, ix) * by * xv;
+               }
+            }
+         }
+      }
+   });
+}
+
+}
+
 // Apply to x corresponding to DOFs in H^1 (domain) the (topological) gradient
 // to get a dof in H(curl) (range). You can think of the range as the "test" space
 // and the domain as the "trial" space, but there's no integration.
@@ -1950,4 +2159,266 @@ void IdentityInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
    }
 }
 
+void CurlInterpolator::AssemblePA(const FiniteElementSpace &dom_fes,
+                                  const FiniteElementSpace &ran_fes)
+{
+   Mesh *mesh = dom_fes.GetMesh();
+   dim = mesh->Dimension();
+   ne = dom_fes.GetNE();
+   pa_mode_2d = 0;
+   MFEM_VERIFY(ne == ran_fes.GetNE(),
+               "Different meshes for domain and range spaces");
+
+   if (dim == 2)
+   {
+      pa_data.SetSize(0);
+      const FiniteElement *dom_fel = dom_fes.GetTypicalFE();
+      const FiniteElement *ran_fel = ran_fes.GetTypicalFE();
+      const bool hcurl_to_scalar =
+         dynamic_cast<const VectorTensorFiniteElement*>(dom_fel) != NULL &&
+         dom_fel->GetDerivType() == FiniteElement::CURL &&
+         dynamic_cast<const TensorBasisElement*>(ran_fel) != NULL &&
+         ran_fel->GetRangeType() == FiniteElement::SCALAR;
+      const bool scalar_to_hdiv =
+         dynamic_cast<const TensorBasisElement*>(dom_fel) != NULL &&
+         dom_fel->GetRangeType() == FiniteElement::SCALAR &&
+         dynamic_cast<const VectorTensorFiniteElement*>(ran_fel) != NULL &&
+         ran_fel->GetDerivType() == FiniteElement::DIV;
+
+      MFEM_VERIFY(hcurl_to_scalar || scalar_to_hdiv,
+                  "2D CurlInterpolator PA supports H(curl)->scalar and scalar->H(div) only.");
+
+      int closed_basis_type = -1;
+      int open_basis_type = -1;
+      if (hcurl_to_scalar)
+      {
+         const auto *trial_fec = dynamic_cast<const ND_FECollection*>(dom_fes.FEColl());
+         const auto *range_fec = dynamic_cast<const L2_FECollection*>(ran_fes.FEColl());
+         MFEM_VERIFY(trial_fec != NULL, "H(curl) domain must use ND_FECollection.");
+         MFEM_VERIFY(range_fec != NULL, "Scalar range must use L2_FECollection.");
+         MFEM_VERIFY(ran_fel->GetMapType() == FiniteElement::INTEGRAL,
+                     "2D H(curl)->scalar CurlInterpolator PA supports integral-map scalar range spaces only.");
+         closed_basis_type = trial_fec->GetClosedBasisType();
+         open_basis_type = trial_fec->GetOpenBasisType();
+         MFEM_VERIFY(range_fec->GetBasisType() == open_basis_type,
+                     "Domain/range open basis types do not match.");
+         pa_mode_2d = 1;
+      }
+      else
+      {
+         const auto *trial_fec = dynamic_cast<const H1_FECollection*>(dom_fes.FEColl());
+         const auto *range_fec = dynamic_cast<const RT_FECollection*>(ran_fes.FEColl());
+         MFEM_VERIFY(trial_fec != NULL, "Scalar domain must use H1_FECollection.");
+         MFEM_VERIFY(range_fec != NULL, "H(div) range must use RT_FECollection.");
+         closed_basis_type = trial_fec->GetBasisType();
+         open_basis_type = range_fec->GetOpenBasisType();
+         MFEM_VERIFY(range_fec->GetClosedBasisType() == closed_basis_type,
+                     "Domain/range closed basis types do not match.");
+         pa_mode_2d = 2;
+      }
+
+      const int order = hcurl_to_scalar
+                        ? dynamic_cast<const VectorTensorFiniteElement*>(dom_fel)->GetOrder()
+                        : dynamic_cast<const NodalTensorFiniteElement*>(dom_fel)->GetOrder();
+      c_dofs1D = order + 1;
+      o_dofs1D = order;
+
+      closed_dofquad_fe.reset(new H1_SegmentElement(order, closed_basis_type));
+      open_dofquad_fe.reset(new L2_SegmentElement(order - 1, open_basis_type));
+
+      mfem::QuadratureFunctions1D qf1d;
+      mfem::IntegrationRule closed_ir;
+      closed_ir.SetSize(c_dofs1D);
+      qf1d.GaussLobatto(c_dofs1D, &closed_ir);
+
+      mfem::IntegrationRule open_ir;
+      open_ir.SetSize(o_dofs1D);
+      qf1d.GaussLegendre(o_dofs1D, &open_ir);
+
+      maps_C_C = &closed_dofquad_fe->GetDofToQuad(closed_ir, DofToQuad::TENSOR);
+      maps_O_C = &closed_dofquad_fe->GetDofToQuad(open_ir, DofToQuad::TENSOR);
+      maps_O_O = &open_dofquad_fe->GetDofToQuad(open_ir, DofToQuad::TENSOR);
+
+      MFEM_VERIFY(maps_C_C->ndof == c_dofs1D && maps_C_C->nqpt == c_dofs1D, "");
+      MFEM_VERIFY(maps_O_C->ndof == c_dofs1D && maps_O_C->nqpt == o_dofs1D, "");
+      MFEM_VERIFY(maps_O_O->ndof == o_dofs1D && maps_O_O->nqpt == o_dofs1D, "");
+      return;
+   }
+
+   closed_dofquad_fe.reset();
+   open_dofquad_fe.reset();
+   maps_C_C = nullptr;
+   maps_O_C = nullptr;
+   maps_O_O = nullptr;
+
+   const VectorTensorFiniteElement *dom_el =
+      dynamic_cast<const VectorTensorFiniteElement *>(dom_fes.GetTypicalFE());
+   const VectorTensorFiniteElement *ran_el =
+      dynamic_cast<const VectorTensorFiniteElement *>(ran_fes.GetTypicalFE());
+   MFEM_VERIFY(dom_el != NULL, "Only VectorTensorFiniteElement is supported!");
+   MFEM_VERIFY(ran_el != NULL, "Only VectorTensorFiniteElement is supported!");
+   MFEM_VERIFY(dom_el->GetDerivType() == FiniteElement::CURL,
+               "Domain space must be H(curl)");
+   MFEM_VERIFY(ran_el->GetDerivType() == FiniteElement::DIV,
+               "Range space must be H(div)");
+
+   const int dims = dom_el->GetDim();
+   MFEM_VERIFY(dims == 3, "");
+
+   ndof_o = dom_el->GetOrder();
+   int ndof_c = ndof_o + 1;
+   nquad_o = ran_el->GetOrder();
+   int nquad_c = nquad_o + 1;
+
+   // extract the tensor product range dof locations
+   std::vector<real_t> qc(nquad_c);
+   std::vector<real_t> qo(nquad_o);
+   {
+      const IntegrationRule &ran_nodes = ran_el->GetNodes();
+      const Array<int> &quad_map = ran_el->GetDofMap();
+      for (int i = 0; i < nquad_c; ++i)
+      {
+         int idx = UnsignIndex(quad_map[i]);
+         qc[i] = ran_nodes.IntPoint(idx).x;
+      }
+      int offset = ndof_c * ndof_o * ndof_o;
+      for (int i = 0; i < nquad_o; ++i)
+      {
+         int idx = UnsignIndex(quad_map[i + offset]);
+         qo[i] = ran_nodes.IntPoint(idx).x;
+      }
+   }
+
+   // evaluate closed/open 1D basis (and their derivatives) at closed and
+   // open quads
+   // storage order: GCO, BCC, BOO
+   pa_data.SetSize(ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
+   auto ptr = pa_data.HostWrite();
+   auto &cbasis1d = dom_el->GetBasis1D();
+   auto &obasis1d = dom_el->GetOpenBasis1D();
+   Vector b, g;
+   b.SetSize(ndof_c);
+   g.SetSize(ndof_c);
+   for (int j = 0; j < nquad_o; ++j)
+   {
+      cbasis1d.Eval(qo[j], b, g);
+      for (int i = 0; i < ndof_c; ++i)
+      {
+         ptr[j + i * nquad_o] = g[i];
+      }
+   }
+   ptr += nquad_o * ndof_c;
+
+   for (int j = 0; j < nquad_c; ++j)
+   {
+      cbasis1d.Eval(qc[j], b);
+      for (int i = 0; i < ndof_c; ++i)
+      {
+         ptr[j + i * nquad_c] = b[i];
+      }
+   }
+   ptr += ndof_c * nquad_c;
+
+   b.SetSize(ndof_o);
+   for (int j = 0; j < nquad_o; ++j)
+   {
+      obasis1d.Eval(qo[j], b);
+      for (int i = 0; i < ndof_o; ++i)
+      {
+         ptr[j + i * nquad_o] = b[i];
+      }
+   }
+}
+
+CurlInterpolator::Kernels::Kernels()
+{
+   CurlInterpolator::AddSpecialization<3, 1, 1>();
+   CurlInterpolator::AddSpecialization<3, 2, 2>();
+   CurlInterpolator::AddSpecialization<3, 3, 3>();
+   CurlInterpolator::AddSpecialization<3, 4, 4>();
+   CurlInterpolator::AddSpecialization<3, 5, 5>();
+}
+
+CurlInterpolator::CurlInterpolator() { static Kernels kernels{}; }
+
+void CurlInterpolator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (dim == 2)
+   {
+      MFEM_VERIFY(maps_C_C != nullptr && maps_O_C != nullptr,
+                  "2D CurlInterpolator PA data is not assembled.");
+      if (pa_mode_2d == 1)
+      {
+         MFEM_VERIFY(maps_O_O != nullptr,
+                     "2D CurlInterpolator scalar curl map is not assembled.");
+         PAHcurlApplyCurl2D(c_dofs1D, o_dofs1D, ne, maps_O_O->B, maps_O_C->G,
+                            x, y);
+      }
+      else if (pa_mode_2d == 2)
+      {
+         PAHdivApplyCurl2D(c_dofs1D, o_dofs1D, ne, maps_C_C->B, maps_O_C->G,
+                           x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported 2D CurlInterpolator mode.");
+      }
+      return;
+   }
+
+   ApplyPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x, y);
+}
+
+void CurlInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
+{
+   if (dim == 2)
+   {
+      MFEM_VERIFY(maps_C_C != nullptr && maps_O_C != nullptr,
+                  "2D CurlInterpolator PA data is not assembled.");
+      if (pa_mode_2d == 1)
+      {
+         MFEM_VERIFY(maps_O_O != nullptr,
+                     "2D CurlInterpolator scalar curl map is not assembled.");
+         PAHcurlApplyCurlTranspose2D(c_dofs1D, o_dofs1D, ne, maps_O_O->B,
+                                     maps_O_C->G, x, y);
+      }
+      else if (pa_mode_2d == 2)
+      {
+         PAHdivApplyCurlTranspose2D(c_dofs1D, o_dofs1D, ne, maps_C_C->B,
+                                    maps_O_C->G, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported 2D CurlInterpolator mode.");
+      }
+      return;
+   }
+
+   ApplyTPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x, y);
+}
+
+/// \cond DO_NOT_DOCUMENT
+
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 3)
+   {
+      return internal::CurlInterpolatorApply3DSmem<0, 0>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyTPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 3)
+   {
+      return internal::CurlInterpolatorTApply3DSmem<0, 0>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
 } // namespace mfem

fem/integ/bilininteg_vecmass_pa.cpp

@@ -22,6 +22,8 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
+   MFEM_VERIFY(el.GetMapType() == FiniteElement::VALUE,
+               "Only value map type supported");
    ElementTransformation &Trans = *mesh->GetTypicalElementTransformation();
    const auto *ir = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, Trans);
 

fem/kernel_dispatch.hpp

@@ -48,11 +48,7 @@ namespace mfem
 
 #define MFEM_REGISTER_KERNELS(KernelName, KernelType, ...)                     \
    MFEM_EXPAND(MFEM_EXPAND(MFEM_REGISTER_KERNELS_N(__VA_ARGS__,2,1,))          \
-      (MFEM_EXPORT, KernelName, KernelType, __VA_ARGS__))
-
-#define MFEM_REGISTER_KERNELS_HEADER_ONLY(KernelName, KernelType, ...)         \
-   MFEM_EXPAND(MFEM_EXPAND(MFEM_REGISTER_KERNELS_N(__VA_ARGS__,2,1,))          \
-      (, KernelName, KernelType, __VA_ARGS__))
+      (KernelName,KernelType,__VA_ARGS__))
 
 #define MFEM_REGISTER_KERNELS_N(_1, _2, N, ...) MFEM_REGISTER_KERNELS_##N
 
@@ -62,19 +58,19 @@ namespace mfem
 
 // Version of MFEM_REGISTER_KERNELS without any "optional" (non-dispatch)
 // parameters.
-#define MFEM_REGISTER_KERNELS_1(ExportMacro, KernelName, KernelType, Params)   \
-   MFEM_REGISTER_KERNELS_(ExportMacro, KernelName, KernelType, Params, (), Params)
+#define MFEM_REGISTER_KERNELS_1(KernelName, KernelType, Params)                \
+   MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, (), Params)
 
 // Version of MFEM_REGISTER_KERNELS with optional (non-dispatch)
 // parameters (e.g. NBZ).
-#define MFEM_REGISTER_KERNELS_2(ExportMacro, KernelName, KernelType, Params, OptParams) \
-   MFEM_REGISTER_KERNELS_(ExportMacro, KernelName, KernelType, Params, OptParams,       \
+#define MFEM_REGISTER_KERNELS_2(KernelName, KernelType, Params, OptParams)     \
+   MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, OptParams,           \
                           (MFEM_PARAM_LIST Params, MFEM_PARAM_LIST OptParams))
 
 // P1 are the parameters, P2 are the optional (non-dispatch parameters), and P3
 // is the concatenation of P1 and P2. We need to pass it as a separate argument
 // to avoid a trailing comma in the case that P2 is empty.
-#define MFEM_REGISTER_KERNELS_(ExportMacro, KernelName, KernelType, P1, P2, P3) \
+#define MFEM_REGISTER_KERNELS_(KernelName, KernelType, P1, P2, P3)             \
   class KernelName                                                             \
       : public ::mfem::KernelDispatchTable<                                    \
             KernelName, KernelType,                                            \
@@ -84,8 +80,8 @@ namespace mfem
     const char *kernel_name = MFEM_KERNEL_NAME(KernelName);                    \
     using KernelSignature = KernelType;                                        \
     template <MFEM_PARAM_LIST P3> static KernelSignature Kernel();             \
-    static ExportMacro KernelSignature Fallback(MFEM_PARAM_LIST P1);           \
-    static ExportMacro KernelName &Get() {                                     \
+    static MFEM_EXPORT KernelSignature Fallback(MFEM_PARAM_LIST P1);           \
+    static MFEM_EXPORT KernelName &Get() {                                     \
       static KernelName table;                                                 \
       return table;                                                            \
     }                                                                          \

fem/lor/lor_ams.cpp

@@ -94,10 +94,10 @@ void BatchedLOR_AMS::Form2DEdgeToVertex_RT(Array<int> &edge2vert)
          const int iv0 = ix + iy*op1;
          const int iv1 = ix1 + iy1*op1;
 
-         // Rotated gradient in 2D (-dy, dx), so flip the sign for the first
-         // component (c == 0).
-         e2v(0, iedge) = (c == 1) ? iv0 : iv1;
-         e2v(1, iedge) = (c == 1) ? iv1 : iv0;
+         // 2D curl (dy, -dx), so flip the sign for the second
+         // component (c == 1).
+         e2v(0, iedge) = (c == 0) ? iv0 : iv1;
+         e2v(1, iedge) = (c == 0) ? iv1 : iv0;
       }
    }
 }

fem/lor/lor_batched.cpp

@@ -142,8 +142,6 @@ static MFEM_HOST_DEVICE int GetAndIncrementNnzIndex(const int i_L, int* I)
 
 int BatchedLORAssembly::FillI(SparseMatrix &A) const
 {
-   static constexpr int Max = 16;
-
    const int nvdof = fes_ho.GetVSize();
 
    const int ndof_per_el = fes_ho.GetTypicalFE()->GetDof();
@@ -165,6 +163,8 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
    const auto K = dof_glob2loc_offsets_.Read();
    const auto map = Reshape(sparse_mapping.Read(), nnz_per_row, ndof_per_el);
 
+   Array<int> ij_elts(dof_glob2loc_.Size() * 2);
+   auto d_ij_elts = Reshape(ij_elts.Write(), dof_glob2loc_.Size(), 2);
 
    auto I = A.WriteI();
 
@@ -176,10 +176,10 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
       const int sii = el_dof_lex(ii_el, iel_ho);
       const int ii = (sii >= 0) ? sii : -1 -sii;
       // Get number and list of elements containing this DOF
-      int i_elts[Max];
       const int i_offset = K[ii];
       const int i_next_offset = K[ii+1];
       const int i_ne = i_next_offset - i_offset;
+      int *i_elts = &d_ij_elts(i_offset, 0);
       for (int e_i = 0; e_i < i_ne; ++e_i)
       {
          const int si_E = dof_glob2loc[i_offset+e_i]; // signed
@@ -202,7 +202,7 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
          }
          else // assembly required
          {
-            int j_elts[Max];
+            int *j_elts = &d_ij_elts(j_offset, 1);
             for (int e_j = 0; e_j < j_ne; ++e_j)
             {
                const int sj_E = dof_glob2loc[j_offset+e_j]; // signed
@@ -269,7 +269,8 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
       mfem::forall(nvdof + 1, [=] MFEM_HOST_DEVICE (int i) { I[i] = I2[i]; });
    }
 
-   static constexpr int Max = 16;
+   Array<int> ij_B_el(dof_glob2loc_.Size() * 4);
+   auto d_ij_B_el = Reshape(ij_B_el.Write(), dof_glob2loc_.Size(), 4);
 
    mfem::forall(ndof_per_el*nel_ho, [=] MFEM_HOST_DEVICE (int i)
    {
@@ -279,11 +280,13 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
       const int sii = el_dof_lex(ii_el, iel_ho); // signed
       const int ii = (sii >= 0) ? sii : -1 - sii;
       // Get number and list of elements containing this DOF
-      int i_elts[Max];
-      int i_B[Max];
       const int i_offset = K[ii];
       const int i_next_offset = K[ii+1];
       const int i_ne = i_next_offset - i_offset;
+
+      int *i_elts = &d_ij_B_el(i_offset, 0);
+      int *i_B = &d_ij_B_el(i_offset, 1);
+
       for (int e_i = 0; e_i < i_ne; ++e_i)
       {
          const int si_E = dof_glob2loc[i_offset+e_i]; // signed
@@ -312,8 +315,8 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
          }
          else // assembly required
          {
-            int j_elts[Max];
-            int j_B[Max];
+            int *j_elts = &d_ij_B_el(j_offset, 2);
+            int *j_B = &d_ij_B_el(j_offset, 3);
             for (int e_j = 0; e_j < j_ne; ++e_j)
             {
                const int sj_E = dof_glob2loc[j_offset+e_j]; // signed

fem/nonlinearform.hpp

@@ -224,9 +224,6 @@ public:
    /** @see GetGradient(const Vector &) */
    Operator &GetGradient(const Vector &x, bool finalize) const;
 
-   /// Suppress a warning about hiding overloaded virtual function.
-   using Operator::GetGradient;
-
    /// Update the NonlinearForm to propagate updates of the associated FE space.
    /** After calling this method, the essential boundary conditions need to be
        set again. */

fem/pfespace.cpp

@@ -26,6 +26,8 @@
 
 #include <limits>
 #include <list>
+#include <unordered_map>
+#include <unordered_set>
 
 namespace mfem
 {
@@ -1285,6 +1287,342 @@ void ParFiniteElementSpace::GetExteriorVDofs(Array<int> &ext_dofs,
    Synchronize(ext_dofs);
 }
 
+void ParFiniteElementSpace::GetBoundaryLoopEdgeDofs(
+   const Array<int> &boundary_element_indices,
+   Array<int> &ess_tdof_list,
+   Array<int> &boundary_edge_dofs_out,
+   Array<int> *ldof_marker,
+   Array<int> *dof_edges,
+   Array<int> *dof_boundary_elements,
+   Array<int> *ess_edge_list)
+{
+   MFEM_VERIFY(!pmesh->Nonconforming(),
+               "GetBoundaryLoopEdgeDofs does not support nonconforming meshes");
+   MFEM_VERIFY(pmesh->Dimension() >= 2,
+               "GetBoundaryLoopEdgeDofs requires 2D or 3D meshes to find 1D edge objects");
+
+   // Call the serial version, then rebuild scratch maps/set from the returned
+   // arrays for the O(1) lookups the parallel reconciliation below needs.
+   Array<int> loc_dofs, loc_edges, loc_belems;
+   FiniteElementSpace::GetBoundaryLoopEdgeDofs(boundary_element_indices, loc_dofs,
+                                               &loc_edges, &loc_belems);
+
+   std::unordered_set<int> boundary_edge_dofs;
+   std::unordered_map<int, int> dof_to_edge_map;
+   std::unordered_map<int, int> dof_to_boundary_element;
+   boundary_edge_dofs.reserve(loc_dofs.Size());
+   dof_to_edge_map.reserve(loc_dofs.Size());
+   dof_to_boundary_element.reserve(loc_dofs.Size());
+   for (int i = 0; i < loc_dofs.Size(); i++)
+   {
+      const int dof = loc_dofs[i];
+      boundary_edge_dofs.insert(dof);
+      dof_to_edge_map[dof] = loc_edges[i];
+      dof_to_boundary_element[dof] = loc_belems[i];
+   }
+
+   // Parallel processing: Build edge sharing lookup table
+   std::unordered_map<int, int> edge_to_group_size;
+   int num_groups = pmesh->GetNGroups();
+
+   int total_shared_edges = 0;
+   for (int group = 1; group < num_groups; group++)
+   {
+      total_shared_edges += pmesh->GroupNEdges(group);
+   }
+   edge_to_group_size.reserve(total_shared_edges);
+
+   for (int group = 1; group < num_groups; group++)
+   {
+      int group_size = pmesh->gtopo.GetGroupSize(group);
+      int num_edges_in_group = pmesh->GroupNEdges(group);
+
+      for (int i = 0; i < num_edges_in_group; i++)
+      {
+         edge_to_group_size.emplace(pmesh->GroupEdge(group, i), group_size);
+      }
+   }
+
+   // Get global indices
+   Array<HYPRE_BigInt> global_edge_indices;
+   pmesh->GetGlobalEdgeIndices(global_edge_indices);
+
+   // Handle dimension-specific boundary element relationships
+   Array<HYPRE_BigInt> global_face_indices;
+   std::unordered_map<int, int> boundary_element_to_companion;
+   std::unordered_set<int> dofs_to_remove;
+
+   const int dim = pmesh->Dimension();
+   if (dim == 3)
+   {
+      // In 3D: boundary elements are faces, we track which face each boundary element is
+      pmesh->GetGlobalFaceIndices(global_face_indices);
+      for (int boundary_element_idx : boundary_element_indices)
+      {
+         int face_index, face_orientation;
+         pmesh->GetBdrElementFace(boundary_element_idx, &face_index, &face_orientation);
+         boundary_element_to_companion[boundary_element_idx] = face_index;
+      }
+
+      std::vector<HYPRE_BigInt> local_data;
+      local_data.reserve(boundary_edge_dofs.size() * 2);
+
+      std::unordered_set<int> processed_edges;
+      processed_edges.reserve(boundary_edge_dofs.size());
+
+      for (const auto& [dof, local_edge] : dof_to_edge_map)
+      {
+         // Skip if already processed this edge
+         if (!processed_edges.insert(local_edge).second) { continue; }
+
+         // Check if edge is shared (fast lookup)
+         auto it = edge_to_group_size.find(local_edge);
+         if (it != edge_to_group_size.end() && it->second > 1)
+         {
+            // Get boundary element and companion index directly from pre-computed map
+            int boundary_element_idx = dof_to_boundary_element[dof];
+            int companion_index = boundary_element_to_companion[boundary_element_idx];
+
+            // Store edge-face pair for 3D artificial boundary detection
+            local_data.push_back(global_edge_indices[local_edge]);
+            local_data.push_back(global_face_indices[companion_index]);
+         }
+      }
+
+      // MPI communication for 3D artificial boundary detection
+      int num_procs = pmesh->GetNRanks();
+      int local_size = local_data.size();
+
+      std::vector<int> mpi_arrays(num_procs * 4);
+      int* all_sizes = mpi_arrays.data();
+      int* displs = all_sizes + num_procs;
+      int* byte_sizes = displs + num_procs;
+      int* byte_displs = byte_sizes + num_procs;
+
+      MPI_Allgather(&local_size, 1, MPI_INT, all_sizes, 1, MPI_INT, pmesh->GetComm());
+
+      int total_size = 0;
+      constexpr int hypre_size = sizeof(HYPRE_BigInt);
+      for (int i = 0; i < num_procs; i++)
+      {
+         displs[i] = total_size;
+         byte_displs[i] = total_size * hypre_size;
+         total_size += all_sizes[i];
+         byte_sizes[i] = all_sizes[i] * hypre_size;
+      }
+
+      if (total_size > 0)
+      {
+         std::vector<HYPRE_BigInt> all_data(total_size);
+         MPI_Allgatherv(local_data.data(), local_size * hypre_size, MPI_BYTE,
+                        all_data.data(), byte_sizes, byte_displs, MPI_BYTE, pmesh->GetComm());
+
+         // Build global-to-local edge mapping
+         std::unordered_map<HYPRE_BigInt, int> global_to_local_edge;
+         global_to_local_edge.reserve(global_edge_indices.Size());
+         for (int i = 0; i < global_edge_indices.Size(); ++i)
+         {
+            global_to_local_edge[global_edge_indices[i]] = i;
+         }
+
+         // Process collected data to find edges in multiple faces (artificial boundaries)
+         std::unordered_map<HYPRE_BigInt, std::unordered_set<HYPRE_BigInt>>edge_to_faces;
+         edge_to_faces.reserve(total_size / 2);
+
+         for (size_t i = 0; i < all_data.size(); i += 2)
+         {
+            edge_to_faces[all_data[i]].insert(all_data[i + 1]);
+         }
+
+         // Mark DOFs from artificial edges for removal
+         dofs_to_remove.reserve(local_data.size() / 4);
+
+         for (size_t i = 0; i < local_data.size(); i += 2)
+         {
+            HYPRE_BigInt global_edge_id = local_data[i];
+
+            // If this edge appears in 2+ distinct faces, it's artificial
+            if (edge_to_faces[global_edge_id].size() >= 2)
+            {
+               int local_edge = global_to_local_edge[global_edge_id];
+               Array<int> local_edge_dofs;
+               GetEdgeDofs(local_edge, local_edge_dofs);
+
+               // Mark boundary DOFs of this edge for removal
+               for (int k = 0; k < local_edge_dofs.Size(); ++k)
+               {
+                  int dof = local_edge_dofs[k];
+                  if (boundary_edge_dofs.count(dof))
+                  {
+                     dofs_to_remove.insert(dof);
+                  }
+               }
+            }
+         }
+      }
+   }
+   else if (dim == 2)
+   {
+      // In 2D the boundary elements are themselves the edges, so there are no
+      // artificial boundary edges to detect. However, for collections with
+      // vertex DOFs (e.g. ND_R2D), a vertex shared by two boundary segments is
+      // interior to the boundary curve and must be dropped. The serial code
+      // does this by erasing a DOF on its second occurrence, which only sees
+      // the occurrences local to this rank. When the two segments meeting at a
+      // vertex live on different ranks, each rank sees a single occurrence and
+      // wrongly keeps the DOF. Reconcile the occurrence parity across each
+      // sharing group: membership in boundary_edge_dofs is the local parity,
+      // and the parities sum (mod 2) to the global occurrence parity.
+      Array<int> boundary_dof_count(GetVSize());
+      boundary_dof_count = 0;
+      for (const int dof : boundary_edge_dofs)
+      {
+         boundary_dof_count[dof] = 1;
+      }
+
+      // implement allreduce(+) as reduce(+) + broadcast
+      gcomm->Reduce<int>(boundary_dof_count, GroupCommunicator::Sum);
+      gcomm->Bcast(boundary_dof_count);
+
+      for (const int dof : boundary_edge_dofs)
+      {
+         if (boundary_dof_count[dof] % 2 == 0)
+         {
+            dofs_to_remove.insert(dof);
+         }
+      }
+   }
+
+   // Remove artificial DOFs
+   for (int dof : dofs_to_remove)
+   {
+      boundary_edge_dofs.erase(dof);
+      dof_to_edge_map.erase(dof);
+      dof_to_boundary_element.erase(dof);
+   }
+
+   // Convert to true DOFs and output
+   ess_tdof_list.SetSize(0);
+   ess_tdof_list.Reserve(boundary_edge_dofs.size());
+   if (ess_edge_list)
+   {
+      // Reset as well, so that it stays in correspondence with ess_tdof_list
+      // when the same output array is reused across calls.
+      ess_edge_list->SetSize(0);
+      ess_edge_list->Reserve(boundary_edge_dofs.size());
+   }
+   // Marker of the boundary edge DOFs. Always computed locally because the
+   // parallel reconciliation below needs it; only copied to the caller's output
+   // if requested (see the ldof_marker parameter).
+   Array<int> local_ldof_marker(GetVSize());
+   local_ldof_marker = 0;
+
+   for (int dof : boundary_edge_dofs)
+   {
+      local_ldof_marker[dof] = 1; // Mark all boundary edge dofs
+   }
+
+   // Make sure that a selected shared DOF is marked on every rank of its
+   // sharing group, including ranks holding none of the selected boundary
+   // elements. Only the group master owns the corresponding true DOF, so
+   // without this the true DOF would be emitted by no rank at all: the
+   // non-master ranks get -1 from GetLocalTDofNumber(), while the master may
+   // not have selected the DOF locally.
+   Synchronize(local_ldof_marker);
+
+   // A DOF marked only through the synchronization above has no local
+   // dof_to_edge_map entry, but the shared edge carrying it is still present in
+   // the local mesh. Build the missing DOF -> edge entries from the shared
+   // edges of the groups, so that ess_edge_list stays in correspondence with
+   // ess_tdof_list. Note that a vertex DOF is not associated with a unique
+   // edge, so it is only resolved when it is an interior DOF of an edge.
+   std::unordered_map<int, int> shared_dof_to_edge;
+   Array<int> shared_edge_dofs;
+   for (int group = 1; group < num_groups; group++)
+   {
+      const int num_edges_in_group = pmesh->GroupNEdges(group);
+      for (int i = 0; i < num_edges_in_group; i++)
+      {
+         const int edge = pmesh->GroupEdge(group, i);
+         GetEdgeInteriorDofs(edge, shared_edge_dofs);
+         for (int k = 0; k < shared_edge_dofs.Size(); k++)
+         {
+            shared_dof_to_edge.emplace(shared_edge_dofs[k], edge);
+         }
+      }
+   }
+
+   // Build parallel arrays for DOFs and corresponding edges
+   std::vector<std::pair<int, int>> tdof_edge_pairs;
+   tdof_edge_pairs.reserve(boundary_edge_dofs.size());
+
+   for (int dof = 0; dof < local_ldof_marker.Size(); dof++)
+   {
+      if (!local_ldof_marker[dof]) { continue; }
+
+      const int tdof = GetLocalTDofNumber(dof);
+      if (tdof < 0) { continue; } // tdof == -1 means not owned by this rank
+
+      int edge = -1;
+      auto it = dof_to_edge_map.find(dof);
+      if (it != dof_to_edge_map.end())
+      {
+         edge = it->second;
+      }
+      else
+      {
+         auto shared_it = shared_dof_to_edge.find(dof);
+         if (shared_it != shared_dof_to_edge.end())
+         {
+            edge = shared_it->second;
+         }
+      }
+      tdof_edge_pairs.push_back({tdof, edge});
+   }
+
+   // Sort by true DOF index to maintain consistent ordering
+   std::sort(tdof_edge_pairs.begin(), tdof_edge_pairs.end());
+
+   // Extract sorted true DOFs and edges
+   for (const auto& pair : tdof_edge_pairs)
+   {
+      ess_tdof_list.Append(pair.first);
+      if (ess_edge_list)
+      {
+         ess_edge_list->Append(pair.second);
+      }
+   }
+
+   // Emit the local boundary-loop DOFs in a deterministic (increasing DOF
+   // index) order shared by all output arrays.
+   std::vector<int> kept(boundary_edge_dofs.begin(), boundary_edge_dofs.end());
+   std::sort(kept.begin(), kept.end());
+
+   boundary_edge_dofs_out.SetSize(0);
+   boundary_edge_dofs_out.Reserve(static_cast<int>(kept.size()));
+   if (dof_edges)
+   {
+      dof_edges->SetSize(0);
+      dof_edges->Reserve(static_cast<int>(kept.size()));
+   }
+   if (dof_boundary_elements)
+   {
+      dof_boundary_elements->SetSize(0);
+      dof_boundary_elements->Reserve(static_cast<int>(kept.size()));
+   }
+   for (int dof : kept)
+   {
+      boundary_edge_dofs_out.Append(dof);
+      if (dof_edges) { dof_edges->Append(dof_to_edge_map[dof]); }
+      if (dof_boundary_elements)
+      {
+         dof_boundary_elements->Append(dof_to_boundary_element[dof]);
+      }
+   }
+
+   if (ldof_marker) { ldof_marker->Swap(local_ldof_marker); }
+}
+
 void ParFiniteElementSpace::GetExteriorTrueDofs(Array<int> &ext_tdof_list,
                                                 int component) const
 {

fem/pfespace.hpp

@@ -460,6 +460,41 @@ public:
    void GetExteriorTrueDofs(Array<int> &ext_tdof_list,
                             int component = -1) const override;
 
+   /** @brief Extract the edge degrees of freedom of a boundary "loop" on a
+       parallel mesh (see the serial FiniteElementSpace::GetBoundaryLoopEdgeDofs
+       for the definition of a loop). This version removes the artificial
+       boundary edges that appear at processor boundaries, so the selected DOFs
+       are independent of the mesh partitioning.
+
+       As in the serial version, the @a boundary_edge_dofs_out, @a dof_edges and
+       @a dof_boundary_elements outputs share a single indexing describing the
+       same local DOF at each position.
+
+       Requirements:
+       - Mesh must be conforming (no hanging nodes)
+       - Mesh dimension must be >= 2
+       @param[in]  boundary_element_indices Array of boundary element indices.
+       @param[out] ess_tdof_list Essential true DOF indices, sorted ascending.
+       @param[out] boundary_edge_dofs_out Local boundary-loop DOF indices.
+       @param[out] ldof_marker Optional; marker of the boundary edge DOFs,
+                   derivable from @a boundary_edge_dofs_out via ListToMarker().
+       @param[out] dof_edges Optional; local edge index of each DOF.
+       @param[out] dof_boundary_elements Optional; a boundary element containing
+                   each DOF.
+       @param[out] ess_edge_list Optional array of edge indices, in one-to-one
+                                 correspondence with @a ess_tdof_list. An entry
+                                 is -1 when the true DOF is owned by this rank
+                                 but no local edge can be associated with it,
+                                 which can happen for a shared vertex DOF whose
+                                 boundary elements are all on other ranks. */
+   void GetBoundaryLoopEdgeDofs(const Array<int> &boundary_element_indices,
+                                Array<int> &ess_tdof_list,
+                                Array<int> &boundary_edge_dofs_out,
+                                Array<int> *ldof_marker = nullptr,
+                                Array<int> *dof_edges = nullptr,
+                                Array<int> *dof_boundary_elements = nullptr,
+                                Array<int> *ess_edge_list = nullptr);
+
    /** If the given ldof is owned by the current processor, return its local
        tdof number, otherwise return -1 */
    int GetLocalTDofNumber(int ldof) const;

fem/qfunction.hpp

@@ -79,27 +79,6 @@ public:
       Vector::operator=(orig);
    }
 
-   /** @brief Construct a QuadratureFunction on the given
-       VectorQuadratureSpace, @a vqspace.
-
-       After construction, the QuadratureFunction does not need the
-       VectorQuadratureSpace object. Instead, it uses directly its underlying
-       QuadratureSpaceBase object. */
-   QuadratureFunction(VectorQuadratureSpace &vqspace)
-      : QuadratureFunction(*vqspace.GetSpace(), vqspace.GetVDim())
-   { }
-
-   /** @brief Construct a QuadratureFunction on the given
-       VectorQuadratureSpace, @a vqspace, with the given MemoryType, @a mt, used
-       for the underlying Vector object.
-
-       After construction, the QuadratureFunction does not need the
-       VectorQuadratureSpace object. Instead, it uses directly its underlying
-       QuadratureSpaceBase object. */
-   QuadratureFunction(VectorQuadratureSpace &vqspace, MemoryType mt)
-      : QuadratureFunction(*vqspace.GetSpace(), mt, vqspace.GetVDim())
-   { }
-
    /// Read a QuadratureFunction from the stream @a in.
    /** The QuadratureFunction assumes ownership of the read QuadratureSpace. */
    QuadratureFunction(Mesh *mesh, std::istream &in);

fem/qinterp/eval_transpose.cpp

@@ -1,44 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../quadinterpolator.hpp"
-#include "eval_transpose.hpp"
-
-namespace mfem
-{
-
-/// @cond Suppress_Doxygen_warnings
-
-QuadratureInterpolator::TensorEvalTransposeKernelType
-QuadratureInterpolator::TensorEvalTransposeKernels::Fallback(
-   int DIM, QVectorLayout Q_LAYOUT, int, int, int)
-{
-   using namespace internal::quadrature_interpolator;
-
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      if (DIM == 1) { return ValuesTranspose1D<QVectorLayout::byNODES>; }
-      else if (DIM == 2) { return ValuesTranspose2D<QVectorLayout::byNODES>; }
-      else if (DIM == 3) { return ValuesTranspose3D<QVectorLayout::byNODES>; }
-   }
-   else
-   {
-      if (DIM == 1) { return ValuesTranspose1D<QVectorLayout::byVDIM>; }
-      else if (DIM == 2) { return ValuesTranspose2D<QVectorLayout::byVDIM>; }
-      else if (DIM == 3) { return ValuesTranspose3D<QVectorLayout::byVDIM>; }
-   }
-   MFEM_ABORT("Invalid dimension");
-   return nullptr;
-}
-
-/// @endcond
-
-} // namespace mfem

fem/qinterp/eval_transpose.hpp

@@ -1,304 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#pragma once
-
-#include "../quadinterpolator.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../linalg/kernels.hpp"
-#include "../kernels.hpp"
-
-namespace mfem
-{
-namespace internal
-{
-namespace quadrature_interpolator
-{
-
-template<QVectorLayout Q_LAYOUT>
-static void ValuesTranspose1D(const int NE,
-                              const real_t *b_,
-                              const real_t *q_,
-                              real_t *e_,
-                              const int vdim,
-                              const int d1d,
-                              const int q1d)
-{
-   const auto b = Reshape(b_, q1d, d1d);
-   const auto qd = Q_LAYOUT == QVectorLayout::byNODES ?
-                   Reshape(q_, q1d, vdim, NE) :
-                   Reshape(q_, vdim, q1d, NE);
-   auto e = Reshape(e_, d1d, vdim, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int el)
-   {
-      for (int c = 0; c < vdim; c++)
-      {
-         for (int d = 0; d < d1d; d++)
-         {
-            real_t u = 0.0;
-            for (int q = 0; q < q1d; q++)
-            {
-               const real_t qval = Q_LAYOUT == QVectorLayout::byVDIM ?
-                                   qd(c, q, el) : qd(q, c, el);
-               u += b(q, d) * qval;
-            }
-            e(d, c, el) += u;
-         }
-      }
-   });
-}
-
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int T_NBZ = 1>
-static void ValuesTranspose2D(const int NE,
-                              const real_t *b_,
-                              const real_t *q_,
-                              real_t *e_,
-                              const int vdim = 0,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
-                  Reshape(q_, Q1D, Q1D, VDIM, NE) :
-                  Reshape(q_, VDIM, Q1D, Q1D, NE);
-   auto e = Reshape(e_, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D_batch(NE, D1D, D1D, NBZ, [=] MFEM_HOST_DEVICE (int el)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      const int tidz = MFEM_THREAD_ID(z);
-
-      MFEM_SHARED real_t sB[MQ1*MD1];
-      MFEM_SHARED real_t sm0[NBZ][MDQ*MDQ];
-      MFEM_SHARED real_t sm1[NBZ][MDQ*MDQ];
-
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
-
-      ConstDeviceMatrix B(sB, D1D, Q1D);
-      DeviceMatrix QQ(sm0[tidz], MQ1, MQ1);
-      DeviceMatrix DQ(sm1[tidz], MD1, MQ1);
-      DeviceMatrix DD(sm0[tidz], MD1, MD1);
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         // Load Q data
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               QQ(qx,qy) = Q_LAYOUT == QVectorLayout::byVDIM ?
-                           q(c,qx,qy,el) : q(qx,qy,c,el);
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Transpose in y: QQ -> DQ (apply B^T in y-direction)
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += B(dy,qy) * QQ(qx,qy);
-               }
-               DQ(dy,qx) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Transpose in x: DQ -> DD (apply B^T in x-direction)
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += B(dx,qx) * DQ(dy,qx);
-               }
-               DD(dx,dy) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Store result
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               e(dx,dy,c,el) += DD(dx,dy);
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
-static void ValuesTranspose3D(const int NE,
-                              const real_t *b_,
-                              const real_t *q_,
-                              real_t *e_,
-                              const int vdim = 0,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
-                  Reshape(q_, Q1D, Q1D, Q1D, VDIM, NE) :
-                  Reshape(q_, VDIM, Q1D, Q1D, Q1D, NE);
-   auto e = Reshape(e_, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int el)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_INTERP_1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_INTERP_1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-
-      MFEM_SHARED real_t sB[MQ1*MD1];
-      MFEM_SHARED real_t sm0[MDQ*MDQ*MDQ];
-      MFEM_SHARED real_t sm1[MDQ*MDQ*MDQ];
-
-      kernels::internal::LoadB<MD1,MQ1>(D1D,Q1D,b,sB);
-
-      ConstDeviceMatrix B(sB, D1D, Q1D);
-      DeviceCube QQQ(sm0, MQ1, MQ1, MQ1);
-      DeviceCube DQQ(sm1, MD1, MQ1, MQ1);
-      DeviceCube DDQ(sm0, MD1, MD1, MQ1);
-      DeviceCube DDD(sm1, MD1, MD1, MD1);
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         // Load Q data
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  QQQ(qx,qy,qz) = Q_LAYOUT == QVectorLayout::byVDIM ?
-                                  q(c,qx,qy,qz,el) : q(qx,qy,qz,c,el);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Transpose in z
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t u = 0.0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u += B(dz,qz) * QQQ(qx,qy,qz);
-                  }
-                  DQQ(dz,qx,qy) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Transpose in y
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t u = 0.0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     u += B(dy,qy) * DQQ(dz,qx,qy);
-                  }
-                  DDQ(dz,dy,qx) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Transpose in x
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     u += B(dx,qx) * DDQ(dz,dy,qx);
-                  }
-                  DDD(dx,dy,dz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-} // namespace quadrature_interpolator
-} // namespace internal
-
-/// \cond DO_NOT_DOCUMENT
-
-template<int DIM, QVectorLayout Q_LAYOUT,
-         int VDIM, int D1D, int Q1D, int NBZ>
-QuadratureInterpolator::TensorEvalTransposeKernelType
-QuadratureInterpolator::TensorEvalTransposeKernels::Kernel()
-{
-   if (DIM == 1) { return internal::quadrature_interpolator::ValuesTranspose1D<Q_LAYOUT>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::ValuesTranspose2D<Q_LAYOUT, VDIM, D1D, Q1D, NBZ>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::ValuesTranspose3D<Q_LAYOUT, VDIM, D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/qinterp/eval_transpose_by_vdim.cpp

@@ -1,61 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../quadinterpolator.hpp"
-#include "eval_transpose.hpp"
-
-namespace mfem
-{
-namespace internal
-{
-namespace quadrature_interpolator
-{
-
-void InitEvalTransposeByVDimKernels()
-{
-   using k = QuadratureInterpolator::TensorEvalTransposeKernels;
-   constexpr auto L = QVectorLayout::byVDIM;
-
-   // 2D
-   k::Specialization<2,L,1,2,4>::Opt<8>::Add();
-   k::Specialization<2,L,1,3,6>::Opt<4>::Add();
-   k::Specialization<2,L,1,4,8>::Opt<2>::Add();
-
-   k::Specialization<2,L,2,2,4>::Opt<8>::Add();
-   k::Specialization<2,L,2,3,4>::Opt<8>::Add();
-   k::Specialization<2,L,2,3,6>::Opt<4>::Add();
-   k::Specialization<2,L,2,4,6>::Opt<2>::Add();
-   k::Specialization<2,L,2,4,8>::Opt<2>::Add();
-
-   // 3D
-   k::Specialization<3,L,1,2,4>::Opt<1>::Add();
-   k::Specialization<3,L,1,3,6>::Opt<1>::Add();
-   k::Specialization<3,L,1,4,8>::Opt<1>::Add();
-   k::Specialization<3,L,3,2,4>::Opt<1>::Add();
-   k::Specialization<3,L,3,3,6>::Opt<1>::Add();
-   k::Specialization<3,L,3,4,8>::Opt<1>::Add();
-
-   k::Specialization<3,L,3,2,2>::Opt<1>::Add();
-   k::Specialization<3,L,3,3,3>::Opt<1>::Add();
-   k::Specialization<3,L,3,4,4>::Opt<1>::Add();
-   k::Specialization<3,L,3,5,5>::Opt<1>::Add();
-   k::Specialization<3,L,3,6,6>::Opt<1>::Add();
-   k::Specialization<3,L,3,7,7>::Opt<1>::Add();
-   k::Specialization<3,L,3,8,8>::Opt<1>::Add();
-   k::Specialization<3,L,3,9,9>::Opt<1>::Add();
-
-   k::Specialization<3,L,3,4,6>::Opt<1>::Add();
-   k::Specialization<3,L,3,3,4>::Opt<1>::Add();
-}
-
-} // namespace quadrature_interpolator
-} // namespace internal
-} // namespace mfem

fem/qinterp/grad_transpose.cpp

@@ -1,62 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../quadinterpolator.hpp"
-#include "grad_transpose.hpp"
-
-namespace mfem
-{
-
-/// @cond Suppress_Doxygen_warnings
-
-QuadratureInterpolator::GradTransposeKernelType
-QuadratureInterpolator::GradTransposeKernels::Fallback(
-   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int, int)
-{
-   using namespace internal::quadrature_interpolator;
-
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      if (GRAD_PHYS)
-      {
-         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byNODES, true>; }
-         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byNODES, true>; }
-         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byNODES, true>; }
-      }
-      else
-      {
-         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byNODES, false>; }
-         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byNODES, false>; }
-         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byNODES, false>; }
-      }
-   }
-   else
-   {
-      if (GRAD_PHYS)
-      {
-         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byVDIM, true>; }
-         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byVDIM, true>; }
-         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byVDIM, true>; }
-      }
-      else
-      {
-         if (DIM == 1) { return DerivativesTranspose1D<QVectorLayout::byVDIM, false>; }
-         else if (DIM == 2) { return DerivativesTranspose2D<QVectorLayout::byVDIM, false>; }
-         else if (DIM == 3) { return DerivativesTranspose3D<QVectorLayout::byVDIM, false>; }
-      }
-   }
-   MFEM_ABORT("Invalid dimension");
-   return nullptr;
-}
-
-/// @endcond
-
-} // namespace mfem

fem/qinterp/grad_transpose.hpp

@@ -1,741 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#pragma once
-
-#include "../quadinterpolator.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../linalg/kernels.hpp"
-#include "../kernels.hpp"
-
-namespace mfem
-{
-namespace internal
-{
-namespace quadrature_interpolator
-{
-
-// Transpose gradient operation: integrate against shape function derivatives
-// This is the adjoint of the Derivatives operation
-
-template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
-static void DerivativesTranspose1D(const int NE,
-                                   const real_t *b_,
-                                   const real_t *g_,
-                                   const real_t *j_,
-                                   const real_t *q_,
-                                   real_t *e_,
-                                   const int sdim,
-                                   const int vdim,
-                                   const int d1d,
-                                   const int q1d)
-{
-   MFEM_CONTRACT_VAR(b_);
-   const int SDIM = GRAD_PHYS ? sdim : 1;
-   const auto g = Reshape(g_, q1d, d1d);
-   const auto j = Reshape(j_, q1d, SDIM, NE);
-   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
-                  Reshape(q_, q1d, vdim, SDIM, NE):
-                  Reshape(q_, vdim, SDIM, q1d, NE);
-   auto e = Reshape(e_, d1d, vdim, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int el)
-   {
-      for (int c = 0; c < vdim; c++)
-      {
-         for (int d = 0; d < d1d; d++)
-         {
-            real_t u = 0.0;
-            for (int qx = 0; qx < q1d; qx++)
-            {
-               // Load gradient from q-vector
-               real_t dq[3] = {0.0, 0.0, 0.0};
-               for (int s = 0; s < SDIM; ++s)
-               {
-                  if (Q_LAYOUT == QVectorLayout::byVDIM)  { dq[s] = q(c, s, qx, el); }
-                  if (Q_LAYOUT == QVectorLayout::byNODES) { dq[s] = q(qx, c, s, el); }
-               }
-
-               // Apply inverse Jacobian transpose (adjoint of physical gradient)
-               real_t du = dq[0];
-               if (GRAD_PHYS)
-               {
-                  if (SDIM == 1) { du = dq[0] / j(qx, 0, el); }
-                  else if (SDIM == 2)
-                  {
-                     const real_t Jloc[2] = {j(qx,0,el), j(qx,1,el)};
-                     real_t Jinv[3];
-                     kernels::CalcLeftInverse<2,1>(Jloc, Jinv);
-                     du = Jinv[0]*dq[0] + Jinv[1]*dq[1];
-                  }
-                  else // SDIM == 3
-                  {
-                     const real_t Jloc[3] = {j(qx,0,el), j(qx,1,el), j(qx,2,el)};
-                     real_t Jinv[3];
-                     kernels::CalcLeftInverse<3,1>(Jloc, Jinv);
-                     du = Jinv[0]*dq[0] + Jinv[1]*dq[1] + Jinv[2]*dq[2];
-                  }
-               }
-
-               // Accumulate contribution (transpose of G matrix)
-               u += g(qx, d) * du;
-            }
-            e(d, c, el) += u;
-         }
-      }
-   });
-}
-
-template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int T_NBZ = 1>
-static void DerivativesTranspose2D(const int NE,
-                                   const real_t *b_,
-                                   const real_t *g_,
-                                   const real_t *j_,
-                                   const real_t *q_,
-                                   real_t *e_,
-                                   const int sdim = 2,
-                                   const int vdim = 0,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   const int SDIM = GRAD_PHYS ? sdim : 2;
-   static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto g = Reshape(g_, Q1D, D1D);
-   const auto j = Reshape(j_, Q1D, Q1D, SDIM, 2, NE);
-   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
-                  Reshape(q_, Q1D, Q1D, VDIM, SDIM, NE):
-                  Reshape(q_, VDIM, SDIM, Q1D, Q1D, NE);
-   auto e = Reshape(e_, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D_batch(NE, D1D, D1D, NBZ, [=] MFEM_HOST_DEVICE (int el)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      const int tidz = MFEM_THREAD_ID(z);
-
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-      DeviceMatrix B(BG[0], D1D, Q1D);
-      DeviceMatrix G(BG[1], D1D, Q1D);
-
-      MFEM_SHARED real_t sm0[NBZ][MDQ*MDQ];
-      MFEM_SHARED real_t sm1[NBZ][MDQ*MDQ];
-
-      DeviceMatrix QQ(sm0[tidz], MQ1, MQ1);
-      DeviceMatrix DQ0(sm1[tidz], MD1, MQ1);
-      DeviceMatrix DQ1(sm1[tidz], MD1, MQ1);  // Reuse sm1 after DQ0 is done
-      DeviceMatrix DD(sm0[tidz], MD1, MD1);   // Reuse sm0 after QQ is done
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         // Load Q data and apply inverse Jacobian
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               // Load gradient components
-               real_t dq[3] = {0.0, 0.0, 0.0};
-               for (int d = 0; d < SDIM; ++d)
-               {
-                  if (Q_LAYOUT == QVectorLayout::byVDIM) { dq[d] = q(c, d, qx, qy, el); }
-                  else { dq[d] = q(qx, qy, c, d, el); }
-               }
-
-               // Apply inverse Jacobian transpose (adjoint of physical gradient)
-               real_t du[2] = {dq[0], dq[1]};
-               if (GRAD_PHYS)
-               {
-                  if (SDIM == 2)
-                  {
-                     real_t Jloc[4], Jinv[4];
-                     Jloc[0] = j(qx,qy,0,0,el);
-                     Jloc[1] = j(qx,qy,1,0,el);
-                     Jloc[2] = j(qx,qy,0,1,el);
-                     Jloc[3] = j(qx,qy,1,1,el);
-                     kernels::CalcInverse<2>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1];
-                     du[0] = U;
-                     du[1] = V;
-                  }
-                  else // SDIM == 3
-                  {
-                     real_t Jloc[6], Jinv[6];
-                     Jloc[0] = j(qx,qy,0,0,el);
-                     Jloc[1] = j(qx,qy,1,0,el);
-                     Jloc[2] = j(qx,qy,2,0,el);
-                     Jloc[3] = j(qx,qy,0,1,el);
-                     Jloc[4] = j(qx,qy,1,1,el);
-                     Jloc[5] = j(qx,qy,2,1,el);
-                     kernels::CalcLeftInverse<3,2>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1] + Jinv[4]*dq[2];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1] + Jinv[5]*dq[2];
-                     du[0] = U;
-                     du[1] = V;
-                  }
-               }
-               QQ(qx, qy) = du[0];  // Store du/dx component
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in y-direction: QQ -> DQ0
-         // (Transpose of d/dx which uses DQ1(dy,qx)*B(dy,qy))
-         // Must produce DQ0(dy,qx) to match forward's DQ1 indexing
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += B(dy,qy) * QQ(qx,qy);
-               }
-               DQ0(dy,qx) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply G^T in x-direction: DQ0 -> DD
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += G(dx,qx) * DQ0(dy,qx);
-               }
-               DD(dx,dy) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Accumulate to output
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               e(dx,dy,c,el) += DD(dx,dy);
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Now process du/dy component
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               // Load gradient components
-               real_t dq[3] = {0.0, 0.0, 0.0};
-               for (int d = 0; d < SDIM; ++d)
-               {
-                  if (Q_LAYOUT == QVectorLayout::byVDIM) { dq[d] = q(c, d, qx, qy, el); }
-                  else { dq[d] = q(qx, qy, c, d, el); }
-               }
-
-               // Apply inverse Jacobian transpose
-               real_t du[2] = {dq[0], dq[1]};
-               if (GRAD_PHYS)
-               {
-                  if (SDIM == 2)
-                  {
-                     real_t Jloc[4], Jinv[4];
-                     Jloc[0] = j(qx,qy,0,0,el);
-                     Jloc[1] = j(qx,qy,1,0,el);
-                     Jloc[2] = j(qx,qy,0,1,el);
-                     Jloc[3] = j(qx,qy,1,1,el);
-                     kernels::CalcInverse<2>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1];
-                     du[0] = U;
-                     du[1] = V;
-                  }
-                  else // SDIM == 3
-                  {
-                     real_t Jloc[6], Jinv[6];
-                     Jloc[0] = j(qx,qy,0,0,el);
-                     Jloc[1] = j(qx,qy,1,0,el);
-                     Jloc[2] = j(qx,qy,2,0,el);
-                     Jloc[3] = j(qx,qy,0,1,el);
-                     Jloc[4] = j(qx,qy,1,1,el);
-                     Jloc[5] = j(qx,qy,2,1,el);
-                     kernels::CalcLeftInverse<3,2>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[2]*dq[1] + Jinv[4]*dq[2];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[3]*dq[1] + Jinv[5]*dq[2];
-                     du[0] = U;
-                     du[1] = V;
-                  }
-               }
-               QQ(qx, qy) = du[1];  // Store du/dy component
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply G^T in y-direction: QQ -> DQ1
-         // (Transpose of d/dy which uses DQ0(dy,qx)*G(dy,qy))
-         // Must produce DQ1(dy,qx) to match forward's DQ0 indexing
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += G(dy,qy) * QQ(qx,qy);
-               }
-               DQ1(dy,qx) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in x-direction: DQ1 -> DD
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += B(dx,qx) * DQ1(dy,qx);
-               }
-               DD(dx,dy) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Accumulate to output
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               e(dx,dy,c,el) += DD(dx,dy);
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
-static void DerivativesTranspose3D(const int NE,
-                                   const real_t *b_,
-                                   const real_t *g_,
-                                   const real_t *j_,
-                                   const real_t *q_,    // q_der
-                                   real_t *e_,          // e_vec
-                                   const int sdim = 3,
-                                   const int vdim = 0,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto g = Reshape(g_, Q1D, D1D);
-   const auto j = Reshape(j_, Q1D, Q1D, Q1D, 3, 3, NE);
-   const auto q = Q_LAYOUT == QVectorLayout::byNODES ?
-                  Reshape(q_, Q1D, Q1D, Q1D, VDIM, 3, NE):
-                  Reshape(q_, VDIM, 3, Q1D, Q1D, Q1D, NE);
-   auto e = Reshape(e_, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int el)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_INTERP_1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_INTERP_1D;
-
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,b,g,BG);
-      DeviceMatrix B(BG[0], D1D, Q1D);
-      DeviceMatrix G(BG[1], D1D, Q1D);
-
-      MFEM_SHARED real_t sm0[1][MQ1*MQ1*MQ1];
-      MFEM_SHARED real_t sm1[1][MQ1*MQ1*MQ1];
-      DeviceCube QQQ(sm0[0], MQ1, MQ1, MQ1);
-      DeviceCube DQQ(sm1[0], MD1, MQ1, MQ1);
-      DeviceCube DDQ(sm0[0], MD1, MD1, MQ1);
-      DeviceCube DDD(sm1[0], MD1, MD1, MD1);
-
-      for (int c = 0; c < VDIM; c++)
-      {
-         // Process du/dx component
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t dq[3];
-                  if (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     dq[0] = q(c,0,qx,qy,qz,el);
-                     dq[1] = q(c,1,qx,qy,qz,el);
-                     dq[2] = q(c,2,qx,qy,qz,el);
-                  }
-                  else
-                  {
-                     dq[0] = q(qx,qy,qz,c,0,el);
-                     dq[1] = q(qx,qy,qz,c,1,el);
-                     dq[2] = q(qx,qy,qz,c,2,el);
-                  }
-
-                  real_t du[3] = {dq[0], dq[1], dq[2]};
-                  if (GRAD_PHYS)
-                  {
-                     real_t Jloc[9], Jinv[9];
-                     for (int col = 0; col < 3; col++)
-                     {
-                        for (int row = 0; row < 3; row++)
-                        {
-                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
-                        }
-                     }
-                     kernels::CalcInverse<3>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
-                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
-                     du[0] = U; du[1] = V; du[2] = W;
-                  }
-                  QQQ(qx,qy,qz) = du[0];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply G^T in x: QQQ -> DQQ (transpose of G⊗B⊗B)
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     u += G(dx,qx) * QQQ(qx,qy,qz);
-                  }
-                  DQQ(dx,qy,qz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in y: DQQ -> DDQ
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     u += B(dy,qy) * DQQ(dx,qy,qz);
-                  }
-                  DDQ(dx,dy,qz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in z: DDQ -> DDD
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u += B(dz,qz) * DDQ(dx,dy,qz);
-                  }
-                  DDD(dx,dy,dz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Accumulate result
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Process du/dy component
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t dq[3];
-                  if (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     dq[0] = q(c,0,qx,qy,qz,el);
-                     dq[1] = q(c,1,qx,qy,qz,el);
-                     dq[2] = q(c,2,qx,qy,qz,el);
-                  }
-                  else
-                  {
-                     dq[0] = q(qx,qy,qz,c,0,el);
-                     dq[1] = q(qx,qy,qz,c,1,el);
-                     dq[2] = q(qx,qy,qz,c,2,el);
-                  }
-
-                  real_t du[3] = {dq[0], dq[1], dq[2]};
-                  if (GRAD_PHYS)
-                  {
-                     real_t Jloc[9], Jinv[9];
-                     for (int col = 0; col < 3; col++)
-                     {
-                        for (int row = 0; row < 3; row++)
-                        {
-                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
-                        }
-                     }
-                     kernels::CalcInverse<3>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
-                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
-                     du[0] = U; du[1] = V; du[2] = W;
-                  }
-                  QQQ(qx,qy,qz) = du[1];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in x: QQQ -> DQQ (transpose of B⊗G⊗B)
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     u += B(dx,qx) * QQQ(qx,qy,qz);
-                  }
-                  DQQ(dx,qy,qz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply G^T in y: DQQ -> DDQ
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     u += G(dy,qy) * DQQ(dx,qy,qz);
-                  }
-                  DDQ(dx,dy,qz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in z: DDQ -> DDD
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u += B(dz,qz) * DDQ(dx,dy,qz);
-                  }
-                  DDD(dx,dy,dz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Accumulate result
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Process du/dz component
-         MFEM_FOREACH_THREAD(qz,z,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t dq[3];
-                  if (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     dq[0] = q(c,0,qx,qy,qz,el);
-                     dq[1] = q(c,1,qx,qy,qz,el);
-                     dq[2] = q(c,2,qx,qy,qz,el);
-                  }
-                  else
-                  {
-                     dq[0] = q(qx,qy,qz,c,0,el);
-                     dq[1] = q(qx,qy,qz,c,1,el);
-                     dq[2] = q(qx,qy,qz,c,2,el);
-                  }
-
-                  real_t du[3] = {dq[0], dq[1], dq[2]};
-                  if (GRAD_PHYS)
-                  {
-                     real_t Jloc[9], Jinv[9];
-                     for (int col = 0; col < 3; col++)
-                     {
-                        for (int row = 0; row < 3; row++)
-                        {
-                           Jloc[row+3*col] = j(qx,qy,qz,row,col,el);
-                        }
-                     }
-                     kernels::CalcInverse<3>(Jloc, Jinv);
-                     const real_t U = Jinv[0]*dq[0] + Jinv[3]*dq[1] + Jinv[6]*dq[2];
-                     const real_t V = Jinv[1]*dq[0] + Jinv[4]*dq[1] + Jinv[7]*dq[2];
-                     const real_t W = Jinv[2]*dq[0] + Jinv[5]*dq[1] + Jinv[8]*dq[2];
-                     du[0] = U; du[1] = V; du[2] = W;
-                  }
-                  QQQ(qx,qy,qz) = du[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply G^T in z: QQQ -> DQQ (transpose of B⊗B⊗G)
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t u = 0.0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u += G(dz,qz) * QQQ(qx,qy,qz);
-                  }
-                  DQQ(dz,qy,qx) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in y: DQQ -> DDQ
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(qx,x,Q1D)
-               {
-                  real_t u = 0.0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     u += B(dy,qy) * DQQ(dz,qy,qx);
-                  }
-                  DDQ(dz,dy,qx) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Apply B^T in x: DDQ -> DDD
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     u += B(dx,qx) * DDQ(dz,dy,qx);
-                  }
-                  DDD(dx,dy,dz) = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         // Accumulate result
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               MFEM_FOREACH_THREAD(dx,x,D1D)
-               {
-                  e(dx,dy,dz,c,el) += DDD(dx,dy,dz);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-} // namespace quadrature_interpolator
-} // namespace internal
-
-/// \cond DO_NOT_DOCUMENT
-
-template<int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
-         int Q1D, int NBZ>
-QuadratureInterpolator::GradTransposeKernelType
-QuadratureInterpolator::GradTransposeKernels::Kernel()
-{
-   if (DIM == 1) { return internal::quadrature_interpolator::DerivativesTranspose1D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 2) { return internal::quadrature_interpolator::DerivativesTranspose2D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D, NBZ>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::DerivativesTranspose3D<Q_LAYOUT, GRAD_PHYS, VDIM, D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/qinterp/grad_transpose_by_nodes.cpp

@@ -1,50 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../quadinterpolator.hpp"
-#include "grad_transpose.hpp"
-
-namespace mfem
-{
-namespace internal
-{
-namespace quadrature_interpolator
-{
-
-template <bool P>
-void InitGradTransposeByNodesKernels()
-{
-   using k = QuadratureInterpolator::GradTransposeKernels;
-   constexpr auto L = QVectorLayout::byNODES;
-   // 2D
-   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
-
-   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
-   // 3D
-   k::Specialization<3,L,P,1,3,4>::Add();
-   k::Specialization<3,L,P,1,4,6>::Add();
-   k::Specialization<3,L,P,1,5,8>::Add();
-   k::Specialization<3,L,P,3,3,4>::Add();
-   k::Specialization<3,L,P,3,4,6>::Add();
-   k::Specialization<3,L,P,3,5,8>::Add();
-}
-
-template void InitGradTransposeByNodesKernels<false>();
-template void InitGradTransposeByNodesKernels<true>();
-
-} // namespace quadrature_interpolator
-} // namespace internal
-} // namespace mfem

fem/qinterp/grad_transpose_by_vdim.cpp

@@ -1,50 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../quadinterpolator.hpp"
-#include "grad_transpose.hpp"
-
-namespace mfem
-{
-namespace internal
-{
-namespace quadrature_interpolator
-{
-
-template <bool P>
-void InitGradTransposeByVDimKernels()
-{
-   using k = QuadratureInterpolator::GradTransposeKernels;
-   constexpr auto L = QVectorLayout::byVDIM;
-   // 2D
-   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
-
-   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
-   // 3D
-   k::Specialization<3,L,P,1,3,4>::Add();
-   k::Specialization<3,L,P,1,4,6>::Add();
-   k::Specialization<3,L,P,1,5,8>::Add();
-   k::Specialization<3,L,P,3,3,4>::Add();
-   k::Specialization<3,L,P,3,4,6>::Add();
-   k::Specialization<3,L,P,3,5,8>::Add();
-}
-
-template void InitGradTransposeByVDimKernels<false>();
-template void InitGradTransposeByVDimKernels<true>();
-
-} // namespace quadrature_interpolator
-} // namespace internal
-} // namespace mfem

fem/qspace.hpp

@@ -27,11 +27,7 @@ enum class QSpaceOffsetStorage
 
 /// Abstract base class for QuadratureSpace and FaceQuadratureSpace.
 /** This class represents the storage layout for QuadratureFunction%s, that may
-    be defined either on mesh elements or mesh faces.
-
-    This class represents the layout for a single scalar value at each
-    quadrature point. Use class VectorQuadratureSpace to represent a space with
-    multiple (vector) values at each quadrature point. */
+    be defined either on mesh elements or mesh faces. */
 class QuadratureSpaceBase
 {
 protected:
@@ -162,62 +158,8 @@ public:
    virtual ~QuadratureSpaceBase() { }
 };
 
-/// Vector version of the scalar class QuadratureSpaceBase.
-class VectorQuadratureSpace
-{
-protected:
-   /// Points to an external object provided during construction. Not owned.
-   QuadratureSpaceBase *qspace;
-   /// Vector dimension.
-   int vdim;
-
-public:
-   /** @brief Construct a VectorQuadratureSpace on the given
-       QuadratureSpaceBase, @a qspace_, with the given vector dimension,
-       @a vdim_.
-
-       The VectorQuadratureSpace does not assume ownership of the
-       QuadratureSpaceBase, @a qspace_. */
-   VectorQuadratureSpace(QuadratureSpaceBase &qspace_, int vdim_)
-      : qspace(&qspace_),
-        vdim(vdim_)
-   { }
-
-   /// Copy constructor: default.
-   VectorQuadratureSpace(const VectorQuadratureSpace &) = default;
-
-   /// Copy assignment: default.
-   VectorQuadratureSpace &operator=(const VectorQuadratureSpace &) = default;
-
-   /// Move construction is not allowed.
-   VectorQuadratureSpace(VectorQuadratureSpace &&) = delete;
-
-   /// Move assignment is not allowed.
-   VectorQuadratureSpace &operator=(VectorQuadratureSpace &&) = delete;
-
-   /// Destructor: default.
-   ~VectorQuadratureSpace() = default;
-
-   /// Get the associated scalar QuadratureSpaceBase object.
-   QuadratureSpaceBase *GetSpace() { return qspace; }
-
-   /// Get the associated scalar QuadratureSpaceBase object (const version).
-   const QuadratureSpaceBase *GetSpace() const { return qspace; }
-
-   /// Get the vector dimension.
-   int GetVDim() const { return vdim; }
-
-   /** @brief Get the total size (on this MPI-rank in parallel) of the
-       VectorQuadratureSpace. */
-   int GetVSize() const { return qspace->GetSize() * vdim; }
-};
-
 /// Class representing the storage layout of a QuadratureFunction.
-/** Multiple QuadratureFunction%s can share the same QuadratureSpace.
-
-    This class represents the layout for a single scalar value at each
-    quadrature point. Use class VectorQuadratureSpace to represent a space with
-    multiple (vector) values at each quadrature point. */
+/** Multiple QuadratureFunction%s can share the same QuadratureSpace. */
 class QuadratureSpace : public QuadratureSpaceBase
 {
 protected:
@@ -267,11 +209,7 @@ public:
 
 /// Class representing the storage layout of a FaceQuadratureFunction.
 /** FaceQuadratureSpace is defined on either the interior or boundary faces
-    of a mesh, depending on the provided FaceType.
-
-    This class represents the layout for a single scalar value at each
-    quadrature point. Use class VectorQuadratureSpace to represent a space with
-    multiple (vector) values at each quadrature point. */
+    of a mesh, depending on the provided FaceType. */
 class FaceQuadratureSpace : public QuadratureSpaceBase
 {
    FaceType face_type; ///< Is the space defined on interior or boundary faces?

fem/quadinterpolator.cpp

@@ -11,7 +11,6 @@
 
 #include "quadinterpolator.hpp"
 #include "qinterp/grad.hpp"
-#include "qinterp/grad_transpose.hpp"
 #include "qinterp/eval.hpp"
 #include "qspace.hpp"
 #include "../general/forall.hpp"
@@ -31,10 +30,7 @@ void InitEvalKernels();
 void InitDetKernels();
 template <bool P> void InitGradByNodesKernels();
 template <bool P> void InitGradByVDimKernels();
-template <bool P> void InitGradTransposeByNodesKernels();
-template <bool P> void InitGradTransposeByVDimKernels();
 void InitTensorEvalHDivKernels();
-void InitEvalTransposeByVDimKernels();
 struct Kernels
 {
    Kernels()
@@ -49,19 +45,12 @@ struct Kernels
       // Phys grad kernels
       InitGradByNodesKernels<true>();
       InitGradByVDimKernels<true>();
-      // Non-phys grad transpose kernels
-      InitGradTransposeByNodesKernels<false>();
-      InitGradTransposeByVDimKernels<false>();
-      // Phys grad transpose kernels
-      InitGradTransposeByNodesKernels<true>();
-      InitGradTransposeByVDimKernels<true>();
       // Determinants
       InitDetKernels();
       // Non-tensor
       InitEvalKernels();
       // Tensor (quad,hex) H(div)
       InitTensorEvalHDivKernels();
-      InitEvalTransposeByVDimKernels();
    }
 };
 }
@@ -419,59 +408,16 @@ void QuadratureInterpolator::MultHDiv(const Vector &e_vec,
    MFEM_CONTRACT_VAR(q_div);
 }
 
-void QuadratureInterpolator::AddMultTranspose(unsigned eval_flags,
-                                              const Vector &q_val,
-                                              const Vector &q_der,
-                                              Vector &e_vec) const
+void QuadratureInterpolator::MultTranspose(unsigned eval_flags,
+                                           const Vector &q_val,
+                                           const Vector &q_der,
+                                           Vector &e_vec) const
 {
-   const int ne = fespace->GetNE();
-   if (ne == 0) { return; }
-   const FiniteElement *fe = fespace->GetFE(0);
-   const int vdim = fespace->GetVDim();
-   const int sdim = fespace->GetMesh()->SpaceDimension();
-
-   const bool use_tensor_eval =
-      use_tensor_products &&
-      dynamic_cast<const TensorBasisElement*>(fe) != nullptr;
-   const IntegrationRule *ir =
-      IntRule ? IntRule : &qspace->GetElementIntRule(0);
-   const DofToQuad::Mode mode =
-      use_tensor_eval ? DofToQuad::TENSOR : DofToQuad::FULL;
-   const DofToQuad &maps = fe->GetDofToQuad(*ir, mode);
-   const int dim = maps.FE->GetDim();
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-
-   const GeometricFactors *geom = nullptr;
-   if (eval_flags & PHYSICAL_DERIVATIVES)
-   {
-      const int jacobians = GeometricFactors::JACOBIANS;
-      geom = fespace->GetMesh()->GetGeometricFactors(*ir, jacobians);
-   }
-
-   if (use_tensor_eval)
-   {
-      if (eval_flags & (VALUES | PHYSICAL_VALUES))
-      {
-         TensorEvalTransposeKernels::Run(dim, q_layout, vdim, nd, nq, ne,
-                                         maps.B.Read(), q_val.Read(),
-                                         e_vec.ReadWrite(), vdim, nd, nq);
-      }
-      if (eval_flags & (DERIVATIVES | PHYSICAL_DERIVATIVES))
-      {
-         const bool phys = (eval_flags & PHYSICAL_DERIVATIVES);
-         const real_t *J = phys ? geom->J.Read() : nullptr;
-         const int s_dim = phys ? sdim : dim;
-         GradTransposeKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
-                                   maps.B.Read(), maps.G.Read(), J,
-                                   q_der.Read(), e_vec.ReadWrite(),
-                                   s_dim, vdim, nd, nq);
-      }
-   }
-   else
-   {
-      MFEM_ABORT("Non-tensor MultTranspose not yet implemented");
-   }
+   MFEM_CONTRACT_VAR(eval_flags);
+   MFEM_CONTRACT_VAR(q_val);
+   MFEM_CONTRACT_VAR(q_der);
+   MFEM_CONTRACT_VAR(e_vec);
+   MFEM_ABORT("this method is not implemented yet");
 }
 
 void QuadratureInterpolator::Values(const Vector &e_vec,

fem/quadinterpolator.hpp

@@ -156,8 +156,8 @@ public:
    void Determinants(const Vector &e_vec, Vector &q_det) const;
 
    /// Perform the transpose operation of Mult(). (TODO)
-   void AddMultTranspose(unsigned eval_flags, const Vector &q_val,
-                         const Vector &q_der, Vector &e_vec) const;
+   void MultTranspose(unsigned eval_flags, const Vector &q_val,
+                      const Vector &q_der, Vector &e_vec) const;
 
    /// @brief Returns true if the given finite element space is supported by
    /// QuadratureInterpolator.
@@ -204,13 +204,6 @@ public:
    using TensorEvalHDivKernelType =
       void(*)(const int, const real_t *, const real_t *, const real_t *,
               const real_t *, real_t *, const int, const int);
-   using TensorEvalTransposeKernelType = void(*)(const int, const real_t *,
-                                                 const real_t *,
-                                                 real_t *, const int, const int, const int);
-   using GradTransposeKernelType = void(*)(const int, const real_t *,
-                                           const real_t *, const real_t *,
-                                           const real_t *, real_t *,
-                                           const int, const int, const int, const int);
 
    // value-type mapping
    MFEM_REGISTER_KERNELS(TensorEvalKernels, TensorEvalKernelType,
@@ -229,10 +222,6 @@ public:
 
    MFEM_REGISTER_KERNELS(TensorEvalHDivKernels, TensorEvalHDivKernelType,
                          (int, QVectorLayout, unsigned, int, int));
-   MFEM_REGISTER_KERNELS(TensorEvalTransposeKernels, TensorEvalTransposeKernelType,
-                         (int, QVectorLayout, int, int, int), (int));
-   MFEM_REGISTER_KERNELS(GradTransposeKernels, GradTransposeKernelType,
-                         (int, QVectorLayout, bool, int, int, int), (int));
 
    /// Adds specializations for TensorEvalKernels
    template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D,

fem/restriction.cpp

@@ -299,17 +299,6 @@ void ElementRestriction::FillSparseMatrix(const Vector &mat_ea,
    FillJAndData(mat_ea, mat);
 }
 
-void ElementRestriction::FillSparseMatrix(
-   const Vector &mat_ea, SparseMatrix &mat,
-   const ElementRestriction &trial_restr) const
-{
-   mat.GetMemoryI().New(mat.Height()+1, mat.GetMemoryI().GetMemoryType());
-   const int nnz = FillI(mat, trial_restr);
-   mat.GetMemoryJ().New(nnz, mat.GetMemoryJ().GetMemoryType());
-   mat.GetMemoryData().New(nnz, mat.GetMemoryData().GetMemoryType());
-   FillJAndData(mat_ea, mat, trial_restr);
-}
-
 static MFEM_HOST_DEVICE int GetMinElt(const int *my_elts, const int nbElts,
                                       const int *nbr_elts, const int nbrNbElts)
 {
@@ -339,23 +328,6 @@ static MFEM_HOST_DEVICE int GetAndIncrementNnzIndex(const int i_L, int* I)
    return ind;
 }
 
-static MFEM_HOST_DEVICE int DofToVDof(const int dof, const int c,
-                                      const int ndofs, const int vdim,
-                                      const bool byvdim)
-{
-   return byvdim ? dof*vdim + c : c*ndofs + dof;
-}
-
-static MFEM_HOST_DEVICE int SignedIndexAbs(const int i)
-{
-   return (i >= 0) ? i : -1 - i;
-}
-
-static MFEM_HOST_DEVICE int SignedIndexSign(const int i)
-{
-   return (i >= 0) ? 1 : -1;
-}
-
 int ElementRestriction::FillI(SparseMatrix &mat) const
 {
    const int all_dofs = ndofs;
@@ -433,98 +405,6 @@ int ElementRestriction::FillI(SparseMatrix &mat) const
    return h_I[nTdofs];
 }
 
-int ElementRestriction::FillI(SparseMatrix &mat,
-                              const ElementRestriction &trial_restr) const
-{
-   MFEM_VERIFY(ne == trial_restr.ne,
-               "ElementRestriction::FillI: test/trial NE mismatch");
-   const int test_all_dofs = ndofs;
-   const int test_vd = vdim;
-   const int trial_vd = trial_restr.vdim;
-   const int test_elt_dofs = dof;
-   const int trial_elt_dofs = trial_restr.dof;
-   const bool test_byvdim = byvdim;
-   auto I = mat.ReadWriteI();
-   auto test_offsets = offsets.Read();
-   auto test_indices = indices.Read();
-   auto test_gather_map = gather_map.Read();
-   auto trial_offsets = trial_restr.offsets.Read();
-   auto trial_indices = trial_restr.indices.Read();
-   auto trial_gather_map = trial_restr.gather_map.Read();
-
-   Array<int> test_elts(indices.Size());
-   Array<int> trial_elts(trial_restr.indices.Size());
-   auto d_test_elts = test_elts.Write();
-   auto d_trial_elts = trial_elts.Write();
-
-   mfem::forall(test_vd*test_all_dofs+1, [=] MFEM_HOST_DEVICE (int i_L)
-   {
-      I[i_L] = 0;
-   });
-   mfem::forall(ne*test_elt_dofs*test_vd, [=] MFEM_HOST_DEVICE (int iE)
-   {
-      const int e = iE/(test_elt_dofs*test_vd);
-      const int it = iE%(test_elt_dofs*test_vd);
-      const int i = it%test_elt_dofs;
-      const int test_c = it/test_elt_dofs;
-
-      const int i_gm = e*test_elt_dofs + i;
-      const int i_dof = SignedIndexAbs(test_gather_map[i_gm]);
-      const int i_L = DofToVDof(i_dof, test_c, test_all_dofs, test_vd,
-                                test_byvdim);
-      const int i_offset = test_offsets[i_dof];
-      const int i_next_offset = test_offsets[i_dof+1];
-      const int i_nbElts = i_next_offset - i_offset;
-
-      int *i_elts = &d_test_elts[i_offset];
-      for (int e_i = 0; e_i < i_nbElts; ++e_i)
-      {
-         const int i_loc = SignedIndexAbs(test_indices[i_offset+e_i]);
-         i_elts[e_i] = i_loc/test_elt_dofs;
-      }
-      for (int trial_c = 0; trial_c < trial_vd; ++trial_c)
-      {
-         MFEM_CONTRACT_VAR(trial_c);
-         for (int j = 0; j < trial_elt_dofs; j++)
-         {
-            const int j_gm = e*trial_elt_dofs + j;
-            const int j_dof = SignedIndexAbs(trial_gather_map[j_gm]);
-            const int j_offset = trial_offsets[j_dof];
-            const int j_next_offset = trial_offsets[j_dof+1];
-            const int j_nbElts = j_next_offset - j_offset;
-            if (i_nbElts == 1 || j_nbElts == 1)
-            {
-               GetAndIncrementNnzIndex(i_L, I);
-            }
-            else
-            {
-               int *j_elts = &d_trial_elts[j_offset];
-               for (int e_j = 0; e_j < j_nbElts; ++e_j)
-               {
-                  const int j_loc = SignedIndexAbs(trial_indices[j_offset+e_j]);
-                  j_elts[e_j] = j_loc/trial_elt_dofs;
-               }
-               const int min_e = GetMinElt(i_elts, i_nbElts,
-                                           j_elts, j_nbElts);
-               if (e == min_e) { GetAndIncrementNnzIndex(i_L, I); }
-            }
-         }
-      }
-   });
-
-   auto h_I = mat.HostReadWriteI();
-   const int nTdofs = test_vd*test_all_dofs;
-   int sum = 0;
-   for (int i = 0; i < nTdofs; i++)
-   {
-      const int nnz = h_I[i];
-      h_I[i] = sum;
-      sum += nnz;
-   }
-   h_I[nTdofs] = sum;
-   return h_I[nTdofs];
-}
-
 void ElementRestriction::FillJAndData(const Vector &ea_data,
                                       SparseMatrix &mat) const
 {
@@ -621,135 +501,6 @@ void ElementRestriction::FillJAndData(const Vector &ea_data,
    h_I[0] = 0;
 }
 
-void ElementRestriction::FillJAndData(
-   const Vector &ea_data, SparseMatrix &mat,
-   const ElementRestriction &trial_restr) const
-{
-   MFEM_VERIFY(ne == trial_restr.ne,
-               "ElementRestriction::FillJAndData: test/trial NE mismatch");
-   const int test_all_dofs = ndofs;
-   const int trial_all_dofs = trial_restr.ndofs;
-   const int test_vd = vdim;
-   const int trial_vd = trial_restr.vdim;
-   const int test_elt_dofs = dof;
-   const int trial_elt_dofs = trial_restr.dof;
-   const bool test_byvdim = byvdim;
-   const bool trial_byvdim = trial_restr.byvdim;
-   auto I = mat.ReadWriteI();
-   auto J = mat.WriteJ();
-   auto Data = mat.WriteData();
-   auto test_offsets = offsets.Read();
-   auto test_indices = indices.Read();
-   auto test_gather_map = gather_map.Read();
-   auto trial_offsets = trial_restr.offsets.Read();
-   auto trial_indices = trial_restr.indices.Read();
-   auto trial_gather_map = trial_restr.gather_map.Read();
-   auto mat_ea = Reshape(ea_data.Read(), test_elt_dofs, test_vd,
-                         trial_elt_dofs, trial_vd, ne);
-
-   Array<int> test_el(indices.Size() * 3);
-   Array<int> trial_el(trial_restr.indices.Size() * 3);
-   auto d_test_el = Reshape(test_el.Write(), indices.Size(), 3);
-   auto d_trial_el = Reshape(trial_el.Write(), trial_restr.indices.Size(), 3);
-
-   mfem::forall(ne*test_elt_dofs*test_vd, [=] MFEM_HOST_DEVICE (int iE)
-   {
-      const int e = iE/(test_elt_dofs*test_vd);
-      const int it = iE%(test_elt_dofs*test_vd);
-      const int i = it%test_elt_dofs;
-      const int test_c = it/test_elt_dofs;
-
-      const int i_gm = e*test_elt_dofs + i;
-      const int i_gm_s = test_gather_map[i_gm];
-      const int i_dof = SignedIndexAbs(i_gm_s);
-      const int i_sgn = SignedIndexSign(i_gm_s);
-      const int i_L = DofToVDof(i_dof, test_c, test_all_dofs, test_vd,
-                                test_byvdim);
-      const int i_offset = test_offsets[i_dof];
-      const int i_next_offset = test_offsets[i_dof+1];
-      const int i_nbElts = i_next_offset - i_offset;
-
-      int *i_elts = &d_test_el(i_offset, 0);
-      int *i_B = &d_test_el(i_offset, 1);
-      int *i_sgns = &d_test_el(i_offset, 2);
-      for (int e_i = 0; e_i < i_nbElts; ++e_i)
-      {
-         const int i_idx_s = test_indices[i_offset+e_i];
-         const int i_idx = SignedIndexAbs(i_idx_s);
-         i_elts[e_i] = i_idx/test_elt_dofs;
-         i_B[e_i] = i_idx%test_elt_dofs;
-         i_sgns[e_i] = SignedIndexSign(i_idx_s);
-      }
-      for (int trial_c = 0; trial_c < trial_vd; ++trial_c)
-      {
-         for (int j = 0; j < trial_elt_dofs; j++)
-         {
-            const int j_gm = e*trial_elt_dofs + j;
-            const int j_gm_s = trial_gather_map[j_gm];
-            const int j_dof = SignedIndexAbs(j_gm_s);
-            const int j_sgn = SignedIndexSign(j_gm_s);
-            const int j_L = DofToVDof(j_dof, trial_c, trial_all_dofs,
-                                      trial_vd, trial_byvdim);
-            const int j_offset = trial_offsets[j_dof];
-            const int j_next_offset = trial_offsets[j_dof+1];
-            const int j_nbElts = j_next_offset - j_offset;
-            if (i_nbElts == 1 || j_nbElts == 1)
-            {
-               const int nnz = GetAndIncrementNnzIndex(i_L, I);
-               J[nnz] = j_L;
-               Data[nnz] = i_sgn*j_sgn*mat_ea(i, test_c, j, trial_c, e);
-            }
-            else
-            {
-               int *j_elts = &d_trial_el(j_offset, 0);
-               int *j_B = &d_trial_el(j_offset, 1);
-               int *j_sgns = &d_trial_el(j_offset, 2);
-               for (int e_j = 0; e_j < j_nbElts; ++e_j)
-               {
-                  const int j_idx_s = trial_indices[j_offset+e_j];
-                  const int j_idx = SignedIndexAbs(j_idx_s);
-                  j_elts[e_j] = j_idx/trial_elt_dofs;
-                  j_B[e_j] = j_idx%trial_elt_dofs;
-                  j_sgns[e_j] = SignedIndexSign(j_idx_s);
-               }
-               const int min_e = GetMinElt(i_elts, i_nbElts,
-                                           j_elts, j_nbElts);
-               if (e == min_e)
-               {
-                  real_t val = 0.0;
-                  for (int k = 0; k < i_nbElts; k++)
-                  {
-                     const int e_i = i_elts[k];
-                     const int i_Bloc = i_B[k];
-                     for (int l = 0; l < j_nbElts; l++)
-                     {
-                        const int e_j = j_elts[l];
-                        const int j_Bloc = j_B[l];
-                        if (e_i == e_j)
-                        {
-                           val += i_sgns[k]*j_sgns[l]*
-                                  mat_ea(i_Bloc, test_c, j_Bloc, trial_c, e_i);
-                        }
-                     }
-                  }
-                  const int nnz = GetAndIncrementNnzIndex(i_L, I);
-                  J[nnz] = j_L;
-                  Data[nnz] = val;
-               }
-            }
-         }
-      }
-   });
-
-   auto h_I = mat.HostReadWriteI();
-   const int size = test_vd*test_all_dofs;
-   for (int i = 0; i < size; i++)
-   {
-      h_I[size-i] = h_I[size-(i+1)];
-   }
-   h_I[0] = 0;
-}
-
 L2ElementRestriction::L2ElementRestriction(const FiniteElementSpace &fes)
    : ne(fes.GetNE()),
      vdim(fes.GetVDim()),

fem/restriction.hpp

@@ -87,25 +87,12 @@ public:
    /// Fill a Sparse Matrix with Element Matrices.
    void FillSparseMatrix(const Vector &mat_ea, SparseMatrix &mat) const;
 
-   /** Fill a SparseMatrix with element matrices for this (test) restriction and
-       the given trial restriction. The element matrix layout is
-       test_dof x test_vdim x trial_dof x trial_vdim x ne. */
-   void FillSparseMatrix(const Vector &mat_ea, SparseMatrix &mat,
-                         const ElementRestriction &trial_restr) const;
-
    /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
        given by this ElementRestriction. */
    int FillI(SparseMatrix &mat) const;
-   /** Fill the I array for this (test) and the given trial restriction. */
-   int FillI(SparseMatrix &mat, const ElementRestriction &trial_restr) const;
    /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
        pattern given by this ElementRestriction, and the values of ea_data. */
    void FillJAndData(const Vector &ea_data, SparseMatrix &mat) const;
-   /** Fill the J and Data arrays for this (test) and the given trial
-       restriction, using element matrix layout
-       test_dof x test_vdim x trial_dof x trial_vdim x ne. */
-   void FillJAndData(const Vector &ea_data, SparseMatrix &mat,
-                     const ElementRestriction &trial_restr) const;
    /// @private Not part of the public interface (device kernel limitation).
    ///
    /// Performs either MultTranspose or AddMultTranspose depending on the

fem/transfer.cpp

@@ -231,9 +231,11 @@ const Operator &InterpolationGridTransfer::BackwardOperator()
 
 L2ProjectionGridTransfer::L2Projection::L2Projection(
    const FiniteElementSpace &fes_ho_, const FiniteElementSpace &fes_lor_,
+   CoefficientWithOrder coeff_ho_, CoefficientWithOrder coeff_lor_,
    MemoryType d_mt_)
    : Operator(fes_lor_.GetVSize(), fes_ho_.GetVSize()),
-     fes_ho(fes_ho_), fes_lor(fes_lor_), d_mt(d_mt_)
+     fes_ho(fes_ho_), fes_lor(fes_lor_), coeff_ho(coeff_ho_),
+     coeff_lor(coeff_lor_), d_mt(d_mt_)
 { }
 
 void L2ProjectionGridTransfer::L2Projection::BuildHo2Lor(
@@ -263,12 +265,13 @@ void L2ProjectionGridTransfer::L2Projection::ElemMixedMass(
    IntegrationPointTransformation& ip_tr,
    DenseMatrix& M_mixed_el) const
 {
-   int order = fe_lor.GetOrder() + fe_ho.GetOrder() + tr_lor->OrderW();
-   const IntegrationRule* ir = &IntRules.Get(geom, order);
+   int order = fe_lor.GetOrder() + fe_ho.GetOrder() + tr_lor->OrderW() +
+               coeff_ho.order;
+   const IntegrationRule &ir = IntRules.Get(geom, order);
    M_mixed_el = 0.0;
-   for (int i = 0; i < ir->GetNPoints(); i++)
+   for (int i = 0; i < ir.GetNPoints(); i++)
    {
-      const IntegrationPoint& ip_lor = ir->IntPoint(i);
+      const IntegrationPoint& ip_lor = ir.IntPoint(i);
       IntegrationPoint ip_ho;
       ip_tr.Transform(ip_lor, ip_ho);
       Vector shape_lor(fe_lor.GetDof());
@@ -284,23 +287,23 @@ void L2ProjectionGridTransfer::L2Projection::ElemMixedMass(
       {
          w *= tr_lor->Weight();
       }
+      if (coeff_ho)
+      {
+         w *= coeff_ho.coeff->Eval(*tr_ho, ip_ho);
+      }
       shape_lor *= w;
       AddMultVWt(shape_lor, shape_ho, M_mixed_el);
    }
 }
 
-void L2ProjectionGridTransfer::L2Projection::ElemMixedMass(
-   Geometry::Type geom, const FiniteElement& fe_ho,
-   const FiniteElement& fe_lor, ElementTransformation* el_tr,
-   IntegrationPointTransformation& ip_tr,
+void L2ProjectionGridTransfer::L2Projection::ElemMixedEvaluation(
+   Geometry::Type geom, const FiniteElement& fe_ho, const FiniteElement& fe_lor,
+   IntegrationPointTransformation& ip_tr, const IntegrationRule& ir,
    DenseMatrix& B_L, DenseMatrix& B_H) const
 {
-   int order = fe_lor.GetOrder() + fe_ho.GetOrder() + el_tr->OrderW();
-   const IntegrationRule* ir = &IntRules.Get(geom, order);
-
-   for (int i = 0; i < ir->GetNPoints(); i++)
+   for (int i = 0; i < ir.GetNPoints(); i++)
    {
-      const IntegrationPoint& ip_lor = ir->IntPoint(i);
+      const IntegrationPoint& ip_lor = ir.IntPoint(i);
       IntegrationPoint ip_ho;
 
       // maps integration point ip_lor -> ip_ho
@@ -320,7 +323,6 @@ void L2ProjectionGridTransfer::L2Projection::ElemMixedMass(
          B_H(i, j) = shape_ho(j);
       }
    }
-
 }
 
 void L2ProjectionGridTransfer::L2Projection::MixedMassEA(
@@ -328,10 +330,11 @@ void L2ProjectionGridTransfer::L2Projection::MixedMassEA(
    const FiniteElementSpace& fes_lor_ea,
    Vector &M_LH, MemoryType d_mt_)
 {
-   Mesh* mesh_ho = fes_ho_ea.GetMesh();
-   Mesh* mesh_lor = fes_lor_ea.GetMesh();
-   int nel_ho = mesh_ho->GetNE();
-   int nel_lor = mesh_lor->GetNE();
+   Mesh &mesh_ho = *fes_ho_ea.GetMesh();
+   Mesh &mesh_lor = *fes_lor_ea.GetMesh();
+
+   const int nel_ho = mesh_ho.GetNE();
+   const int nel_lor = mesh_lor.GetNE();
 
    if (nel_ho == 0)
    {
@@ -339,11 +342,11 @@ void L2ProjectionGridTransfer::L2Projection::MixedMassEA(
       return;
    }
 
-   const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();
+   const CoarseFineTransformations& cf_tr = mesh_lor.GetRefinementTransforms();
 
    int nref_max = 0;
    Array<Geometry::Type> geoms;
-   mesh_ho->GetGeometries(mesh_ho->Dimension(), geoms);
+   mesh_ho.GetGeometries(mesh_ho.Dimension(), geoms);
    for (int ig = 0; ig < geoms.Size(); ++ig)
    {
       Geometry::Type geom = geoms[ig];
@@ -360,130 +363,226 @@ void L2ProjectionGridTransfer::L2Projection::MixedMassEA(
    {
       // Assume all HO elements are LOR in the same way
       const int iho = 0;
-      {
-         Array<int> lor_els;
-         ho2lor.GetRow(iho, lor_els);
-         int nref = ho2lor.RowSize(iho);
-
-         Geometry::Type geom = mesh_ho->GetElementBaseGeometry(iho);
-         const FiniteElement &fe_ho = *fes_ho_ea.GetFE(iho);
-         const FiniteElement &fe_lor = *fes_lor_ea.GetFE(lor_els[0]);
-
-         // Allocate space for DenseTensors
-         ElementTransformation *el_tr = fes_lor_ea.GetElementTransformation(0);
-         int order = fe_lor.GetOrder() + fe_ho.GetOrder() + el_tr->OrderW();
-         const IntegrationRule* ir_ea = &IntRules.Get(geom, order);
-         int qPts = ir_ea->GetNPoints();
-
-         // Containers for the basis functions sampled at quadrature points
-         B_L.SetSize(qPts, fe_lor.GetDof(), nref, d_mt);
-         B_H.SetSize(qPts, fe_ho.GetDof(), nref, d_mt);
-         D.SetSize(qPts, nref, nel_ho, d_mt);
-
-         const GeometricFactors *geo_facts =
-            mesh_lor->GetGeometricFactors(*ir_ea, GeometricFactors::DETERMINANTS);
-
-         MFEM_ASSERT(nel_ho*nref == nel_lor, "we expect nel_ho*nref == nel_lor");
-
-         // Setup data at quadrature points
-         // TODO add support for user coefficient
-         const auto W = Reshape(ir_ea->GetWeights().Read(), qPts);
-         const auto J = Reshape(geo_facts->detJ.Read(), qPts, nel_lor);
-         const auto d_D = Reshape(D.Write(), qPts, nref, nel_ho);
-
-         mfem::forall(qPts * nref * nel_ho, [=] MFEM_HOST_DEVICE (int tid)
-         {
-            const int q    = tid % qPts;
-            const int iref = (tid / qPts) % nref;
-            const int iho  = (tid / (qPts * nref)) % nel_ho;
-
-            const int lo_el_id = iref + nref*iho;
-            const real_t detJ = J(q, lo_el_id);
-
-            d_D(q, iref, iho) = W(q) * detJ;
-
-         });
-
-         emb_tr.SetIdentityTransformation(geom);
-         const DenseTensor &pmats = cf_tr.point_matrices[geom];
-
-         // Collect the basis functions
-         for (int iref = 0; iref < nref; ++iref)
-         {
-            int ilor = lor_els[iref];
-            // Now assemble the block-row of the mixed mass matrix associated
-            // with integrating HO functions against LOR functions on the LOR
-            // sub-element.
-
-            // Create the transformation that embeds the fine low-order element
-            // within the coarse high-order element in reference space
-            emb_tr.SetPointMat(pmats(cf_tr.embeddings[ilor].matrix));
-
-            DenseMatrix &b_lo = B_L(ilor);
-            DenseMatrix &b_ho = B_H(ilor);
-
-            ElemMixedMass(geom, fe_ho, fe_lor, el_tr, ip_tr, b_lo, b_ho);
-
-         } // loop over subcells of ho element
-         // end of quadrature point setup
-      }
-
-   } // completed setup of basis function and quadrature point
-
-   // Assemble mixed mass matrix
-   {
-      int iho = 0;
       Array<int> lor_els;
       ho2lor.GetRow(iho, lor_els);
-      int nref = ho2lor.RowSize(iho);
+      const int nref = ho2lor.RowSize(iho);
+      MFEM_VERIFY(nel_ho*nref == nel_lor, "we expect nel_ho*nref == nel_lor");
+
+      Geometry::Type geom = mesh_ho.GetElementBaseGeometry(iho);
+
+      emb_tr.SetIdentityTransformation(geom);
+      const DenseTensor &pmats = cf_tr.point_matrices[geom];
 
       const FiniteElement &fe_ho = *fes_ho_ea.GetFE(iho);
       const FiniteElement &fe_lor = *fes_lor_ea.GetFE(lor_els[0]);
-      const int ndof_ho = fe_ho.GetDof();
-      const int ndof_lor = fe_lor.GetDof();
 
-      const int qPts = D.SizeI();
+      // Allocate space for DenseTensors
+      ElementTransformation &el_tr = *mesh_lor.GetTypicalElementTransformation();
+      const int order = fe_lor.GetOrder() + fe_ho.GetOrder() + el_tr.OrderW()
+                        + coeff_ho.order;
+      const IntegrationRule &ir_ea = IntRules.Get(geom, order);
+      const int qPts = ir_ea.GetNPoints();
 
-      M_LH.SetSize(ndof_lor*ndof_ho*nref*nel_ho, d_mt);
+      // Containers for the basis functions sampled at quadrature points
+      B_L.SetSize(qPts, fe_lor.GetDof(), nref, d_mt);
+      B_H.SetSize(qPts, fe_ho.GetDof(), nref, d_mt);
+      D.SetSize(qPts, nref, nel_ho, d_mt);
 
-      // Rows x columns
-      // Recall MFEM is column major
-      // rows x columns is inverted  - matrix is ndof_lor x ndof_ho
-      auto v_M_LH = Reshape(M_LH.Write(), ndof_lor, ndof_ho, nref,
-                            nel_ho);
+      const GeometricFactors *geo_facts =
+         mesh_lor.GetGeometricFactors(ir_ea, GeometricFactors::DETERMINANTS);
 
-      const int fe_ho_ndof = fe_ho.GetDof();
-      const int fe_lor_ndof = fe_lor.GetDof();
+      Vector coeff_vec(qPts*nel_lor);
+      coeff_vec.UseDevice(true);
 
-      auto d_B_L = Reshape(B_L.Read(), qPts, fe_lor_ndof, nref);
-      auto d_B_H = Reshape(B_H.Read(), qPts, fe_ho_ndof, nref);
-      auto d_D   = Reshape(D.Read(), qPts, nref, nel_ho);
+      const int dim = mesh_ho.Dimension();
+      const int nq1d = (int)floor(pow(ir_ea.Size(), 1.0/dim) + 0.5);
+      const int nref_1d = (int)floor(pow(nref, 1.0/dim) + 0.5);
 
-      mfem::forall(fe_ho_ndof*nref*nel_ho, [=] MFEM_HOST_DEVICE (int idx)
+      if (!coeff_ho)
       {
-         const int bh = idx % fe_ho_ndof;
-         const int iref = (idx / fe_ho_ndof) % nref;
-         const int iho = idx / fe_ho_ndof / nref;
-         // (B_lo_dofs x Q) x (Q x B_ho_dofs)
-         for (int bl = 0; bl < fe_lor_ndof; ++bl)
+         coeff_vec = 1.0;
+      }
+      else if (UsesTensorBasis(fes_ho) &&
+               nq1d*nref_1d <= DeviceDofQuadLimits::Get().MAX_Q1D)
+      {
+         // Fast coefficient evaluation for tensor-product case. We create a
+         // "composite" quadrature rule in the high-order element that is the
+         // union of the quadrature rules within each of the low-order-refined
+         // subelements.
+         //
+         // NOTE: if the integration rule order is high and there are many LOR
+         // subelements, this can create a very big quadrature rule. That is
+         // why we need to check that we do not exceed MAX_Q1D. If we do, then
+         // we fall back on the slower "legacy" evaluation.
+
+         // Construct the composite rule as a tensor-product of the 1D LOR rule.
+         IntegrationRule ir_ho = [&]()
          {
-            real_t dot = 0.0;
-            for (int qi=0; qi<qPts; ++qi)
+            IntegrationRule ir_ho_1d(nq1d * nref_1d);
+            for (int iref = 0; iref < nref_1d; ++iref)
             {
-               dot += d_B_L(qi, bl, iref) *  d_D(qi, iref, iho) * d_B_H(qi, bh, iref);
+               const real_t a = pmats(cf_tr.embeddings[iref].matrix)(0,0);
+               const real_t b = pmats(cf_tr.embeddings[iref].matrix)(0,1);
+               for (int iq = 0; iq < nq1d; ++iq)
+               {
+                  ir_ho_1d[iq + iref*nq1d].x = a + ir_ea[iq].x*(b - a);
+               }
+            }
+            if (dim == 1) { return ir_ho_1d; }
+            else if (dim == 2) { return IntegrationRule(ir_ho_1d, ir_ho_1d); }
+            else { return IntegrationRule(ir_ho_1d, ir_ho_1d, ir_ho_1d); }
+         }();
+
+         // Project the high-order coefficient on the high-order composite rule.
+         QuadratureSpace qs(mesh_ho, ir_ho);
+         CoefficientVector coeff_vec_ho(*coeff_ho.coeff, qs);
+
+         // Permute the coefficient values to the expected LOR ordering.
+         const int nq_ho = ir_ho.Size();
+         const auto d_Q_ho = Reshape(coeff_vec_ho.Read(), nq_ho, nel_ho);
+         const auto d_Q = Reshape(coeff_vec.Write(), qPts, nel_lor);
+
+         mfem::forall(nq_ho * nel_ho, [=] MFEM_HOST_DEVICE (int ii)
+         {
+            const int e_ho = ii / nq_ho;
+            const int iq_ho = ii % nq_ho;
+
+            int iq_tensor = iq_ho;
+            int iq_lor = 0;
+            int iref = 0;
+            int iq_stride = 1;
+            int iref_stride = 1;
+            const int nq_ho_1d = nq1d*nref_1d;
+            for (int d = 0; d < dim; ++d)
+            {
+               const int iq_ho_1d = iq_tensor % nq_ho_1d;
+               iq_tensor /= nq_ho_1d;
+
+               iq_lor += (iq_ho_1d % nq1d)*iq_stride;
+               iref += (iq_ho_1d / nq1d)*iref_stride;
+               iq_stride *= nq1d;
+               iref_stride *= nref_1d;
+            }
+            const int e_lor = iref + e_ho*nref;
+
+            d_Q(iq_lor, e_lor) = d_Q_ho(iq_ho, e_ho);
+         });
+      }
+      else
+      {
+         // Legacy/fallback coefficient evaluation for non-tensor-product cases
+         // or when the number of quadrature points is too large for the device
+         // kernels.
+         IntegrationPoint ip_ho;
+         for (int e_ho = 0; e_ho < nel_ho; ++e_ho)
+         {
+            ElementTransformation &ho_tr = *mesh_ho.GetElementTransformation(e_ho);
+            for (int iref = 0; iref < nref; ++iref)
+            {
+               const int e_lor = iref + e_ho*nref;
+               emb_tr.SetPointMat(pmats(cf_tr.embeddings[e_lor].matrix));
+
+               for (int iq = 0; iq < qPts; ++iq)
+               {
+                  const IntegrationPoint &ip_lor = ir_ea[iq];
+                  ip_tr.Transform(ip_lor, ip_ho);
+                  ho_tr.SetIntPoint(&ip_ho);
+                  coeff_vec[iq + e_lor*qPts] = coeff_ho.coeff->Eval(ho_tr, ip_ho);
+               }
             }
-            // column major storage
-            v_M_LH(bl, bh, iref, iho) = dot;
          }
+      }
+
+      // Setup data at quadrature points
+      const auto W = Reshape(ir_ea.GetWeights().Read(), qPts);
+      const auto J = Reshape(geo_facts->detJ.Read(), qPts, nel_lor);
+      const auto d_D = Reshape(D.Write(), qPts, nref, nel_ho);
+      const auto d_Q = Reshape(coeff_vec.Read(), qPts, nel_lor);
+
+      mfem::forall(qPts * nref * nel_ho, [=] MFEM_HOST_DEVICE (int tid)
+      {
+         const int q    = tid % qPts;
+         const int iref = (tid / qPts) % nref;
+         const int iho  = (tid / (qPts * nref)) % nel_ho;
+
+         const int lo_el_id = iref + nref*iho;
+         const real_t detJ = J(q, lo_el_id);
+
+         d_D(q, iref, iho) = W(q) * d_Q(q, lo_el_id) * detJ;
       });
-   } // end of mixed assembly mass matrix
+
+      // Collect the basis functions
+      for (int iref = 0; iref < nref; ++iref)
+      {
+         int ilor = lor_els[iref];
+         // Now assemble the block-row of the mixed mass matrix associated
+         // with integrating HO functions against LOR functions on the LOR
+         // sub-element.
+
+         // Create the transformation that embeds the fine low-order element
+         // within the coarse high-order element in reference space
+         emb_tr.SetPointMat(pmats(cf_tr.embeddings[ilor].matrix));
+
+         DenseMatrix &b_lo = B_L(ilor);
+         DenseMatrix &b_ho = B_H(ilor);
+
+         ElemMixedEvaluation(geom, fe_ho, fe_lor, ip_tr, ir_ea, b_lo, b_ho);
+      } // loop over subcells of ho element
+      // end of quadrature point setup
+   } // completed setup of basis function and quadrature point
+
+   // Assemble mixed mass matrix
+   int iho = 0;
+   Array<int> lor_els;
+   ho2lor.GetRow(iho, lor_els);
+   int nref = ho2lor.RowSize(iho);
+
+   const FiniteElement &fe_ho = *fes_ho_ea.GetFE(iho);
+   const FiniteElement &fe_lor = *fes_lor_ea.GetFE(lor_els[0]);
+   const int ndof_ho = fe_ho.GetDof();
+   const int ndof_lor = fe_lor.GetDof();
+
+   const int qPts = D.SizeI();
+
+   M_LH.SetSize(ndof_lor*ndof_ho*nref*nel_ho, d_mt);
+
+   // Rows x columns
+   // Recall MFEM is column major
+   // rows x columns is inverted  - matrix is ndof_lor x ndof_ho
+   auto v_M_LH = Reshape(M_LH.Write(), ndof_lor, ndof_ho, nref,
+                         nel_ho);
+
+   const int fe_ho_ndof = fe_ho.GetDof();
+   const int fe_lor_ndof = fe_lor.GetDof();
+
+   auto d_B_L = Reshape(B_L.Read(), qPts, fe_lor_ndof, nref);
+   auto d_B_H = Reshape(B_H.Read(), qPts, fe_ho_ndof, nref);
+   auto d_D   = Reshape(D.Read(), qPts, nref, nel_ho);
+
+   mfem::forall(fe_ho_ndof*nref*nel_ho, [=] MFEM_HOST_DEVICE (int idx)
+   {
+      const int bh = idx % fe_ho_ndof;
+      const int iref = (idx / fe_ho_ndof) % nref;
+      const int iho = idx / fe_ho_ndof / nref;
+      // (B_lo_dofs x Q) x (Q x B_ho_dofs)
+      for (int bl = 0; bl < fe_lor_ndof; ++bl)
+      {
+         real_t dot = 0.0;
+         for (int qi=0; qi<qPts; ++qi)
+         {
+            dot += d_B_L(qi, bl, iref) *  d_D(qi, iref, iho) * d_B_H(qi, bh, iref);
+         }
+         // column major storage
+         v_M_LH(bl, bh, iref, iho) = dot;
+      }
+   });
 }
 
 L2ProjectionGridTransfer::L2ProjectionL2Space::L2ProjectionL2Space
 (const FiniteElementSpace &fes_ho_, const FiniteElementSpace &fes_lor_,
+ CoefficientWithOrder coeff_ho_, CoefficientWithOrder coeff_lor_,
  const bool use_ea_, MemoryType d_mt_)
-   : L2Projection(fes_ho_, fes_lor_, d_mt_),
-     use_ea(use_ea_)
+   : L2Projection(fes_ho_, fes_lor_, coeff_ho_, coeff_lor_, d_mt_), use_ea(use_ea_)
 {
    if (use_ea)
    {
@@ -559,7 +658,11 @@ L2ProjectionGridTransfer::L2ProjectionL2Space::L2ProjectionL2Space
       DenseMatrix Minv_lor(ndof_lor*nref, ndof_lor*nref);
       DenseMatrix M_mixed(ndof_lor*nref, ndof_ho);
 
-      MassIntegrator mi;
+      MassIntegrator mi = [&]()
+      {
+         return coeff_lor ? MassIntegrator(*coeff_lor.coeff) : MassIntegrator();
+      }();
+
       DenseMatrix M_lor_el(ndof_lor, ndof_lor);
       DenseMatrixInverse Minv_lor_el(&M_lor_el);
       DenseMatrix M_lor(ndof_lor*nref, ndof_lor*nref);
@@ -577,6 +680,10 @@ L2ProjectionGridTransfer::L2ProjectionL2Space::L2ProjectionL2Space
          // Assemble the low-order refined mass matrix and invert locally
          int ilor = lor_els[iref];
          ElementTransformation *tr_lor = fes_lor.GetElementTransformation(ilor);
+
+         const int order = 2*fe_lor.GetOrder() + tr_lor->OrderW() + coeff_lor.order;
+         mi.SetIntegrationRule(IntRules.Get(geom, order));
+
          mi.AssembleElementMatrix(fe_lor, *tr_lor, M_lor_el);
          M_lor.CopyMN(M_lor_el, iref*ndof_lor, iref*ndof_lor);
          Minv_lor_el.Factor();
@@ -668,25 +775,22 @@ void L2ProjectionGridTransfer::L2ProjectionL2Space::EAL2ProjectionL2Space()
    // Need to compute M_L
    // Note: Using user-inputted M_LH IntegrationRule ir
    // (higher order than needed) in order to re-use coeff
-   MassIntegrator mi;
+   MassIntegrator mi = [&]()
+   {
+      return coeff_lor ? MassIntegrator(*coeff_lor.coeff) : MassIntegrator();
+   }();
+
+   const int order = 2*fes_lor.GetMaxElementOrder()
+                     + mesh_lor->GetTypicalElementTransformation()->OrderW()
+                     + coeff_lor.order;
+   mi.SetIntegrationRule(
+      IntRules.Get(mesh_lor->GetTypicalElementGeometry(), order));
 
    Vector M_ea_lor;
-   int ndof_lor;
-   int ndof_ho;
-   int nref;
-   {
-      int iho = 0;
-      Array<int> lor_els;
-      ho2lor.GetRow(iho, lor_els);
-      nref = ho2lor.RowSize(iho);
-
-      const FiniteElement &fe_ho = *fes_ho.GetFE(iho);
-      const FiniteElement &fe_lor = *fes_lor.GetFE(lor_els[0]);
-      ndof_ho = fe_ho.GetDof();
-      ndof_lor = fe_lor.GetDof();
-
-      M_ea_lor.SetSize(ndof_lor*ndof_lor*nel_lor, d_mt);
-   }
+   const int ndof_lor = fes_lor.GetTypicalFE()->GetDof();
+   const int ndof_ho = fes_ho.GetTypicalFE()->GetDof();
+   const int nref = ho2lor.RowSize(0);
+   M_ea_lor.SetSize(ndof_lor*ndof_lor*nel_lor, d_mt);
 
    const bool add = false;
    mi.AssembleEA(fes_lor, M_ea_lor, add);
@@ -1032,8 +1136,9 @@ void L2ProjectionGridTransfer::L2ProjectionL2Space::EAProlongateTranspose(
 
 L2ProjectionGridTransfer::L2ProjectionH1Space::L2ProjectionH1Space(
    const FiniteElementSpace& fes_ho_, const FiniteElementSpace& fes_lor_,
+   CoefficientWithOrder coeff_ho_, CoefficientWithOrder coeff_lor_,
    const bool use_ea_, MemoryType d_mt_)
-   : L2Projection(fes_ho_, fes_lor_, d_mt_),
+   : L2Projection(fes_ho_, fes_lor_, coeff_ho_, coeff_lor_, d_mt_),
      use_ea(use_ea_)
 {
 
@@ -1092,8 +1197,9 @@ L2ProjectionGridTransfer::L2ProjectionH1Space::L2ProjectionH1Space(
 
 L2ProjectionGridTransfer::L2ProjectionH1Space::L2ProjectionH1Space(
    const ParFiniteElementSpace& pfes_ho, const ParFiniteElementSpace& pfes_lor,
+   CoefficientWithOrder coeff_ho_, CoefficientWithOrder coeff_lor_,
    const bool use_ea_, MemoryType d_mt_)
-   : L2Projection(pfes_ho, pfes_lor, d_mt_),
+   : L2Projection(pfes_ho, pfes_lor, coeff_ho_, coeff_lor_, d_mt_),
      use_ea(use_ea_), pcg(pfes_ho.GetComm())
 {
 
@@ -1165,12 +1271,12 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::SetupPCG()
 
 void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
 {
-   Mesh* mesh_ho = fes_ho.GetMesh();
-   Mesh* mesh_lor = fes_lor.GetMesh();
-   int nel_ho = mesh_ho->GetNE();
-   int nel_lor = mesh_lor->GetNE();
-   int ndof_ho = fes_ho.GetNDofs();
-   int ndof_lor = fes_lor.GetNDofs();
+   Mesh &mesh_ho = *fes_ho.GetMesh();
+   Mesh &mesh_lor = *fes_lor.GetMesh();
+   const int nel_ho = mesh_ho.GetNE();
+   const int nel_lor = mesh_lor.GetNE();
+   const int ndof_ho = fes_ho.GetNDofs();
+   const int ndof_lor = fes_lor.GetNDofs();
 
    // If the local mesh is empty, skip all computations
    if (nel_ho == 0)
@@ -1178,11 +1284,11 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
       return;
    }
 
-   const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();
+   const CoarseFineTransformations& cf_tr = mesh_lor.GetRefinementTransforms();
 
    int nref_max = 0;
    Array<Geometry::Type> geoms;
-   mesh_ho->GetGeometries(mesh_ho->Dimension(), geoms);
+   mesh_ho.GetGeometries(mesh_ho.Dimension(), geoms);
    for (int ig = 0; ig < geoms.Size(); ++ig)
    {
       Geometry::Type geom = geoms[ig];
@@ -1205,7 +1311,8 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
 
    BilinearForm Mho(fes_ho_scalar.get());
    Mho.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   Mho.AddDomainIntegrator(new MassIntegrator);
+   Mho.AddDomainIntegrator(coeff_ho ? new MassIntegrator(*coeff_ho.coeff)
+                           : new MassIntegrator);
    Mho.Assemble();
 
    // Processor local lumped Mass
@@ -1215,7 +1322,16 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
 
    BilinearForm Mlor(fes_lor_scalar.get());
    Mlor.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   Mlor.AddDomainIntegrator(new MassIntegrator);
+   {
+      MassIntegrator *mi = coeff_lor ? new MassIntegrator(*coeff_lor.coeff)
+                           : new MassIntegrator;
+      const int order = 2*fes_lor.GetMaxElementOrder()
+                        + mesh_lor.GetTypicalElementTransformation()->OrderW()
+                        + coeff_lor.order;
+      mi->SetIntegrationRule(
+         IntRules.Get(mesh_lor.GetTypicalElementGeometry(), order));
+      Mlor.AddDomainIntegrator(mi);
+   }
    Mlor.Assemble();
 
    Vector ones_lor(Mlor.Width()); ones_lor = 1.0;
@@ -1228,15 +1344,14 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
    MixedMassEA(fes_ho, fes_lor, M_LH_ea, d_mt);
 
    // Set ownership
-   M_LH_local_op = new H1SpaceMixedMassOperator(fes_ho_scalar.get(),
-                                                fes_lor_scalar.get(),
-                                                &ho2lor,
-                                                &M_LH_ea);
+   M_LH.reset(new H1SpaceMixedMassOperator(fes_ho_scalar.get(),
+                                           fes_lor_scalar.get(),
+                                           &ho2lor,
+                                           &M_LH_ea));
 
    ML_inv_vea.reset(new H1SpaceLumpedMassOperator(fes_ho_scalar.get(),
                                                   fes_lor_scalar.get(),
                                                   ML_inv_ea));
-   M_LH.reset(M_LH_local_op);
    R.reset(new ProductOperator(ML_inv_vea.get(), M_LH.get(), false,
                                false));
 
@@ -1253,18 +1368,18 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
 void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space
 (const ParFiniteElementSpace& pfes_ho, const ParFiniteElementSpace& pfes_lor)
 {
-   Mesh* mesh_ho = pfes_ho.GetParMesh();
-   Mesh* mesh_lor = pfes_lor.GetParMesh();
-   int nel_ho = mesh_ho->GetNE();
-   int nel_lor = mesh_lor->GetNE();
+   Mesh &mesh_ho = *pfes_ho.GetParMesh();
+   Mesh &mesh_lor = *pfes_lor.GetParMesh();
+   int nel_ho = mesh_ho.GetNE();
+   int nel_lor = mesh_lor.GetNE();
    int ndof_ho = pfes_ho.GetNDofs();
    int ndof_lor = pfes_lor.GetNDofs();
 
-   const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();
+   const CoarseFineTransformations& cf_tr = mesh_lor.GetRefinementTransforms();
 
    int nref_max = 0;
    Array<Geometry::Type> geoms;
-   mesh_ho->GetGeometries(mesh_ho->Dimension(), geoms);
+   mesh_ho.GetGeometries(mesh_ho.Dimension(), geoms);
    for (int ig = 0; ig < geoms.Size(); ++ig)
    {
       Geometry::Type geom = geoms[ig];
@@ -1287,7 +1402,8 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space
 
    ParBilinearForm pMho(pfes_ho_scalar.get());
    pMho.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   pMho.AddDomainIntegrator(new MassIntegrator);
+   pMho.AddDomainIntegrator(coeff_ho ? new MassIntegrator(*coeff_ho.coeff)
+                            : new MassIntegrator);
    pMho.Assemble();
 
    // Processor local lumped Mass
@@ -1297,7 +1413,16 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space
 
    ParBilinearForm pMlor(pfes_lor_scalar.get());
    pMlor.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   pMlor.AddDomainIntegrator(new MassIntegrator);
+   {
+      MassIntegrator *mi = coeff_lor ? new MassIntegrator(*coeff_lor.coeff)
+                           : new MassIntegrator;
+      const int order = 2*fes_lor.GetMaxElementOrder()
+                        + mesh_lor.GetTypicalElementTransformation()->OrderW()
+                        + coeff_lor.order;
+      mi->SetIntegrationRule(
+         IntRules.Get(mesh_lor.GetTypicalElementGeometry(), order));
+      pMlor.AddDomainIntegrator(mi);
+   }
    pMlor.Assemble();
 
    Vector ones_lor(pMlor.Width()); ones_lor = 1.0;
@@ -1570,7 +1695,7 @@ std::unique_ptr<SparseMatrix>>
          int ilor = lor_els[iref];
          ElementTransformation* el_tr = fes_lor.GetElementTransformation(ilor);
 
-         int order = 2 * fe_lor.GetOrder() + el_tr->OrderW();
+         int order = 2 * fe_lor.GetOrder() + el_tr->OrderW() + coeff_lor.order;
          const IntegrationRule* ir = &IntRules.Get(geom, order);
          ML_el = 0.0;
          for (int i = 0; i < ir->GetNPoints(); ++i)
@@ -1578,7 +1703,13 @@ std::unique_ptr<SparseMatrix>>
             const IntegrationPoint& ip_lor = ir->IntPoint(i);
             fe_lor.CalcShape(ip_lor, shape_lor);
             el_tr->SetIntPoint(&ip_lor);
-            ML_el += (shape_lor *= (el_tr->Weight() * ip_lor.weight));
+            real_t w = ip_lor.weight;
+            if (coeff_lor)
+            {
+               w *= coeff_lor.coeff->Eval(*el_tr, ip_lor);
+            }
+            shape_lor *= el_tr->Weight() * w;
+            ML_el += shape_lor;
          }
          fes_lor.GetElementDofs(ilor, dofs_lor);
          ML_inv.AddElementVector(dofs_lor, ML_el);
@@ -2024,8 +2155,8 @@ void L2ProjectionGridTransfer::BuildF()
    {
       if (!Parallel())
       {
-         F = new L2ProjectionH1Space(dom_fes, ran_fes,
-                                     use_ea, d_mt);
+         F = new L2ProjectionH1Space(
+            dom_fes, ran_fes, coeff_ho, coeff_lor, use_ea, d_mt);
       }
       else
       {
@@ -2034,15 +2165,15 @@ void L2ProjectionGridTransfer::BuildF()
             static_cast<mfem::ParFiniteElementSpace&>(dom_fes);
          const mfem::ParFiniteElementSpace& ran_pfes =
             static_cast<mfem::ParFiniteElementSpace&>(ran_fes);
-         F = new L2ProjectionH1Space(dom_pfes, ran_pfes,
-                                     use_ea, d_mt);
+         F = new L2ProjectionH1Space(
+            dom_pfes, ran_pfes, coeff_ho, coeff_lor, use_ea, d_mt);
 #endif
       }
    }
    else
    {
-      F = new L2ProjectionL2Space(dom_fes, ran_fes,
-                                  use_ea, d_mt);
+      F = new L2ProjectionL2Space(
+         dom_fes, ran_fes, coeff_ho, coeff_lor, use_ea, d_mt);
    }
 }
 

fem/transfer.hpp

@@ -19,6 +19,8 @@
 #include "pfespace.hpp"
 #endif
 
+#include <cstddef>
+
 namespace mfem
 {
 
@@ -162,6 +164,18 @@ public:
 };
 
 
+struct CoefficientWithOrder
+{
+   Coefficient *coeff;
+   int order;
+   CoefficientWithOrder() : coeff(nullptr), order(0) { }
+   CoefficientWithOrder(std::nullptr_t) : coeff(nullptr), order(0) { }
+   CoefficientWithOrder(Coefficient &coeff_) : coeff(&coeff_), order(1) { }
+   CoefficientWithOrder(Coefficient &coeff_, int order_)
+      : coeff(&coeff_), order(order_) { }
+   operator bool() const { return coeff != nullptr; }
+};
+
 /** @brief Transfer data in L2 and H1 finite element spaces between a coarse
     mesh and an embedded refined mesh using L2 projection. */
 /** The forward, coarse-to-fine, transfer uses L2 projection. The backward,
@@ -207,6 +221,8 @@ public:
    protected:
       const FiniteElementSpace& fes_ho;
       const FiniteElementSpace& fes_lor;
+      CoefficientWithOrder coeff_ho;
+      CoefficientWithOrder coeff_lor;
 
       MemoryType d_mt;
       Array<int> offsets;
@@ -214,8 +230,15 @@ public:
 
       L2Projection(const FiniteElementSpace& fes_ho_,
                    const FiniteElementSpace& fes_lor_,
+                   CoefficientWithOrder coeff_ho_,
+                   CoefficientWithOrder coeff_lor_,
                    MemoryType d_mt_ = Device::GetHostMemoryType());
 
+      L2Projection(const FiniteElementSpace& fes_ho_,
+                   const FiniteElementSpace& fes_lor_,
+                   MemoryType d_mt_ = Device::GetHostMemoryType())
+         : L2Projection(fes_ho_, fes_lor_, nullptr, nullptr, d_mt_) { }
+
       void BuildHo2Lor(int nel_ho, int nel_lor,
                        const CoarseFineTransformations& cf_tr);
 
@@ -225,11 +248,11 @@ public:
                          IntegrationPointTransformation& ip_tr,
                          DenseMatrix& M_mixed_el) const;
 
-      void ElemMixedMass(Geometry::Type geom, const FiniteElement& fe_ho,
-                         const FiniteElement& fe_lor,
-                         ElementTransformation* el_tr,
-                         IntegrationPointTransformation& ip_tr,
-                         DenseMatrix& B_L, DenseMatrix& B_H) const;
+      void ElemMixedEvaluation(Geometry::Type geom, const FiniteElement& fe_ho,
+                               const FiniteElement& fe_lor,
+                               IntegrationPointTransformation& ip_tr,
+                               const IntegrationRule& ir,
+                               DenseMatrix& B_L, DenseMatrix& B_H) const;
    public:
       /* Returns the Mixed Mass M_LH via device element assembly by building the
       basis functions and data at the quadrature points. */
@@ -287,9 +310,17 @@ public:
    public:
       L2ProjectionL2Space(const FiniteElementSpace& fes_ho_,
                           const FiniteElementSpace& fes_lor_,
+                          CoefficientWithOrder coeff_ho_,
+                          CoefficientWithOrder coeff_lor_,
                           const bool use_ea_,
                           MemoryType d_mt_ = Device::GetHostMemoryType());
 
+      L2ProjectionL2Space(const FiniteElementSpace& fes_ho_,
+                          const FiniteElementSpace& fes_lor_,
+                          const bool use_ea_,
+                          MemoryType d_mt_ = Device::GetHostMemoryType())
+         : L2ProjectionL2Space(fes_ho_, fes_lor_, nullptr, nullptr, use_ea_, d_mt_) { }
+
       /*Same as above but assembles and stores R_ea, P_ea */
       void EAL2ProjectionL2Space();
 
@@ -356,13 +387,30 @@ public:
    public:
       L2ProjectionH1Space(const FiniteElementSpace &fes_ho_,
                           const FiniteElementSpace &fes_lor_,
+                          CoefficientWithOrder coeff_ho_,
+                          CoefficientWithOrder coeff_lor_,
                           const bool use_ea_,
                           MemoryType d_mt_ = Device::GetHostMemoryType());
+
+      L2ProjectionH1Space(const FiniteElementSpace& fes_ho_,
+                          const FiniteElementSpace& fes_lor_,
+                          const bool use_ea_,
+                          MemoryType d_mt_ = Device::GetHostMemoryType())
+         : L2ProjectionH1Space(fes_ho_, fes_lor_, nullptr, nullptr, use_ea_, d_mt_) { }
+
 #ifdef MFEM_USE_MPI
       L2ProjectionH1Space(const ParFiniteElementSpace &pfes_ho_,
                           const ParFiniteElementSpace &pfes_lor_,
+                          CoefficientWithOrder coeff_ho_,
+                          CoefficientWithOrder coeff_lor_,
                           const bool use_ea_,
                           MemoryType d_mt_ = Device::GetHostMemoryType());
+
+      L2ProjectionH1Space(const ParFiniteElementSpace& fes_ho_,
+                          const ParFiniteElementSpace& fes_lor_,
+                          const bool use_ea_,
+                          MemoryType d_mt_ = Device::GetHostMemoryType())
+         : L2ProjectionH1Space(fes_ho_, fes_lor_, nullptr, nullptr, use_ea_, d_mt_) { }
 #endif
       /// Same as above but assembles action of R through 4 parts:
       ///   ( )  inv( lumped(M_L) ), which is a diagonal matrix (essentially a vector)
@@ -508,18 +556,38 @@ public:
       virtual ~L2Prolongation() { }
    };
 
+   /// Coefficient for the mixed L2 inner product.
+   CoefficientWithOrder coeff_ho;
+   /// Coefficient for the low-order L2 inner product.
+   CoefficientWithOrder coeff_lor;
    L2Projection   *F; ///< Forward, coarse-to-fine, operator
    L2Prolongation *B; ///< Backward, fine-to-coarse, operator
    bool force_l2_space;
 
 public:
+   /// Construct the unweighted L2 projection grid transfer.
    L2ProjectionGridTransfer(FiniteElementSpace &coarse_fes_,
                             FiniteElementSpace &fine_fes_,
                             bool force_l2_space_ = false,
                             MemoryType d_mt_ = Device::GetHostMemoryType()) // move to method
       : GridTransfer(coarse_fes_, fine_fes_),
-        F(NULL), B(NULL), force_l2_space(force_l2_space_)
-   { }
+        coeff_ho(nullptr), coeff_lor(nullptr), F(nullptr), B(nullptr),
+        force_l2_space(force_l2_space_) { }
+
+   /// @brief Construct the weighted L2 projection grid transfer.
+   ///
+   /// The low-order inner product is weighted by @a coeff_lor, and the mixed
+   /// inner product is weighted by @a coeff_ho.
+   L2ProjectionGridTransfer(FiniteElementSpace &coarse_fes_,
+                            FiniteElementSpace &fine_fes_,
+                            CoefficientWithOrder coeff_ho_,
+                            CoefficientWithOrder coeff_lor_,
+                            bool force_l2_space_ = false,
+                            MemoryType d_mt_ = Device::GetHostMemoryType()) // move to method
+      : GridTransfer(coarse_fes_, fine_fes_),
+        coeff_ho(coeff_ho_), coeff_lor(coeff_lor_), F(nullptr), B(nullptr),
+        force_l2_space(force_l2_space_) { }
+
    virtual ~L2ProjectionGridTransfer();
 
    const Operator &ForwardOperator() override;
@@ -527,6 +595,7 @@ public:
    const Operator &BackwardOperator() override;
 
    bool SupportsBackwardsOperator() const override;
+
 private:
    void BuildF();
 };

general/backends.hpp

@@ -14,7 +14,7 @@
 
 #include "../config/config.hpp"
 
-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#if defined(MFEM_USE_CUDA)
 #include <cusparse.h>
 #include <library_types.h>
 #include <cuda_runtime.h>
@@ -22,7 +22,7 @@
 #endif
 #include "cuda.hpp"
 
-#if defined(MFEM_USE_HIP) && defined(__HIP__)
+#if defined(MFEM_USE_HIP)
 #include <hip/hip_runtime.h>
 #endif
 #include "hip.hpp"
@@ -45,15 +45,17 @@
 #endif
 
 #if !defined(MFEM_USE_CUDA_OR_HIP)
-constexpr bool mfem_use_gpu = false;
-#define MFEM_DEVICE
-#define MFEM_HOST
-#define MFEM_LAMBDA
-// #define MFEM_HOST_DEVICE // defined in config/config.hpp
 // MFEM_DEVICE_SYNC is made available for debugging purposes
 #define MFEM_DEVICE_SYNC
 // MFEM_STREAM_SYNC is used for UVM and MPI GPU-Aware kernels
 #define MFEM_STREAM_SYNC
+#endif
+
+#if !defined(MFEM_USE_CUDA_OR_HIP_LANG)
+#define MFEM_DEVICE
+#define MFEM_HOST
+#define MFEM_LAMBDA
+// #define MFEM_HOST_DEVICE // defined in config/config.hpp
 #define MFEM_LAUNCH_BOUNDS(...)
 #endif
 
@@ -126,4 +128,23 @@ MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
 #endif
 }
 
+namespace mfem::internal
+{
+
+#if defined(MFEM_USE_CUDA_OR_HIP) && !defined(MFEM_USE_CUDA_OR_HIP_LANG)
+static constexpr bool can_compile_kernels = false;
+#else
+static constexpr bool can_compile_kernels = true;
+#endif
+
+template <bool can_compile_kernels = can_compile_kernels>
+void RequireKernelCompilation()
+{
+   static_assert(
+      can_compile_kernels,
+      "The calling function needs to be compiled with CUDA/HIP language!");
+}
+
+}
+
 #endif // MFEM_BACKENDS_HPP

general/communication.cpp

@@ -1108,6 +1108,126 @@ void GroupCommunicator::ReduceEnd(T *ldata, int layout,
    num_requests = 0;
 }
 
+template <class T>
+void GroupCommunicator::ReduceMarked(T *ldata, const Array<int> &marker,
+                                     int layout,
+                                     void (*Op)(OpData<T>)) const
+{
+   if (comm_lock == 0) { return; }
+   // The above also handles the case (group_buf_size == 0).
+   MFEM_VERIFY(comm_lock == 2, "object is NOT locked for Reduce");
+
+   switch (mode)
+   {
+      case byGroup: // ***** Communication by groups *****
+      {
+         OpData<T> opd;
+         opd.ldata = ldata;
+         Array<int> group_num_req(group_ldof.Size());
+         for (int gr = 1; gr < group_ldof.Size(); gr++)
+         {
+            group_num_req[gr] =
+               gtopo.IAmMaster(gr) ? gtopo.GetGroupSize(gr)-1 : 0;
+         }
+         int idx;
+         while (MPI_Waitany(num_requests, requests, &idx, MPI_STATUS_IGNORE),
+                idx != MPI_UNDEFINED)
+         {
+            int gr = request_marker[idx];
+            if (gr == -1) { continue; } // skip send requests
+
+            // Delay the processing of a group until all receive requests, for
+            // that group, are done:
+            if ((--group_num_req[gr]) != 0) { continue; }
+
+            opd.nldofs = group_ldof.RowSize(gr);
+            // groups without dofs are skipped, so here nldofs > 0.
+
+            opd.buf = (T *)group_buf.GetData() + buf_offsets[gr];
+            opd.ldofs = (layout == 0) ?
+                        group_ldof.GetRow(gr) : group_ltdof.GetRow(gr);
+            opd.nb = gtopo.GetGroupSize(gr)-1;
+
+            // Apply operation only to marked DOFs. The receive buffer is
+            // neighbor-major with stride opd.nldofs, i.e. the contributions to
+            // DOF i are buf[j*opd.nldofs + i] for j = 0 ... opd.nb-1. Setting
+            // nldofs = 1 for a single DOF changes that stride to 1, so the
+            // strided values must first be gathered into a contiguous buffer.
+            Array<T> single_buf(opd.nb);
+            for (int i = 0; i < opd.nldofs; i++)
+            {
+               if (marker[opd.ldofs[i]])
+               {
+                  for (int j = 0; j < opd.nb; j++)
+                  {
+                     single_buf[j] = opd.buf[j*opd.nldofs + i];
+                  }
+
+                  // Create a temporary OpData with just this one DOF
+                  OpData<T> single_opd;
+                  single_opd.ldata = ldata;
+                  single_opd.buf = single_buf.GetData();
+                  single_opd.ldofs = opd.ldofs + i;
+                  single_opd.nldofs = 1;
+                  single_opd.nb = opd.nb;
+
+                  // Apply the operation
+                  Op(single_opd);
+               }
+            }
+         }
+         break;
+      }
+
+      case byNeighbor: // ***** Communication by neighbors *****
+      {
+         MPI_Waitall(num_requests, requests, MPI_STATUSES_IGNORE);
+
+         for (int nbr = 1; nbr < nbr_send_groups.Size(); nbr++)
+         {
+            // In Reduce operation: send_groups <--> recv_groups
+            const int num_recv_groups = nbr_send_groups.RowSize(nbr);
+            if (num_recv_groups > 0)
+            {
+               const int *grp_list = nbr_send_groups.GetRow(nbr);
+               const T *buf = (T*)group_buf.GetData() + buf_offsets[nbr];
+               for (int i = 0; i < num_recv_groups; i++)
+               {
+                  // Custom version of ReduceGroupFromBuffer that checks marker
+                  int gr = grp_list[i];
+                  const int *ldofs = (layout == 0) ?
+                                     group_ldof.GetRow(gr) : group_ltdof.GetRow(gr);
+                  const int nldofs = group_ldof.RowSize(gr);
+
+                  for (int j = 0; j < nldofs; j++)
+                  {
+                     if (marker[ldofs[j]])
+                     {
+                        // Create a temporary OpData with just this one DOF
+                        OpData<T> opd;
+                        opd.ldata = ldata;
+                        opd.buf = const_cast<T*>(buf) + j;
+                        opd.ldofs = ldofs + j;
+                        opd.nldofs = 1;
+                        opd.nb = 1;
+
+                        // Apply the operation
+                        Op(opd);
+                     }
+                  }
+
+                  buf += nldofs;
+               }
+            }
+         }
+         break;
+      }
+   }
+
+   comm_lock = 0; // 0 - no lock
+   num_requests = 0;
+}
+
 template <class T>
 void GroupCommunicator::Sum(OpData<T> opd)
 {
@@ -1171,6 +1291,8 @@ void GroupCommunicator::Max(OpData<T> opd)
 template <class T>
 void GroupCommunicator::BitOR(OpData<T> opd)
 {
+   static_assert(std::is_integral<T>::value,
+                 "BitOR reduction requires an integral type.");
    for (int i = 0; i < opd.nldofs; i++)
    {
       T data = opd.ldata[opd.ldofs[i]];
@@ -1182,6 +1304,33 @@ void GroupCommunicator::BitOR(OpData<T> opd)
    }
 }
 
+template <class T>
+void GroupCommunicator::MaxAbs(OpData<T> opd)
+{
+   for (int i = 0; i < opd.nldofs; i++)
+   {
+      T data = opd.ldata[opd.ldofs[i]];
+      T abs_data = std::abs(data);
+
+      for (int j = 0; j < opd.nb; j++)
+      {
+         T b = opd.buf[j*opd.nldofs+i];
+         T abs_b = std::abs(b);
+
+         // On an equal-magnitude tie keep the more positive value, so
+         // opposite-sign ties resolve deterministically to the positive one.
+         if (abs_data < abs_b || (abs_data == abs_b && data < b))
+         {
+            data = b;
+            abs_data = abs_b;
+         }
+      }
+
+      opd.ldata[opd.ldofs[i]] = data;
+   }
+}
+
+
 void GroupCommunicator::PrintInfo(std::ostream &os) const
 {
    char c = '\0';
@@ -1318,18 +1467,24 @@ template void GroupCommunicator::BcastEnd<int>(int *, int) const;
 template void GroupCommunicator::ReduceBegin<int>(const int *) const;
 template void GroupCommunicator::ReduceEnd<int>(
    int *, int, void (*)(OpData<int>)) const;
+template void GroupCommunicator::ReduceMarked<int>(
+   int*, const Array<int>&, int, void (*)(OpData<int>)) const;
 
 template void GroupCommunicator::BcastBegin<double>(double *, int) const;
 template void GroupCommunicator::BcastEnd<double>(double *, int) const;
 template void GroupCommunicator::ReduceBegin<double>(const double *) const;
 template void GroupCommunicator::ReduceEnd<double>(
    double *, int, void (*)(OpData<double>)) const;
+template void GroupCommunicator::ReduceMarked<double>(
+   double*, const Array<int>&, int, void (*)(OpData<double>)) const;
 
 template void GroupCommunicator::BcastBegin<float>(float *, int) const;
 template void GroupCommunicator::BcastEnd<float>(float *, int) const;
 template void GroupCommunicator::ReduceBegin<float>(const float *) const;
 template void GroupCommunicator::ReduceEnd<float>(
    float *, int, void (*)(OpData<float>)) const;
+template void GroupCommunicator::ReduceMarked<float>(
+   float*, const Array<int>&, int, void (*)(OpData<float>)) const;
 
 // @endcond
 
@@ -1338,14 +1493,17 @@ template void GroupCommunicator::Sum<int>(OpData<int>);
 template void GroupCommunicator::Min<int>(OpData<int>);
 template void GroupCommunicator::Max<int>(OpData<int>);
 template void GroupCommunicator::BitOR<int>(OpData<int>);
+template void GroupCommunicator::MaxAbs<int>(OpData<int>);
 
 template void GroupCommunicator::Sum<double>(OpData<double>);
 template void GroupCommunicator::Min<double>(OpData<double>);
 template void GroupCommunicator::Max<double>(OpData<double>);
+template void GroupCommunicator::MaxAbs<double>(OpData<double>);
 
 template void GroupCommunicator::Sum<float>(OpData<float>);
 template void GroupCommunicator::Min<float>(OpData<float>);
 template void GroupCommunicator::Max<float>(OpData<float>);
+template void GroupCommunicator::MaxAbs<float>(OpData<float>);
 
 
 #ifdef __bgq__

general/communication.hpp

@@ -22,6 +22,7 @@
 #include "globals.hpp"
 #include <mpi.h>
 #include <cstdint>
+#include <type_traits>
 
 // can't directly use MPI_CXX_BOOL because Microsoft's MPI implementation
 // doesn't include MPI_CXX_BOOL. Fallback to MPI_C_BOOL if unavailable.
@@ -408,14 +409,38 @@ public:
    template <class T> void Reduce(Array<T> &ldata, void (*Op)(OpData<T>)) const
    { Reduce<T>((T *)ldata, Op); }
 
-   /// Reduce operation Sum, instantiated for int and double
+   /// Reduce operation Sum, instantiated for int, double and float
    template <class T> static void Sum(OpData<T>);
-   /// Reduce operation Min, instantiated for int and double
+   /// Reduce operation Min, instantiated for int, double and float
    template <class T> static void Min(OpData<T>);
-   /// Reduce operation Max, instantiated for int and double
+   /// Reduce operation Max, instantiated for int, double and float
    template <class T> static void Max(OpData<T>);
    /// Reduce operation bitwise OR, instantiated for int only
    template <class T> static void BitOR(OpData<T>);
+   /// Reduce operation selecting the signed value with the largest absolute
+   /// value, instantiated for int, double and float. The result keeps its sign;
+   /// it is not the non-negative absolute value. Equal-magnitude ties are
+   /// broken deterministically toward the more positive value, so opposite-sign
+   /// ties resolve to the positive one regardless of accumulation order.
+   template <class T> static void MaxAbs(OpData<T>);
+
+   /** @brief Finalize reduction operation started with ReduceBegin(), but only apply
+       the reduction to DOFs marked in the marker array.
+
+       @note The reduction is carried out in the signed type @a T, so the result
+       is signed even for bitwise operations.
+   */
+   template <class T>
+   void ReduceMarked(T *ldata, const Array<int> &marker, int layout,
+                     void (*Op)(OpData<T>)) const;
+
+   /** @brief Reduce within each group where the master is the root, but only for marked DOFs. */
+   template <class T>
+   void Reduce(T *ldata, const Array<int> &marker, void (*Op)(OpData<T>)) const
+   {
+      ReduceBegin(ldata);
+      ReduceMarked(ldata, marker, 0, Op);
+   }
 
    /// Print information about the GroupCommunicator from all MPI ranks.
    void PrintInfo(std::ostream &out = mfem::out) const;

general/cuda.hpp

@@ -18,14 +18,8 @@
 // CUDA block size used by MFEM.
 #define MFEM_CUDA_BLOCKS 256
 
-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#if defined(MFEM_USE_CUDA)
 #define MFEM_USE_CUDA_OR_HIP
-constexpr bool mfem_use_gpu = true;
-#define MFEM_DEVICE __device__
-#define MFEM_HOST __host__
-#define MFEM_LAMBDA __host__
-#define MFEM_LAUNCH_BOUNDS __launch_bounds__
-// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
 #define MFEM_DEVICE_SYNC MFEM_GPU_CHECK(cudaDeviceSynchronize())
 #define MFEM_STREAM_SYNC MFEM_GPU_CHECK(cudaStreamSynchronize(0))
 // Define a CUDA error check macro, MFEM_GPU_CHECK(x), where x returns/is of
@@ -40,6 +34,15 @@ constexpr bool mfem_use_gpu = true;
     }                                                                          \
   } while (0)
 
+// Macros defined only when compiling with CUDA language
+#if defined(__CUDACC__)
+#define MFEM_USE_CUDA_OR_HIP_LANG
+#define MFEM_DEVICE __device__
+#define MFEM_HOST __host__
+#define MFEM_LAMBDA __host__
+#define MFEM_LAUNCH_BOUNDS __launch_bounds__
+// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
+
 // Define the MFEM inner threading macros
 #if defined(__CUDA_ARCH__)
 #define MFEM_SHARED __shared__
@@ -67,12 +70,13 @@ constexpr bool mfem_use_gpu = true;
    if (int ix = threadIdx.k % (OX), iy = threadIdx.k / (OX), iz = iy / (OY);   \
        (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__CUDA_ARCH__)
-#endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#endif // defined(__CUDACC__)
+#endif // defined(MFEM_USE_CUDA)
 
 namespace mfem
 {
 
-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#if defined(MFEM_USE_CUDA)
 // Function used by the macro MFEM_GPU_CHECK.
 void mfem_cuda_error(cudaError_t err, const char *expr, const char *func,
                      const char *file, int line);