- deleted Qfunction

# Conflicts:
#	linalg/tensor.hpp

.github/workflows/builds-and-tests.yml

@@ -142,10 +142,6 @@ jobs:
 
     continue-on-error: ${{ matrix.enzyme && true || false }}
 
-    # Enable ccache for all jobs except Windows (would need sccache).
-    env:
-      USE_CCACHE: ${{ matrix.os != 'windows-latest' }}
-
     steps:
       # Fix 'No space left on device' errors for Ubuntu builds.
       - name: Run Actions Cleaner
@@ -294,52 +290,6 @@ jobs:
           echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
           echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
 
-      # Restore the compiler cache (ccache). The key embeds the run id, so new
-      # runs save a fresh snapshot; the restore-keys prefix warm-starts from the
-      # most recent prior run (incl. the base branch for PRs).
-      - name: cache ccache
-        if: ${{ env.USE_CCACHE == 'true' }}
-        uses: actions/cache@v5
-        with:
-          path: .ccache
-          key: ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-${{ github.run_id }}
-          restore-keys: |
-            ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-
-
-      # Configure ccache and select how it is injected into the MFEM build:
-      # - make:  set CXX="ccache g++"; for MPI, OMPI_CXX="ccache g++" so mpicxx
-      #          runs ccache around g++ (not ccache around the mpicxx wrapper).
-      # - cmake: set CMAKE_<LANG>_COMPILER_LAUNCHER=ccache.
-      # - enzyme: wrap the brew clang++ via OMPI_CXX.
-      # The chosen options are passed through build-mfem's 'config-options'
-      # input (see the build step below).
-      - name: configure ccache
-        if: ${{ env.USE_CCACHE == 'true' }}
-        run: |
-          command -v ccache >/dev/null 2>&1 || {
-            if [[ "${{ runner.os }}" == "Linux" ]]; then
-              sudo apt-get update && sudo apt-get install -y ccache
-            else
-              brew install ccache
-            fi
-          }
-          echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
-          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
-          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
-          # Ignore header timestamps (restamped by each checkout) so direct mode hits.
-          echo "CCACHE_SLOPPINESS=include_file_mtime,include_file_ctime,time_macros" >> $GITHUB_ENV
-          # Hash absolute paths relative to the workspace.
-          echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
-          if [[ "${{ matrix.enzyme }}" == "true" ]]; then
-            echo "OMPI_CXX=ccache $LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
-          elif [[ "${{ matrix.build-system }}" == "cmake" ]]; then
-            echo 'CCACHE_CONFIG_OPTS=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache' >> $GITHUB_ENV
-          else
-            echo "OMPI_CXX=ccache g++" >> $GITHUB_ENV
-            echo 'CCACHE_CONFIG_OPTS=CXX="ccache g++" MPICXX="mpicxx"' >> $GITHUB_ENV
-          fi
-        shell: bash
-
       # MFEM build and test
       - name: build
         uses: mfem/github-actions/build-mfem@v2.7
@@ -355,14 +305,9 @@ jobs:
           metis-dir: ${{ env.METIS_TOP_DIR }}
           mfem-dir: ${{ env.MFEM_TOP_DIR }}
           precision: ${{ matrix.precision }}
-          config-options: ${{ matrix.config-opts }} ${{ env.CCACHE_CONFIG_OPTS }}
+          config-options: ${{ matrix.config-opts }}
           library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
 
-      - name: ccache stats
-        if: ${{ env.USE_CCACHE == 'true' }}
-        run: ccache -s
-        shell: bash
-
       # Run checks (and only checks) on debug targets
       - name: checks
         if: matrix.build-system == 'make' && matrix.target == 'dbg'

.github/workflows/cleanup-pr-caches.yml

@@ -1,42 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-# A closed PR's caches can never be restored again, so delete them to free
-# space against the 10 GB per-repo cache limit.
-name: Cleanup PR caches
-
-on:
-  pull_request:
-    types: [closed]
-
-permissions:
-  actions: write
-
-jobs:
-  cleanup:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Delete caches for the closed PR
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GH_REPO: ${{ github.repository }}
-          PR_REF: refs/pull/${{ github.event.pull_request.number }}/merge
-        run: |
-          echo "Deleting caches for $PR_REF"
-          while :; do
-            ids=$(gh cache list --ref "$PR_REF" --limit 100 --json id --jq '.[].id')
-            [ -n "$ids" ] || break
-            echo "$ids" | while read -r id; do
-              [ -n "$id" ] || continue
-              echo "Deleting cache $id"
-              gh cache delete "$id" || echo "  (already gone)"
-            done
-          done

.github/workflows/repo-check.yml

@@ -13,7 +13,6 @@ name: "Checks"
 
 permissions:
   actions: write
-  pull-requests: read
 
 on:
   push:
@@ -30,11 +29,6 @@ concurrency:
 # by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
 # and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
 # github.repository").
-#
-# The logic for the branch-history check is slightly different, since that check
-# also inspects the PR's labels to allow for overriding failures. In this case,
-# we run on all 'pull_request' triggers, but only run for 'push' triggers that
-# do not correspond to any open PRs.
 
 jobs:
   file-headers-check:
@@ -134,7 +128,10 @@ jobs:
 
   branch-history:
     if: |
-      github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
+      github.ref != 'refs/heads/next' &&
+      github.ref != 'refs/heads/master' &&
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
     runs-on: ubuntu-latest
     steps:
       - name: checkout mfem
@@ -142,27 +139,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: check for pull request
-        id: check_pr
-        if: github.event_name == 'push'
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          pr_exists=$(gh pr list --repo "$GITHUB_REPOSITORY" \
-            --head "$GITHUB_REF_NAME" \
-            --state open \
-            --json number \
-            --jq 'length > 0')
-          echo "pr_exists=$pr_exists" >> "$GITHUB_OUTPUT"
-
       - name: branch-history
-        id: branch_history
-        if: |
-          (github.event_name == 'pull_request' ||
-           github.event_name == 'workflow_dispatch' ||
-           steps.check_pr.outputs.pr_exists == 'false')
-        continue-on-error: ${{ contains(github.event.pull_request.labels.*.name,
-                                        'branch-history-override') }}
         run: |
           # We override origin to make sure we point to the main repo.
           # This is to have consistent test results on PRs from forks.
@@ -170,9 +147,3 @@ jobs:
           git remote add origin https://github.com/mfem/mfem.git
           git checkout -b gh-actions-branch-history
           ./config/githooks/pre-push --history
-
-      - name: report branch-history override
-        if: steps.branch_history.outcome == 'failure'
-        run: |
-          echo "::warning::branch-history check failed, but the" \
-            "'branch-history-override' label is set."

.gitignore

@@ -260,7 +260,6 @@ miniapps/meshing/polar-nc
 miniapps/meshing/mesh-quality
 miniapps/meshing/hpref
 miniapps/meshing/phpref
-miniapps/meshing/pref321
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh

.gitlab/dane-baseline.yml

@@ -102,14 +102,12 @@ report_baseline:
         mkdir -p ${MACHINE_NAME}
         rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
         rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        status=0
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir} || { status=1; }
+        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
         printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
           >> ${rundir}/pipeline.txt
         # We create an autotest-email.html file, because that's how we signal
         # that there was an error / diff (temporary).
-        if [[ $status -ne 0 ]] || \
-           [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
+        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
            [[ -f ${rundir}/${BASELINE_TEST}-${MACHINE_NAME}.diff ]]; then
           cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
         fi

CHANGELOG

@@ -44,31 +44,15 @@ Discretization improvements
 - Added methods to estimate function extremum using piecewise linear bounds plus
   recursive subdivision.
 
+<<<<<<<<< Temporary merge branch 1
+- Added class VectorQuadratureSpace that represents a vector (multi-component)
+  version of the scalar (single-component) class QuadratureSpaceBase.
+=========
 - Extend FindPointsGSLIB to support surface meshes.
-
-- Added support for complex-valued mixed bilinear forms via the new classes
-  MixedSesquilinearForm and ParMixedSesquilinearForm, mirroring the existing
-  SesquilinearForm classes. Rectangular complex operators are now also
-  handled correctly by ComplexSparseMatrix::GetSystemMatrix and
-  ComplexHypreParMatrix::GetSystemMatrix, which previously assumed equal
-  trial and test spaces.
-
-- Added FiniteElementSpace::GetBoundaryLoopEdgeDofs to extract the edge DOFs on
-  the perimeter loop of a set of boundary elements, with a ParFiniteElementSpace
-  overload that reconciles the selection across processor boundaries so the
-  result is partition invariant. This is useful for imposing boundary conditions
-  on boundary edge DOFs.
-
-- Added a MaxAbs reduction to GroupCommunicator that selects the signed value of
-  largest magnitude across a group, keeping its sign. Equal-magnitude ties
-  resolve deterministically to the positive value.
+>>>>>>>>> Temporary merge branch 2
 
 Meshing improvements
 --------------------
-- Added support for nonuniform anisotropic mesh refinement on parallel quad/hex
-  meshes with arbitrary spacing in each direction. This enables in particular
-  3:1 refinement in parallel, as demonstrated in the new meshing miniapp pref321.
-
 - Added option to guarantee mesh validity during TMOP-based r-adaptivity, using
   bounds on the determinant of the mesh transformation Jacobian.
 
@@ -89,55 +73,16 @@ Linear and nonlinear solvers
   PRefinement multigrid methods for problems posed on trace spaces (see e.g. the
   DPG miniapps).
 
-- Added new class MultiVector: an array of Vectors of different sizes where each
-  Vector can be allocated independently. Also, added associated methods in class
-  Operator: MultMV, MultTransposeMV, and GetGradientMV, that use MultiVector
-  objects for input and/or output parameters. [PR #5249]
-
 GPU computing
 -------------
-- Improved partial assembly for VectorDivergenceIntegrator with shared-memory
-  kernels, kernel registration, and transpose support.
-
-- Improved partial-assembly diagonal kernels for VectorMassIntegrator (shared-
-  memory specializations) and ElasticityIntegrator (no scratch Q-vector).
-
-- Added PA gradient and diagonal support for VectorConvectionNLFIntegrator
- (AssembleGradPA, AddMultGradPA, AssembleGradDiagonalPA).
-
 - Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
 
-- Added partial assembly support for MixedScalarWeakGradientIntegrator.
-
-- Added partial assembly support for MixedDotProductIntegrator.
-
-- Added partial assembly support for MixedScalarCrossProductIntegrator.
-
-- Added partial assembly support for MixedScalarWeakCrossProductIntegrator.
-
-- Added partial assembly support for MixedVectorGradientIntegrator for H1->RT.
-
-- Added support for device partial assembly CurlInterpolator.
-  This supports 2D and 3D variants:
-    2D H1 (out-of-plane) to RT (in-plane)
-    2D ND (in-plane) to Integral L2 (out-of-plane)
-    3D ND to RT
-
 - Added NVIDIA cuDSS library interface. Implementation examples have been
   added to ex1 and ex1p. See https://developer.nvidia.com/cudss for more
   details. Supported versions >= 0.6.0.
 
 - Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
 
-- Changed VectorFEMassIntegrator to use kernel specialization dispatch for
-  partial assembly.
-
-- Added support for FiniteElement::MapType::INTEGRAL spaces to
-  QuadratureInterpolator.
-
-- Added support for FiniteElement::MapType::INTEGRAL spaces to
-  MixedScalarCurlIntegrator.
-
 New and updated examples and miniapps
 -------------------------------------
 - The Lorentz miniapp (in miniapps/electromagnetics) has been updated to
@@ -152,20 +97,6 @@ Miscellaneous
   using the new method ApplyDofSigns() in class ParFiniteElementSpace: the
   method will return immediately if no sign flips are needed.
 
-- Added support for coefficient-weighted LOR transfer in
-  L2ProjectionGridTransfer. The transfer conserves the weighted mass, for
-  example when transferring velocity while conserving density-weighted momentum.
-  This is illustrated in the lor-transfer and plor-transfer miniapps.
-
-- Added support for saving DataCollection output on the node-local storage,
-  instead of requiring that the filesystem is shared among all the ranks.
-
-API changes
------------
-- Removed ProjectGrad from 2D RT elements. Users should use ProjectCurl instead.
-  This also fixes a bug where ProjectCurl was returning the negative curl,
-  identical to ProjectGrad.
-
 
 Version 4.9, released on Dec 11, 2025
 =====================================

CMakeLists.txt

@@ -88,9 +88,18 @@ if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
-# RAJA requires C++20:
-if ((MFEM_USE_UMPIRE OR MFEM_USE_RAJA) AND ("${CMAKE_CXX_STANDARD}" LESS "20"))
-  set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard to use." FORCE)
+# Ginkgo requires C++17:
+if ((MFEM_USE_GINKGO) AND ("${CMAKE_CXX_STANDARD}" LESS "17"))
+  set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to use." FORCE)
+# Google Benchmark, SUNDIALS, STRUMPACK, Tribol, RAJA and Umpire require C++14:
+elseif ((MFEM_USE_BENCHMARK OR
+         MFEM_USE_SUNDIALS OR
+         MFEM_USE_STRUMPACK OR
+         MFEM_USE_TRIBOL OR
+         MFEM_USE_RAJA OR
+         MFEM_USE_UMPIRE) AND
+    ("${CMAKE_CXX_STANDARD}" LESS "14"))
+  set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ standard to use." FORCE)
 endif()
 
 # Include xSDK default CMake file.
@@ -230,13 +239,6 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
-# Shadow warnings for clang only; GCC's -Wshadow flags more.
-if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall -Wshadow")
-elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall")
-endif()
-
 # Shared build on Windows
 if (WIN32 AND BUILD_SHARED_LIBS)
   # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?
@@ -599,6 +601,13 @@ if (MFEM_USE_ENZYME)
     set(ENZYME_INCLUDE_DIRS ${ENZYME_DIR}/include)
 endif()
 
+if (MFEM_USE_PROTEUS)
+    enable_language(C)
+    find_package(proteus REQUIRED PATHS "${PROTEUS_DIR}")
+    message(STATUS "${PROTEUS_DIR}/include")
+    include_directories("${PROTEUS_DIR}/include")
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -735,6 +744,16 @@ mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
+
+if (MFEM_USE_PROTEUS)
+    add_library(ClangProteusFlags INTERFACE IMPORTED)
+    set_target_properties(ClangProteusFlags PROPERTIES
+        INTERFACE_COMPILE_OPTIONS "-fpass-plugin=$<TARGET_FILE:ProteusPass>"
+    )
+    target_link_libraries(mfem PUBLIC ClangProteusFlags)
+    target_link_libraries(mfem PUBLIC proteus)
+endif()
+
 if (TPL_TARGETS)
   add_dependencies(mfem ${TPL_TARGETS})
 endif()
@@ -771,7 +790,7 @@ set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
 # If building out-of-source, define MFEM_CONFIG_FILE to point to the config file
 # inside the build directory.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
-  target_compile_definitions(mfem PRIVATE
+  target_compile_definitions(mfem PUBLIC
     "MFEM_CONFIG_FILE=\"${PROJECT_BINARY_DIR}/config/_config.hpp\"")
 endif()
 
@@ -831,16 +850,15 @@ if (MFEM_ENABLE_TESTING)
   add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
   add_subdirectory(tests EXCLUDE_FROM_ALL)
 
-  if (MFEM_USE_BENCHMARK)
-    # Create a target for all benchmarks and, optionally, enable it.
-    set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
-    add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
-                    ${MFEM_ENABLE_BENCHMARKS})
-    if (MFEM_ENABLE_BENCHMARKS)
-      add_subdirectory(tests/benchmarks) #install benchmarks if enabled
-    else()
-      add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
-    endif()
+  # Create a target for all benchmarks and, optionally, enable it. Some simple
+  # timer-based benchmarks in tests/benchmarks do not require Google Benchmark.
+  set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
+  add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
+                  ${MFEM_ENABLE_BENCHMARKS})
+  if (MFEM_ENABLE_BENCHMARKS)
+    add_subdirectory(tests/benchmarks) #install benchmarks if enabled
+  else()
+    add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
   endif()
 endif()
 

config/config.hpp

@@ -18,9 +18,9 @@
 #define MFEM_CONFIG_HPP
 
 #ifdef MFEM_CONFIG_FILE
-#include MFEM_CONFIG_FILE
+#include MFEM_CONFIG_FILE // IWYU pragma: export
 #else
-#include "_config.hpp"
+#include "_config.hpp" // IWYU pragma: export
 #endif
 
 #include <cstdint>

config/defaults.mk

@@ -27,10 +27,7 @@ MPICXX = mpicxx
 
 BASE_FLAGS  = -std=c++17
 OPTIM_FLAGS = -O3 $(BASE_FLAGS)
-
-# The variable WARNING_FLAGS depends on which compiler is used, and is defined
-# later in this file.
-DEBUG_FLAGS = $(strip -g $(addprefix $(XCOMPILER),$(WARNING_FLAGS)) $(BASE_FLAGS))
+DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS)
 
 # Prefixes for passing flags to the compiler and linker when using CXX or MPICXX
 CXX_XCOMPILER =
@@ -49,10 +46,6 @@ SHARED = NO
 #
 # If you set MFEM_USE_ENZYME=YES, must use CUDA_CXX=clang++
 CUDA_CXX = nvcc
-# CUDA compute capability used during compilation, e.g. sm_60. Multiple
-# architectures can be requested as a comma-separated list, e.g. sm_70,sm_80.
-# A single value may also be one of the nvcc special values "all",
-# "all-major", or "native".
 CUDA_ARCH = sm_60
 # Base CUDA install directory, only needed if building with clang+cuda:
 # The default setting is:
@@ -61,23 +54,11 @@ CUDA_ARCH = sm_60
 # 3. Use /usr/local/cuda
 CUDA_DIR = $(or $(CUDA_HOME),$(patsubst %/,%,$(dir \
  $(patsubst %/,%,$(dir $(shell command -v nvcc))))),/usr/local/cuda)
-# Derive nvcc/clang architecture flags from CUDA_ARCH. A comma-separated list
-# expands into one -gencode / --cuda-gpu-arch flag per architecture; otherwise
-# use the -arch / --cuda-gpu-arch shorthand.
-MFEM_COMMA := ,
-CUDA_ARCH_NUMS = $(patsubst sm_%,%,$(subst $(MFEM_COMMA), ,$(CUDA_ARCH)))
-NVCC_ARCH_FLAGS = $(strip $(if $(findstring $(MFEM_COMMA),$(CUDA_ARCH)),\
- $(foreach arch,$(CUDA_ARCH_NUMS),\
- -gencode arch=compute_$(arch)$(MFEM_COMMA)code=sm_$(arch)),\
- -arch=$(CUDA_ARCH)))
-CLANG_ARCH_FLAGS = $(strip $(if $(findstring $(MFEM_COMMA),$(CUDA_ARCH)),\
- $(foreach arch,$(CUDA_ARCH_NUMS),--cuda-gpu-arch=sm_$(arch)),\
- --cuda-gpu-arch=$(CUDA_ARCH)))
 # flags for clang+cuda
-CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) $(CLANG_ARCH_FLAGS)
+CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) --cuda-gpu-arch=$(CUDA_ARCH)
 # flags for nvcc
 NVCC_FLAGS = -x=cu --expt-extended-lambda --expt-relaxed-constexpr \
- $(NVCC_ARCH_FLAGS) -isystem "$(CUDA_DIR)/include"
+ -arch=$(CUDA_ARCH) -isystem "$(CUDA_DIR)/include"
 # Prefixes for passing flags to the host compiler and linker when using
 # CUDA_CXX=nvcc
 CUDA_XCOMPILER = -Xcompiler=
@@ -395,7 +376,7 @@ CUDSS_LIBRARY_DIR = $(CUDSS_DIR)/lib
 CUDSS_OPT         = -I$(CUDSS_INCLUDE_DIR)
 CUDSS_LIB         = \
  $(XLINKER)-rpath,$(CUDSS_LIBRARY_DIR) -L$(CUDSS_LIBRARY_DIR) -lcudss
-# The cuDSS communication and threading libraries.
+# The cuDSS communication and threading libraries. 
 MFEM_CUDSS_COMM_LIB = $(abspath $(wildcard $(or $(CUDSS_COMM_LIB),\
    $(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so))))
 MFEM_CUDSS_THREADING_LIB = $(abspath $(wildcard $(or $(CUDSS_THREADING_LIB),\
@@ -678,15 +659,3 @@ VERBOSE = NO
 
 # Optional build tag
 MFEM_BUILD_TAG = $(shell uname -snm)
-
-# Enable -pedantic flag only for gcc or clang. nvcc complains with -pedantic
-# because of line directives.
-PEDANTIC_FLAG = $(if \
-   $(findstring NVIDIA,$(shell $(MFEM_CXX) --version 2>&1)),, \
-   $(if $(or \
-      $(findstring gcc version,$(shell $(MFEM_CXX) -v 2>&1)), \
-      $(findstring clang version,$(shell $(MFEM_CXX) -v 2>&1))),-pedantic,))
-# Enable shadow warnings for clang only; GCC's -Wshadow flags more.
-SHADOW_WARNING_FLAG = $(if $(findstring clang,\
- $(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
-WARNING_FLAGS = $(PEDANTIC_FLAG) -Wall $(SHADOW_WARNING_FLAG)

config/githooks/README.md

@@ -39,8 +39,3 @@ when a picture was added for documentation.
 If that is the case, make sure the failure is indeed justified, and rerun the
 push command with the `--no-verify` option. This will skip the hooks, allowing
 you to push those changes.
-
-The `branch-history` check is run automatically through GitHub Actions. If a
-branch is known to have a large number of changes that are legitimate, the
-check can be overridden by setting  the label 'branch-history-override' on the
-pull request.

data/nested_cubes.geo

@@ -1,131 +0,0 @@
-// Define the cube sizes
-L_outer = 1.0;
-L_inner = 0.5;
-
-// Set mesh size and algorithm
-mesh_size = 0.4;
-Mesh.Algorithm3D = 1; // Delaunay algorithm for 3D mesh
-Mesh.CharacteristicLengthFactor = 1.0;
-Mesh.MshFileVersion = 2.2;
-
-// Define center point for concentric cubes
-cx = 0.5;
-cy = 0.5;
-cz = 0.5;
-
-// Define the points (vertices of the outer cube)
-Point(1) = {cx-L_outer/2, cy-L_outer/2, cz-L_outer/2, mesh_size};
-Point(2) = {cx+L_outer/2, cy-L_outer/2, cz-L_outer/2, mesh_size};
-Point(3) = {cx+L_outer/2, cy+L_outer/2, cz-L_outer/2, mesh_size};
-Point(4) = {cx-L_outer/2, cy+L_outer/2, cz-L_outer/2, mesh_size};
-Point(5) = {cx-L_outer/2, cy-L_outer/2, cz+L_outer/2, mesh_size};
-Point(6) = {cx+L_outer/2, cy-L_outer/2, cz+L_outer/2, mesh_size};
-Point(7) = {cx+L_outer/2, cy+L_outer/2, cz+L_outer/2, mesh_size};
-Point(8) = {cx-L_outer/2, cy+L_outer/2, cz+L_outer/2, mesh_size};
-
-// Define the points (vertices of the inner cube)
-Point(9) = {cx-L_inner/2, cy-L_inner/2, cz-L_inner/2, mesh_size};
-Point(10) = {cx+L_inner/2, cy-L_inner/2, cz-L_inner/2, mesh_size};
-Point(11) = {cx+L_inner/2, cy+L_inner/2, cz-L_inner/2, mesh_size};
-Point(12) = {cx-L_inner/2, cy+L_inner/2, cz-L_inner/2, mesh_size};
-Point(13) = {cx-L_inner/2, cy-L_inner/2, cz+L_inner/2, mesh_size};
-Point(14) = {cx+L_inner/2, cy-L_inner/2, cz+L_inner/2, mesh_size};
-Point(15) = {cx+L_inner/2, cy+L_inner/2, cz+L_inner/2, mesh_size};
-Point(16) = {cx-L_inner/2, cy+L_inner/2, cz+L_inner/2, mesh_size};
-
-// Define the lines (edges of the outer cube)
-Line(1) = {1, 2};
-Line(2) = {2, 3};
-Line(3) = {3, 4};
-Line(4) = {4, 1};
-Line(5) = {5, 6};
-Line(6) = {6, 7};
-Line(7) = {7, 8};
-Line(8) = {8, 5};
-Line(9) = {1, 5};
-Line(10) = {2, 6};
-Line(11) = {3, 7};
-Line(12) = {4, 8};
-
-// Define the lines (edges of the inner cube)
-Line(13) = {9, 10};
-Line(14) = {10, 11};
-Line(15) = {11, 12};
-Line(16) = {12, 9};
-Line(17) = {13, 14};
-Line(18) = {14, 15};
-Line(19) = {15, 16};
-Line(20) = {16, 13};
-Line(21) = {9, 13};
-Line(22) = {10, 14};
-Line(23) = {11, 15};
-Line(24) = {12, 16};
-
-// Define the surfaces (faces of the outer cube)
-Line Loop(1) = {1, 2, 3, 4};
-Plane Surface(1) = {1};
-
-Line Loop(2) = {5, 6, 7, 8};
-Plane Surface(2) = {2};
-
-Line Loop(3) = {9, 5, -10, -1};
-Plane Surface(3) = {3};
-
-Line Loop(4) = {10, 6, -11, -2};
-Plane Surface(4) = {4};
-
-Line Loop(5) = {11, 7, -12, -3};
-Plane Surface(5) = {5};
-
-Line Loop(6) = {12, 8, -9, -4};
-Plane Surface(6) = {6};
-
-// Define the surfaces (faces of the inner cube)
-Line Loop(7) = {13, 14, 15, 16};
-Plane Surface(7) = {7};
-
-Line Loop(8) = {17, 18, 19, 20};
-Plane Surface(8) = {8};
-
-Line Loop(9) = {21, 17, -22, -13};
-Plane Surface(9) = {9};
-
-Line Loop(10) = {22, 18, -23, -14};
-Plane Surface(10) = {10};
-
-Line Loop(11) = {23, 19, -24, -15};
-Plane Surface(11) = {11};
-
-Line Loop(12) = {24, 20, -21, -16};
-Plane Surface(12) = {12};
-
-// Define the volumes
-Surface Loop(1) = {1, 2, 3, 4, 5, 6};
-Surface Loop(2) = {7, 8, 9, 10, 11, 12};
-Volume(1) = {1, 2}; // Outer volume with inner hole
-Volume(2) = {2};    // Inner volume
-
-// Assign physical groups
-Physical Volume(1) = {1}; // Outer volume
-Physical Volume(2) = {2}; // Inner volume
-
-// Outer cube surfaces
-Physical Surface(1) = {1}; // Outer bottom
-Physical Surface(2) = {2}; // Outer top
-Physical Surface(3) = {3}; // Outer front
-Physical Surface(4) = {4}; // Outer right
-Physical Surface(5) = {5}; // Outer back
-Physical Surface(6) = {6}; // Outer left
-
-// Inner cube surfaces
-Physical Surface(7) = {7};  // Inner bottom (-xy)
-Physical Surface(8) = {8};  // Inner top (+xy)
-Physical Surface(9) = {9};  // Inner front (-xz)
-Physical Surface(10) = {10}; // Inner right (+yz)
-Physical Surface(11) = {11}; // Inner back (+xz)
-Physical Surface(12) = {12}; // Inner left (-yz)
-
-// Mesh control
-Mesh.OptimizeNetgen = 1;
-Mesh.Optimize = 1;
-Mesh.ElementOrder = 1;

data/nested_cubes.msh

@@ -1,907 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-138
-1 0 0 0
-2 1 0 0
-3 1 1 0
-4 0 1 0
-5 0 0 1
-6 1 0 1
-7 1 1 1
-8 0 1 1
-9 0.25 0.25 0.25
-10 0.75 0.25 0.25
-11 0.75 0.75 0.25
-12 0.25 0.75 0.25
-13 0.25 0.25 0.75
-14 0.75 0.25 0.75
-15 0.75 0.75 0.75
-16 0.25 0.75 0.75
-17 0.3333333333325025 0 0
-18 0.6666666666657889 0 0
-19 1 0.3333333333325025 0
-20 1 0.6666666666657889 0
-21 0.6666666666675911 1 0
-22 0.3333333333347203 1 0
-23 0 0.6666666666675911 0
-24 0 0.3333333333347203 0
-25 0.3333333333325025 0 1
-26 0.6666666666657889 0 1
-27 1 0.3333333333325025 1
-28 1 0.6666666666657889 1
-29 0.6666666666675911 1 1
-30 0.3333333333347203 1 1
-31 0 0.6666666666675911 1
-32 0 0.3333333333347203 1
-33 0 0 0.3333333333325025
-34 0 0 0.6666666666657889
-35 1 0 0.3333333333325025
-36 1 0 0.6666666666657889
-37 1 1 0.3333333333325025
-38 1 1 0.6666666666657889
-39 0 1 0.3333333333325025
-40 0 1 0.6666666666657889
-41 0.5000000000003468 0.25 0.25
-42 0.75 0.5000000000003468 0.25
-43 0.5000000000013763 0.75 0.25
-44 0.25 0.5000000000013763 0.25
-45 0.5000000000003468 0.25 0.75
-46 0.75 0.5000000000003468 0.75
-47 0.5000000000013763 0.75 0.75
-48 0.25 0.5000000000013763 0.75
-49 0.25 0.25 0.5000000000003468
-50 0.75 0.25 0.5000000000003468
-51 0.75 0.75 0.5000000000003468
-52 0.25 0.75 0.5000000000003468
-53 0.7113248654055673 0.4999999999991457 0
-54 0.2886751345942123 0.5000000000011557 0
-55 0.5000000000006117 0.7525600817161773 0
-56 0.4999999999993867 0.2474399182839603 0
-57 0.2423197548524782 0.7576802451481532 0
-58 0.757680245147464 0.2423197548520695 0
-59 0.2423197548507857 0.2423197548513912 0
-60 0.7576802451491019 0.7576802451486099 0
-61 0.7113248654055673 0.4999999999991457 1
-62 0.2886751345942123 0.5000000000011557 1
-63 0.5000000000006117 0.7525600817161773 1
-64 0.4999999999993867 0.2474399182839603 1
-65 0.2423197548524782 0.7576802451481532 1
-66 0.757680245147464 0.2423197548520695 1
-67 0.2423197548507857 0.2423197548513912 1
-68 0.7576802451491019 0.7576802451486099 1
-69 0.4999999999993203 0 0.301447615129799
-70 0.4999999999992795 0 0.7028666213189801
-71 0.7525600817158393 0 0.5007190394076877
-72 0.2474399182836191 0 0.5007190394076877
-73 0.7576802451479793 0 0.7576802451479793
-74 0.2423197548517962 0 0.7576802451477375
-75 0.7576802451484569 0 0.2423197548510767
-76 0.2423197548513188 0 0.2423197548513187
-77 1 0.4999999999993203 0.301447615129799
-78 1 0.4999999999992795 0.7028666213189801
-79 1 0.7525600817158394 0.5007190394076877
-80 1 0.2474399182836191 0.5007190394076877
-81 1 0.7576802451479794 0.7576802451479794
-82 1 0.2423197548517962 0.7576802451477376
-83 1 0.7576802451484569 0.2423197548510768
-84 1 0.2423197548513188 0.2423197548513188
-85 0.5000000000008327 1 0.3014476151298047
-86 0.500000000000961 1 0.7028666213191928
-87 0.2474399182842484 1 0.5007190394077241
-88 0.7525600817164384 1 0.5007190394078933
-89 0.2423197548520873 1 0.7576802451480517
-90 0.7576802451481496 1 0.2423197548518761
-91 0.2423197548516099 1 0.2423197548510044
-92 0.7576802451486874 1 0.7576802451481952
-93 0 0.5000000000008327 0.3014476151298047
-94 0 0.500000000000961 0.7028666213191928
-95 0 0.2474399182842484 0.5007190394077241
-96 0 0.7525600817164384 0.5007190394078933
-97 0 0.2423197548520873 0.7576802451480517
-98 0 0.7576802451481496 0.2423197548518761
-99 0 0.2423197548516099 0.2423197548510044
-100 0 0.7576802451486874 0.7576802451481952
-101 0.3968750000003409 0.603125000000244 0.25
-102 0.4374999999998713 0.4375000000001287 0.25
-103 0.5739583333335919 0.5718750000001767 0.25
-104 0.6093749999999631 0.3906250000003402 0.25
-105 0.3968750000003409 0.603125000000244 0.75
-106 0.4374999999998713 0.4375000000001287 0.75
-107 0.5739583333335919 0.5718750000001767 0.75
-108 0.6093749999999631 0.3906250000003402 0.75
-109 0.3806942419826734 0.25 0.3806942419826734
-110 0.5625000000001735 0.25 0.4375000000000001
-111 0.4254282069971791 0.25 0.5712615403304835
-112 0.6093749999998808 0.25 0.6093749999998808
-113 0.75 0.3806942419826734 0.3806942419826734
-114 0.75 0.5625000000001735 0.4375000000000001
-115 0.75 0.4254282069971791 0.5712615403304835
-116 0.75 0.6093749999998808 0.6093749999998808
-117 0.3968750000004991 0.75 0.3968750000002804
-118 0.4375000000000001 0.75 0.5625000000004308
-119 0.5739583333336153 0.75 0.4281250000000707
-120 0.6093749999998166 0.75 0.6093749999993661
-121 0.25 0.3968750000004991 0.3968750000002804
-122 0.25 0.4375000000000001 0.5625000000004308
-123 0.25 0.5739583333336153 0.4281250000000707
-124 0.25 0.6093749999998166 0.6093749999993661
-125 0.4962939304035875 0.5214350017087855 0.4925553109323813
-126 0.3432581985549767 0.6471275530923523 0.3554206606168006
-127 0.6442168181713744 0.5929232373214773 0.3593785632300704
-128 0.625174517421737 0.3491579444372839 0.3604636129462503
-129 0.6130544111091688 0.6576364353993367 0.5046720648819553
-130 0.4281518698369243 0.3632662294430484 0.3548726263205102
-131 0.3639383531355198 0.3520949250221278 0.4978389662613994
-132 0.629585530087249 0.3489878162230438 0.5124654846339122
-133 0.3710853378652663 0.6517586292698121 0.6382643241302075
-134 0.5917018263727056 0.6522525456211955 0.6390764961119443
-135 0.3530810314228338 0.4582477062424107 0.6430239639637545
-136 0.6571010904289998 0.5300774811423468 0.6603970500567977
-137 0.6484596018596915 0.361399676127967 0.6360267588157177
-138 0.4782020887035478 0.3534611476388013 0.6141275027013793
-$EndNodes
-$Elements
-760
-1 2 2 1 1 1 17 59
-2 2 2 1 1 24 1 59
-3 2 2 1 1 18 2 58
-4 2 2 1 1 2 19 58
-5 2 2 1 1 20 3 60
-6 2 2 1 1 3 21 60
-7 2 2 1 1 22 4 57
-8 2 2 1 1 4 23 57
-9 2 2 1 1 17 18 56
-10 2 2 1 1 17 56 59
-11 2 2 1 1 56 18 58
-12 2 2 1 1 19 20 53
-13 2 2 1 1 19 53 58
-14 2 2 1 1 53 20 60
-15 2 2 1 1 21 22 55
-16 2 2 1 1 21 55 60
-17 2 2 1 1 55 22 57
-18 2 2 1 1 23 24 54
-19 2 2 1 1 23 54 57
-20 2 2 1 1 54 24 59
-21 2 2 1 1 54 53 55
-22 2 2 1 1 53 54 56
-23 2 2 1 1 55 53 60
-24 2 2 1 1 53 56 58
-25 2 2 1 1 54 55 57
-26 2 2 1 1 56 54 59
-27 2 2 2 2 5 25 67
-28 2 2 2 2 32 5 67
-29 2 2 2 2 26 6 66
-30 2 2 2 2 6 27 66
-31 2 2 2 2 28 7 68
-32 2 2 2 2 7 29 68
-33 2 2 2 2 30 8 65
-34 2 2 2 2 8 31 65
-35 2 2 2 2 25 26 64
-36 2 2 2 2 25 64 67
-37 2 2 2 2 64 26 66
-38 2 2 2 2 27 28 61
-39 2 2 2 2 27 61 66
-40 2 2 2 2 61 28 68
-41 2 2 2 2 29 30 63
-42 2 2 2 2 29 63 68
-43 2 2 2 2 63 30 65
-44 2 2 2 2 31 32 62
-45 2 2 2 2 31 62 65
-46 2 2 2 2 62 32 67
-47 2 2 2 2 62 61 63
-48 2 2 2 2 61 62 64
-49 2 2 2 2 63 61 68
-50 2 2 2 2 61 64 66
-51 2 2 2 2 62 63 65
-52 2 2 2 2 64 62 67
-53 2 2 3 3 17 1 76
-54 2 2 3 3 1 33 76
-55 2 2 3 3 2 18 75
-56 2 2 3 3 35 2 75
-57 2 2 3 3 5 25 74
-58 2 2 3 3 34 5 74
-59 2 2 3 3 26 6 73
-60 2 2 3 3 6 36 73
-61 2 2 3 3 18 17 69
-62 2 2 3 3 69 17 76
-63 2 2 3 3 18 69 75
-64 2 2 3 3 25 26 70
-65 2 2 3 3 25 70 74
-66 2 2 3 3 70 26 73
-67 2 2 3 3 33 34 72
-68 2 2 3 3 33 72 76
-69 2 2 3 3 72 34 74
-70 2 2 3 3 36 35 71
-71 2 2 3 3 71 35 75
-72 2 2 3 3 36 71 73
-73 2 2 3 3 69 70 71
-74 2 2 3 3 70 69 72
-75 2 2 3 3 69 71 75
-76 2 2 3 3 72 69 76
-77 2 2 3 3 71 70 73
-78 2 2 3 3 70 72 74
-79 2 2 4 4 19 2 84
-80 2 2 4 4 2 35 84
-81 2 2 4 4 3 20 83
-82 2 2 4 4 37 3 83
-83 2 2 4 4 6 27 82
-84 2 2 4 4 36 6 82
-85 2 2 4 4 28 7 81
-86 2 2 4 4 7 38 81
-87 2 2 4 4 20 19 77
-88 2 2 4 4 77 19 84
-89 2 2 4 4 20 77 83
-90 2 2 4 4 27 28 78
-91 2 2 4 4 27 78 82
-92 2 2 4 4 78 28 81
-93 2 2 4 4 35 36 80
-94 2 2 4 4 35 80 84
-95 2 2 4 4 80 36 82
-96 2 2 4 4 38 37 79
-97 2 2 4 4 79 37 83
-98 2 2 4 4 38 79 81
-99 2 2 4 4 77 78 79
-100 2 2 4 4 78 77 80
-101 2 2 4 4 77 79 83
-102 2 2 4 4 80 77 84
-103 2 2 4 4 79 78 81
-104 2 2 4 4 78 80 82
-105 2 2 5 5 21 3 90
-106 2 2 5 5 3 37 90
-107 2 2 5 5 4 22 91
-108 2 2 5 5 39 4 91
-109 2 2 5 5 7 29 92
-110 2 2 5 5 38 7 92
-111 2 2 5 5 30 8 89
-112 2 2 5 5 8 40 89
-113 2 2 5 5 22 21 85
-114 2 2 5 5 85 21 90
-115 2 2 5 5 22 85 91
-116 2 2 5 5 29 30 86
-117 2 2 5 5 29 86 92
-118 2 2 5 5 86 30 89
-119 2 2 5 5 37 38 88
-120 2 2 5 5 37 88 90
-121 2 2 5 5 88 38 92
-122 2 2 5 5 40 39 87
-123 2 2 5 5 87 39 91
-124 2 2 5 5 40 87 89
-125 2 2 5 5 85 86 87
-126 2 2 5 5 86 85 88
-127 2 2 5 5 85 87 91
-128 2 2 5 5 88 85 90
-129 2 2 5 5 87 86 89
-130 2 2 5 5 86 88 92
-131 2 2 6 6 1 24 99
-132 2 2 6 6 33 1 99
-133 2 2 6 6 23 4 98
-134 2 2 6 6 4 39 98
-135 2 2 6 6 32 5 97
-136 2 2 6 6 5 34 97
-137 2 2 6 6 8 31 100
-138 2 2 6 6 40 8 100
-139 2 2 6 6 24 23 93
-140 2 2 6 6 93 23 98
-141 2 2 6 6 24 93 99
-142 2 2 6 6 31 32 94
-143 2 2 6 6 31 94 100
-144 2 2 6 6 94 32 97
-145 2 2 6 6 34 33 95
-146 2 2 6 6 95 33 99
-147 2 2 6 6 34 95 97
-148 2 2 6 6 39 40 96
-149 2 2 6 6 39 96 98
-150 2 2 6 6 96 40 100
-151 2 2 6 6 93 94 95
-152 2 2 6 6 94 93 96
-153 2 2 6 6 93 95 99
-154 2 2 6 6 96 93 98
-155 2 2 6 6 95 94 97
-156 2 2 6 6 94 96 100
-157 2 2 7 7 9 41 102
-158 2 2 7 7 44 9 102
-159 2 2 7 7 41 10 104
-160 2 2 7 7 10 42 104
-161 2 2 7 7 42 11 103
-162 2 2 7 7 11 43 103
-163 2 2 7 7 43 12 101
-164 2 2 7 7 12 44 101
-165 2 2 7 7 102 41 104
-166 2 2 7 7 42 103 104
-167 2 2 7 7 43 101 103
-168 2 2 7 7 101 44 102
-169 2 2 7 7 101 102 103
-170 2 2 7 7 103 102 104
-171 2 2 8 8 13 45 106
-172 2 2 8 8 48 13 106
-173 2 2 8 8 45 14 108
-174 2 2 8 8 14 46 108
-175 2 2 8 8 46 15 107
-176 2 2 8 8 15 47 107
-177 2 2 8 8 47 16 105
-178 2 2 8 8 16 48 105
-179 2 2 8 8 106 45 108
-180 2 2 8 8 46 107 108
-181 2 2 8 8 47 105 107
-182 2 2 8 8 105 48 106
-183 2 2 8 8 105 106 107
-184 2 2 8 8 107 106 108
-185 2 2 9 9 41 9 109
-186 2 2 9 9 9 49 109
-187 2 2 9 9 10 41 110
-188 2 2 9 9 50 10 110
-189 2 2 9 9 13 45 111
-190 2 2 9 9 49 13 111
-191 2 2 9 9 45 14 112
-192 2 2 9 9 14 50 112
-193 2 2 9 9 41 109 110
-194 2 2 9 9 111 45 112
-195 2 2 9 9 109 49 111
-196 2 2 9 9 50 110 112
-197 2 2 9 9 110 109 111
-198 2 2 9 9 110 111 112
-199 2 2 10 10 42 10 113
-200 2 2 10 10 10 50 113
-201 2 2 10 10 11 42 114
-202 2 2 10 10 51 11 114
-203 2 2 10 10 14 46 115
-204 2 2 10 10 50 14 115
-205 2 2 10 10 46 15 116
-206 2 2 10 10 15 51 116
-207 2 2 10 10 42 113 114
-208 2 2 10 10 115 46 116
-209 2 2 10 10 113 50 115
-210 2 2 10 10 51 114 116
-211 2 2 10 10 114 113 115
-212 2 2 10 10 114 115 116
-213 2 2 11 11 43 11 119
-214 2 2 11 11 11 51 119
-215 2 2 11 11 12 43 117
-216 2 2 11 11 52 12 117
-217 2 2 11 11 15 47 120
-218 2 2 11 11 51 15 120
-219 2 2 11 11 47 16 118
-220 2 2 11 11 16 52 118
-221 2 2 11 11 117 43 119
-222 2 2 11 11 47 118 120
-223 2 2 11 11 119 51 120
-224 2 2 11 11 52 117 118
-225 2 2 11 11 118 117 119
-226 2 2 11 11 118 119 120
-227 2 2 12 12 9 44 121
-228 2 2 12 12 49 9 121
-229 2 2 12 12 44 12 123
-230 2 2 12 12 12 52 123
-231 2 2 12 12 48 13 122
-232 2 2 12 12 13 49 122
-233 2 2 12 12 16 48 124
-234 2 2 12 12 52 16 124
-235 2 2 12 12 121 44 123
-236 2 2 12 12 48 122 124
-237 2 2 12 12 49 121 122
-238 2 2 12 12 123 52 124
-239 2 2 12 12 122 121 123
-240 2 2 12 12 122 123 124
-241 4 2 1 1 105 62 106 107
-242 4 2 1 1 102 54 101 103
-243 4 2 1 1 118 86 120 119
-244 4 2 1 1 124 94 123 122
-245 4 2 1 1 52 39 12 96
-246 4 2 1 1 52 12 39 87
-247 4 2 1 1 88 38 15 51
-248 4 2 1 1 79 15 38 51
-249 4 2 1 1 80 14 50 36
-250 4 2 1 1 71 50 14 36
-251 4 2 1 1 120 88 15 51
-252 4 2 1 1 116 15 79 51
-253 4 2 1 1 50 14 112 71
-254 4 2 1 1 52 117 12 87
-255 4 2 1 1 111 69 109 110
-256 4 2 1 1 114 77 115 113
-257 4 2 1 1 124 123 94 96
-258 4 2 1 1 120 86 88 119
-259 4 2 1 1 14 45 64 108
-260 4 2 1 1 103 54 101 55
-261 4 2 1 1 62 105 63 107
-262 4 2 1 1 63 29 15 47
-263 4 2 1 1 14 45 26 64
-264 4 2 1 1 51 11 88 37
-265 4 2 1 1 80 50 10 35
-266 4 2 1 1 49 33 9 72
-267 4 2 1 1 71 10 50 35
-268 4 2 1 1 79 11 51 37
-269 4 2 1 1 49 9 33 95
-270 4 2 1 1 12 123 52 96
-271 4 2 1 1 96 16 52 40
-272 4 2 1 1 49 13 34 72
-273 4 2 1 1 87 52 16 40
-274 4 2 1 1 13 49 34 95
-275 4 2 1 1 43 101 12 55
-276 4 2 1 1 43 12 22 55
-277 4 2 1 1 106 61 108 107
-278 4 2 1 1 102 103 104 53
-279 4 2 1 1 50 115 14 80
-280 4 2 1 1 80 10 50 113
-281 4 2 1 1 9 109 49 72
-282 4 2 1 1 47 15 63 107
-283 4 2 1 1 124 16 52 96
-284 4 2 1 1 49 121 9 95
-285 4 2 1 1 15 28 68 46
-286 4 2 1 1 81 28 15 46
-287 4 2 1 1 92 15 29 47
-288 4 2 1 1 27 82 14 46
-289 4 2 1 1 14 66 27 46
-290 4 2 1 1 26 45 14 73
-291 4 2 1 1 78 115 116 114
-292 4 2 1 1 112 70 111 110
-293 4 2 1 1 79 11 114 51
-294 4 2 1 1 71 50 10 110
-295 4 2 1 1 10 41 104 56
-296 4 2 1 1 77 80 115 113
-297 4 2 1 1 69 109 72 111
-298 4 2 1 1 63 16 30 47
-299 4 2 1 1 25 45 13 64
-300 4 2 1 1 52 16 118 87
-301 4 2 1 1 95 13 49 122
-302 4 2 1 1 57 12 23 44
-303 4 2 1 1 119 11 88 51
-304 4 2 1 1 41 17 9 56
-305 4 2 1 1 18 41 10 56
-306 4 2 1 1 21 11 43 55
-307 4 2 1 1 63 105 16 47
-308 4 2 1 1 13 45 106 64
-309 4 2 1 1 9 102 41 56
-310 4 2 1 1 72 49 13 111
-311 4 2 1 1 11 103 43 55
-312 4 2 1 1 70 112 71 110
-313 4 2 1 1 116 79 78 114
-314 4 2 1 1 32 13 48 67
-315 4 2 1 1 16 89 30 47
-316 4 2 1 1 32 48 13 97
-317 4 2 1 1 48 16 100 31
-318 4 2 1 1 121 123 93 122
-319 4 2 1 1 119 117 118 85
-320 4 2 1 1 10 19 42 84
-321 4 2 1 1 17 9 76 41
-322 4 2 1 1 24 9 59 44
-323 4 2 1 1 42 19 10 58
-324 4 2 1 1 10 41 18 75
-325 4 2 1 1 20 83 11 42
-326 4 2 1 1 90 11 43 21
-327 4 2 1 1 53 104 102 56
-328 4 2 1 1 106 61 64 108
-329 4 2 1 1 87 118 117 85
-330 4 2 1 1 95 121 93 122
-331 4 2 1 1 80 50 115 113
-332 4 2 1 1 109 49 72 111
-333 4 2 1 1 78 27 28 46
-334 4 2 1 1 31 94 48 32
-335 4 2 1 1 19 77 20 42
-336 4 2 1 1 23 24 44 93
-337 4 2 1 1 123 124 52 96
-338 4 2 1 1 88 120 119 51
-339 4 2 1 1 91 43 85 117
-340 4 2 1 1 39 98 12 96
-341 4 2 1 1 12 91 39 87
-342 4 2 1 1 15 92 38 88
-343 4 2 1 1 38 81 15 79
-344 4 2 1 1 36 80 14 82
-345 4 2 1 1 14 71 36 73
-346 4 2 1 1 103 102 54 53
-347 4 2 1 1 61 106 62 107
-348 4 2 1 1 86 29 30 47
-349 4 2 1 1 26 45 70 25
-350 4 2 1 1 21 85 22 43
-351 4 2 1 1 94 93 123 122
-352 4 2 1 1 119 118 86 85
-353 4 2 1 1 12 96 93 123
-354 4 2 1 1 115 78 77 114
-355 4 2 1 1 69 111 70 110
-356 4 2 1 1 28 27 61 46
-357 4 2 1 1 48 62 31 32
-358 4 2 1 1 88 15 92 120
-359 4 2 1 1 79 81 15 116
-360 4 2 1 1 71 14 112 73
-361 4 2 1 1 114 116 79 51
-362 4 2 1 1 112 50 71 110
-363 4 2 1 1 91 12 117 87
-364 4 2 1 1 80 115 14 82
-365 4 2 1 1 63 15 29 68
-366 4 2 1 1 64 26 14 66
-367 4 2 1 1 47 63 105 107
-368 4 2 1 1 43 103 101 55
-369 4 2 1 1 35 10 80 84
-370 4 2 1 1 33 76 9 72
-371 4 2 1 1 37 11 88 90
-372 4 2 1 1 35 71 10 75
-373 4 2 1 1 37 79 11 83
-374 4 2 1 1 9 99 33 95
-375 4 2 1 1 68 63 15 107
-376 4 2 1 1 99 44 93 121
-377 4 2 1 1 18 69 41 17
-378 4 2 1 1 10 35 2 84
-379 4 2 1 1 15 7 28 81
-380 4 2 1 1 3 37 90 11
-381 4 2 1 1 108 64 14 66
-382 4 2 1 1 51 38 79 37
-383 4 2 1 1 12 4 23 98
-384 4 2 1 1 12 39 91 4
-385 4 2 1 1 49 33 72 34
-386 4 2 1 1 10 71 110 75
-387 4 2 1 1 114 11 79 83
-388 4 2 1 1 40 96 16 100
-389 4 2 1 1 34 13 74 72
-390 4 2 1 1 16 87 40 89
-391 4 2 1 1 34 97 13 95
-392 4 2 1 1 20 19 42 53
-393 4 2 1 1 54 23 24 44
-394 4 2 1 1 87 16 118 89
-395 4 2 1 1 95 97 13 122
-396 4 2 1 1 57 22 12 55
-397 4 2 1 1 41 102 104 56
-398 4 2 1 1 64 45 106 108
-399 4 2 1 1 101 57 12 55
-400 4 2 1 1 74 13 5 25
-401 4 2 1 1 13 34 74 5
-402 4 2 1 1 8 40 16 100
-403 4 2 1 1 16 31 65 8
-404 4 2 1 1 82 36 6 14
-405 4 2 1 1 102 9 59 56
-406 4 2 1 1 106 67 13 64
-407 4 2 1 1 119 88 11 90
-408 4 2 1 1 95 49 121 122
-409 4 2 1 1 52 118 117 87
-410 4 2 1 1 124 16 96 100
-411 4 2 1 1 80 10 113 84
-412 4 2 1 1 109 9 76 72
-413 4 2 1 1 72 13 70 111
-414 4 2 1 1 99 9 121 95
-415 4 2 1 1 30 65 16 63
-416 4 2 1 1 13 67 25 64
-417 4 2 1 1 11 55 53 103
-418 4 2 1 1 123 12 44 93
-419 4 2 1 1 56 104 10 58
-420 4 2 1 1 10 18 56 58
-421 4 2 1 1 21 60 11 55
-422 4 2 1 1 17 59 9 56
-423 4 2 1 1 16 63 62 105
-424 4 2 1 1 48 105 16 62
-425 4 2 1 1 111 45 13 70
-426 4 2 1 1 119 86 88 85
-427 4 2 1 1 123 93 94 96
-428 4 2 1 1 115 78 80 77
-429 4 2 1 1 111 72 69 70
-430 4 2 1 1 55 103 54 53
-431 4 2 1 1 63 61 62 107
-432 4 2 1 1 14 27 66 6
-433 4 2 1 1 99 9 1 24
-434 4 2 1 1 49 33 34 95
-435 4 2 1 1 96 52 39 40
-436 4 2 1 1 22 12 91 4
-437 4 2 1 1 20 11 83 3
-438 4 2 1 1 29 15 92 7
-439 4 2 1 1 18 10 75 2
-440 4 2 1 1 36 50 80 35
-441 4 2 1 1 18 17 41 56
-442 4 2 1 1 51 88 38 37
-443 4 2 1 1 62 31 16 48
-444 4 2 1 1 62 16 31 65
-445 4 2 1 1 26 45 25 64
-446 4 2 1 1 104 58 42 10
-447 4 2 1 1 29 63 30 47
-448 4 2 1 1 77 78 79 114
-449 4 2 1 1 71 69 70 110
-450 4 2 1 1 42 103 11 53
-451 4 2 1 1 93 94 95 122
-452 4 2 1 1 118 87 86 85
-453 4 2 1 1 61 62 106 64
-454 4 2 1 1 53 102 54 56
-455 4 2 1 1 78 82 27 46
-456 4 2 1 1 94 48 32 97
-457 4 2 1 1 86 30 89 47
-458 4 2 1 1 78 28 81 46
-459 4 2 1 1 26 70 45 73
-460 4 2 1 1 100 94 48 31
-461 4 2 1 1 86 92 29 47
-462 4 2 1 1 76 69 17 41
-463 4 2 1 1 42 19 77 84
-464 4 2 1 1 77 83 20 42
-465 4 2 1 1 18 41 69 75
-466 4 2 1 1 43 85 90 21
-467 4 2 1 1 91 43 117 12
-468 4 2 1 1 48 62 32 67
-469 4 2 1 1 27 66 61 46
-470 4 2 1 1 68 28 61 46
-471 4 2 1 1 54 24 59 44
-472 4 2 1 1 19 42 53 58
-473 4 2 1 1 23 54 57 44
-474 4 2 1 1 12 93 96 98
-475 4 2 1 1 99 44 121 9
-476 4 2 1 1 42 58 104 53
-477 4 2 1 1 21 43 22 55
-478 4 2 1 1 53 20 11 42
-479 4 2 1 1 53 11 20 60
-480 4 2 1 1 106 67 48 13
-481 4 2 1 1 36 71 50 35
-482 4 2 1 1 52 87 39 40
-483 4 2 1 1 48 67 106 62
-484 4 2 1 1 72 70 13 74
-485 4 2 1 1 9 33 99 1
-486 4 2 1 1 3 37 11 83
-487 4 2 1 1 13 34 5 97
-488 4 2 1 1 12 39 4 98
-489 4 2 1 1 14 36 6 73
-490 4 2 1 1 15 38 92 7
-491 4 2 1 1 8 40 89 16
-492 4 2 1 1 75 35 2 10
-493 4 2 1 1 30 16 65 8
-494 4 2 1 1 14 27 6 82
-495 4 2 1 1 26 14 66 6
-496 4 2 1 1 16 31 8 100
-497 4 2 1 1 32 13 67 5
-498 4 2 1 1 13 67 5 25
-499 4 2 1 1 68 7 28 15
-500 4 2 1 1 29 15 7 68
-501 4 2 1 1 57 4 23 12
-502 4 2 1 1 22 12 4 57
-503 4 2 1 1 10 19 84 2
-504 4 2 1 1 9 59 1 24
-505 4 2 1 1 17 9 59 1
-506 4 2 1 1 18 10 2 58
-507 4 2 1 1 11 21 90 3
-508 4 2 1 1 20 11 3 60
-509 4 2 1 1 16 62 63 65
-510 4 2 1 1 11 53 55 60
-511 4 2 1 1 78 81 79 116
-512 4 2 1 1 71 112 70 73
-513 4 2 1 1 86 88 92 120
-514 4 2 1 1 96 94 124 100
-515 4 2 1 1 94 97 95 122
-516 4 2 1 1 89 118 87 86
-517 4 2 1 1 15 38 7 81
-518 4 2 1 1 102 44 59 9
-519 4 2 1 1 59 44 102 54
-520 4 2 1 1 110 71 69 75
-521 4 2 1 1 79 77 114 83
-522 4 2 1 1 76 69 109 72
-523 4 2 1 1 113 77 80 84
-524 4 2 1 1 93 99 121 95
-525 4 2 1 1 87 117 91 85
-526 4 2 1 1 90 119 43 11
-527 4 2 1 1 80 78 115 82
-528 4 2 1 1 88 119 85 90
-529 4 2 1 1 90 43 119 85
-530 4 2 1 1 46 108 66 61
-531 4 2 1 1 46 66 108 14
-532 4 2 1 1 26 14 6 73
-533 4 2 1 1 44 101 57 12
-534 4 2 1 1 44 57 101 54
-535 4 2 1 1 46 107 68 15
-536 4 2 1 1 46 68 107 61
-537 4 2 1 1 9 33 1 76
-538 4 2 1 1 100 124 48 94
-539 4 2 1 1 100 48 124 16
-540 4 2 1 1 25 70 13 45
-541 4 2 1 1 56 53 104 58
-542 4 2 1 1 61 64 108 66
-543 4 2 1 1 106 62 67 64
-544 4 2 1 1 102 59 54 56
-545 4 2 1 1 32 13 5 97
-546 4 2 1 1 30 16 8 89
-547 4 2 1 1 13 70 25 74
-548 4 2 1 1 97 122 48 13
-549 4 2 1 1 75 110 41 69
-550 4 2 1 1 54 57 101 55
-551 4 2 1 1 97 48 122 94
-552 4 2 1 1 47 118 89 86
-553 4 2 1 1 47 89 118 16
-554 4 2 1 1 17 9 1 76
-555 4 2 1 1 11 21 3 60
-556 4 2 1 1 10 19 2 58
-557 4 2 1 1 75 41 110 10
-558 4 2 1 1 61 63 68 107
-559 4 2 1 1 70 111 45 112
-560 4 2 1 1 78 115 46 116
-561 4 2 1 1 77 114 42 113
-562 4 2 1 1 69 41 109 110
-563 4 2 1 1 107 61 108 46
-564 4 2 1 1 103 42 104 53
-565 4 2 1 1 118 86 47 120
-566 4 2 1 1 94 124 48 122
-567 4 2 1 1 117 119 43 85
-568 4 2 1 1 123 44 121 93
-569 4 2 1 1 101 54 102 44
-570 4 2 1 1 48 105 62 106
-571 4 2 1 1 91 43 12 22
-572 4 2 1 1 91 43 22 85
-573 4 2 1 1 93 12 23 98
-574 4 2 1 1 93 23 12 44
-575 4 2 1 1 46 81 116 15
-576 4 2 1 1 46 116 81 78
-577 4 2 1 1 92 47 120 15
-578 4 2 1 1 120 47 92 86
-579 4 2 1 1 46 82 115 78
-580 4 2 1 1 46 115 82 14
-581 4 2 1 1 73 45 112 70
-582 4 2 1 1 73 112 45 14
-583 4 2 1 1 99 44 9 24
-584 4 2 1 1 99 44 24 93
-585 4 2 1 1 84 42 113 77
-586 4 2 1 1 84 113 42 10
-587 4 2 1 1 76 41 109 69
-588 4 2 1 1 109 41 76 9
-589 4 2 1 1 42 83 114 77
-590 4 2 1 1 42 114 83 11
-591 4 2 2 2 135 13 122 131
-592 4 2 2 2 132 138 125 110
-593 4 2 2 2 138 108 125 106
-594 4 2 2 2 136 108 125 137
-595 4 2 2 2 110 138 125 131
-596 4 2 2 2 136 107 134 125
-597 4 2 2 2 122 135 131 125
-598 4 2 2 2 13 49 122 131
-599 4 2 2 2 136 108 137 46
-600 4 2 2 2 110 138 131 111
-601 4 2 2 2 132 138 110 112
-602 4 2 2 2 106 107 108 125
-603 4 2 2 2 138 112 137 45
-604 4 2 2 2 104 127 125 128
-605 4 2 2 2 113 115 114 125
-606 4 2 2 2 131 122 125 121
-607 4 2 2 2 112 45 14 137
-608 4 2 2 2 110 131 125 130
-609 4 2 2 2 115 116 114 125
-610 4 2 2 2 132 138 112 137
-611 4 2 2 2 50 137 115 132
-612 4 2 2 2 104 127 128 42
-613 4 2 2 2 105 134 125 133
-614 4 2 2 2 129 120 51 119
-615 4 2 2 2 104 130 128 125
-616 4 2 2 2 105 134 133 47
-617 4 2 2 2 135 106 138 125
-618 4 2 2 2 106 105 107 125
-619 4 2 2 2 138 108 106 45
-620 4 2 2 2 104 130 125 102
-621 4 2 2 2 104 127 42 103
-622 4 2 2 2 109 131 49 111
-623 4 2 2 2 106 135 48 105
-624 4 2 2 2 126 101 43 125
-625 4 2 2 2 138 137 125 108
-626 4 2 2 2 104 127 103 125
-627 4 2 2 2 116 114 129 51
-628 4 2 2 2 136 108 46 107
-629 4 2 2 2 136 108 107 125
-630 4 2 2 2 52 124 123 125
-631 4 2 2 2 132 112 110 50
-632 4 2 2 2 44 121 125 123
-633 4 2 2 2 131 122 121 49
-634 4 2 2 2 105 134 47 107
-635 4 2 2 2 106 13 135 138
-636 4 2 2 2 52 133 125 118
-637 4 2 2 2 103 43 101 125
-638 4 2 2 2 117 43 119 125
-639 4 2 2 2 105 134 107 125
-640 4 2 2 2 137 115 132 125
-641 4 2 2 2 50 137 132 112
-642 4 2 2 2 106 135 105 125
-643 4 2 2 2 102 101 44 125
-644 4 2 2 2 104 130 102 41
-645 4 2 2 2 138 45 13 111
-646 4 2 2 2 131 138 135 13
-647 4 2 2 2 138 112 45 111
-648 4 2 2 2 131 138 13 111
-649 4 2 2 2 107 134 15 136
-650 4 2 2 2 118 52 117 125
-651 4 2 2 2 101 126 12 44
-652 4 2 2 2 104 130 41 128
-653 4 2 2 2 138 137 108 45
-654 4 2 2 2 133 118 16 47
-655 4 2 2 2 44 101 126 125
-656 4 2 2 2 133 134 118 47
-657 4 2 2 2 133 134 125 118
-658 4 2 2 2 124 122 123 125
-659 4 2 2 2 129 118 119 125
-660 4 2 2 2 102 104 103 125
-661 4 2 2 2 135 48 13 106
-662 4 2 2 2 123 122 121 125
-663 4 2 2 2 103 127 43 125
-664 4 2 2 2 106 45 13 138
-665 4 2 2 2 103 11 127 42
-666 4 2 2 2 127 51 129 114
-667 4 2 2 2 13 49 131 111
-668 4 2 2 2 137 108 14 46
-669 4 2 2 2 15 47 134 107
-670 4 2 2 2 10 41 128 104
-671 4 2 2 2 16 118 133 52
-672 4 2 2 2 46 107 15 136
-673 4 2 2 2 41 102 9 130
-674 4 2 2 2 128 50 132 110
-675 4 2 2 2 10 41 110 128
-676 4 2 2 2 50 137 112 14
-677 4 2 2 2 130 102 9 44
-678 4 2 2 2 105 133 16 47
-679 4 2 2 2 127 11 103 43
-680 4 2 2 2 128 130 41 110
-681 4 2 2 2 116 134 15 51
-682 4 2 2 2 137 45 14 108
-683 4 2 2 2 12 126 101 43
-684 4 2 2 2 133 48 135 105
-685 4 2 2 2 128 42 10 104
-686 4 2 2 2 131 9 109 49
-687 4 2 2 2 118 117 119 125
-688 4 2 2 2 102 103 101 125
-689 4 2 2 2 129 118 125 134
-690 4 2 2 2 125 129 116 114
-691 4 2 2 2 117 126 43 125
-692 4 2 2 2 126 117 12 52
-693 4 2 2 2 52 126 117 125
-694 4 2 2 2 12 117 126 43
-695 4 2 2 2 127 51 114 11
-696 4 2 2 2 114 129 127 125
-697 4 2 2 2 127 119 43 125
-698 4 2 2 2 119 11 127 43
-699 4 2 2 2 127 113 114 125
-700 4 2 2 2 127 113 42 114
-701 4 2 2 2 128 130 110 125
-702 4 2 2 2 15 47 120 134
-703 4 2 2 2 127 11 114 42
-704 4 2 2 2 120 47 118 134
-705 4 2 2 2 44 130 102 125
-706 4 2 2 2 44 126 12 123
-707 4 2 2 2 123 44 126 125
-708 4 2 2 2 129 118 134 120
-709 4 2 2 2 110 132 128 125
-710 4 2 2 2 128 127 125 113
-711 4 2 2 2 128 127 113 42
-712 4 2 2 2 128 50 110 10
-713 4 2 2 2 113 42 10 128
-714 4 2 2 2 129 134 125 116
-715 4 2 2 2 127 51 11 119
-716 4 2 2 2 129 134 116 51
-717 4 2 2 2 127 51 119 129
-718 4 2 2 2 129 119 127 125
-719 4 2 2 2 110 131 130 109
-720 4 2 2 2 109 130 9 131
-721 4 2 2 2 121 130 9 44
-722 4 2 2 2 122 48 13 135
-723 4 2 2 2 44 121 130 125
-724 4 2 2 2 110 138 111 112
-725 4 2 2 2 131 138 125 135
-726 4 2 2 2 115 137 14 46
-727 4 2 2 2 126 52 12 123
-728 4 2 2 2 123 126 52 125
-729 4 2 2 2 50 137 14 115
-730 4 2 2 2 136 137 115 46
-731 4 2 2 2 121 9 131 49
-732 4 2 2 2 121 131 130 125
-733 4 2 2 2 131 130 9 121
-734 4 2 2 2 113 132 115 125
-735 4 2 2 2 113 50 115 132
-736 4 2 2 2 128 50 10 113
-737 4 2 2 2 132 113 128 125
-738 4 2 2 2 128 50 113 132
-739 4 2 2 2 133 105 135 125
-740 4 2 2 2 52 124 125 133
-741 4 2 2 2 133 48 105 16
-742 4 2 2 2 16 133 124 52
-743 4 2 2 2 136 137 125 115
-744 4 2 2 2 132 138 137 125
-745 4 2 2 2 41 130 9 109
-746 4 2 2 2 134 120 15 51
-747 4 2 2 2 129 118 120 119
-748 4 2 2 2 129 120 134 51
-749 4 2 2 2 135 122 124 125
-750 4 2 2 2 135 48 124 122
-751 4 2 2 2 133 48 16 124
-752 4 2 2 2 133 135 124 125
-753 4 2 2 2 133 48 124 135
-754 4 2 2 2 116 136 134 125
-755 4 2 2 2 115 136 116 125
-756 4 2 2 2 46 115 136 116
-757 4 2 2 2 136 134 15 116
-758 4 2 2 2 46 136 15 116
-759 4 2 2 2 109 41 130 110
-760 4 2 2 2 110 131 109 111
-$EndElements

data/square_in_square.geo

@@ -1,77 +0,0 @@
-// Square-in-square 2D geometry for MFEM
-// Creates concentric squares with different material attributes
-
-// Define the square sizes
-L_outer = 2.0;
-L_inner = 0.5;
-
-// Set mesh size and algorithm
-mesh_size = 1.0;
-Mesh.Algorithm = 6; // Frontal-Delaunay for 2D triangular mesh
-Mesh.CharacteristicLengthFactor = 1.0;
-Mesh.MshFileVersion = 2.2;
-
-// Define center point for concentric squares
-cx = 0.0;
-cy = 0.0;
-
-// Define the points (vertices of the outer square)
-Point(1) = {cx-L_outer/2, cy-L_outer/2, 0, mesh_size}; // bottom-left outer
-Point(2) = {cx+L_outer/2, cy-L_outer/2, 0, mesh_size}; // bottom-right outer
-Point(3) = {cx+L_outer/2, cy+L_outer/2, 0, mesh_size}; // top-right outer
-Point(4) = {cx-L_outer/2, cy+L_outer/2, 0, mesh_size}; // top-left outer
-
-// Define the points (vertices of the inner square)
-Point(5) = {cx-L_inner/2, cy-L_inner/2, 0, mesh_size}; // bottom-left inner
-Point(6) = {cx+L_inner/2, cy-L_inner/2, 0, mesh_size}; // bottom-right inner
-Point(7) = {cx+L_inner/2, cy+L_inner/2, 0, mesh_size}; // top-right inner
-Point(8) = {cx-L_inner/2, cy+L_inner/2, 0, mesh_size}; // top-left inner
-
-// Define the lines (edges of the outer square)
-Line(1) = {1, 2}; // bottom edge
-Line(2) = {2, 3}; // right edge
-Line(3) = {3, 4}; // top edge
-Line(4) = {4, 1}; // left edge
-
-// Define the lines (edges of the inner square)
-Line(5) = {5, 6}; // bottom edge
-Line(6) = {6, 7}; // right edge
-Line(7) = {7, 8}; // top edge
-Line(8) = {8, 5}; // left edge
-
-// Define the surfaces
-// Outer square boundary
-Line Loop(1) = {1, 2, 3, 4};
-
-// Inner square boundary (hole in the outer region)
-Line Loop(2) = {5, 6, 7, 8};
-
-// Define the surface areas
-// Outer region (annular region between squares)
-Plane Surface(1) = {1, 2}; // Outer loop minus inner loop (creates hole)
-
-// Inner region (solid inner square)
-Plane Surface(2) = {2}; // Inner loop only
-
-// Assign physical groups for materials
-Physical Surface(1) = {1}; // Outer material (annular region)
-Physical Surface(2) = {2}; // Inner material (solid square)
-
-// Physical lines for boundary conditions
-// Outer square boundary edges
-Physical Line(1) = {1}; // outer bottom
-Physical Line(2) = {2}; // outer right
-Physical Line(3) = {3}; // outer top
-Physical Line(4) = {4}; // outer left
-
-// Inner square boundary edges
-Physical Line(5) = {5}; // inner bottom
-Physical Line(6) = {6}; // inner right
-Physical Line(7) = {7}; // inner top
-Physical Line(8) = {8}; // inner left
-
-// Mesh control for quality
-Mesh.OptimizeNetgen = 1;
-Mesh.Optimize = 1;
-Mesh.ElementOrder = 1;
-Mesh.RecombineAll = 0; // Keep triangular elements (don't recombine to quads)

data/square_in_square.msh

@@ -1,50 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-13
-1 -1 -1 0
-2 1 -1 0
-3 1 1 0
-4 -1 1 0
-5 -0.25 -0.25 0
-6 0.25 -0.25 0
-7 0.25 0.25 0
-8 -0.25 0.25 0
-9 -2.752797989558076e-12 -1 0
-10 1 -2.752797989558076e-12 0
-11 2.752797989558076e-12 1 0
-12 -1 2.752797989558076e-12 0
-13 0 0 0
-$EndNodes
-$Elements
-28
-1 1 2 1 1 1 9
-2 1 2 1 1 9 2
-3 1 2 2 2 2 10
-4 1 2 2 2 10 3
-5 1 2 3 3 3 11
-6 1 2 3 3 11 4
-7 1 2 4 4 4 12
-8 1 2 4 4 12 1
-9 1 2 5 5 5 6
-10 1 2 6 6 6 7
-11 1 2 7 7 7 8
-12 1 2 8 8 8 5
-13 2 2 1 1 6 5 9
-14 2 2 1 1 5 8 12
-15 2 2 1 1 7 6 10
-16 2 2 1 1 8 7 11
-17 2 2 1 1 9 5 1
-18 2 2 1 1 5 12 1
-19 2 2 1 1 6 9 2
-20 2 2 1 1 10 6 2
-21 2 2 1 1 7 10 3
-22 2 2 1 1 11 7 3
-23 2 2 1 1 8 11 4
-24 2 2 1 1 8 4 12
-25 2 2 2 2 5 6 13
-26 2 2 2 2 8 5 13
-27 2 2 2 2 6 7 13
-28 2 2 2 2 7 8 13
-$EndElements

data/tinyzoo-2d.mesh

@@ -1,38 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-
-dimension
-2
-
-elements
-2
-1 3 0 1 4 3
-1 2 1 2 4
-
-boundary
-5
-1 1 0 1
-1 1 1 2
-1 1 2 4
-1 1 4 3
-1 1 3 0
-
-vertices
-5
-2
-0 0
-1 0
-2 0
-0 1
-1 1

doc/CodeDocumentation.conf.in

@@ -1083,8 +1083,7 @@ EXCLUDE_PATTERNS       =
 # ANamespace::AClass, ANamespace::*Test
 
 EXCLUDE_SYMBOLS        = mfem::internal \
-                         mfem::kernels::internal \
-                         mfem::future::detail
+                         mfem::kernels::internal
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include

doc/CodeDocumentation.dox

@@ -201,7 +201,6 @@ namespace mfem {
  * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
  * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
  * - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
- * - <a class="el" href="nurbs__surface_8cpp_source.html">NURBS Surface</a>: interpolate a 3D Surface in a NURBS Patch
  *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -246,9 +245,6 @@ namespace mfem {
  * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
  * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
  * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
- * - <a class="el" href="reflector_8cpp_source.html">Reflector Miniapp</a>: reflect a mesh about a plane
- * - <a class="el" href="ref321_8cpp_source.html">3:1 Refinement Miniapp</a>: perform 3:1 anisotropic mesh refinements
- * - <a class="el" href="pref321_8cpp_source.html">3:1 Refinement Miniapp</a>: parallel 3:1 anisotropic mesh refinements
  *
  * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
  */

examples/CMakeLists.txt

@@ -47,6 +47,7 @@ list(APPEND ALL_EXE_SRCS
   ex39.cpp
   ex40.cpp
   ex41.cpp
+  # jitplayground.cpp
   )
 
 if (MFEM_USE_MPI)

examples/jitplayground.cpp

@@ -0,0 +1,548 @@
+#include <mfem.hpp>
+
+#ifdef MFEM_USE_PROTEUS
+
+#include "../fem/dfem/util.hpp"
+
+#include <proteus/CppJitModule.h>
+
+#include "jitplayground.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cctype>
+#include <cmath>
+#include <fstream>
+#include <initializer_list>
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace util
+{
+constexpr std::string_view Dirname(std::string_view path)
+{
+   const size_t last_sep = path.find_last_of("/\\");
+   if (last_sep == std::string_view::npos) { return {}; }
+   return path.substr(0, last_sep);
+}
+
+constexpr std::string_view thisFileDir = Dirname(__FILE__);
+}
+
+template <typename T>
+static std::string TypeNameString()
+{
+   return std::string(mfem::future::get_type_name<T>());
+}
+
+template <typename Tuple, size_t... Is>
+static auto ParamTypeStringsImpl(std::index_sequence<Is...>)
+{
+   return std::array<std::string, sizeof...(Is)>
+   {
+      TypeNameString<std::remove_reference_t<decltype(mfem::future::get<Is>(std::declval<Tuple&>()))>>()...
+   };
+}
+
+template <typename Tuple>
+static auto ParamTypeStrings()
+{
+   return ParamTypeStringsImpl<Tuple>(
+             std::make_index_sequence<mfem::future::tuple_size<Tuple>::value> {});
+}
+
+static std::string_view Trim(std::string_view s)
+{
+   size_t begin = 0;
+   while (begin < s.size() && std::isspace(static_cast<unsigned char>(s[begin])))
+   {
+      ++begin;
+   }
+   size_t end = s.size();
+   while (end > begin &&
+          std::isspace(static_cast<unsigned char>(s[end - 1])))
+   {
+      --end;
+   }
+   return s.substr(begin, end - begin);
+}
+
+static bool IsValidIdentifier(std::string_view s)
+{
+   if (s.empty()) { return false; }
+   const unsigned char c0 = static_cast<unsigned char>(s[0]);
+   if (!(std::isalpha(c0) || c0 == '_')) { return false; }
+   for (size_t i = 1; i < s.size(); ++i)
+   {
+      const unsigned char c = static_cast<unsigned char>(s[i]);
+      if (!(std::isalnum(c) || c == '_')) { return false; }
+   }
+   return true;
+}
+
+static bool ParseJitDirective(std::string_view line,
+                              std::string &type,
+                              std::string &var,
+                              std::string &kind)
+{
+   const size_t jit_pos = line.find("$JIT");
+   if (jit_pos == std::string_view::npos) { return false; }
+
+   const size_t open = line.find('[', jit_pos);
+   const size_t close = line.find(']', jit_pos);
+   MFEM_VERIFY(open != std::string_view::npos &&
+               close != std::string_view::npos &&
+               close > open,
+               "malformed $JIT directive (expected brackets): " << line);
+
+   const std::string_view payload = line.substr(open + 1, close - open - 1);
+   const size_t comma1 = payload.find(',');
+   const size_t comma2 = (comma1 == std::string_view::npos)
+                         ? std::string_view::npos
+                         : payload.find(',', comma1 + 1);
+   MFEM_VERIFY(comma1 != std::string_view::npos &&
+               comma2 != std::string_view::npos,
+               "malformed $JIT directive (expected 3 comma-separated fields): "
+               << line);
+
+   const std::string_view f0 = Trim(payload.substr(0, comma1));
+   const std::string_view f1 = Trim(payload.substr(comma1 + 1,
+                                                   comma2 - comma1 - 1));
+   const std::string_view f2 = Trim(payload.substr(comma2 + 1));
+   MFEM_VERIFY(!f0.empty() && !f1.empty() && !f2.empty(),
+               "malformed $JIT directive (empty field): " << line);
+
+   type.assign(f0);
+   var.assign(f1);
+   kind.assign(f2);
+   return true;
+}
+
+static std::string ReadFileOrEmpty(const std::string &fn)
+{
+   std::ifstream file(fn);
+   if (!file.is_open())
+   {
+      std::cerr << "could not open file " << fn << "\n";
+      return {};
+   }
+   std::stringstream buffer;
+   buffer << file.rdbuf();
+   return buffer.str();
+}
+
+static std::vector<std::string> ExtractJitVarNames(const std::string
+                                                   &kernel_code)
+{
+   std::stringstream ss(kernel_code);
+   std::string line;
+   std::vector<std::string> var_names;
+   std::unordered_set<std::string> seen_vars;
+
+   while (std::getline(ss, line))
+   {
+      std::string type, var, kind;
+      if (ParseJitDirective(line, type, var, kind))
+      {
+         MFEM_VERIFY(IsValidIdentifier(var),
+                     "$JIT variable must be a valid identifier: " << var);
+         MFEM_VERIFY(seen_vars.insert(var).second,
+                     "duplicate $JIT variable name: " << var);
+         var_names.push_back(var);
+      }
+   }
+   return var_names;
+}
+
+static std::string RewriteKernelForJit(std::string kernel_code,
+                                       const std::vector<std::string> &jit_values)
+{
+   std::stringstream ss(kernel_code);
+   std::string line;
+
+   std::string out;
+   out.reserve(kernel_code.size() + 128);
+
+   bool have_pending = false;
+   size_t pending_index = 0;
+   std::string pending_type;
+   std::string pending_var;
+   std::unordered_set<std::string> seen_vars;
+
+   while (std::getline(ss, line))
+   {
+      line.push_back('\n');
+
+      if (have_pending)
+      {
+         MFEM_VERIFY(pending_index < jit_values.size(),
+                     "not enough JIT values provided");
+         const size_t indent_end = line.find_first_not_of(" \t");
+         const std::string indent =
+            (indent_end == std::string::npos) ? std::string() :
+            line.substr(0, indent_end);
+         out += indent + "const " + pending_type + " " + pending_var + " = " +
+                jit_values[pending_index] + ";\n";
+         have_pending = false;
+         ++pending_index;
+         continue;
+      }
+
+      std::string type, var, kind;
+      if (ParseJitDirective(line, type, var, kind))
+      {
+         MFEM_VERIFY(IsValidIdentifier(var),
+                     "$JIT variable must be a valid identifier: " << var);
+         MFEM_VERIFY(kind == "generic",
+                     "unsupported $JIT kind: " << kind);
+         MFEM_VERIFY(seen_vars.insert(var).second,
+                     "duplicate $JIT variable name: " << var);
+
+         pending_type = std::move(type);
+         pending_var = std::move(var);
+         have_pending = true;
+         continue; // drop directive line
+      }
+
+      out += line;
+   }
+
+   MFEM_VERIFY(!have_pending,
+               "$JIT directive must annotate a following line");
+   MFEM_VERIFY(jit_values.size() == pending_index,
+               "JIT value count must match number of $JIT directives");
+   return out;
+}
+
+static std::string GeneratedOutputPath(std::string_view original_path)
+{
+   const size_t last_sep = original_path.find_last_of("/\\");
+   const size_t dot = original_path.find_last_of('.');
+   const bool dot_in_filename =
+      (dot != std::string_view::npos) &&
+      (last_sep == std::string_view::npos || dot > last_sep);
+
+   const std::string_view base =
+      dot_in_filename ? original_path.substr(0, dot) : original_path;
+   return std::string(base) + "_generated.hpp";
+}
+
+static void WriteFileOrWarn(const std::string &path,
+                            const std::string &contents)
+{
+   std::ofstream out(path);
+   if (!out.is_open())
+   {
+      std::cerr << "could not write generated file " << path << "\n";
+      return;
+   }
+   out << contents;
+}
+
+class JitQFunction
+{
+public:
+   template <typename ImplT, size_t N>
+   JitQFunction(ImplT, const std::string &fn,
+                const std::array<bool, N> &activity_map)
+   {
+      using qf_signature = typename
+                           mfem::future::get_function_signature<
+                           decltype(&ImplT::operator())>::type;
+      using qf_param_ts = typename qf_signature::parameter_ts;
+      constexpr size_t nparams = mfem::future::tuple_size<qf_param_ts>::value;
+      static_assert(N == nparams, "activity_map size must match qfunc arity");
+
+      this->fn = fn;
+      this->nparams = nparams;
+      this->activity_map.reserve(N);
+      for (size_t i = 0; i < N; ++i)
+      {
+         this->activity_map.push_back(activity_map[i]);
+      }
+      {
+         const auto param_types_arr = ParamTypeStrings<qf_param_ts>();
+         this->param_types.assign(param_types_arr.begin(), param_types_arr.end());
+      }
+      this->return_type = TypeNameString<typename qf_signature::return_t>();
+      this->return_is_void = std::is_same_v<typename qf_signature::return_t, void>;
+      this->impl_type_name = TypeNameString<ImplT>();
+      this->jit_var_names = ExtractJitVarNames(ReadFileOrEmpty(fn));
+   }
+
+   template <typename ReturnT, typename... Args>
+   ReturnT run(std::string_view name,
+               std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
+               Args&&... args)
+   {
+      auto ordered_values = MatchJitValues(jit_values);
+      auto &mod = GetOrCreateModule(ordered_values);
+      auto &instance = mod.instantiate(std::string(name), std::string());
+      return instance.template run<ReturnT>(std::forward<Args>(args)...);
+   }
+
+   template <typename ReturnT, typename... Args>
+   ReturnT run_primal(
+      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
+      Args&&... args)
+   {
+      return run<ReturnT>(qfunc_name, jit_values,
+                          std::forward<Args>(args)...);
+   }
+
+   template <typename ReturnT, typename... Args>
+   ReturnT run_derivative(
+      std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
+      Args&&... args)
+   {
+      return run<ReturnT>(qfunc_name + "_fwddiff", jit_values,
+                          std::forward<Args>(args)...);
+   }
+
+private:
+   std::vector<std::string_view> MatchJitValues(
+      std::initializer_list<std::pair<std::string_view, std::string_view>>
+      named_values) const
+   {
+      std::unordered_map<std::string_view, std::string_view> value_map;
+      for (const auto &[name, value] : named_values)
+      {
+         value_map[name] = value;
+      }
+
+      std::vector<std::string_view> ordered_values;
+      ordered_values.reserve(jit_var_names.size());
+      for (const auto &var_name : jit_var_names)
+      {
+         auto it = value_map.find(var_name);
+         MFEM_VERIFY(it != value_map.end(),
+                     "missing JIT value for variable: " << var_name);
+         ordered_values.push_back(it->second);
+      }
+
+      MFEM_VERIFY(ordered_values.size() == named_values.size(),
+                  "provided " << named_values.size() << " JIT values but expected "
+                  << jit_var_names.size());
+      return ordered_values;
+   }
+
+
+   std::string BuildModuleCode(const std::vector<std::string> &jit_values) const
+   {
+      std::string module_code =
+         RewriteKernelForJit(ReadFileOrEmpty(fn), jit_values);
+      module_code += "\n\n";
+      module_code += "// --- generated ---\n";
+      module_code +=
+         "template <typename return_type, typename... Args>\n"
+         "return_type __enzyme_fwddiff(Args...);\n"
+         "\n"
+         "extern int enzyme_const;\n"
+         "extern int enzyme_dup;\n"
+         "\n";
+
+      // Generate a primal wrapper with the requested symbol name, so the kernel
+      // header can just define the qfunc as a functor.
+      //
+      // Note: Proteus instantiates entrypoints via `qfunc_wrapper<>(...)` even
+      // when there are no user template args, so keep the wrapper itself a
+      // template (with a default parameter) while still doing literal `$JIT`
+      // replacements in the kernel code.
+      module_code += "template <typename = void>\n";
+      module_code += return_type + " " +
+                     std::string(qfunc_name) + "(";
+      bool first = true;
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (!first) { module_code += ", "; }
+         first = false;
+         module_code += param_types[i] + " Arg" + std::to_string(i);
+      }
+      module_code += ")\n";
+      module_code += "{\n";
+      module_code += "   " + impl_type_name + " qf;\n";
+      if (return_is_void)
+      {
+         module_code += "   ";
+      }
+      else
+      {
+         module_code += "   return ";
+      }
+      module_code += "qf(";
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (i) { module_code += ", "; }
+         module_code += "Arg" + std::to_string(i);
+      }
+      module_code += ");\n";
+      module_code += "}\n\n";
+
+      module_code += "template <typename = void>\n";
+      module_code += return_type + " " +
+                     std::string(qfunc_name) + "_fwddiff(";
+
+      first = true;
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (!first) { module_code += ", "; }
+         first = false;
+         module_code += param_types[i] + " Arg" + std::to_string(i);
+         if (activity_map[i])
+         {
+            module_code += ", " + param_types[i] + " dArg" + std::to_string(i);
+         }
+      }
+      module_code += ")\n";
+      module_code += "{\n";
+      if (return_is_void)
+      {
+         module_code += "   __enzyme_fwddiff<void>(\n";
+      }
+      else
+      {
+         module_code += "   return __enzyme_fwddiff<" +
+                        return_type + ">(\n";
+      }
+      module_code += "      (void*)" + std::string(qfunc_name) + "<>";
+      module_code += ",\n";
+      for (size_t i = 0; i < nparams; ++i)
+      {
+         if (activity_map[i])
+         {
+            module_code += "      enzyme_dup, Arg" + std::to_string(i) +
+                           ", dArg" + std::to_string(i);
+         }
+         else
+         {
+            module_code += "      enzyme_const, Arg" + std::to_string(i);
+         }
+         module_code += (i + 1 == nparams) ? ");\n" : ",\n";
+      }
+      module_code += "}\n";
+
+      WriteFileOrWarn(GeneratedOutputPath(fn), module_code);
+      return module_code;
+   }
+
+   proteus::CppJitModule &GetOrCreateModule(
+      const std::vector<std::string_view> &jit_values)
+   {
+      std::string key;
+      for (const auto &val : jit_values)
+      {
+         if (!key.empty()) { key += ","; }
+         key += val;
+      }
+
+      auto it = modules.find(key);
+      if (it != modules.end())
+      {
+         return *it->second;
+      }
+
+      std::vector<std::string> values(jit_values.begin(), jit_values.end());
+      std::string code = BuildModuleCode(values);
+      auto mod = std::make_unique<proteus::CppJitModule>("host", code,
+                                                         DefaultExtraArgs());
+      auto [inserted, ok] = modules.emplace(key, std::move(mod));
+      MFEM_VERIFY(ok, "failed to cache JIT module");
+      return *inserted->second;
+   }
+
+   static std::vector<std::string> DefaultExtraArgs()
+   {
+      return {"-fplugin=/Users/andrej1/local/enzyme/lib/ClangEnzyme-20.dylib"};
+   }
+
+   std::string qfunc_name = "qfunc_wrapper";
+   std::string fn;
+   size_t nparams = 0;
+   std::vector<bool> activity_map;
+   std::vector<std::string> param_types;
+   std::string return_type;
+   bool return_is_void = false;
+   std::string impl_type_name;
+   std::vector<std::string> jit_var_names;
+   std::unordered_map<std::string, std::unique_ptr<proteus::CppJitModule>> modules;
+};
+
+int main()
+{
+   const size_t N = 4;
+   const size_t M = 5;
+   const double A = 123.4;
+
+   std::vector<double> X(N);
+   std::vector<double> Y(N);
+   for (size_t i = 0; i < N; ++i)
+   {
+      X[i] = static_cast<double>(i + 1);
+      Y[i] = static_cast<double>(N - i);
+   }
+
+   // // >>> user interface calls
+   // const std::string kernel_path = std::string(util::thisFileDir) +
+   //                                 "/jitplayground.hpp";
+   // JitQFunction qf(daxpy_op{}, kernel_path, std::array{false, true, false});
+   // // <<< user interface calls
+
+   // // this will happen internally in dFEM
+
+   daxpy_op op;
+   printf("\n\nfunction call\n");
+   op(&A, X.data(), Y.data(), &N);
+
+   // reset X for the derivative test
+   for (size_t i = 0; i < N; ++i)
+   {
+      X[i] = static_cast<double>(i + 1);
+      Y[i] = static_cast<double>(N - i);
+   }
+
+   std::vector<double> dX(N, 1.0);
+   printf("\n\nforward diff call\n");
+   daxpy_op_fwddiff(&A, X.data(), dX.data(), Y.data(), &N);
+
+   std::vector<double> dX_manual(N, A);
+
+   printf("\n\nderivative checks\n");
+   std::cout << "dX: ";
+   for (size_t i = 0; i < N; ++i)
+   {
+      std::cout << dX[i] << (i + 1 == N ? '\n' : ' ');
+   }
+
+   std::cout << "dX_manual: ";
+   for (size_t i = 0; i < N; ++i)
+   {
+      std::cout << dX_manual[i] << (i + 1 == N ? '\n' : ' ');
+   }
+
+   double max_abs_err = 0.0;
+   for (size_t i = 0; i < N; ++i)
+   {
+      max_abs_err = std::max(max_abs_err, std::abs(dX[i] - dX_manual[i]));
+   }
+   std::cout << "max |dX - dX_manual| = " << max_abs_err << "\n";
+
+   return 0;
+}
+
+#else
+
+int main()
+{
+   std::cerr << "Proteus is required to run this example.\n";
+   return EXIT_FAILURE;
+}
+
+#endif // MFEM_USE_PROTEUS

examples/jitplayground.hpp

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <cstddef>
+#include <vector>
+#include <type_traits>
+
+#include "proteus/JitInterface.h"
+
+struct daxpy_op
+{
+   void operator()(
+      const double *a,
+      double *x,
+      const double *y,
+      const size_t *N) const
+   {
+      const size_t n = *N;
+      auto lam = [=, n = proteus::jit_variable(n)]
+                 () __attribute__((annotate("jit")))
+      {
+         printf("N = %zu\n", n);
+         for (size_t i = 0; i < n; ++i)
+         {
+            printf("x[%zu] = %f, y[%zu] = %f\n", i, x[i], i, y[i]);
+            x[i] = *a * x[i] + y[i];
+            printf("updated x[%zu] = %f\n", i, x[i]);
+         }
+      };
+
+      proteus::register_lambda(lam);
+
+      lam();
+   }
+};
+
+template <typename return_type, typename... Args>
+return_type __enzyme_fwddiff(Args...);
+
+extern int enzyme_const;
+extern int enzyme_dup;
+
+void daxpy_op_wrapper(const double * Arg0, double * Arg1,
+                      const double * Arg2, const size_t *Arg3)
+{
+   daxpy_op qf;
+   qf(Arg0, Arg1, Arg2, Arg3);
+}
+
+void daxpy_op_fwddiff(const double * Arg0, double * Arg1,
+                      double * dArg1, const double * Arg2, const size_t *Arg3)
+{
+   __enzyme_fwddiff<void>(
+      (void*)daxpy_op_wrapper,
+      enzyme_const, Arg0,
+      enzyme_dup, Arg1, dArg1,
+      enzyme_const, Arg2,
+      enzyme_const, Arg3);
+}

fem/CMakeLists.txt

@@ -57,8 +57,6 @@ set(SRCS
   integ/lininteg_domain_grad.cpp
   integ/lininteg_domain_vectorfe.cpp
   integ/nonlininteg_vecconvection_pa.cpp
-  integ/nonlininteg_vecconvection_pa_diag.cpp
-  integ/nonlininteg_vecconvection_pa_grad.cpp
   integ/nonlininteg_vecconvection_mf.cpp
   coefficient.cpp
   complex_fem.cpp
@@ -67,6 +65,7 @@ set(SRCS
   dgmassinv.cpp
   doftrans.cpp
   dfem/doperator.cpp
+  dfem/backends/local_qf/kernels.cpp
   eltrans.cpp
   batchitrans.cpp
   estimators.cpp
@@ -123,6 +122,11 @@ set(SRCS
   qinterp/eval_hdiv.cpp
   qinterp/grad_by_nodes.cpp
   qinterp/grad_by_vdim.cpp
+  qinterp/grad_transpose.cpp
+  qinterp/grad_transpose_by_nodes.cpp
+  qinterp/grad_transpose_by_vdim.cpp
+  qinterp/eval_transpose.cpp
+  qinterp/eval_transpose_by_vdim.cpp
   qspace.cpp
   quadinterpolator.cpp
   quadinterpolator_face.cpp
@@ -206,11 +210,7 @@ set(HDRS
   integ/bilininteg_mass_kernels.hpp
   integ/bilininteg_mass_pa_simplices.hpp
   integ/bilininteg_vecdiffusion_pa.hpp
-  integ/bilininteg_vecdiv_pa.hpp
   integ/bilininteg_vecmass_pa.hpp
-  integ/nonlininteg_vecconvection_pa.hpp
-  integ/nonlininteg_vecconvection_pa_diag.hpp
-  integ/nonlininteg_vecconvection_pa_grad.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp
@@ -219,13 +219,31 @@ set(HDRS
   dgmassinv_kernels.hpp
   doftrans.hpp
   dfem/doperator.hpp
+  dfem/fielddescriptor.hpp
   dfem/fieldoperator.hpp
-  dfem/integrate.hpp
+  dfem/integrator_ctx.hpp
   dfem/parameterspace.hpp
-  dfem/qfunction_apply.hpp
-  dfem/qfunction_transform.hpp
   dfem/tuple.hpp
   dfem/util.hpp
+  dfem/backends/util.hpp
+  dfem/backends/global_qf/action.hpp
+  dfem/backends/global_qf/derivative_action.hpp
+  dfem/backends/global_qf/derivative_apply.hpp
+  dfem/backends/global_qf/derivative_apply_transpose.hpp
+  dfem/backends/global_qf/derivative_setup.hpp
+  dfem/backends/global_qf/prelude.hpp
+  dfem/backends/local_qf/action.hpp
+  dfem/backends/local_qf/derivative_action.hpp
+  dfem/backends/local_qf/derivative_apply.hpp
+  dfem/backends/local_qf/derivative_apply_transpose.hpp
+  dfem/backends/local_qf/derivative_assemble.hpp
+  dfem/backends/local_qf/derivative_assemble_diagonal.hpp
+  dfem/backends/local_qf/derivative_setup.hpp
+  dfem/backends/local_qf/kernels.hpp
+  dfem/backends/local_qf/kernels_ho.hpp
+  dfem/backends/local_qf/kernels_lo.hpp
+  dfem/backends/local_qf/prelude.hpp
+  dfem/backends/local_qf/util.hpp
   eltrans.hpp
   estimators.hpp
   fe.hpp
@@ -290,8 +308,10 @@ set(HDRS
   qfunction.hpp
   qinterp/det.hpp
   qinterp/eval.hpp
+  qinterp/eval_transpose.hpp
   qinterp/eval_hdiv.hpp
   qinterp/grad.hpp
+  qinterp/grad_transpose.hpp
   qspace.hpp
   quadinterpolator.hpp
   quadinterpolator_face.hpp
@@ -326,36 +346,36 @@ set(HDRS
   )
 
 if (MFEM_USE_SIDRE)
-  list(APPEND SRCS sidredatacollection.cpp)
-  list(APPEND HDRS sidredatacollection.hpp)
+    list(APPEND SRCS sidredatacollection.cpp)
+    list(APPEND HDRS sidredatacollection.hpp)
 endif()
 
 if (MFEM_USE_CONDUIT)
-  list(APPEND SRCS conduitdatacollection.cpp)
-  list(APPEND HDRS conduitdatacollection.hpp)
+    list(APPEND SRCS conduitdatacollection.cpp)
+    list(APPEND HDRS conduitdatacollection.hpp)
 endif()
 
 if (MFEM_USE_ADIOS2)
-  list(APPEND SRCS adios2datacollection.cpp)
-  list(APPEND HDRS adios2datacollection.hpp)
+    list(APPEND SRCS adios2datacollection.cpp)
+    list(APPEND HDRS adios2datacollection.hpp)
 endif()
 
 if (MFEM_USE_FMS)
-  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
-  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
+    list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
+    list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
 endif()
 
 if (MFEM_USE_MPI)
-  list(APPEND SRCS
+    list(APPEND SRCS
     pbilinearform.cpp
     pfespace.cpp
     pgridfunc.cpp
     plinearform.cpp
     pnonlinearform.cpp
     prestriction.cpp)
-  # If this list (HDRS -> HEADERS) is used for install, we probably want the
-  # headers added all the time.
-  list(APPEND HDRS
+    # If this list (HDRS -> HEADERS) is used for install, we probably want the
+    # headers added all the time.
+    list(APPEND HDRS
     pbilinearform.hpp
     pfespace.hpp
     pgridfunc.hpp

fem/bilinearform.cpp

@@ -1255,31 +1255,6 @@ void BilinearForm::Mult(const Vector &x, Vector &y) const
    }
 }
 
-void BilinearForm::AddMult(const Vector &x, Vector &y, const real_t a) const
-{
-   if (ext)
-   {
-      ext->AddMult(x, y, a);
-   }
-   else
-   {
-      mat->AddMult(x, y, a);
-   }
-}
-
-void BilinearForm::AddMultTranspose(const Vector &x, Vector &y,
-                                    const real_t a) const
-{
-   if (ext)
-   {
-      ext->AddMultTranspose(x, y, a);
-   }
-   else
-   {
-      mat->AddMultTranspose(x, y, a);
-   }
-}
-
 void BilinearForm::MultTranspose(const Vector & x, Vector & y) const
 {
    if (ext)

fem/bilinearform.hpp

@@ -307,8 +307,8 @@ public:
    { mat->Mult(x, y); mat_e->AddMult(x, y); }
 
    /// Add the matrix vector multiple to a vector:  $ y += a M x $
-   void AddMult(const Vector &x, Vector &y,
-                const real_t a = 1.0) const override;
+   void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const override
+   { mat -> AddMult (x, y, a); }
 
    /** @brief Add the original uneliminated matrix vector multiple to a vector.
        The original matrix is $ M + Me $ so we have:
@@ -318,7 +318,8 @@ public:
 
    /// Add the matrix transpose vector multiplication:  $ y += a M^T x $
    void AddMultTranspose(const Vector & x, Vector & y,
-                         const real_t a = 1.0) const override;
+                         const real_t a = 1.0) const override
+   { mat->AddMultTranspose(x, y, a); }
 
    /** @brief Add the original uneliminated matrix transpose vector
        multiple to a vector. The original matrix is $ M + M_e $

fem/bilinearform_ext.cpp

@@ -1997,11 +1997,7 @@ void PADiscreteLinearOperatorExtension::Assemble()
    }
    else
    {
-      const L2ElementRestriction* l2_elem_restrict =
-         dynamic_cast<const L2ElementRestriction*>(elem_restrict_test);
-      MFEM_VERIFY(l2_elem_restrict,
-                  "A real ElementRestriction is required in this setting!");
-      test_multiplicity = 1.0;
+      mfem_error("A real ElementRestriction is required in this setting!");
    }
 
    auto tm = test_multiplicity.ReadWrite();
@@ -2040,13 +2036,7 @@ void PADiscreteLinearOperatorExtension::AddMult(
    }
    else
    {
-      const L2ElementRestriction* l2_elem_restrict =
-         dynamic_cast<const L2ElementRestriction*>(elem_restrict_test);
-      MFEM_VERIFY(l2_elem_restrict,
-                  "In this setting you need a real ElementRestriction!");
-      tempY.SetSize(y.Size());
-      l2_elem_restrict->MultTranspose(localTest, tempY);
-      y += tempY;
+      mfem_error("In this setting you need a real ElementRestriction!");
    }
 }
 

fem/coefficient.hpp

@@ -1055,8 +1055,7 @@ public:
 
 typedef VectorCoefficient DiagonalMatrixCoefficient;
 
-/** Base class for matrix-valued coefficients that optionally depend on time
-    and space. */
+/// Base class for Matrix Coefficients that optionally depend on time and space.
 class MatrixCoefficient
 {
 protected:
@@ -1103,9 +1102,6 @@ public:
    /// the quadrature points. The matrix will be transposed or not according to
    /// the boolean argument @a transpose.
    ///
-   /// The stored entries use the same row/column convention as `Eval()`,
-   /// unless `transpose == true`, in which case `K^T` is stored instead.
-   ///
    /// The @a vdim of the QuadratureFunction should be equal to the height times
    /// the width of the matrix.
    virtual void Project(QuadratureFunction &qf, bool transpose=false);

fem/complex_fem.hpp

@@ -166,75 +166,6 @@ public:
       return sqrt(err_r * err_r + err_i * err_i);
    }
 
-   /// @brief Returns Max|u_ex - u_h| error for complex-valued H1 or L2 elements
-   ///
-   /// Compute the $L_\infty$ error across the entire domain.
-   ///
-   /// @param[in] exsolr  Coefficient object reproducing the real part of the
-   ///                    anticipated values of the scalar field, Re(u_ex).
-   /// @param[in] exsoli  Coefficient object reproducing the imaginary part of
-   ///                    the anticipated values of the scalar field, Im(u_ex).
-   /// @param[in] irs        Optional pointer to an array of custom integration
-   ///                       rules e.g. higher order than the default rules. If
-   ///                       present the array will be indexed by
-   ///                       Geometry::Type.
-   ///
-   /// @note Uses ComputeLpError internally. See the ComputeLpError
-   ///       documentation for generalizations of this error computation.
-   ///
-   /// @note If an array of integration rules is provided through @a irs, be
-   ///       sure to include valid rules for each element type that may occur
-   ///       in the list of elements.
-   ///
-   virtual real_t ComputeMaxError(Coefficient &exsolr,
-                                  Coefficient &exsoli,
-                                  const IntegrationRule *irs[] = NULL) const
-   {
-      return ComputeLpError(infinity(), exsolr, exsoli, NULL, irs);
-   }
-
-   /// @brief Returns ||u_ex - u_h||_Lp for complex-valued H1 or L2 elements
-   ///
-   /// Computes:
-   ///    $$(\sum_{elems} \int_{elem} w \, |u_{ex} - u_h|^p)^{1/p}$$
-   /// Where:
-   ///    $$|u_{ex} - u_h| = \sqrt{Re(u_{ex} - u_h)^2 + Im(u_{ex} - u_h)^2}$$
-   ///
-   /// @param[in] p       Real value indicating the exponent of the $L^p$ norm.
-   ///                    To avoid domain errors p should have a positive value,
-   ///                    either finite or infinite.
-   /// @param[in] exsolr  Coefficient object reproducing the real part of the
-   ///                    anticipated values of the scalar field, Re(u_ex).
-   /// @param[in] exsoli  Coefficient object reproducing the imaginary part of
-   ///                    the anticipated values of the scalar field, Im(u_ex).
-   /// @param[in] weight  Optional pointer to a Coefficient object reproducing
-   ///                    a weighting function, w.
-   /// @param[in] irs     Optional pointer to an array of custom integration
-   ///                    rules e.g. higher order than the default rules. If
-   ///                    present the array will be indexed by Geometry::Type.
-   /// @param[in] elems   Optional pointer to a marker array, with a length
-   ///                    equal to the number of local elements, indicating
-   ///                    which elements to integrate over. Only those elements
-   ///                    corresponding to non-zero entries in @a elems will
-   ///                    contribute to the computed L2 error.
-   ///
-   /// @note If an array of integration rules is provided through @a irs, be
-   ///       sure to include valid rules for each element type that may occur
-   ///       in the list of elements.
-   ///
-   /// @note Quadratures with negative weights (as in some simplex integration
-   ///       rules in MFEM) can produce negative integrals even with
-   ///       non-negative integrands. To avoid returning negative errors this
-   ///       function uses the absolute values of the element-wise integrals.
-   ///       This may lead to results which are not entirely consistent with
-   ///       such integration rules.
-   virtual real_t ComputeLpError(const real_t p,
-                                 Coefficient &exsolr,
-                                 Coefficient &exsoli,
-                                 Coefficient *weight = NULL,
-                                 const IntegrationRule *irs[] = NULL,
-                                 const Array<int> *elems = NULL) const;
-
    /// Save the ComplexGridFunction to an output stream.
    virtual void Save(std::ostream &out) const;
 
@@ -392,9 +323,6 @@ private:
    bool RealInteg();
    bool ImagInteg();
 
-   void BuildComplexOperator(OperatorHandle &A_r, OperatorHandle &A_i,
-                             OperatorHandle &A) const;
-
 public:
    SesquilinearForm(FiniteElementSpace *fes,
                     ComplexOperator::Convention
@@ -508,186 +436,6 @@ public:
    virtual ~SesquilinearForm();
 };
 
-/** Class for a mixed sesquilinear form
-
-    A mixed sesquilinear form is a generalization of a mixed bilinear form to
-    complex-valued fields. Mixed sesquilinear forms are linear in the second
-    argument but the first argument involves a complex conjugate in the sense
-    that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the MixedSesquilinearForm either the real or
-    imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class MixedSesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   MixedBilinearForm * mblfr;
-   MixedBilinearForm * mblfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are not
-      empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   MixedSesquilinearForm(
-      FiniteElementSpace * trial_fes,
-      FiniteElementSpace * test_fes,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a MixedSesquilinearForm on the given trial and test
-       FiniteElementSpaces, using the same integrators as the
-       MixedBilinearForms @a bfr and @a bfi.
-
-       The FiniteElementSpace pointers are not owned by the newly constructed
-       object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed MixedSesquilinearForm. */
-   MixedSesquilinearForm(
-      FiniteElementSpace * trial_fes,
-      FiniteElementSpace * test_fes,
-      MixedBilinearForm * bfr,
-      MixedBilinearForm * bfi,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
-
-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::LEGACY (default)
-       - AssemblyLevel::FULL
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      mblfr->SetAssemblyLevel(assembly_level);
-      mblfi->SetAssemblyLevel(assembly_level);
-   }
-
-   MixedBilinearForm & real() { return *mblfr; }
-   MixedBilinearForm & imag() { return *mblfi; }
-   const MixedBilinearForm & real() const { return *mblfr; }
-   const MixedBilinearForm & imag() const { return *mblfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag,
-                            Array<int> & elem_marker);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag,
-                              Array<int> & bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag,
-                             Array<int> & bdr_marker);
-
-   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
-
-       This type of integrator assembles terms over all faces of the mesh using
-       the face FE from the trial space and the two adjacent volume FEs from
-       the test space. */
-   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                               BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag,
-                                  Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Updates the internal mixed forms with the new finite element space.
-   virtual void Update();
-
-   /** @brief Return a ComplexSparseMatrix wrapping the local (L-dof) real
-       and imaginary matrices of the form.
-
-       The returned wrapper has to be deleted by the caller, but it does not
-       own the wrapped real and imaginary matrices, which remain owned by
-       this form. */
-   ComplexSparseMatrix *AssembleComplexSparseMatrix();
-
-   /// Return the trial FE space associated with the MixedSesquilinearForm.
-   FiniteElementSpace *TrialFESpace() { return mblfr->TrialFESpace(); }
-
-   /// Read-only access to the associated trial FiniteElementSpace.
-   const FiniteElementSpace *TrialFESpace() const { return mblfr->TrialFESpace(); }
-
-   /// Return the test FE space associated with the MixedSesquilinearForm.
-   FiniteElementSpace *TestFESpace() { return mblfr->TestFESpace(); }
-
-   /// Read-only access to the associated test FiniteElementSpace.
-   const FiniteElementSpace *TestFESpace() const { return mblfr->TestFESpace(); }
-
-
-   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    Vector & x,
-                                    Vector & b,
-                                    OperatorHandle & A,
-                                    Vector & X,
-                                    Vector & B);
-
-   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    OperatorHandle & A);
-
-   virtual ~MixedSesquilinearForm();
-};
-
 #ifdef MFEM_USE_MPI
 
 /// Class for parallel complex-valued grid function - real + imaginary part
@@ -989,12 +737,6 @@ private:
    bool RealInteg();
    bool ImagInteg();
 
-   void SetImaginaryEssentialDiagonalToZero(
-      const Array<int> &ess_tdof_list, OperatorHandle &A);
-
-   void BuildComplexOperator(OperatorHandle &A_r, OperatorHandle &A_i,
-                             OperatorHandle &A) const;
-
 public:
    ParSesquilinearForm(ParFiniteElementSpace *pf,
                        ComplexOperator::Convention
@@ -1110,169 +852,6 @@ public:
    virtual ~ParSesquilinearForm();
 };
 
-/** Class for a parallel mixed sesquilinear form
-
-    A mixed sesquilinear form is a generalization of a mixed bilinear form to
-    complex-valued fields. Mixed sesquilinear forms are linear in the second
-    argument but the first argument involves a complex conjugate in the sense
-    that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the ParMixedSesquilinearForm either the real
-    or imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class ParMixedSesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   ParMixedBilinearForm * pmblfr;
-   ParMixedBilinearForm * pmblfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are
-    not empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   ParMixedSesquilinearForm(
-      ParFiniteElementSpace * trial_fes,
-      ParFiniteElementSpace * test_fes,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a ParMixedSesquilinearForm on the given trial and test
-       ParFiniteElementSpaces, using the same integrators as the
-       ParMixedBilinearForms @a pbfr and @a pbfi.
-
-       The ParFiniteElementSpace pointers are not owned by the newly
-       constructed object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed ParMixedSesquilinearForm. */
-   ParMixedSesquilinearForm(
-      ParFiniteElementSpace * trial_fes,
-      ParFiniteElementSpace * test_fes,
-      ParMixedBilinearForm * pbfr,
-      ParMixedBilinearForm * pbfi,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
-
-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::LEGACY (default)
-       - AssemblyLevel::FULL
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      pmblfr->SetAssemblyLevel(assembly_level);
-      pmblfi->SetAssemblyLevel(assembly_level);
-   }
-
-   ParMixedBilinearForm & real() { return *pmblfr; }
-   ParMixedBilinearForm & imag() { return *pmblfi; }
-   const ParMixedBilinearForm & real() const { return *pmblfr; }
-   const ParMixedBilinearForm & imag() const { return *pmblfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag,
-                            Array<int> & elem_marker);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag,
-                              Array<int> & bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag,
-                             Array<int> & bdr_marker);
-
-   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
-
-      This type of integrator assembles terms over all faces of the mesh using
-      the face FE from the trial space and the two adjacent volume FEs from
-      the test space. */
-   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                               BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag,
-                                  Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Updates the internal mixed forms with the new finite element space.
-   virtual void Update();
-
-   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
-   /** The returned matrix has to be deleted by the caller. */
-   ComplexHypreParMatrix * ParallelAssemble();
-
-   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    Vector & x,
-                                    Vector & b,
-                                    OperatorHandle & A,
-                                    Vector & X,
-                                    Vector & B);
-
-   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    OperatorHandle & A);
-
-   virtual ~ParMixedSesquilinearForm();
-};
-
 #endif // MFEM_USE_MPI
 
 }

fem/datacollection.cpp

@@ -38,24 +38,9 @@ int DataCollection::create_directory(const std::string &dir_name,
    // create directories recursively
    const char path_delim = '/';
    std::string::size_type pos = 0;
-   int err_flag = 0;
+   int err_flag;
 #ifdef MFEM_USE_MPI
    const ParMesh *pmesh = dynamic_cast<const ParMesh*>(mesh);
-   // In addition to the global root, let the lowest rank on each shared-memory
-   // node create the directory too, so that node-local (non-shared) filesystems
-   // get it on every node rather than only where the global root lives. On a
-   // shared filesystem the extra mkdir() hits EEXIST and is tolerated below.
-   bool node_root = true;
-   if (pmesh)
-   {
-      MPI_Comm node_comm;
-      MPI_Comm_split_type(pmesh->GetComm(), MPI_COMM_TYPE_SHARED, myid,
-                          MPI_INFO_NULL, &node_comm);
-      int node_rank;
-      MPI_Comm_rank(node_comm, &node_rank);
-      node_root = (node_rank == 0);
-      MPI_Comm_free(&node_comm);
-   }
 #endif
 
    do
@@ -67,7 +52,7 @@ int DataCollection::create_directory(const std::string &dir_name,
       err_flag = mkdir(subdir.c_str(), 0777);
       err_flag = (err_flag && (errno != EEXIST)) ? 1 : 0;
 #else
-      if (node_root || pmesh == NULL)
+      if (myid == 0 || pmesh == NULL)
       {
          err_flag = mkdir(subdir.c_str(), 0777);
          err_flag = (err_flag && (errno != EEXIST)) ? 1 : 0;
@@ -79,8 +64,7 @@ int DataCollection::create_directory(const std::string &dir_name,
 #ifdef MFEM_USE_MPI
    if (pmesh)
    {
-      MPI_Allreduce(MPI_IN_PLACE, &err_flag, 1, MPI_INT, MPI_MAX,
-                    pmesh->GetComm());
+      MPI_Bcast(&err_flag, 1, MPI_INT, 0, pmesh->GetComm());
    }
 #endif
 
@@ -825,7 +809,7 @@ ParaViewDataCollectionBase::ParaViewDataCollectionBase(
 
 void ParaViewDataCollectionBase::SetLevelsOfDetail(int levels_of_detail_)
 {
-   levels_of_detail = std::max(levels_of_detail_, 1);
+   levels_of_detail = levels_of_detail_;
 }
 
 void ParaViewDataCollectionBase::SetHighOrderOutput(bool high_order_output_)
@@ -1197,14 +1181,12 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,
    DenseMatrix vval, pmat;
    std::vector<char> buf;
    int vec_dim = it->second->VectorDim();
-   int map_type = it->second->FESpace()->GetTypicalFE()->GetMapType();
    os << "<DataArray type=\"" << GetDataTypeString()
       << "\" Name=\"" << it->first
       << "\" NumberOfComponents=\"" << vec_dim << "\" "
       << VTKComponentLabels(vec_dim) << " "
       << "format=\"" << GetDataFormatString() << "\" >" << '\n';
-   if (vec_dim == 1 && (map_type == FiniteElement::VALUE ||
-                        map_type == FiniteElement::INTEGRAL))
+   if (vec_dim == 1)
    {
       for (int i = 0; i < mesh->GetNE(); i++)
       {

fem/dfem/assemble.hpp

@@ -1,403 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "util.hpp"
-
-namespace mfem::future
-{
-
-/// @brief Assemble element matrix for three dimensional data.
-///
-/// Note: In the below layouts, total_trial_op_dim is > 1 if
-/// there are more than one inputs dependent on the derivative variable.
-///
-/// @param A Memory for one element matrix with layout
-/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
-/// @param fhat Memory to hold the residual computation with layout
-/// [test_vdim, test_op_dim, nqp].
-/// @param qpdc The quadrature point data cache with data layout
-/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
-/// @param itod Input Trial Operator Dimension array. If the trial
-/// operator is not dependent, the dimension is 0 to indicate that.
-/// @param inputs The input field operator types.
-/// @param output The output field operator types.
-/// @param input_dtqmaps The input DofToQuad maps.
-/// @param output_dtqmap The output DofToQuad maps.
-/// @param scratch_shmem Scratch shared memory for computations.
-/// @param q1d The number of quadrature points in one dimension.
-/// @param td1d The number of trial dofs in one dimension.
-template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
-MFEM_HOST_DEVICE void assemble_element_mat_t3d(
-   const DeviceTensor<4, real_t>& A,
-   const DeviceTensor<3, real_t>& fhat,
-   const DeviceTensor<5, const real_t>& qpdc,
-   const DeviceTensor<1, const real_t>& itod,
-   const input_fop_ts& inputs,
-   const output_fop_t& output,
-   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
-   const DofToQuadMap& output_dtqmap,
-   std::array<DeviceTensor<1>, 6>& scratch_shmem,
-   const int& q1d,
-   const int& td1d)
-{
-   constexpr int dimension = 3;
-
-   // [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
-   const int test_vdim = qpdc.GetShape()[0];
-   const int test_op_dim = qpdc.GetShape()[1];
-   const int trial_vdim = qpdc.GetShape()[2];
-
-   // [num_test_dof, ...]
-   const auto num_test_dof = A.GetShape()[0];
-
-   for (int Jx = 0; Jx < td1d; Jx++)
-   {
-      for (int Jy = 0; Jy < td1d; Jy++)
-      {
-         for (int Jz = 0; Jz < td1d; Jz++)
-         {
-            const int J = Jx + td1d * (Jy + td1d * Jz);
-
-            for (int j = 0; j < trial_vdim; j++)
-            {
-               for (int tv = 0; tv < test_vdim; tv++)
-               {
-                  for (int tod = 0; tod < test_op_dim; tod++)
-                  {
-                     MFEM_FOREACH_THREAD(qx, x, q1d)
-                     {
-                        MFEM_FOREACH_THREAD(qy, y, q1d)
-                        {
-                           MFEM_FOREACH_THREAD(qz, z, q1d)
-                           {
-                              const int q = qx + q1d * (qy + q1d * qz);
-                              fhat(tv, tod, q) = 0.0;
-                           }
-                        }
-                     }
-                  }
-               }
-
-               // MSVC lambda capture workaround
-               [[maybe_unused]] const auto& inputs_ref = inputs;
-
-               int m_offset = 0;
-               for_constexpr<num_inputs>([&](auto s)
-               {
-                  using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
-
-                  const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
-                  if (trial_op_dim == 0)
-                  {
-                     // This is inside a lambda so we have to return
-                     // instead of idiomatic 'continue'.
-                     return;
-                  }
-
-                  auto& B = input_dtqmaps[s].B;
-                  auto& G = input_dtqmaps[s].G;
-
-                  if constexpr (is_value_fop<fop_t>::value)
-                  {
-                     MFEM_FOREACH_THREAD(qx, x, q1d)
-                     {
-                        MFEM_FOREACH_THREAD(qy, y, q1d)
-                        {
-                           MFEM_FOREACH_THREAD(qz, z, q1d)
-                           {
-                              const int q = qx + q1d * (qy + q1d * qz);
-
-                              for (int m = 0; m < trial_op_dim; m++)
-                              {
-                                 for (int i = 0; i < test_vdim; i++)
-                                 {
-                                    for (int k = 0; k < test_op_dim; k++)
-                                    {
-                                       const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                       fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
-                                    }
-                                 }
-                              }
-                           }
-                        }
-                     }
-                  }
-                  else if constexpr (is_gradient_fop<fop_t>::value)
-                  {
-                     MFEM_FOREACH_THREAD(qx, x, q1d)
-                     {
-                        MFEM_FOREACH_THREAD(qy, y, q1d)
-                        {
-                           MFEM_FOREACH_THREAD(qz, z, q1d)
-                           {
-                              const int q = qx + q1d * (qy + q1d * qz);
-                              for (int m = 0; m < trial_op_dim; m++)
-                              {
-                                 for (int i = 0; i < test_vdim; i++)
-                                 {
-                                    for (int k = 0; k < test_op_dim; k++)
-                                    {
-                                       const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                       if (m == 0)
-                                       {
-                                          fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
-                                       }
-                                       else if (m == 1)
-                                       {
-                                          fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy) * B(qz, 0, Jz);
-                                       }
-                                       else if (m == 2)
-                                       {
-                                          fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * G(qz, 0, Jz);
-                                       }
-                                    }
-                                 }
-                              }
-                           }
-                        }
-                     }
-                  }
-                  else
-                  {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-                     MFEM_ABORT("sum factorized sparse matrix assemble routine "
-                                "not implemented for field operator");
-#endif
-                  }
-                  MFEM_SYNC_THREAD;
-                  m_offset += trial_op_dim;
-               });
-
-               auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
-               map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
-                                             scratch_shmem, dimension, true);
-            }
-         }
-      }
-   }
-}
-
-/// @brief Assemble element matrix for two dimensional data.
-///
-/// Note: In the below layouts, total_trial_op_dim is > 1 if
-/// there are more than one inputs dependent on the derivative variable.
-///
-/// @param A Memory for one element matrix with layout
-/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
-/// @param fhat Memory to hold the residual computation with layout
-/// [test_vdim, test_op_dim, nqp].
-/// @param qpdc The quadrature point data cache with data layout
-/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
-/// @param itod Input Trial Operator Dimension array. If the trial
-/// operator is not dependent, the dimension is 0 to indicate that.
-/// @param inputs The input field operator types.
-/// @param output The output field operator types.
-/// @param input_dtqmaps The input DofToQuad maps.
-/// @param output_dtqmap The output DofToQuad maps.
-/// @param scratch_shmem Scratch shared memory for computations.
-/// @param q1d The number of quadrature points in one dimension.
-/// @param td1d The number of trial dofs in one dimension.
-template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
-MFEM_HOST_DEVICE void assemble_element_mat_t2d(
-   const DeviceTensor<4, real_t>& A,
-   const DeviceTensor<3, real_t>& fhat,
-   const DeviceTensor<5, const real_t>& qpdc,
-   const DeviceTensor<1, const real_t>& itod,
-   const input_fop_ts& inputs,
-   const output_fop_t& output,
-   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
-   const DofToQuadMap& output_dtqmap,
-   std::array<DeviceTensor<1>, 6>& scratch_shmem,
-   const int& q1d,
-   const int& td1d)
-{
-   constexpr int dimension = 2;
-
-   // [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
-   const int test_vdim = qpdc.GetShape()[0];
-   const int test_op_dim = qpdc.GetShape()[1];
-   const int trial_vdim = qpdc.GetShape()[2];
-
-   // [num_test_dof, ...]
-   const auto num_test_dof = A.GetShape()[0];
-
-   for (int Jx = 0; Jx < td1d; Jx++)
-   {
-      for (int Jy = 0; Jy < td1d; Jy++)
-      {
-         const int J = Jy + Jx * td1d;
-
-         for (int j = 0; j < trial_vdim; j++)
-         {
-            for (int tv = 0; tv < test_vdim; tv++)
-            {
-               for (int tod = 0; tod < test_op_dim; tod++)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, q1d)
-                  {
-                     MFEM_FOREACH_THREAD(qy, y, q1d)
-                     {
-                        const int q = qy + qx * q1d;
-                        fhat(tv, tod, q) = 0.0;
-                     }
-                  }
-               }
-            }
-
-            // MSVC lambda capture workaround
-            [[maybe_unused]] const auto& inputs_ref = inputs;
-
-            int m_offset = 0;
-            for_constexpr<num_inputs>([&](auto s)
-            {
-               using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
-
-               const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
-               if (trial_op_dim == 0)
-               {
-                  // This is inside a lambda so we have to return
-                  // instead of idiomatic 'continue'.
-                  return;
-               }
-
-               auto& B = input_dtqmaps[s].B;
-               auto& G = input_dtqmaps[s].G;
-
-               if constexpr (is_value_fop<fop_t>::value)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, q1d)
-                  {
-                     MFEM_FOREACH_THREAD(qy, y, q1d)
-                     {
-                        const int q = qy + qx * q1d;
-                        for (int m = 0; m < trial_op_dim; m++)
-                        {
-                           for (int i = 0; i < test_vdim; i++)
-                           {
-                              for (int k = 0; k < test_op_dim; k++)
-                              {
-                                 const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                 fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy);
-                              }
-                           }
-                        }
-                     }
-                  }
-               }
-               else if constexpr (is_gradient_fop<fop_t>::value)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, q1d)
-                  {
-                     MFEM_FOREACH_THREAD(qy, y, q1d)
-                     {
-                        const int q = qy + qx * q1d;
-                        for (int m = 0; m < trial_op_dim; m++)
-                        {
-                           for (int i = 0; i < test_vdim; i++)
-                           {
-                              for (int k = 0; k < test_op_dim; k++)
-                              {
-                                 const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                 if (m == 0)
-                                 {
-                                    fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy);
-                                 }
-                                 else
-                                 {
-                                    fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy);
-                                 }
-                              }
-                           }
-                        }
-                     }
-                  }
-               }
-               else
-               {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-                  MFEM_ABORT("sum factorized sparse matrix assemble routine "
-                             "not implemented for field operator");
-#endif
-               }
-               MFEM_SYNC_THREAD;
-               m_offset += trial_op_dim;
-            });
-
-            auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
-            map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
-                                          scratch_shmem, dimension, true);
-         }
-      }
-   }
-}
-
-/// @brief Assemble element matrix for two or three dimensional data.
-///
-/// Note: In the below layouts, total_trial_op_dim is > 1 if
-/// there are more than one inputs dependent on the derivative variable.
-///
-/// @param A Memory for one element matrix with layout
-/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
-/// @param fhat Memory to hold the residual computation with layout
-/// [test_vdim, test_op_dim, nqp].
-/// @param qpdc The quadrature point data cache with data layout
-/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
-/// @param itod Input Trial Operator Dimension array. If the trial
-/// operator is not dependent, the dimension is 0 to indicate that.
-/// @param inputs The input field operator types.
-/// @param output The output field operator types.
-/// @param input_dtqmaps The input DofToQuad maps.
-/// @param output_dtqmap The output DofToQuad maps.
-/// @param scratch_shmem Scratch shared memory for computations.
-/// @param dimension The spatial dimension.
-/// @param q1d The number of quadrature points in one dimension.
-/// @param td1d The number of trial dofs in one dimension.
-/// @param use_sum_factorization Indicator if sum factorization is used.
-template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
-MFEM_HOST_DEVICE void assemble_element_mat_naive(
-   const DeviceTensor<4, real_t>& A,
-   const DeviceTensor<3, real_t>& fhat,
-   const DeviceTensor<5, const real_t>& qpdc,
-   const DeviceTensor<1, const real_t>& itod,
-   const input_fop_ts& inputs,
-   const output_fop_t& output,
-   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
-   const DofToQuadMap& output_dtqmap,
-   std::array<DeviceTensor<1>, 6>& scratch_shmem,
-   const int& dimension,
-   const int& q1d,
-   const int& td1d,
-   const bool& use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 2)
-      {
-         assemble_element_mat_t2d(A, fhat, qpdc, itod, inputs, output,
-                                  input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
-      }
-      else if (dimension == 3)
-      {
-         assemble_element_mat_t3d(A, fhat, qpdc, itod, inputs, output,
-                                  input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
-      }
-   }
-   else
-   {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-      MFEM_ABORT("element matrix assemble not implemented for non tensor "
-                 "product basis");
-#endif
-   }
-}
-
-} // namespace mfem::future

fem/dfem/backends/global_qf/action.hpp

@@ -0,0 +1,122 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../util.hpp"
+#include "../../integrator_ctx.hpp"
+
+#include <utility>
+
+namespace mfem::future
+{
+
+namespace GlobalQFImpl
+{
+
+template<
+   typename qfunc_t,
+   typename inputs_t,
+   typename outputs_t,
+   size_t ninputs = tuple_size<inputs_t>::value,
+   size_t noutputs = tuple_size<outputs_t>::value>
+struct Action
+{
+   Action(
+      IntegratorContext ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs) :
+      ctx(ctx),
+      qfunc(qfunc),
+      inputs(inputs),
+      outputs(outputs)
+   {
+      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(inputs, input_to_infd, ctx.infds);
+      check_consistency(outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
+      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
+
+      const int nqp = ctx.ir.GetNPoints();
+      gnqp = nqp * ctx.nentities;
+
+      // prepare xq and yq BlockVectors
+      xq_offsets.SetSize(ninputs + 1);
+      xq_offsets[0] = 0;
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         const auto input = get<i>(inputs);
+         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
+      });
+      xq_offsets.PartialSum();
+      InitBlockVector(xq, xq_offsets);
+
+      yq_offsets.SetSize(noutputs + 1);
+      yq_offsets[0] = 0;
+      constexpr_for<0, noutputs>([&](auto i)
+      {
+         const auto output = get<i>(outputs);
+         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
+      });
+      yq_offsets.PartialSum();
+      InitBlockVector(yq, yq_offsets);
+   }
+
+   void operator()(
+      const std::vector<Vector *> &xe,
+      std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      // E -> Q
+      interpolate(input_to_infd, input_bases, xe, xq);
+
+      // Q -> Q
+      static_assert(
+         detail::supports_tensor_array_qfunc<qfunc_t, inputs_t, outputs_t>::value,
+         "qfunc signature not supported by default backend Action");
+
+      detail::call_qfunc(
+         qfunc, xq, yq, gnqp, input_qlayouts, output_qlayouts,
+         std::make_index_sequence<ninputs> {},
+         std::make_index_sequence<noutputs> {});
+
+      // Q -> E
+      integrate(output_to_outfd, output_bases, yq, ye);
+   }
+
+   IntegratorContext ctx;
+   qfunc_t qfunc;
+   inputs_t inputs;
+   outputs_t outputs;
+
+   std::array<size_t, ninputs> input_to_infd;
+   std::array<size_t, noutputs> output_to_outfd;
+
+   std::array<FieldBasis, ninputs> input_bases;
+   std::array<FieldBasis, noutputs> output_bases;
+
+   std::array<std::vector<int>, ninputs>  input_qlayouts;
+   std::array<std::vector<int>, noutputs> output_qlayouts;
+
+   int gnqp = 0;
+   Array<int> xq_offsets, yq_offsets;
+   mutable BlockVector xq, yq;
+};
+
+}
+}

fem/dfem/backends/global_qf/derivative_action.hpp

@@ -0,0 +1,180 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "../util.hpp"
+#include <utility>
+
+namespace mfem::future::GlobalQFImpl
+{
+
+template<
+   int derivative_id,
+   typename qfunc_t,
+   typename inputs_t,
+   typename outputs_t,
+   size_t ninputs = tuple_size<inputs_t>::value,
+   size_t noutputs = tuple_size<outputs_t>::value>
+struct DerivativeAction
+{
+   using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
+
+   DerivativeAction(
+      IntegratorContext ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs):
+      ctx(ctx), qfunc(qfunc),
+      qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
+      inputs(inputs), outputs(outputs)
+   {
+      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(inputs, input_to_infd, ctx.infds);
+      check_consistency(outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(
+         outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
+      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
+
+      const int nqp = ctx.ir.GetNPoints();
+      gnqp = nqp * ctx.nentities;
+
+      xq_offsets.SetSize(ninputs + 1);
+      xq_offsets[0] = 0;
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         const auto input = get<i>(inputs);
+         xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
+      });
+      xq_offsets.PartialSum();
+      InitBlockVector(xq, xq_offsets);
+
+      yq_offsets.SetSize(noutputs + 1);
+      yq_offsets[0] = 0;
+      constexpr_for<0, noutputs>([&](auto i)
+      {
+         const auto output = get<i>(outputs);
+         yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
+      });
+      yq_offsets.PartialSum();
+      InitBlockVector(yq, yq_offsets);
+
+      // Shadow blocks use the same offsets as xq so tensor_array views
+      shadow_xq_offsets.SetSize(xq_offsets.Size());
+      shadow_xq_offsets = xq_offsets;
+      InitBlockVector(shadow_xq, shadow_xq_offsets);
+
+      dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
+
+      const size_t direction_fd_idx = FindIdx(derivative_id, ctx.infds);
+      MFEM_ASSERT(direction_fd_idx != SIZE_MAX,
+                  "derivative direction field not found in infds");
+      direction_fd = ctx.infds[direction_fd_idx];
+   }
+
+   void operator()(
+      const std::vector<Vector *> &xe,
+      const Vector *de,
+      std::vector<Vector *> &ye)
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      // E -> Q
+      interpolate(input_to_infd, input_bases, xe, xq);
+
+      constexpr auto input_active =
+         detail::make_activity_map<derivative_id>(inputs_t{});
+
+      MFEM_ASSERT(de != nullptr, "derivative direction vector is null");
+      restriction(direction_fd, direction_rcache, *de, direction_e,
+                  dof_ordering);
+
+      shadow_xq = 0.0;
+      shadow_xq.SyncToBlocks();
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         if (!input_active[i]) { return; }
+         input_bases[i].forward(direction_e, shadow_xq.GetBlock(i));
+      });
+
+      static_assert(detail::supports_tensor_array_qfunc<qfunc_t,
+                    inputs_t,
+                    outputs_t>::value,
+                    "qfunc signature not supported by default backend Action");
+
+      // Q -> Q
+      yq = 0.0;
+      yq.SyncToBlocks();
+      if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
+      {
+         detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t, inputs_t,
+                outputs_t>(
+                   qfunc,
+                   qfunc_shadow,
+                   xq,
+                   shadow_xq,
+                   yq,
+                   gnqp,
+                   input_qlayouts,
+                   output_qlayouts,
+                   std::make_index_sequence<ninputs> {},
+                   std::make_index_sequence<noutputs> {});
+      }
+      else
+      {
+         detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
+            qfunc,
+            xq,
+            shadow_xq,
+            yq,
+            gnqp,
+            input_qlayouts,
+            output_qlayouts,
+            std::make_index_sequence<ninputs> {},
+            std::make_index_sequence<noutputs> {});
+      }
+
+      // Q -> E
+      integrate(output_to_outfd, output_bases, yq, ye);
+   }
+
+   IntegratorContext ctx;
+   qfunc_t qfunc;
+   qfunc_shadow_t qfunc_shadow;
+   inputs_t inputs;
+   outputs_t outputs;
+
+   std::array<size_t, ninputs> input_to_infd;
+   std::array<size_t, noutputs> output_to_outfd;
+
+   std::array<FieldBasis, ninputs> input_bases;
+   std::array<FieldBasis, noutputs> output_bases;
+
+   std::array<std::vector<int>, ninputs> input_qlayouts;
+   std::array<std::vector<int>, noutputs> output_qlayouts;
+
+   int gnqp = 0;
+   Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
+   mutable BlockVector xq, shadow_xq, yq;
+
+   FieldDescriptor direction_fd;
+   ElementDofOrdering dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   mutable Vector direction_e;
+   mutable RestrictionCache<Entity::Element> direction_rcache;
+};
+
+} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_apply.hpp

@@ -0,0 +1,242 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "../util.hpp"
+
+#include <array>
+#include <utility>
+#include <vector>
+
+namespace mfem::future::GlobalQFImpl
+{
+
+// Q-function-shape-agnostic cached forward apply (J·v)
+template<
+   int derivative_id,
+   typename inputs_t,
+   typename outputs_t>
+struct DerivativeApply
+{
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+
+   DerivativeApply(
+      IntegratorContext ctx,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache):
+      ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
+      qp_cache(qp_cache)
+   {
+      create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(this->inputs, input_to_infd, ctx.infds);
+      check_consistency(this->outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(
+         this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(
+         this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      const int nqp = ctx.ir.GetNPoints();
+      const int ne = ctx.nentities;
+      num_qp = nqp;
+      gnqp = nqp * ne;
+
+      // Precompute Q-space BlockVector layouts
+      dir_q_offsets.SetSize(n_inputs + 1);
+      dir_q_offsets[0] = 0;
+      constexpr_for<0, n_inputs>([&](auto i)
+      {
+         dir_q_offsets[i + 1] =
+            dir_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
+      });
+      InitBlockVector(dir_q_local, dir_q_offsets);
+
+      result_q_offsets.SetSize(n_outputs + 1);
+      result_q_offsets[0] = 0;
+      constexpr_for<0, n_outputs>([&](auto i)
+      {
+         result_q_offsets[i + 1] =
+            result_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
+      });
+      InitBlockVector(result_q_local, result_q_offsets);
+
+      // Cache layout metadata (must match DerivativeSetup)
+      residual_size_on_qp = 0;
+      trial_vdim = 0;
+      total_trial_op_dim = 0;
+
+      constexpr auto activity =
+         detail::make_activity_map<derivative_id>(inputs_t{});
+
+      constexpr_for<0, n_inputs>([&](auto i)
+      {
+         if (!activity[i]) { return; }
+         const auto &fop = get<i>(this->inputs);
+         trial_vdim = fop.vdim;
+         total_trial_op_dim += fop.size_on_qp / fop.vdim;
+      });
+
+      constexpr_for<0, n_outputs>([&](auto i)
+      { residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
+      residual_size_on_qp *= trial_vdim * total_trial_op_dim;
+   }
+
+   void operator()(
+      const std::vector<Vector *> & /*xe*/,
+      const Vector *direction_l,
+      std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      MFEM_ASSERT(direction_l != nullptr,
+                  "Global DerivativeApply: direction vector is null");
+
+      // Re-zero pre-allocated Q temporaries
+      dir_q_local = 0.0;
+      dir_q_local.SyncToBlocks();
+      result_q_local = 0.0;
+      result_q_local.SyncToBlocks();
+
+      // Restrict trial direction from the derivative field
+      size_t in_fd = SIZE_MAX;
+      constexpr_for<0, n_inputs>([&](auto i)
+      {
+         if (get<i>(inputs).GetFieldId() == derivative_id)
+         {
+            in_fd = input_to_infd[i.value];
+         }
+      });
+      MFEM_ASSERT(in_fd != SIZE_MAX,
+                  "DerivativeApply: derivative field not found among inputs");
+
+      const auto &fd = ctx.infds[in_fd];
+
+      Vector dir_e;
+      restriction(fd, direction_rcache, *direction_l, dir_e,
+                  ElementDofOrdering::LEXICOGRAPHIC);
+
+      // Forward the trial direction into active input Q block
+      constexpr_for<0, n_inputs>([&](auto s)
+      {
+         if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
+         input_bases[s.value].forward(dir_e, dir_q_local.GetBlock(s.value));
+      });
+
+      const real_t *cache_ptr = qp_cache.Read();
+      const int res_sz = residual_size_on_qp;
+      const int gnqp_local = gnqp;
+      const int num_qp_local = num_qp;
+      const int trial_vdim_local = trial_vdim;
+      const int total_trial_op_dim_local = total_trial_op_dim;
+
+      constexpr_for<0, n_outputs>([&](auto o)
+      {
+         const int tv_o = get<o>(outputs).vdim;
+         const int to_o = get<o>(outputs).size_on_qp / tv_o;
+         const int out_base = [&]
+         {
+            int off = 0;
+            constexpr_for<0, o.value>([&](auto prev)
+            { off += get<prev>(outputs).size_on_qp; });
+            return off;
+         }();
+
+         real_t *res_o = result_q_local.GetBlock(o.value).ReadWrite();
+
+         int m_offset = 0;
+         constexpr_for<0, n_inputs>([&](auto s)
+         {
+            if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
+
+            const int tv = get<s>(inputs).vdim;
+            const int to = get<s>(inputs).size_on_qp / tv;
+            const real_t *dir_s = dir_q_local.GetBlock(s.value).Read();
+
+            mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
+            {
+               // Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
+               // read adjacent addresses for a fixed cache_idx.
+               const int cache_base =
+                  (gq % num_qp_local) +
+                  num_qp_local * res_sz * (gq / num_qp_local);
+
+               for (int j = 0; j < tv; ++j)
+               {
+                  for (int m = 0; m < to; ++m)
+                  {
+                     const real_t v = dir_s[(j * to + m) + (tv * to) * gq];
+                     const int m_global = m + m_offset;
+
+                     for (int i = 0; i < tv_o; ++i)
+                     {
+                        for (int k = 0; k < to_o; ++k)
+                        {
+                           const int out_comp = out_base + i * to_o + k;
+
+                           const int cache_idx =
+                              out_comp * trial_vdim_local * total_trial_op_dim_local +
+                              j * total_trial_op_dim_local + m_global;
+
+                           const real_t c =
+                              cache_ptr[cache_base + num_qp_local * cache_idx];
+                           res_o[(i * to_o + k) + (tv_o * to_o) * gq] += c * v;
+                        }
+                     }
+                  }
+               }
+            });
+            m_offset += to;
+         });
+      });
+
+      result_q_local.SyncToBlocks();
+
+      // Map result Q back to output fields
+      constexpr_for<0, n_outputs>([&](auto o)
+      {
+         const size_t out_fd = output_to_outfd[o.value];
+         output_bases[o.value].transpose(result_q_local.GetBlock(o.value),
+                                         *ye[out_fd]);
+      });
+   }
+
+private:
+   IntegratorContext ctx;
+   inputs_t inputs;
+   outputs_t outputs;
+   const Vector &qp_cache;
+
+   std::array<size_t, n_inputs> input_to_infd;
+   std::array<size_t, n_outputs> output_to_outfd;
+
+   std::array<FieldBasis, n_inputs> input_bases;
+   std::array<FieldBasis, n_outputs> output_bases;
+
+   int gnqp = 0;
+   int num_qp = 0;
+
+   Array<int> dir_q_offsets;
+   Array<int> result_q_offsets;
+   mutable BlockVector dir_q_local;
+   mutable BlockVector result_q_local;
+   mutable RestrictionCache<Entity::Element> direction_rcache;
+
+   int residual_size_on_qp = 0;
+   int trial_vdim = 0;
+   int total_trial_op_dim = 0;
+};
+
+} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_apply_transpose.hpp

@@ -0,0 +1,260 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "../util.hpp"
+
+#include <array>
+#include <utility>
+#include <vector>
+
+namespace mfem::future::GlobalQFImpl
+{
+
+// Q-function-shape-agnostic cached transpose apply (Jᵀ·w)
+template<
+   int derivative_id,
+   typename qfunc_t,
+   typename inputs_t,
+   typename outputs_t>
+struct DerivativeApplyTranspose
+{
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+
+   DerivativeApplyTranspose(
+      IntegratorContext ctx,
+      qfunc_t /*qfunc*/,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache):
+      ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
+      qp_cache(qp_cache), dir_out_l(n_outputs), dir_out_e(n_outputs)
+   {
+      create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(this->inputs, input_to_infd, ctx.infds);
+      check_consistency(this->outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(
+         this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(
+         this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      const int nqp = ctx.ir.GetNPoints();
+      const int ne = ctx.nentities;
+      num_qp = nqp;
+      gnqp = nqp * ne;
+
+      // Precompute Q-space BlockVector layouts
+      dir_q_offsets.SetSize(n_outputs + 1);
+      dir_q_offsets[0] = 0;
+      constexpr_for<0, n_outputs>([&](auto i)
+      {
+         dir_q_offsets[i + 1] =
+            dir_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
+      });
+      InitBlockVector(dir_q_local, dir_q_offsets);
+
+      result_q_offsets.SetSize(n_inputs + 1);
+      result_q_offsets[0] = 0;
+      constexpr_for<0, n_inputs>([&](auto i)
+      {
+         result_q_offsets[i + 1] =
+            result_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
+      });
+      InitBlockVector(result_q_local, result_q_offsets);
+
+      // Cache layout metadata
+      residual_size_on_qp = 0;
+      trial_vdim = 0;
+      total_trial_op_dim = 0;
+
+      constexpr auto activity =
+         detail::make_activity_map<derivative_id>(inputs_t{});
+
+      constexpr_for<0, n_inputs>([&](auto i)
+      {
+         if (!activity[i]) { return; }
+         const auto &fop = get<i>(this->inputs);
+         trial_vdim = fop.vdim;
+         total_trial_op_dim += fop.size_on_qp / fop.vdim;
+      });
+
+      constexpr_for<0, n_outputs>([&](auto i)
+      { residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
+      residual_size_on_qp *= trial_vdim * total_trial_op_dim;
+   }
+
+   void operator()(
+      const std::vector<Vector *> & /*xe*/,
+      const Vector *direction_l,
+      std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      MFEM_ASSERT(direction_l != nullptr,
+                  "Global DerivativeApplyTranspose: direction vector is null");
+
+      // Re-zero the pre-allocated Q temporaries
+      dir_q_local = 0.0;
+      result_q_local = 0.0;
+      dir_q_local.SyncToBlocks();
+      result_q_local.SyncToBlocks();
+
+      // Bring test cotangent to quadrature points
+      pull_output_cotangents_to_q(direction_l, dir_q_local);
+
+      // Contract qp_cache with test directions at quadrature points
+      const real_t *cache_ptr = qp_cache.Read();
+      const int res_sz = residual_size_on_qp;
+      const int gnqp_local = gnqp;
+      const int num_qp_local = num_qp;
+      const int trial_vdim_local = trial_vdim;
+      const int total_trial_op_dim_local = total_trial_op_dim;
+
+      constexpr_for<0, n_outputs>([&](auto o)
+      {
+         const int tv_o = get<o>(outputs).vdim;
+         const int to_o = get<o>(outputs).size_on_qp / tv_o;
+         const int out_base = [&]
+         {
+            int off = 0;
+            constexpr_for<0, o.value>([&](auto prev)
+            { off += get<prev>(outputs).size_on_qp; });
+            return off;
+         }();
+
+         const int size_o = get<o>(outputs).size_on_qp;
+         const real_t *dir_o = dir_q_local.GetBlock(o.value).Read();
+
+         int m_offset = 0;
+         constexpr_for<0, n_inputs>([&](auto s)
+         {
+            if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
+
+            const int size_s = get<s>(inputs).size_on_qp;
+            const int to_s = size_s / trial_vdim_local;
+            real_t *res_s = result_q_local.GetBlock(s.value).ReadWrite();
+
+            mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
+            {
+               // Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
+               // read adjacent addresses for a fixed cache_idx.
+               const int cache_base =
+                  (gq % num_qp_local) +
+                  num_qp_local * res_sz * (gq / num_qp_local);
+
+               for (int i = 0; i < tv_o; ++i)
+               {
+                  for (int k = 0; k < to_o; ++k)
+                  {
+                     const int out_comp = out_base + i * to_o + k;
+                     const real_t w = dir_o[(i * to_o + k) + size_o * gq];
+
+                     for (int j = 0; j < trial_vdim_local; ++j)
+                     {
+                        for (int m = 0; m < to_s; ++m)
+                        {
+                           const int m_global = m + m_offset;
+                           const int cache_idx =
+                              out_comp * trial_vdim_local * total_trial_op_dim_local +
+                              j * total_trial_op_dim_local + m_global;
+
+                           const real_t c =
+                              cache_ptr[cache_base + num_qp_local * cache_idx];
+                           res_s[(j * to_s + m) + size_s * gq] += c * w;
+                        }
+                     }
+                  }
+               }
+            });
+            m_offset += to_s;
+         });
+      });
+
+      // Map result Q back to the trial (input) fields
+      constexpr_for<0, n_inputs>([&](auto s)
+      {
+         if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
+
+         const size_t in_fd = input_to_infd[s.value];
+         input_bases[s.value].transpose(
+            result_q_local.GetBlock(s.value), *ye[in_fd]);
+      });
+   }
+
+private:
+   IntegratorContext ctx;
+   inputs_t inputs;
+   outputs_t outputs;
+   const Vector &qp_cache;
+
+   std::array<size_t, n_inputs> input_to_infd;
+   std::array<size_t, n_outputs> output_to_outfd;
+
+   std::array<FieldBasis, n_inputs> input_bases;
+   std::array<FieldBasis, n_outputs> output_bases;
+
+   int gnqp = 0;
+   int num_qp = 0;
+
+   // Pre-allocated Q-space temporaries
+   Array<int> dir_q_offsets;
+   Array<int> result_q_offsets;
+   mutable BlockVector dir_q_local;
+   mutable BlockVector result_q_local;
+
+   // Pre-allocated owning storage for output cotangent temporaries
+   mutable std::array<Vector, n_outputs> dir_out_l_owned;
+   mutable std::array<Vector, n_outputs> dir_out_e_owned;
+   mutable std::vector<Vector *> dir_out_l;
+   mutable std::vector<Vector *> dir_out_e;
+   mutable RestrictionCache<Entity::Element> out_rcache;
+
+   int residual_size_on_qp = 0;
+   int trial_vdim = 0;
+   int total_trial_op_dim = 0;
+
+   /// Pull output cotangents from L-space into the pre-allocated Q BlockVector
+   void pull_output_cotangents_to_q(const Vector *direction_l,
+                                    BlockVector &dir_q) const
+   {
+      int l_offset = 0;
+      constexpr_for<0, n_outputs>([&](auto i)
+      {
+         const size_t outfd = output_to_outfd[i];
+         const auto &fd = ctx.outfds[outfd];
+         const int l_size = GetVSize(fd);
+
+         dir_out_l_owned[i] =
+            Vector(*const_cast<Vector *>(direction_l), l_offset, l_size);
+         dir_out_e_owned[i].SetSize(0);
+         dir_out_e_owned[i].UseDevice(true);
+
+         dir_out_l[i] = &dir_out_l_owned[i];
+         dir_out_e[i] = &dir_out_e_owned[i];
+         l_offset += l_size;
+      });
+
+      restriction(ctx.outfds, out_rcache, dir_out_l, dir_out_e);
+
+      constexpr_for<0, n_outputs>([&](auto i)
+      {
+         output_bases[i.value].forward(*dir_out_e[i], dir_q.GetBlock(i.value));
+      });
+      dir_q.SyncToBlocks();
+   }
+};
+
+} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/derivative_setup.hpp

@@ -0,0 +1,245 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "../util.hpp"
+#include <utility>
+
+namespace mfem::future::GlobalQFImpl
+{
+
+template<
+   int derivative_id,
+   typename qfunc_t,
+   typename inputs_t,
+   typename outputs_t,
+   size_t ninputs = tuple_size<inputs_t>::value,
+   size_t noutputs = tuple_size<outputs_t>::value>
+struct DerivativeSetup
+{
+   using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
+
+   DerivativeSetup(
+      IntegratorContext ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      Vector &qp_cache) :
+      ctx(ctx),
+      qfunc(qfunc),
+      qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
+      inputs(inputs),
+      outputs(outputs),
+      qp_cache(qp_cache)
+   {
+      create_fop_to_fd(inputs, ctx.infds, input_to_infd);
+      create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
+
+      check_consistency(inputs, input_to_infd, ctx.infds);
+      check_consistency(outputs, output_to_outfd, ctx.outfds);
+
+      create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
+      create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
+
+      create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
+      create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
+
+      const int nqp = ctx.ir.GetNPoints();
+      num_qp = nqp;
+      nentities = ctx.nentities;
+      gnqp = nqp * nentities;
+
+      xq_offsets.SetSize(ninputs + 1);
+      xq_offsets[0] = 0;
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         xq_offsets[i + 1] = nqp * get<i>(inputs).size_on_qp * nentities;
+      });
+      xq_offsets.PartialSum();
+      InitBlockVector(xq, xq_offsets);
+
+      shadow_xq_offsets.SetSize(xq_offsets.Size());
+      shadow_xq_offsets = xq_offsets;
+      InitBlockVector(shadow_xq, shadow_xq_offsets);
+
+      yq_offsets.SetSize(noutputs + 1);
+      yq_offsets[0] = 0;
+      constexpr_for<0, noutputs>([&](auto o)
+      {
+         yq_offsets[o + 1] = nqp * get<o>(outputs).size_on_qp * nentities;
+      });
+      yq_offsets.PartialSum();
+      InitBlockVector(yq, yq_offsets);
+
+      total_out_size_on_qp = 0;
+      constexpr_for<0, noutputs>([&](auto o)
+      {
+         total_out_size_on_qp += get<o>(outputs).size_on_qp;
+         out_vdim[o] = get<o>(outputs).vdim;
+         out_op_dim[o] = get<o>(outputs).size_on_qp / get<o>(outputs).vdim;
+      });
+
+      activity_map = detail::make_activity_map<derivative_id>(inputs_t {});
+
+      trial_vdim = 0;
+      total_trial_op_dim = 0;
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         if (!activity_map[i]) { return; }
+         const auto inp = get<i>(inputs);
+         trial_vdim = inp.vdim;
+         total_trial_op_dim += inp.size_on_qp / inp.vdim;
+      });
+
+      constexpr_for<0, ninputs>([&](auto i)
+      {
+         input_size_on_qp_arr[i] = get<i>(inputs).size_on_qp;
+      });
+
+      residual_size_on_qp = total_out_size_on_qp * trial_vdim * total_trial_op_dim;
+      qp_cache.SetSize(residual_size_on_qp * num_qp * nentities);
+      qp_cache.UseDevice(true);
+   }
+
+   void operator()(const std::vector<Vector *> &xe)
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      interpolate(input_to_infd, input_bases, xe, xq);
+
+      const int gnqp_local = gnqp;
+      const int num_qp_local = num_qp;
+      const int trial_vdim_local = trial_vdim;
+      const int total_trial_op_dim_local = total_trial_op_dim;
+      const int residual_size_local = residual_size_on_qp;
+
+      for (int j = 0; j < trial_vdim; j++)
+      {
+         int m_offset = 0;
+         constexpr_for<0, ninputs>([&](auto s)
+         {
+            if (!activity_map[s]) { return; }
+
+            const int input_vdim_s  = get<s>(inputs).vdim;
+            const int input_size_s  = input_size_on_qp_arr[s];
+            const int trial_op_dim_s = input_size_s / input_vdim_s;
+
+            for (int m = 0; m < trial_op_dim_s; m++)
+            {
+               shadow_xq = 0.0;
+               shadow_xq.SyncToBlocks();
+
+               // Set component (j + input_vdim_s * m) to 1 at all QPs
+               const int c_shadow = j + input_vdim_s * m;
+               real_t *shadow_ptr = shadow_xq.GetBlock(s.value).ReadWrite();
+               mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
+               {
+                  shadow_ptr[c_shadow + input_size_s * gq] = 1.0;
+               });
+
+               yq = 0.0;
+               yq.SyncToBlocks();
+
+               if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
+               {
+                  detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t,
+                         inputs_t, outputs_t>(
+                            qfunc, qfunc_shadow, xq, shadow_xq, yq, gnqp,
+                            input_qlayouts, output_qlayouts,
+                            std::make_index_sequence<ninputs> {},
+                            std::make_index_sequence<noutputs> {});
+               }
+               else
+               {
+                  detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
+                     qfunc, xq, shadow_xq, yq, gnqp,
+                     input_qlayouts, output_qlayouts,
+                     std::make_index_sequence<ninputs> {},
+                     std::make_index_sequence<noutputs> {});
+               }
+
+               yq.SyncToBlocks();
+               real_t *cache_d = qp_cache.ReadWrite();
+
+               // Write yq into the cache column
+               const int m_global = m + m_offset;
+               const int j_cur    = j;
+               int out_offset = 0;
+               constexpr_for<0, noutputs>([&](auto o)
+               {
+                  const int test_vdim_o   = out_vdim[o];
+                  const int test_op_dim_o = out_op_dim[o];
+                  const int yq_out_size   = test_vdim_o * test_op_dim_o;
+                  const int out_offset_o  = out_offset;
+                  const real_t *yq_d      = yq.GetBlock(o.value).Read();
+
+                  // The cache is (q, cache_idx, e) with the quadrature index
+                  // fastest, so gq is the fastest-varying thread index to keep
+                  // the stores coalesced.
+                  mfem::forall(gnqp_local * yq_out_size, [=] MFEM_HOST_DEVICE(int idx)
+                  {
+                     const int gq       = idx % gnqp_local;
+                     const int c_out    = idx / gnqp_local;
+                     const int q        = gq % num_qp_local;
+                     const int entity   = gq / num_qp_local;
+                     const int out_comp = out_offset_o + c_out;
+                     const int cache_idx =
+                        out_comp * trial_vdim_local * total_trial_op_dim_local +
+                        j_cur * total_trial_op_dim_local +
+                        m_global;
+                     cache_d[q + num_qp_local *
+                                 (cache_idx + residual_size_local * entity)] =
+                        yq_d[c_out + yq_out_size * gq];
+                  });
+                  out_offset += yq_out_size;
+               });
+            }
+            m_offset += trial_op_dim_s;
+         });
+      }
+   }
+
+   IntegratorContext ctx;
+   qfunc_t qfunc;
+   qfunc_shadow_t qfunc_shadow;
+   inputs_t inputs;
+   outputs_t outputs;
+   Vector &qp_cache;
+
+   std::array<size_t, ninputs> input_to_infd;
+   std::array<size_t, noutputs> output_to_outfd;
+
+   std::array<FieldBasis, ninputs> input_bases;
+   std::array<FieldBasis, noutputs> output_bases;
+
+   std::array<std::vector<int>, ninputs>  input_qlayouts;
+   std::array<std::vector<int>, noutputs> output_qlayouts;
+
+   int gnqp = 0;
+   int num_qp = 0;
+   int nentities = 0;
+
+   Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
+   mutable BlockVector xq, shadow_xq, yq;
+
+   int total_out_size_on_qp = 0;
+   int trial_vdim = 0;
+   int total_trial_op_dim = 0;
+   int residual_size_on_qp = 0;
+
+   std::array<int, noutputs> out_vdim {};
+   std::array<int, noutputs> out_op_dim {};
+   std::array<int, ninputs>  input_size_on_qp_arr {};
+   std::array<bool, ninputs> activity_map {};
+};
+
+} // namespace mfem::future::GlobalQFImpl

fem/dfem/backends/global_qf/prelude.hpp

@@ -0,0 +1,219 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "action.hpp"
+#include "derivative_action.hpp"
+#include "derivative_setup.hpp"
+
+#include "derivative_apply_transpose.hpp"
+
+#include "../local_qf/derivative_apply.hpp"
+#include "../local_qf/derivative_assemble.hpp"
+#include "../local_qf/derivative_assemble_diagonal.hpp"
+
+#include "../scratch_bank.hpp"
+
+namespace mfem::future
+{
+
+namespace detail
+{
+
+template <typename T>
+struct LocalQFShapeArg
+{
+   using type = std::remove_const_t<T>&;
+};
+
+template <typename scalar_t, int ndims, int... tensor_sizes>
+struct LocalQFShapeArg<tensor_ndarray<scalar_t, ndims, tensor_sizes...>>
+{
+   using scalar_type = std::remove_const_t<scalar_t>;
+   using type = std::conditional_t<
+                   sizeof...(tensor_sizes) == 0,
+                   scalar_type,
+                   tensor<scalar_type, tensor_sizes...>>&;
+};
+
+template <typename scalar_t, int... tensor_sizes>
+struct LocalQFShapeArg<tensor<scalar_t, tensor_sizes...>>
+{
+   using scalar_type = std::remove_const_t<scalar_t>;
+   using type = std::conditional_t<
+                   sizeof...(tensor_sizes) == 0,
+                   scalar_type,
+                   tensor<scalar_type, tensor_sizes...>>&;
+};
+
+template <typename qf_param_ts>
+struct LocalQFShapeFunction;
+
+template <typename... qf_param_ts>
+struct LocalQFShapeFunction<tuple<qf_param_ts...>>
+{
+   void operator()(
+      typename LocalQFShapeArg<qf_param_decay_t<qf_param_ts>>::type...) const;
+};
+
+template <typename qfunc_t>
+using LocalQFShapeFunctionFor = LocalQFShapeFunction<
+   typename get_function_signature<qfunc_t>::type::parameter_ts>;
+
+} // namespace detail
+
+struct GlobalQFBackend
+{
+   /**
+    * @brief Make an action for a global Q-function.
+    *
+    * @param ctx The integrator context.
+    * @param args The arguments to the action.
+    * @return The action.
+    */
+   template<
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeAction(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      return GlobalQFImpl::Action(ctx, qfunc, inputs, outputs);
+   }
+
+   /**
+    * @brief Make a derivative action for a global Q-function.
+    *
+    * @tparam derivative_id The id of the derivative.
+    * @param ctx The integrator context.
+    * @param args The arguments to the derivative action.
+    * @return The derivative action.
+    */
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeAction(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      return GlobalQFImpl::DerivativeAction<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs);
+   }
+
+   /**
+    * @brief Make a derivative setup for a global Q-function.
+    *
+    * @tparam derivative_id The id of the derivative.
+    * @param ctx The integrator context.
+    * @param args The arguments to the derivative setup.
+    * @return The derivative setup.
+    */
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeSetup(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      Vector &qp_cache)
+   {
+      return GlobalQFImpl::DerivativeSetup<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeApply(
+      const IntegratorContext &ctx,
+      const qfunc_t & /*qfunc*/,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeApply<
+             derivative_id,
+             detail::LocalQFShapeFunctionFor<qfunc_t>,
+             inputs_t,
+             outputs_t>(ctx,
+                        detail::LocalQFShapeFunctionFor<qfunc_t> {},
+                        inputs,
+                        outputs,
+                        qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeApplyTranspose(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return GlobalQFImpl::DerivativeApplyTranspose<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeAssemble(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeAssemble<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   auto static MakeDerivativeAssembleDiagonal(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeAssembleDiagonal<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs, qp_cache);
+   }
+};
+
+} // namespace mfem::future

fem/dfem/backends/local_qf/action.hpp

@@ -0,0 +1,514 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "fem/dfem/fieldoperator.hpp"
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include "../../integrator_ctx.hpp"
+#include "../util.hpp"
+
+namespace mfem::future::LocalQFImpl
+{
+
+template<typename qfunc_t, typename inputs_t, typename outputs_t>
+class Action
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using args_tuple_t = decay_tuple<qf_param_ts>;
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
+                 "LocalQF: q-function arity must match inputs + outputs");
+
+   const qfunc_t qfunc;
+   const inputs_t inputs;
+   const outputs_t outputs;
+   const IntegratorContext ctx;
+   const std::vector<const DofToQuad *> dtqs;
+   // inputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_inputs> input_dtq;
+   const std::array<size_t, n_inputs> input_idx; // input to field
+   const std::array<const real_t *, n_inputs> input_B, input_G;
+   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
+   // outputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_outputs> output_dtq;
+   const std::array<size_t, n_outputs> output_idx; // output to field
+   const std::array<const real_t *, n_outputs> output_B, output_G;
+   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
+   // other constants
+   const int dim, ne, nq, q1d;
+
+public:
+   ////////////////////////////////////////////////////////
+   Action() = delete;
+
+   Action(IntegratorContext ctx,
+          qfunc_t qfunc,
+          inputs_t inputs,
+          outputs_t outputs):
+      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
+      dtqs(make_dtqs(ctx)),
+      // inputs: dtq, idx, B, G, d1d, q1d, vdim
+      input_dtq(create_dtq_maps<Entity::Element>(
+                   inputs,
+                   dtqs,
+                   create_union_field_map_for_dtq(ctx, inputs),
+                   ctx.unionfds,
+                   ctx.ir)),
+      input_idx(create_input_vector_map(ctx, inputs)),
+      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
+      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
+      input_vdim(get_vdim(inputs)),
+      // outputs: dtq, idx, B, G, d1d, q1d, vdim
+      output_dtq(create_dtq_maps<Entity::Element>(
+                    outputs,
+                    dtqs,
+                    create_union_field_map_for_dtq(ctx, outputs),
+                    ctx.unionfds,
+                    ctx.ir)),
+      output_idx(create_output_vector_map(ctx, outputs)),
+      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
+      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
+      output_vdim(get_vdim(outputs)),
+      // other constants
+      dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
+      q1d(tensor_1d_size(nq, dim))
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+   }
+
+   template<typename Backend>
+   void run_kernels(const std::vector<Vector *> &xe,
+                    std::vector<Vector *> &ye) const
+   {
+      Backend::Run(dim,
+                   q1d,
+                   // arguments
+                   ctx,
+                   qfunc,
+                   // inputs
+                   input_idx,
+                   input_B,
+                   input_G,
+                   input_vdim,
+                   input_d1d,
+                   input_q1d,
+                   // outputs
+                   output_idx,
+                   output_B,
+                   output_G,
+                   output_vdim,
+                   output_d1d,
+                   output_q1d,
+                   // input and output vectors
+                   xe,
+                   ye,
+                   // fallback arguments
+                   dim,
+                   q1d);
+   }
+
+   void operator()(const std::vector<Vector *> &xe,
+                   std::vector<Vector *> &ye) const
+   {
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<ActionLO>(xe, ye);
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<ActionHO>(xe, ye);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+   }
+
+   ////////////////////////////////////////////////////////
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void
+   action_callback(const IntegratorContext &ctx,
+                   const qfunc_t &qfunc,
+                   // inputs: idx, B, G, vdim, d1d, q1d
+                   const std::array<size_t, n_inputs> &in_idx,
+                   const std::array<const real_t *, n_inputs> in_B,
+                   const std::array<const real_t *, n_inputs> in_G,
+                   const std::array<int, n_inputs> &in_vdim,
+                   const std::array<int, n_inputs> &in_d1d,
+                   const std::array<int, n_inputs> &in_q1d,
+                   // outputs: idx, B, G, vdim, d1d, q1d
+                   const std::array<size_t, n_outputs> &out_idx,
+                   const std::array<const real_t *, n_outputs> out_B,
+                   const std::array<const real_t *, n_outputs> out_G,
+                   const std::array<int, n_outputs> &out_vdim,
+                   const std::array<int, n_outputs> &out_d1d,
+                   const std::array<int, n_outputs> &out_q1d,
+                   const std::vector<Vector *> &xe,
+                   std::vector<Vector *> &ye,
+                   // fallback arguments
+                   const int dim,
+                   const int q1d)
+   {
+      if (ctx.attr.Size() == 0) { return; }
+      MFEM_CONTRACT_VAR(dim);
+      MFEM_ASSERT(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+
+      static constexpr auto B2D = backend_t::DIM == 2;
+      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::Q1D;
+      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const int ne = ctx.nentities;
+
+      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
+
+      // --------------------------------------------------
+      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = in_idx[i];
+         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
+         using FOP = tuple_element_t<i, inputs_t>;
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else if constexpr (is_weight_fop_v<FOP>)
+         {
+            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
+                        "tensor-product IR expected");
+            in_XE[i] = Reshape(
+                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      // --------------------------------------------------
+      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
+      for_constexpr<n_outputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = out_idx[i];
+         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
+         using FOP = tuple_element_t<i, outputs_t>;
+         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
+         {
+            MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP> || is_functionalvalue_fop_v<FOP>)
+         {
+            MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else
+         {
+            static_assert(false, "Unsupported FieldOperator");
+         }
+      });
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // -----------------------------------------------
+         // Inputs and outputs argument registers
+         // -----------------------------------------------
+         args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
+
+         // -----------------------------------------------
+         // Shared memory
+         // -----------------------------------------------
+         MFEM_SHARED typename backend_t::Shared smem;
+
+         // -----------------------------------------------
+         // Load inputs
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value;
+            const auto &XE = in_XE[i];
+            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
+            ;
+            const real_t *B = in_B[i], *G = in_G[i];
+            auto &rarg = get<i>(rargs);
+            using FOP = tuple_element_t<i, inputs_t>;
+            if constexpr (is_value_fop<FOP>::value)
+            {
+               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               using XE_t = decltype(XE);
+               using rarg_t = decltype(rarg);
+               using qf_param_t =
+                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
+               backend_t::template LoadGradient<RNK, rarg_t, XE_t, qf_param_t>(
+                  smem, e, d, q, q1d, B, G, XE, rarg);
+            }
+            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
+            {
+               // qp values are read directly from in_XE / IR
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+
+         // -----------------------------------------------
+         // Evaluate the quadrature function
+         // Warning: no 'DIRECT' on the 'Z' direction,
+         // as one backend may need to iterate over it.
+         // -----------------------------------------------
+         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+               {
+                  args_tuple_t qargs;
+
+                  // --------------------------------------
+                  // Pulling arguments from registers to qargs tuple
+                  // --------------------------------------
+                  for_constexpr<n_inputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value;
+                     auto &qarg = get<i>(qargs);
+                     const auto &XE = in_XE[i];
+                     using FOP = tuple_element_t<i, inputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        using DT =
+                           typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+                        if constexpr (qf_param_uses_dual_v<DT>)
+                        {
+                           qarg = backend_t::template identity_qp_pull_dual<DT>(
+                              false, XE, XE, qx, qy, qz, e);
+                        }
+                        else
+                        {
+                           qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
+                        }
+                     }
+                     else if constexpr (is_weight_fop_v<FOP>)
+                     {
+                        qarg = XE(qx, qy, qz, 0, 0);
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        qarg = backend_t::template qp_pull<ARG>(
+                           get<i>(rargs), qx, qy, qz);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+
+                  // --------------------------------------
+                  // Call the quadrature function
+                  // --------------------------------------
+                  call_qfunc_no_move(qfunc, qargs);
+
+                  // --------------------------------------
+                  // Pushing arguments from qargs tuple to registers
+                  // --------------------------------------
+                  for_constexpr<n_outputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value, o = n_inputs + i;
+                     const auto &qarg = get<o>(qargs);
+                     const auto &YE = out_YE[i];
+                     using FOP = tuple_element_t<i, outputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP> ||
+                                   is_functionalvalue_fop_v<FOP>)
+                     {
+                        using DT =
+                           typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
+                        if constexpr (qf_param_uses_dual_v<DT>)
+                        {
+                           backend_t::identity_qp_write_value(
+                              YE, qx, qy, qz, e, qarg);
+                        }
+                        else
+                        {
+                           as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
+                        }
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        auto &rarg = get<o>(rargs);
+                        backend_t::template qp_push<ARG>(
+                           rarg, qx, qy, qz, qarg);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Integrate outputs
+         // -----------------------------------------------
+         for_constexpr<n_outputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value, o = n_inputs + i;
+            const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
+            const auto B = out_B[i], G = out_G[i];
+            const auto &YE = out_YE[i];
+            auto &rarg = get<o>(rargs);
+            using FOP = tuple_element_t<i, outputs_t>;
+            if constexpr (is_value_fop_v<FOP>)
+            {
+               backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               using YE_t = decltype(YE);
+               using rarg_t = decltype(rarg);
+               // Both the rank and the extents have to come from the output
+               // parameter slot o, not from the output index i.
+               using qf_param_t =
+                  typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
+               constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
+               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
+                  smem, e, d, q, Q1D, B, G, YE, rarg);
+            }
+            else if constexpr (is_identity_fop_v<FOP> ||
+                               is_functionalvalue_fop_v<FOP>)
+            {
+               // nothing to do
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+      },
+      ne,
+      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
+                                  q1d, in_d1d, out_d1d)),
+      0,
+      nullptr);
+   }
+   using KernelType = decltype(&Action::action_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionLO, KernelType, (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionHO, KernelType, (int, int) );
+};
+
+// Low Order kernels
+template<typename qfunc_t, typename inputs_t, typename outputs_t>
+template<int DIM, int Q1D>
+inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
+Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Kernel()
+{
+   static_assert(Q1D <= LocalQFLOBackend<DIM>::MQ1);
+   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
+   return action_t::template action_callback<LocalQFLOBackend<DIM, Q1D>>;
+}
+
+// Low Order fallback
+template<typename qfunc_t, typename inputs_t, typename outputs_t>
+inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
+Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Fallback(int dim, int q1d)
+{
+   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
+   using ActionLO = typename action_t::ActionLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<ActionLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<ActionLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+// High Order kernels
+template<typename qfunc_t, typename inputs_t, typename outputs_t>
+template<int DIM, int Q1D>
+inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
+Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Kernel()
+{
+   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
+   return action_t::template action_callback<LocalQFHOBackend<DIM>, Q1D>;
+}
+
+// High Order fallback
+template<typename qfunc_t, typename inputs_t, typename outputs_t>
+inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
+Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Fallback(int dim, int q1d)
+{
+   using action_t = Action<qfunc_t, inputs_t, outputs_t>;
+   using ActionHO = typename action_t::ActionHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<ActionHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<ActionHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_action.hpp

@@ -0,0 +1,763 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+namespace mfem::future::LocalQFImpl
+{
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeAction
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using args_tuple_t = decay_tuple<qf_param_ts>;
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
+                 "LocalQF: q-function arity must match inputs + outputs");
+
+   qfunc_t qfunc;
+   const inputs_t inputs;
+   const outputs_t outputs;
+   const IntegratorContext ctx;
+   const std::vector<const DofToQuad *> dtqs;
+   // inputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_inputs> input_dtq;
+   const std::array<size_t, n_inputs> input_idx;
+   const std::array<const real_t *, n_inputs> input_B, input_G;
+   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
+   // outputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_outputs> output_dtq;
+   const std::array<size_t, n_outputs> output_idx;
+   const std::array<const real_t *, n_outputs> output_B, output_G;
+   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
+   // other constants
+   const int dim, ne, nq, q1d;
+
+   std::array<bool, n_inputs> input_is_dependent;
+   FieldDescriptor direction_fd;
+   mutable Vector direction_e;
+   mutable RestrictionCache<Entity::Element> direction_rcache;
+
+public:
+   //////////////////////////////////////////////////////////////////
+   DerivativeAction() = delete;
+
+   DerivativeAction(IntegratorContext ctx,
+                    qfunc_t qfunc,
+                    inputs_t inputs,
+                    outputs_t outputs):
+      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
+      dtqs(make_dtqs(ctx)),
+      // inputs: dtq, idx, B, G, d1d, q1d, vdim
+      input_dtq(create_dtq_maps<Entity::Element>(
+                   inputs,
+                   dtqs,
+                   create_union_field_map_for_dtq(ctx, inputs),
+                   ctx.unionfds,
+                   ctx.ir)),
+      input_idx(create_input_vector_map(ctx, inputs)),
+      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
+      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
+      input_vdim(get_vdim(inputs)),
+      // outputs: dtq, idx, B, G, d1d, q1d, vdim
+      output_dtq(create_dtq_maps<Entity::Element>(
+                    outputs,
+                    dtqs,
+                    create_union_field_map_for_dtq(ctx, outputs),
+                    ctx.unionfds,
+                    ctx.ir)),
+      output_idx(create_output_vector_map(ctx, outputs)),
+      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
+      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
+      output_vdim(get_vdim(outputs)),
+      // other constants
+      dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
+      q1d(tensor_1d_size(nq, dim))
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+
+      // Determine which inputs are dependent on the derivative direction
+      auto dependency_map = make_dependency_map(inputs);
+      auto it = dependency_map.find(derivative_id);
+      MFEM_ASSERT(it != dependency_map.end(),
+                  "Derivative ID not found in dependency map");
+      input_is_dependent = it->second;
+
+      // Find direction field index
+      int direction_field_idx = -1;
+      for (size_t uf = 0; uf < nfields; uf++)
+      {
+         if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
+         {
+            direction_field_idx = static_cast<int>(uf);
+            break;
+         }
+      }
+      MFEM_ASSERT(
+         direction_field_idx != -1,
+         "LocalQFBackend: derivative direction field not found in unionfds");
+      direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename Backend>
+   void run_kernels(const std::vector<Vector *> &xe,
+                    std::vector<Vector *> &ye)
+   {
+      Backend::Run(dim,
+                   q1d,
+                   // arguments
+                   ctx,
+                   qfunc,
+                   // inputs
+                   input_idx,
+                   input_B,
+                   input_G,
+                   input_vdim,
+                   input_d1d,
+                   input_q1d,
+                   // outputs
+                   output_idx,
+                   output_B,
+                   output_G,
+                   output_vdim,
+                   output_d1d,
+                   output_q1d,
+                   // input and output vectors
+                   xe,
+                   ye,
+                   input_is_dependent,
+                   direction_e,
+                   // fallback arguments
+                   dim,
+                   q1d);
+   }
+
+   //////////////////////////////////////////////////////////////////
+   void operator()(const std::vector<Vector *> &xe,
+                   const Vector *direction_l,
+                   std::vector<Vector *> &ye)
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      MFEM_ASSERT(direction_l != nullptr,
+                  "LocalQF DerivativeAction: direction vector is null");
+
+      restriction(direction_fd,
+                  direction_rcache,
+                  *direction_l,
+                  direction_e,
+                  ElementDofOrdering::LEXICOGRAPHIC);
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<DerivativeActionLO>(xe, ye);
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<DerivativeActionHO>(xe, ye);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void
+   derivative_action_callback(const IntegratorContext &ctx,
+                              qfunc_t &qfunc,
+                              // inputs: idx, B, G, vdim, d1d, q1d
+                              const std::array<size_t, n_inputs> &in_idx,
+                              const std::array<const real_t *, n_inputs> in_B,
+                              const std::array<const real_t *, n_inputs> in_G,
+                              const std::array<int, n_inputs> &in_vdim,
+                              const std::array<int, n_inputs> &in_d1d,
+                              const std::array<int, n_inputs> &in_q1d,
+                              // outputs: idx, B, G, vdim, d1d, q1d
+                              const std::array<size_t, n_outputs> &out_idx,
+                              const std::array<const real_t *, n_outputs> out_B,
+                              const std::array<const real_t *, n_outputs> out_G,
+                              const std::array<int, n_outputs> &out_vdim,
+                              const std::array<int, n_outputs> &out_d1d,
+                              const std::array<int, n_outputs> &out_q1d,
+                              const std::vector<Vector *> &xe,
+                              std::vector<Vector *> &ye,
+                              const std::array<bool, n_inputs> &input_dep,
+                              const Vector &direction_e,
+                              // fallback arguments
+                              const int dim,
+                              const int q1d)
+   {
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+
+      if (ctx.attr.Size() == 0) { return; }
+
+      static constexpr auto B2D = backend_t::DIM == 2;
+      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const int ne = ctx.nentities;
+
+      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
+
+      // --------------------------------------------------
+      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = in_idx[i];
+         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
+         using FOP = tuple_element_t<i, inputs_t>;
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else if constexpr (is_weight_fop_v<FOP>)
+         {
+            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
+                        "tensor-product IR expected");
+            in_XE[i] = Reshape(
+                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      const auto d_direction = direction_e.Read();
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = in_idx[i];
+         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
+         using FOP = tuple_element_t<i, inputs_t>;
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            if (input_dep[i])
+            {
+               MFEM_ASSERT(direction_e.Size() == xe[k]->Size(),
+                           "direction E-vector size mismatch for input " << i);
+               in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
+            }
+            else
+            {
+               in_XE_dir[i] = in_XE[i];
+            }
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            if (input_dep[i])
+            {
+               MFEM_VERIFY(direction_e.Size() == xe[k]->Size(),
+                           "direction E-vector size mismatch (identity input) "
+                           << i);
+               in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
+            }
+            else
+            {
+               in_XE_dir[i] = in_XE[i];
+            }
+         }
+         else if constexpr (is_weight_fop_v<FOP>) { in_XE_dir[i] = in_XE[i]; }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      // --------------------------------------------------
+      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
+      for_constexpr<n_outputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = out_idx[i];
+         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
+         using FOP = tuple_element_t<i, outputs_t>;
+         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
+         {
+            MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else
+         {
+            static_assert(false, "Unsupported FieldOperator");
+         }
+      });
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // -----------------------------------------------
+         // Inputs and outputs argument registers
+         // -----------------------------------------------
+         args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
+         input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1>
+         sargs; // shadow
+
+         // -----------------------------------------------
+         // Shared memory
+         // -----------------------------------------------
+         MFEM_SHARED typename backend_t::Shared smem;
+
+         // -----------------------------------------------
+         // Load primal inputs (rargs)
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value;
+            const auto &XE = in_XE[i];
+            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
+            const real_t *B = in_B[i], *G = in_G[i];
+            auto &rarg = get<i>(rargs);
+            using FOP = tuple_element_t<i, inputs_t>;
+            if constexpr (is_value_fop<FOP>::value)
+            {
+               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
+               using FieldParamT =
+                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+               backend_t::template LoadGradient<RNK,
+                                                decltype(rarg),
+                                                decltype(XE),
+                                                FieldParamT>(
+                                                   smem, e, d, q, Q1D, B, G, XE, rarg);
+            }
+            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
+            {
+               // qp values are read directly from in_XE / IR
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+
+         // -----------------------------------------------
+         // Load tangent directions (sargs)
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value;
+            if (!input_dep[i]) { return; }
+            const auto &XE = in_XE_dir[i];
+            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
+            const real_t *B = in_B[i], *G = in_G[i];
+            auto &sarg = get<i>(sargs); // shadow argument register
+            using FOP = tuple_element_t<i, inputs_t>;
+            if constexpr (is_value_fop<FOP>::value)
+            {
+               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
+               using FieldParamT =
+                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+               backend_t::template LoadGradient<RNK,
+                                                decltype(sarg),
+                                                decltype(XE),
+                                                FieldParamT>(
+                                                   smem, e, d, q, Q1D, B, G, XE, sarg);
+            }
+            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
+            {
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+
+         // -----------------------------------------------
+         // Evaluate the quadrature function
+         // Warning: no 'DIRECT' on the 'Z' direction,
+         // as one backend may need to iterate over it.
+         // -----------------------------------------------
+         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+               {
+#ifdef MFEM_USE_ENZYME
+                  args_tuple_t primal_args {}, shadow_args {};
+
+                  // --------------------------------------
+                  // Pulling arguments from registers to primal and shadow
+                  // tuples
+                  // --------------------------------------
+                  for_constexpr<n_inputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value;
+                     auto &parg = get<i>(primal_args);
+                     auto &targ = get<i>(shadow_args);
+                     const auto &XE = in_XE[i];
+                     const auto &XEd = in_XE_dir[i];
+                     using FOP = tuple_element_t<i, inputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
+                        if (input_dep[i])
+                        {
+                           targ = as_tensor<ARG>(&XEd(0, qx, qy, qz, e));
+                        }
+                        else
+                        {
+                           targ = ARG{};
+                        }
+                     }
+                     else if constexpr (is_weight_fop_v<FOP>)
+                     {
+                        parg = XE(qx, qy, qz, 0, 0);
+                        targ = real_t(0.0);
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        parg = backend_t::template qp_pull<ARG>(
+                           get<i>(rargs), qx, qy, qz);
+                        if (input_dep[i])
+                        {
+                           targ = backend_t::template qp_pull<ARG>(
+                              get<i>(sargs), qx, qy, qz);
+                        }
+                        else
+                        {
+                           targ = ARG{};
+                        }
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+
+                  // --------------------------------------
+                  // Call the quadrature function
+                  // --------------------------------------
+                  call_enzyme_fwddiff(qfunc, primal_args, shadow_args);
+
+                  // --------------------------------------
+                  // Pushing arguments from enzyme_shadow tuple to registers
+                  // --------------------------------------
+                  for_constexpr<n_outputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value, o = n_inputs + i;
+                     const auto &qout = get<o>(shadow_args);
+                     auto &YE = out_YE[i];
+                     using FOP = tuple_element_t<i, outputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qout;
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        auto &rarg = get<o>(rargs);
+                        backend_t::template qp_push_tangent<ARG>(
+                           rarg, qx, qy, qz, qout);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+#else  // MFEM_USE_ENZYME
+                  args_tuple_t qargs;
+
+                  // --------------------------------------
+                  // Pulling arguments from registers to qargs tuple
+                  // --------------------------------------
+                  for_constexpr<n_inputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value;
+                     auto &qarg = get<i>(qargs);
+                     const auto &XE = in_XE[i];
+                     const auto &XEd = in_XE_dir[i];
+                     using FOP = tuple_element_t<i, inputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        using DT =
+                           typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+                        if constexpr (qf_param_uses_dual_v<DT>)
+                        {
+                           qarg = backend_t::template identity_qp_pull_dual<DT>(
+                              input_dep[i], XE, XEd, qx, qy, qz, e);
+                        }
+                        else
+                        {
+                           qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
+                        }
+                     }
+                     else if constexpr (is_weight_fop_v<FOP>)
+                     {
+                        qarg = XE(qx, qy, qz, 0, 0);
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        qarg = backend_t::template qp_pull_directional<ARG>(
+                           get<i>(rargs),
+                           get<i>(sargs),
+                           qx,
+                           qy,
+                           qz,
+                           input_dep[i]);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+
+                  // --------------------------------------
+                  // Call the quadrature function
+                  // --------------------------------------
+                  call_qfunc_no_move(qfunc, qargs);
+
+                  // --------------------------------------
+                  // Pushing arguments from qargs tuple to registers
+                  // --------------------------------------
+                  for_constexpr<n_outputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value, o = n_inputs + i;
+                     const auto &qarg = get<o>(qargs);
+                     const auto &YE = out_YE[i];
+                     using FOP = tuple_element_t<i, outputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        using DT =
+                           typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
+                        if constexpr (qf_param_uses_dual_v<DT>)
+                        {
+                           backend_t::identity_qp_write_tangent(
+                              YE, qx, qy, qz, e, qarg);
+                        }
+                        else
+                        {
+                           as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
+                        }
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        auto &rarg = get<o>(rargs);
+                        backend_t::template qp_push_tangent<ARG>(
+                           rarg, qx, qy, qz, qarg);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+#endif // MFEM_USE_ENZYME
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Integrate outputs
+         // -----------------------------------------------
+         for_constexpr<n_outputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value, o = n_inputs + i;
+            const int d = out_d1d[i], q = out_q1d[i];
+            const auto B = out_B[i], G = out_G[i];
+            auto &YE = out_YE[i];
+            auto &rarg = get<o>(rargs);
+            using FOP = tuple_element_t<i, outputs_t>;
+            if constexpr (is_value_fop_v<FOP>)
+            {
+               backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               using YE_t = decltype(YE);
+               using rarg_t = decltype(rarg);
+               using qf_param_t =
+                  typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
+               constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
+               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
+                  smem, e, d, q, q1d, B, G, YE, rarg);
+            }
+            else if constexpr (is_identity_fop_v<FOP>)
+            {
+               // nothing to do
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+      },
+      ne,
+      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
+                                  q1d, in_d1d, out_d1d)),
+      0,
+      nullptr);
+   }
+
+   using DerivativeKernelType =
+      decltype(&DerivativeAction::derivative_action_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionLO,
+                                     DerivativeKernelType,
+                                     (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionHO,
+                                     DerivativeKernelType,
+                                     (int, int) );
+};
+
+// Low Order kernels
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeKernelType
+DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeActionLO::Kernel()
+{
+   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
+   using derivative_action_t =
+      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return derivative_action_t::template derivative_action_callback<
+             LocalQFLOBackend<DIM, Q1D>>;
+}
+
+// Low Order fallback
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeKernelType
+DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeActionLO::Fallback(int dim, int q1d)
+{
+   using derivative_action_t =
+      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeActionLO =
+      typename derivative_action_t::DerivativeActionLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<DerivativeActionLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<DerivativeActionLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+// High Order kernels
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeKernelType
+DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeActionHO::Kernel()
+{
+   using derivative_action_t =
+      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return derivative_action_t::
+          template derivative_action_callback<LocalQFHOBackend<DIM>, Q1D>;
+}
+
+// High Order fallback
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeKernelType
+DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeActionHO::Fallback(int dim, int q1d)
+{
+   using derivative_action_t =
+      DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeActionHO = typename derivative_action_t::DerivativeActionHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeActionHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeActionHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_apply.hpp

@@ -0,0 +1,738 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include <array>
+
+namespace mfem::future::LocalQFImpl
+{
+
+// Cached Jacobian apply: J·v from qp_cache filled by DerivativeSetup
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeApply
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using args_tuple_t = decay_tuple<qf_param_ts>;
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
+                 "LocalQF: q-function arity must match inputs + outputs");
+
+   const inputs_t inputs;
+   const outputs_t outputs;
+   const IntegratorContext ctx;
+   const Vector &qp_cache;
+   const std::vector<const DofToQuad *> dtqs;
+   // inputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_inputs> input_dtq;
+   const std::array<size_t, n_inputs> input_idx;
+   const std::array<const real_t *, n_inputs> input_B, input_G;
+   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
+   // outputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_outputs> output_dtq;
+   const std::array<size_t, n_outputs> output_idx;
+   const std::array<const real_t *, n_outputs> output_B, output_G;
+   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
+   // Jacobian cache metadata
+   const std::array<bool, n_inputs> input_is_dependent;
+   const std::array<int, n_inputs> input_size_on_qp;
+   const std::array<int, n_outputs> out_op_dim;
+   const std::array<int, n_outputs> out_offsets;
+   const int output_size_on_qp;
+   const int trial_vdim;
+   const int total_trial_op_dim;
+   const int residual_size_on_qp;
+   // other constants
+   const int dim, ne, nq, q1d;
+   FieldDescriptor direction_fd;
+   mutable Vector direction_e;
+   mutable RestrictionCache<Entity::Element> direction_rcache;
+
+   template <std::size_t slot>
+   static constexpr int ParamRank()
+   {
+      using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
+      return qf_param_shape<param_t>::rank;
+   }
+
+   template <std::size_t slot, int dim_idx>
+   static constexpr int ParamExtent()
+   {
+      using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
+      return qf_param_shape<param_t>::extents[dim_idx];
+   }
+
+   template <typename fop_t, std::size_t slot>
+   static constexpr int StaticVDim()
+   {
+      constexpr int rank = ParamRank<slot>();
+      if constexpr (is_gradient_fop_v<fop_t>)
+      {
+         if constexpr (rank <= 1) { return 1; }
+         else { return ParamExtent<slot, 0>(); }
+      }
+      else
+      {
+         if constexpr (rank == 0) { return 1; }
+         else { return ParamExtent<slot, 0>(); }
+      }
+   }
+
+   template <typename fop_t, std::size_t slot>
+   static constexpr int StaticOpDim()
+   {
+      constexpr int rank = ParamRank<slot>();
+      if constexpr (is_gradient_fop_v<fop_t>)
+      {
+         if constexpr (rank == 0) { return 1; }
+         else if constexpr (rank == 1) { return ParamExtent<slot, 0>(); }
+         else { return ParamExtent<slot, 1>(); }
+      }
+      else
+      {
+         if constexpr (rank <= 1) { return 1; }
+         else { return ParamExtent<slot, 1>(); }
+      }
+   }
+
+   template <std::size_t input_slot>
+   static constexpr bool StaticInputDep()
+   {
+      using fop_t = tuple_element_t<input_slot, inputs_t>;
+      return fop_t::GetFieldId() == derivative_id;
+   }
+
+   template <std::size_t input_slot>
+   static constexpr int StaticInputVDim()
+   {
+      using fop_t = tuple_element_t<input_slot, inputs_t>;
+      return StaticVDim<fop_t, input_slot>();
+   }
+
+   template <std::size_t input_slot>
+   static constexpr int StaticInputOpDim()
+   {
+      using fop_t = tuple_element_t<input_slot, inputs_t>;
+      return StaticOpDim<fop_t, input_slot>();
+   }
+
+   template <std::size_t output_slot>
+   static constexpr int StaticOutputVDim()
+   {
+      using fop_t = tuple_element_t<output_slot, outputs_t>;
+      return StaticVDim<fop_t, n_inputs + output_slot>();
+   }
+
+   template <std::size_t output_slot>
+   static constexpr int StaticOutputOpDim()
+   {
+      using fop_t = tuple_element_t<output_slot, outputs_t>;
+      return StaticOpDim<fop_t, n_inputs + output_slot>();
+   }
+
+   template <std::size_t output_slot>
+   static constexpr int StaticOutputOffset()
+   {
+      int offset = 0;
+      for_constexpr<output_slot>([&](auto oc)
+      {
+         constexpr size_t o = oc.value;
+         offset += StaticOutputVDim<o>() * StaticOutputOpDim<o>();
+      });
+      return offset;
+   }
+
+   template <std::size_t input_slot>
+   static constexpr int StaticInputOpOffset()
+   {
+      int offset = 0;
+      for_constexpr<input_slot>([&](auto sc)
+      {
+         constexpr size_t s = sc.value;
+         if constexpr (StaticInputDep<s>()) { offset += StaticInputOpDim<s>(); }
+      });
+      return offset;
+   }
+
+   static constexpr int StaticTrialVDim()
+   {
+      int vdim = 1;
+      for_constexpr<n_inputs>([&](auto sc)
+      {
+         constexpr size_t s = sc.value;
+         if constexpr (StaticInputDep<s>()) { vdim = StaticInputVDim<s>(); }
+      });
+      return vdim;
+   }
+
+   static constexpr int StaticTotalTrialOpDim()
+   {
+      int op_dim = 0;
+      for_constexpr<n_inputs>([&](auto sc)
+      {
+         constexpr size_t s = sc.value;
+         if constexpr (StaticInputDep<s>()) { op_dim += StaticInputOpDim<s>(); }
+      });
+      return op_dim;
+   }
+
+public:
+   DerivativeApply() = delete;
+
+   DerivativeApply(IntegratorContext ctx,
+                   qfunc_t /*qfunc*/,
+                   inputs_t inputs,
+                   outputs_t outputs,
+                   const Vector &qp_cache_in):
+      inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache_in),
+      dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
+                                         inputs,
+                                         dtqs,
+                                         create_union_field_map_for_dtq(ctx, inputs),
+                                         ctx.unionfds,
+                                         ctx.ir)),
+      input_idx(create_input_vector_map(ctx, inputs)),
+      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
+      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
+      input_vdim(get_vdim(inputs)),
+      output_dtq(create_dtq_maps<Entity::Element>(
+                    outputs,
+                    dtqs,
+                    create_union_field_map_for_dtq(ctx, outputs),
+                    ctx.unionfds,
+                    ctx.ir)),
+      output_idx(create_output_vector_map(ctx, outputs)),
+      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
+      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
+      output_vdim(get_vdim(outputs)),
+      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
+      input_size_on_qp(
+         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
+                           out_op_dim(compute_out_op_dim(outputs)),
+                           out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
+                           output_size_on_qp(
+                              [&]
+   {
+      int s = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      { s += get<o>(outputs).size_on_qp; });
+      return s;
+   }()),
+   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
+   total_trial_op_dim(compute_total_trial_op_dim(
+                         inputs, input_is_dependent, input_size_on_qp)),
+   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
+   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
+   q1d(tensor_1d_size(nq, dim))
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+
+      int direction_field_idx = -1;
+      for (size_t uf = 0; uf < nfields; uf++)
+      {
+         if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
+         {
+            direction_field_idx = static_cast<int>(uf);
+            break;
+         }
+      }
+      MFEM_ASSERT(direction_field_idx != -1,
+                  "DerivativeApply: derivative direction field not found");
+
+      direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename Backend>
+   void run_kernels(std::vector<Vector *> &ye) const
+   {
+      Backend::Run(dim,
+                   q1d,
+                   ctx,
+                   qp_cache,
+                   // inputs
+                   input_idx,
+                   input_B,
+                   input_G,
+                   input_vdim,
+                   input_d1d,
+                   input_q1d,
+                   input_size_on_qp,
+                   input_is_dependent,
+                   // outputs
+                   output_idx,
+                   output_B,
+                   output_G,
+                   output_vdim,
+                   output_d1d,
+                   output_q1d,
+                   out_op_dim,
+                   out_offsets,
+                   trial_vdim,
+                   total_trial_op_dim,
+                   residual_size_on_qp,
+                   output_size_on_qp,
+                   direction_e,
+                   ye,
+                   // fallback arguments
+                   dim,
+                   q1d);
+   }
+
+   //////////////////////////////////////////////////////////////////
+   void operator()(const std::vector<Vector *> &,
+                   const Vector *direction_l,
+                   std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      MFEM_ASSERT(direction_l != nullptr,
+                  "LocalQF DerivativeApply: direction vector is null");
+
+      restriction(direction_fd,
+                  direction_rcache,
+                  *direction_l,
+                  direction_e,
+                  ElementDofOrdering::LEXICOGRAPHIC);
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<DerivativeApplyLO>(ye);
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<DerivativeApplyHO>(ye);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void
+   derivative_apply_callback(const IntegratorContext &ctx,
+                             const Vector &qp_cache,
+                             // inputs: idx, B, G, vdim, d1d, q1d
+                             const std::array<size_t, n_inputs> & /*in_idx*/,
+                             const std::array<const real_t *, n_inputs> in_B,
+                             const std::array<const real_t *, n_inputs> in_G,
+                             const std::array<int, n_inputs> &in_vdim,
+                             const std::array<int, n_inputs> &in_d1d,
+                             const std::array<int, n_inputs> &in_q1d,
+                             const std::array<int, n_inputs> &in_size_on_qp,
+                             const std::array<bool, n_inputs> &input_dep,
+                             // outputs: idx, B, G, vdim, d1d, q1d
+                             const std::array<size_t, n_outputs> &out_idx,
+                             const std::array<const real_t *, n_outputs> out_B,
+                             const std::array<const real_t *, n_outputs> out_G,
+                             const std::array<int, n_outputs> &out_vdim,
+                             const std::array<int, n_outputs> &out_d1d,
+                             const std::array<int, n_outputs> &out_q1d,
+                             const std::array<int, n_outputs> &out_op_dim,
+                             const std::array<int, n_outputs> &out_offsets,
+                             const int trial_vdim,
+                             const int total_trial_op_dim,
+                             const int residual_size_on_qp,
+                             const int output_size_on_qp,
+                             const Vector &direction_e,
+                             std::vector<Vector *> &ye,
+                             // fallback arguments
+                             const int dim,
+                             const int q1d)
+   {
+      MFEM_CONTRACT_VAR(input_dep);
+      MFEM_CONTRACT_VAR(in_size_on_qp);
+      MFEM_CONTRACT_VAR(out_vdim);
+      MFEM_CONTRACT_VAR(out_op_dim);
+      MFEM_CONTRACT_VAR(out_offsets);
+      MFEM_CONTRACT_VAR(trial_vdim);
+      MFEM_CONTRACT_VAR(total_trial_op_dim);
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+      if (ctx.attr.Size() == 0) { return; }
+
+      static constexpr auto B2D = backend_t::DIM == 2;
+      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const int ne = ctx.nentities;
+      const int nq = ctx.ir.GetNPoints();
+      MFEM_CONTRACT_VAR(output_size_on_qp);
+
+      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
+
+      // --------------------------------------------------
+      // DIRECTION (trial): XE_dir for the dependent inputs
+      // --------------------------------------------------
+      const auto d_direction = direction_e.Read();
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
+         using FOP = tuple_element_t<i, inputs_t>;
+         if (!input_dep[i]) { return; }
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
+         }
+         else if constexpr (is_weight_fop_v<FOP>) { /* never a direction */ }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      // --------------------------------------------------
+      // OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
+      for_constexpr<n_outputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = out_idx[i];
+         const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
+         using FOP = tuple_element_t<i, outputs_t>;
+         if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
+         {
+            MFEM_VERIFY(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            MFEM_VERIFY(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else
+         {
+            static_assert(false, "Unsupported FieldOperator");
+         }
+      });
+
+      auto cache_tensor = DeviceTensor<3, const real_t>(
+                             qp_cache.Read(), nq, residual_size_on_qp, ne);
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // -----------------------------------------------
+         // Output integration registers, trial direction (shadow) registers
+         // and shared memory. `rargs` only ever holds test-function data, so
+         // it is an output-only bank: slot `o` is q-function parameter
+         // `n_inputs + o`.
+         // -----------------------------------------------
+         output_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
+         input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> sargs;
+         MFEM_SHARED typename backend_t::Shared smem;
+
+         // -----------------------------------------------
+         // Load trial direction (sargs) for the dependent inputs
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value;
+            if constexpr (!StaticInputDep<i>()) { return; }
+            const auto &XE = in_XE_dir[i];
+            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
+            const real_t *B = in_B[i], *G = in_G[i];
+            auto &sarg = get<i>(sargs);
+            using FOP = tuple_element_t<i, inputs_t>;
+            if constexpr (is_value_fop<FOP>::value)
+            {
+               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
+               using FieldParamT =
+                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+               backend_t::template LoadGradient<RNK,
+                                                decltype(sarg),
+                                                decltype(XE),
+                                                FieldParamT>(
+                                                   smem, e, d, q, Q1D, B, G, XE, sarg);
+            }
+            else if constexpr (is_identity_fop_v<FOP> || is_weight_fop_v<FOP>)
+            {
+               // identity read at qp; weight is never a trial direction
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Contract the cached Jacobian with the trial direction at each
+         // quadrature point and push the result into the test registers.
+         // -----------------------------------------------
+         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+
+                  // The trial direction at this quadrature point is the same
+                  // for every test row (i, k), so pull each dependent input
+                  // slot out of the register bank once, here, instead of once
+                  // per row inside the contraction below.
+                  args_tuple_t dvecs {};
+                  for_constexpr<n_inputs>([&](auto sc)
+                  {
+                     constexpr size_t s = sc.value;
+                     if constexpr (StaticInputDep<s>())
+                     {
+                        using SARG =
+                           typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
+                        get<s>(dvecs) = backend_t::template qp_pull<SARG>(
+                                           get<s>(sargs), qx, qy, qz);
+                     }
+                  });
+
+                  for_constexpr<n_outputs>([&](auto oc)
+                  {
+                     constexpr size_t o = oc.value, ao = n_inputs + o;
+                     using FOP = tuple_element_t<o, outputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
+                     constexpr int tv = StaticOutputVDim<o>();
+                     constexpr int to = StaticOutputOpDim<o>();
+                     constexpr int offset_o = StaticOutputOffset<o>();
+                     constexpr int trial_vdim_ct = StaticTrialVDim();
+                     constexpr int total_trial_op_dim_ct = StaticTotalTrialOpDim();
+
+                     ARG fhat{};
+                     MFEM_UNROLL(tv)
+                     for (int i = 0; i < tv; i++)
+                     {
+                        MFEM_UNROLL(to)
+                        for (int k = 0; k < to; k++)
+                        {
+                           const int row = offset_o + i * to + k;
+                           const int cache_row =
+                              row * trial_vdim_ct * total_trial_op_dim_ct;
+                           real_t sum = 0.0;
+                           for_constexpr<n_inputs>([&](auto sc)
+                           {
+                              constexpr size_t s = sc.value;
+                              if constexpr (StaticInputDep<s>())
+                              {
+                                 constexpr int op_dim_s = StaticInputOpDim<s>();
+                                 constexpr int m_offset = StaticInputOpOffset<s>();
+                                 const auto &dvec = get<s>(dvecs);
+                                 MFEM_UNROLL(trial_vdim_ct)
+                                 for (int j = 0; j < trial_vdim_ct; j++)
+                                 {
+                                    MFEM_UNROLL(op_dim_s)
+                                    for (int m = 0; m < op_dim_s; m++)
+                                    {
+                                       const int cache_idx =
+                                          cache_row + j * total_trial_op_dim_ct +
+                                          (m + m_offset);
+                                       sum += cache_tensor(q, cache_idx, e) *
+                                              qf_value_at(dvec, j, m);
+                                    }
+                                 }
+                              }
+                           });
+                           qf_set_value_at(fhat, i, k, sum);
+                        }
+                     }
+
+                     auto &YE = out_YE[o];
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        MFEM_UNROLL(tv)
+                        for (int i = 0; i < tv; i++)
+                        {
+                           MFEM_UNROLL(to)
+                           for (int k = 0; k < to; k++)
+                           {
+                              YE(i + tv * k, qx, qy, qz, e) =
+                                 qf_value_at(fhat, i, k);
+                           }
+                        }
+                     }
+                     else
+                     {
+                        backend_t::template qp_push<ARG>(
+                           get<o>(rargs), qx, qy, qz, fhat);
+                     }
+                  });
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Integrate value / gradient outputs to the test dofs
+         // -----------------------------------------------
+         for_constexpr<n_outputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value, o = n_inputs + i;
+            const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
+            const auto B = out_B[i], G = out_G[i];
+            auto &YE = out_YE[i];
+            auto &rarg = get<i>(rargs);
+            using FOP = tuple_element_t<i, outputs_t>;
+            if constexpr (is_value_fop_v<FOP>)
+            {
+               backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               using YE_t = decltype(YE);
+               using rarg_t = decltype(rarg);
+               using qf_param_t =
+                  typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
+               constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
+               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
+                  smem, e, d, q, Q1D, B, G, YE, rarg);
+            }
+            else if constexpr (is_identity_fop_v<FOP>) { /* written at qp */ }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+      },
+      ne,
+      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
+                                  q1d, in_d1d, out_d1d)),
+      0,
+      nullptr);
+   }
+
+   using ApplyKernelType =
+      decltype(&DerivativeApply::derivative_apply_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyLO,
+                                     ApplyKernelType,
+                                     (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyHO,
+                                     ApplyKernelType,
+                                     (int, int) );
+};
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+ApplyKernelType
+DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeApplyLO::Kernel()
+{
+   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
+   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return apply_t::template derivative_apply_callback<
+             LocalQFLOBackend<DIM, Q1D>>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+ApplyKernelType
+DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeApplyLO::Fallback(int dim, int q1d)
+{
+   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeApplyLO = typename apply_t::DerivativeApplyLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<DerivativeApplyLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<DerivativeApplyLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+ApplyKernelType
+DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeApplyHO::Kernel()
+{
+   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return apply_t::template derivative_apply_callback<LocalQFHOBackend<DIM>,
+                                                      Q1D>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+ApplyKernelType
+DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeApplyHO::Fallback(int dim, int q1d)
+{
+   using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeApplyHO = typename apply_t::DerivativeApplyHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeApplyHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeApplyHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_apply_transpose.hpp

@@ -0,0 +1,637 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include <array>
+
+namespace mfem::future::LocalQFImpl
+{
+
+// Cached transposed Jacobian apply: Jᵀ·w from the qp_cache
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeApplyTranspose
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using args_tuple_t = decay_tuple<qf_param_ts>;
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
+                 "LocalQF: q-function arity must match inputs + outputs");
+
+   // Input tuple slot referencing the derivative field (compile-time)
+   static constexpr size_t deriv_input_idx_ct = []() constexpr
+   {
+      size_t idx = SIZE_MAX;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         using FOP = tuple_element_t<i, inputs_t>;
+         if (FOP::GetFieldId() == derivative_id) { idx = i; }
+      });
+      return idx;
+   }();
+   static_assert(deriv_input_idx_ct < n_inputs,
+                 "DerivativeApplyTranspose: derivative input slot not found");
+
+   const inputs_t inputs;
+   const outputs_t outputs;
+   const IntegratorContext ctx;
+   const Vector &qp_cache; // Jacobian cache from DerivativeSetup
+   const std::vector<const DofToQuad *> dtqs;
+   // inputs: dtq, B, G, d1d, q1d, vdim (trial / derivative fields)
+   const std::array<DofToQuadMap, n_inputs> input_dtq;
+   const std::array<const real_t *, n_inputs> input_B, input_G;
+   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
+   // outputs: dtq, idx, B, G, d1d, q1d, vdim (test / cotangent fields)
+   const std::array<DofToQuadMap, n_outputs> output_dtq;
+   const std::array<size_t, n_outputs> output_idx;
+   const std::array<const real_t *, n_outputs> output_B, output_G;
+   const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
+   // Jacobian cache metadata
+   const std::array<bool, n_inputs> input_is_dependent;
+   const std::array<int, n_inputs> input_size_on_qp;
+   const std::array<int, n_outputs> out_op_dim;
+   const std::array<int, n_outputs> out_offsets;
+   const int output_size_on_qp;
+   const int trial_vdim;
+   const int total_trial_op_dim;
+   const int residual_size_on_qp;
+   // other constants
+   const int dim, ne, nq, q1d;
+   const size_t deriv_infd_idx; // index of the derivative field in ye
+   // output cotangent restriction workspace (blocked by element)
+   std::array<int, n_outputs> out_elem_dof_size;
+   mutable Vector dir_out_e;
+   /// One restriction cache per output field, resolved on first use.
+   mutable std::array<RestrictionCache<Entity::Element>, n_outputs>
+   out_rcaches;
+
+public:
+   //////////////////////////////////////////////////////////////////
+   DerivativeApplyTranspose() = delete;
+
+   DerivativeApplyTranspose(IntegratorContext ctx,
+                            qfunc_t /*qfunc*/,
+                            inputs_t inputs,
+                            outputs_t outputs,
+                            const Vector &qp_cache):
+      inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache),
+      dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
+                                         inputs,
+                                         dtqs,
+                                         create_union_field_map_for_dtq(ctx, inputs),
+                                         ctx.unionfds,
+                                         ctx.ir)),
+      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
+      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
+      input_vdim(get_vdim(inputs)),
+      output_dtq(create_dtq_maps<Entity::Element>(
+                    outputs,
+                    dtqs,
+                    create_union_field_map_for_dtq(ctx, outputs),
+                    ctx.unionfds,
+                    ctx.ir)),
+      output_idx(create_output_vector_map(ctx, outputs)),
+      output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
+      output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
+      output_vdim(get_vdim(outputs)),
+      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
+      input_size_on_qp(
+         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
+                           out_op_dim(compute_out_op_dim(outputs)),
+                           out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
+                           output_size_on_qp(
+                              [&]
+   {
+      int s = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      { s += get<o>(outputs).size_on_qp; });
+      return s;
+   }()),
+   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
+   total_trial_op_dim(compute_total_trial_op_dim(
+                         inputs, input_is_dependent, input_size_on_qp)),
+   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
+   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
+   q1d(tensor_1d_size(nq, dim)),
+   deriv_infd_idx(find_infd_index(ctx, derivative_id)), out_elem_dof_size{}
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+      MFEM_ASSERT(
+         deriv_infd_idx != SIZE_MAX,
+         "DerivativeApplyTranspose: derivative field not found in infds");
+
+      // Size the workspace that holds the output cotangent(s) in element
+      // layout.
+      int total_dir_e_size = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      {
+         const int elem_sz = compute_element_dof_sz(
+            ctx.outfds[output_idx[o]], ne, ElementDofOrdering::LEXICOGRAPHIC);
+         out_elem_dof_size[o] = elem_sz;
+         total_dir_e_size += elem_sz;
+      });
+      dir_out_e.SetSize(total_dir_e_size * ne);
+      dir_out_e.UseDevice(true);
+      dir_out_e.Read();
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename Backend>
+   void run_kernels(std::vector<Vector *> &ye) const
+   {
+      Backend::Run(dim,
+                   q1d,
+                   ctx,
+                   qp_cache,
+                   dir_out_e,
+                   // inputs (integration target metadata)
+                   input_B,
+                   input_G,
+                   input_vdim,
+                   input_d1d,
+                   input_q1d,
+                   input_size_on_qp,
+                   input_is_dependent,
+                   // outputs (direction interpolation metadata)
+                   output_B,
+                   output_G,
+                   output_vdim,
+                   output_d1d,
+                   output_q1d,
+                   out_op_dim,
+                   out_offsets,
+                   trial_vdim,
+                   total_trial_op_dim,
+                   residual_size_on_qp,
+                   output_size_on_qp,
+                   deriv_infd_idx,
+                   ye,
+                   // fallback arguments
+                   dim,
+                   q1d);
+   }
+
+   //////////////////////////////////////////////////////////////////
+   void operator()(const std::vector<Vector *> & /*xe*/,
+                   const Vector *direction_l,
+                   std::vector<Vector *> &ye) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      MFEM_ASSERT(direction_l != nullptr,
+                  "LocalQF DerivativeApplyTranspose: direction vector is null");
+
+      // Restrict output cotangent from L-vectors into element layout
+      // (dir_out_e).
+      int l_offset = 0;
+      int e_offset = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      {
+         const size_t outfd = output_idx[o];
+         const auto &fd = ctx.outfds[outfd];
+         const int l_size = GetVSize(fd);
+         Vector dir_o_l(*const_cast<Vector *>(direction_l), l_offset, l_size);
+         dir_o_l.UseDevice(true);
+         const int elem_sz = out_elem_dof_size[o];
+         Vector dir_o_e(dir_out_e, e_offset, elem_sz * ne);
+         dir_o_e.UseDevice(true);
+         restriction(fd, out_rcaches[o], dir_o_l, dir_o_e,
+                     ElementDofOrdering::LEXICOGRAPHIC);
+         l_offset += l_size;
+         e_offset += elem_sz * ne;
+      });
+
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<DerivativeApplyTransposeLO>(ye);
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<DerivativeApplyTransposeHO>(ye);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void derivative_apply_transpose_callback(
+      const IntegratorContext &ctx,
+      const Vector &qp_cache,
+      const Vector &dir_e, // restricted, concatenated output cotangents
+      // inputs (integration target metadata)
+      const std::array<const real_t *, n_inputs> in_B,
+      const std::array<const real_t *, n_inputs> in_G,
+      const std::array<int, n_inputs> &in_vdim,
+      const std::array<int, n_inputs> &in_d1d,
+      const std::array<int, n_inputs> &in_q1d,
+      const std::array<int, n_inputs> &in_size_on_qp,
+      const std::array<bool, n_inputs> &input_dep,
+      // outputs (direction interpolation metadata)
+      const std::array<const real_t *, n_outputs> out_B,
+      const std::array<const real_t *, n_outputs> out_G,
+      const std::array<int, n_outputs> &out_vdim,
+      const std::array<int, n_outputs> &out_d1d,
+      const std::array<int, n_outputs> &out_q1d,
+      const std::array<int, n_outputs> &out_op_dim,
+      const std::array<int, n_outputs> &out_offsets,
+      const int trial_vdim,
+      const int total_trial_op_dim,
+      const int residual_size_on_qp,
+      const int output_size_on_qp,
+      const size_t deriv_infd_idx,
+      std::vector<Vector *> &ye,
+      // fallback arguments
+      const int dim,
+      const int q1d)
+   {
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+      if (ctx.attr.Size() == 0) { return; }
+
+      static constexpr auto B2D = backend_t::DIM == 2;
+      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const int ne = ctx.nentities;
+      const int nq = ctx.ir.GetNPoints();
+      MFEM_CONTRACT_VAR(output_size_on_qp);
+      MFEM_CONTRACT_VAR(in_q1d);
+
+      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
+
+      // --------------------------------------------------
+      // DIRECTION (test cotangent): out_XE_dir, concatenated per output
+      // --------------------------------------------------
+      const auto d_dir = dir_e.Read();
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_outputs> out_XE_dir;
+      int e_offset = 0;
+      for_constexpr<n_outputs>([&](auto oc)
+      {
+         constexpr size_t o = oc.value;
+         const int d = out_d1d[o], q = out_q1d[o], v = out_vdim[o];
+         using FOP = tuple_element_t<o, outputs_t>;
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            out_XE_dir[o] = Reshape(d_dir + e_offset, d, d, B2D ? 1 : d, v, ne);
+            e_offset += k_dim(d) * v * ne;
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            out_XE_dir[o] = Reshape(d_dir + e_offset, v, q, q, B2D ? 1 : q, ne);
+            e_offset += k_dim(q) * v * ne;
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      // --------------------------------------------------
+      // DERIVATIVE TRIAL FIELD: ye_XE (accumulates Jᵀ w)
+      // --------------------------------------------------
+      const int d_in = in_d1d[deriv_input_idx_ct];
+      const int v_in = in_vdim[deriv_input_idx_ct];
+      auto ye_XE = Reshape(
+                      ye[deriv_infd_idx]->ReadWrite(), d_in, d_in, B2D ? 1 : d_in, v_in, ne);
+
+      auto cache_tensor = DeviceTensor<3, const real_t>(
+                             qp_cache.Read(), nq, residual_size_on_qp, ne);
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // -----------------------------------------------
+         // Output cotangent (direction) registers live in the output slots;
+         // the trial integration data is pushed into the input slots.
+         // -----------------------------------------------
+         args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
+         MFEM_SHARED typename backend_t::Shared smem;
+
+         // -----------------------------------------------
+         // Interpolate the test cotangent to quadrature points (output slots)
+         // -----------------------------------------------
+         for_constexpr<n_outputs>([&](auto oc)
+         {
+            constexpr size_t o = oc.value, ao = n_inputs + o;
+            using FOP = tuple_element_t<o, outputs_t>;
+            const auto &XE = out_XE_dir[o];
+            const int d = out_d1d[o], q = out_q1d[o], Q1D = q1d;
+            const real_t *B = out_B[o], *G = out_G[o];
+            auto &oarg = get<ao>(rargs);
+            if constexpr (is_value_fop_v<FOP>)
+            {
+               backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, oarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               constexpr auto RNK = qf_param_slot<qfunc_t, ao>::extents.size();
+               using FieldParamT =
+                  typename qf_param_slot<qfunc_t, ao>::qf_decay_param_t;
+               backend_t::template LoadGradient<RNK,
+                                                decltype(oarg),
+                                                decltype(XE),
+                                                FieldParamT>(
+                                                   smem, e, d, q, Q1D, B, G, XE, oarg);
+            }
+            else if constexpr (is_identity_fop_v<FOP>)
+            {
+               // identity cotangent is read directly at qp from out_XE_dir
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Contract the transposed cached Jacobian with the test cotangent at
+         // each quadrature point and push the trial result into the dependent
+         // input registers.
+         // -----------------------------------------------
+         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+
+                  // The test cotangent at this quadrature point is the same
+                  // for every trial column (j, m), so pull each interpolated
+                  // output slot out of the register bank once, here, instead
+                  // of once per column inside the contraction below. Identity
+                  // outputs have no register bank and are read from
+                  // out_XE_dir at the point of use.
+                  args_tuple_t wvecs {};
+                  for_constexpr<n_outputs>([&](auto oc)
+                  {
+                     constexpr size_t o = oc.value, ao = n_inputs + o;
+                     using OFOP = tuple_element_t<o, outputs_t>;
+                     if constexpr (is_value_fop_v<OFOP> ||
+                                   is_gradient_fop_v<OFOP>)
+                     {
+                        using OARG =
+                           typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
+                        get<ao>(wvecs) = backend_t::template qp_pull<OARG>(
+                                            get<ao>(rargs), qx, qy, qz);
+                     }
+                  });
+
+                  int m_offset = 0;
+                  for_constexpr<n_inputs>([&](auto sc)
+                  {
+                     constexpr size_t s = sc.value;
+                     if (!input_dep[s]) { return; }
+                     using SARG =
+                        typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
+                     const int vdim_s = in_vdim[s];
+                     const int op_dim_s = in_size_on_qp[s] / vdim_s;
+
+                     SARG fhat{};
+                     for (int j = 0; j < trial_vdim; j++)
+                     {
+                        for (int m = 0; m < op_dim_s; m++)
+                        {
+                           const int col =
+                              j * total_trial_op_dim + (m + m_offset);
+                           real_t sum = 0.0;
+                           for_constexpr<n_outputs>([&](auto oc)
+                           {
+                              constexpr size_t o = oc.value, ao = n_inputs + o;
+                              using OFOP = tuple_element_t<o, outputs_t>;
+                              const int tv = out_vdim[o], to = out_op_dim[o];
+                              const auto offset_o = out_offsets[o];
+                              const auto &cache = cache_tensor;
+                              if constexpr (is_value_fop_v<OFOP> ||
+                                            is_gradient_fop_v<OFOP>)
+                              {
+                                 const auto &wvec = get<ao>(wvecs);
+                                 for (int i = 0; i < tv; i++)
+                                 {
+                                    for (int k = 0; k < to; k++)
+                                    {
+                                       const int row = offset_o + i * to + k;
+                                       const int cache_idx =
+                                          row * trial_vdim *
+                                          total_trial_op_dim +
+                                          col;
+                                       sum += cache(q, cache_idx, e) *
+                                              qf_value_at(wvec, i, k);
+                                    }
+                                 }
+                              }
+                              else if constexpr (is_identity_fop_v<OFOP>)
+                              {
+                                 const auto &XEo = out_XE_dir[o];
+                                 for (int i = 0; i < tv; i++)
+                                 {
+                                    for (int k = 0; k < to; k++)
+                                    {
+                                       const int row = offset_o + i * to + k;
+                                       const int cache_idx =
+                                          row * trial_vdim *
+                                          total_trial_op_dim +
+                                          col;
+                                       sum += cache(q, cache_idx, e) *
+                                              XEo(i + tv * k, qx, qy, qz, e);
+                                    }
+                                 }
+                              }
+                           });
+                           qf_set_value_at(fhat, j, m, sum);
+                        }
+                     }
+                     backend_t::template qp_push<SARG>(
+                        get<s>(rargs), qx, qy, qz, fhat);
+                     m_offset += op_dim_s;
+                  });
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Integrate the trial result into the derivative field dofs. Multiple
+         // dependent input slots (e.g. value and gradient of the same field)
+         // accumulate into ye_XE via the writers' '+=' semantics.
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto sc)
+         {
+            constexpr size_t s = sc.value;
+            if (!input_dep[s]) { return; }
+            using FOP = tuple_element_t<s, inputs_t>;
+            const int d = in_d1d[s], q = in_q1d[s], Q1D = q1d;
+            const real_t *B = in_B[s], *G = in_G[s];
+            auto &sarg = get<s>(rargs);
+            auto &YE = ye_XE;
+            if constexpr (is_value_fop_v<FOP>)
+            {
+               backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, sarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               using YE_t = decltype(YE);
+               using rarg_t = decltype(sarg);
+               using qf_param_t =
+                  typename qf_param_slot<qfunc_t, s>::qf_decay_param_t;
+               constexpr auto RNK = qf_param_slot<qfunc_t, s>::extents.size();
+               backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
+                  smem, e, d, q, Q1D, B, G, YE, sarg);
+            }
+            else
+            {
+               // identity / weight derivative targets are not produced here
+            }
+         });
+      },
+      ne,
+      backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
+                                  q1d, in_d1d, out_d1d)),
+      0,
+      nullptr);
+   }
+
+   using TransposeKernelType =
+      decltype(&DerivativeApplyTranspose::
+               derivative_apply_transpose_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeLO,
+                                     TransposeKernelType,
+                                     (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeHO,
+                                     TransposeKernelType,
+                                     (int, int) );
+};
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeApplyTranspose<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::TransposeKernelType
+       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeApplyTransposeLO::Kernel()
+{
+   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
+   using transpose_t =
+      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return transpose_t::template derivative_apply_transpose_callback<
+             LocalQFLOBackend<DIM, Q1D>>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeApplyTranspose<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::TransposeKernelType
+       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeApplyTransposeLO::Fallback(int dim, int q1d)
+{
+   using transpose_t =
+      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeApplyTransposeLO =
+      typename transpose_t::DerivativeApplyTransposeLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeApplyTranspose<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::TransposeKernelType
+       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeApplyTransposeHO::Kernel()
+{
+   using transpose_t =
+      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return transpose_t::
+          template derivative_apply_transpose_callback<LocalQFHOBackend<DIM>, Q1D>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeApplyTranspose<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::TransposeKernelType
+       DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeApplyTransposeHO::Fallback(int dim, int q1d)
+{
+   using transpose_t =
+      DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeApplyTransposeHO =
+      typename transpose_t::DerivativeApplyTransposeHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_assemble.hpp

@@ -0,0 +1,910 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "../../../kernels.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include <array>
+#include <type_traits>
+
+namespace ker = mfem::kernels::internal;
+
+namespace mfem::future::LocalQFImpl
+{
+
+namespace detail
+{
+
+template<int DIM>
+MFEM_HOST_DEVICE inline int tensor_idx(int x, int y, int z, int N)
+{
+   static_assert(DIM == 2 || DIM == 3);
+   if constexpr (DIM == 2) { assert(z == 0); }
+   return x + N * (y + N * z);
+}
+
+template<int DIM>
+MFEM_HOST_DEVICE inline real_t
+trial_basis_weight_value(const DeviceTensor<3, const real_t> &B,
+                         const int qx,
+                         const int qy,
+                         const int qz,
+                         const int Jx,
+                         const int Jy,
+                         const int Jz)
+{
+   static_assert(DIM == 2 || DIM == 3);
+   return B(qx, 0, Jx) * B(qy, 0, Jy) * ((DIM == 3) ? B(qz, 0, Jz) : 1.0);
+}
+
+template<int DIM>
+MFEM_HOST_DEVICE inline real_t
+trial_basis_weight_gradient(const DeviceTensor<3, const real_t> &B,
+                            const DeviceTensor<3, const real_t> &G,
+                            const int m,
+                            const int qx,
+                            const int qy,
+                            const int qz,
+                            const int Jx,
+                            const int Jy,
+                            const int Jz)
+{
+   const auto Gx = G(qx, 0, Jx), Gy = G(qy, 0, Jy);
+   const auto Bx = B(qx, 0, Jx), By = B(qy, 0, Jy);
+   if constexpr (DIM == 2)
+   {
+      MFEM_CONTRACT_VAR(qz & Jz);
+      return (m == 0) ? Gx * By : Bx * Gy;
+   }
+   else
+   {
+      const auto Bz = B(qz, 0, Jz), Gz = G(qz, 0, Jz);
+      return (m == 0)   ? Gx * By * Bz
+             : (m == 1) ? Bx * Gy * Bz
+             : (m == 2) ? Bx * By * Gz
+             : (assert(false), 0.0);
+   }
+}
+
+template<int DIM, int MQ1, typename Shared, typename output_t>
+MFEM_HOST_DEVICE void
+map_quadrature_data_to_fields(DeviceTensor<2, real_t> &y,
+                              const DeviceTensor<3, real_t> &f,
+                              const output_t &output,
+                              const DofToQuadMap &dtq,
+                              Shared &s,
+                              const int tv_dof = -1)
+{
+   using output_fop_t = std::decay_t<output_t>;
+   const auto B = dtq.B, G = dtq.G;
+   const bool f_slab = (tv_dof >= 0);
+   const int vdim = output.vdim;
+   const int vd_begin = f_slab ? tv_dof : 0;
+   const int vd_end = f_slab ? tv_dof + 1 : vdim;
+
+   if constexpr (is_value_fop_v<output_fop_t>)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      MFEM_CONTRACT_VAR(unused);
+      const int test_dim = output.size_on_qp / vdim;
+      MFEM_CONTRACT_VAR(test_dim);
+      const int f_vdim = f_slab ? 1 : vdim;
+
+      if constexpr (DIM == 2)
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d);
+         auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
+         ker::LoadMatrix(d1d, q1d, B, s.B);
+         ker::s_regs2d_t<MQ1> r_qp, Y;
+         for (int vd = vd_begin; vd < vd_end; vd++)
+         {
+            const int fi = f_slab ? 0 : vd;
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            { r_qp[qy][qx] = fqp(fi, 0, qx, qy); }
+            MFEM_SYNC_THREAD;
+            ker::Eval2d<MQ1, true>(d1d, q1d, s.M, s.B, r_qp, Y);
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            MFEM_FOREACH_THREAD(dx, x, d1d) { yd(dx, dy, vd) += Y[dy][dx]; }
+            MFEM_SYNC_THREAD;
+         }
+      }
+      else
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d, q1d);
+         auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
+         ker::LoadMatrix(d1d, q1d, B, s.B);
+         ker::s_regs3d_t<MQ1> f_qp, Y;
+         for (int vd = vd_begin; vd < vd_end; vd++)
+         {
+            const int fi = f_slab ? 0 : vd;
+            for (int qz = 0; qz < q1d; qz++)
+            {
+               MFEM_FOREACH_THREAD(qy, y, q1d)
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               { f_qp[qz][qy][qx] = fqp(fi, 0, qx, qy, qz); }
+            }
+            MFEM_SYNC_THREAD;
+            ker::Eval3d<MQ1, true>(d1d, q1d, s.M, s.B, f_qp, Y);
+            for (int dz = 0; dz < d1d; dz++)
+            {
+               MFEM_FOREACH_THREAD(dy, y, d1d)
+               MFEM_FOREACH_THREAD(dx, x, d1d)
+               { yd(dx, dy, dz, vd) += Y[dz][dy][dx]; }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   }
+   else if constexpr (is_gradient_fop_v<output_fop_t>)
+   {
+      const auto [q1d, unused, d1d] = G.GetShape();
+      MFEM_CONTRACT_VAR(unused);
+      const int test_dim = output.size_on_qp / vdim;
+      const int f_vdim = f_slab ? 1 : vdim;
+
+      if constexpr (DIM == 2)
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d);
+         auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
+         ker::LoadMatrix(d1d, q1d, B, s.B);
+         ker::LoadMatrix(d1d, q1d, G, s.G);
+         ker::vd_regs2d_t<1, DIM, MQ1> X, Y;
+         for (int vd = vd_begin; vd < vd_end; vd++)
+         {
+            const int fi = f_slab ? 0 : vd;
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            for (int k = 0; k < DIM; k++)
+            {
+               X[0][k][qy][qx] = fqp(fi, k, qx, qy);
+            }
+            MFEM_SYNC_THREAD;
+            ker::Grad2d<1, DIM, MQ1, true>(d1d, q1d, s.M, s.B, s.G, X, Y);
+            MFEM_FOREACH_THREAD(dy, y, d1d)
+            MFEM_FOREACH_THREAD(dx, x, d1d)
+            {
+               real_t u = 0.0;
+               for (int k = 0; k < DIM; k++) { u += Y[0][k][dy][dx]; }
+               yd(dx, dy, vd) += u;
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+      else
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d, q1d);
+         auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
+         ker::LoadMatrix(d1d, q1d, B, s.B);
+         ker::LoadMatrix(d1d, q1d, G, s.G);
+         ker::vd_regs3d_t<1, DIM, MQ1> X, Y;
+         for (int vd = vd_begin; vd < vd_end; vd++)
+         {
+            const int fi = f_slab ? 0 : vd;
+            for (int qz = 0; qz < q1d; qz++)
+            {
+               MFEM_FOREACH_THREAD(qy, y, q1d)
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               for (int k = 0; k < DIM; k++)
+               {
+                  X[0][k][qz][qy][qx] = fqp(fi, k, qx, qy, qz);
+               }
+            }
+            MFEM_SYNC_THREAD;
+            ker::Grad3d<1, DIM, MQ1, true>(d1d, q1d, s.M, s.B, s.G, X, Y);
+            for (int dz = 0; dz < d1d; dz++)
+            {
+               MFEM_FOREACH_THREAD(dy, y, d1d)
+               MFEM_FOREACH_THREAD(dx, x, d1d)
+               {
+                  real_t u = 0.0;
+                  for (int k = 0; k < DIM; k++) { u += Y[0][k][dz][dy][dx]; }
+                  yd(dx, dy, dz, vd) += u;
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   }
+   else if constexpr (is_identity_fop_v<output_fop_t>)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      MFEM_CONTRACT_VAR(unused);
+      MFEM_CONTRACT_VAR(d1d);
+
+      const int f_sq = f_slab ? 1 : output.size_on_qp;
+      const int sq_begin = f_slab ? tv_dof : 0;
+      const int sq_end = f_slab ? tv_dof + 1 : output.size_on_qp;
+
+      if constexpr (DIM == 2)
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_sq, q1d, q1d);
+         auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d);
+         for (int sq = sq_begin; sq < sq_end; sq++)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               int qz = 0;
+               MFEM_CONTRACT_VAR(qz);
+               yqp(sq, qx, qy) = fqp(0, qx, qy);
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+      else
+      {
+         const auto fqp = Reshape(&f(0, 0, 0), f_sq, q1d, q1d, q1d);
+         auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
+         for (int sq = sq_begin; sq < sq_end; sq++)
+         {
+            for (int qz = 0; qz < q1d; qz++)
+            {
+               MFEM_FOREACH_THREAD(qy, y, q1d)
+               MFEM_FOREACH_THREAD(qx, x, q1d)
+               { yqp(sq, qx, qy, qz) = fqp(0, qx, qy, qz); }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
+                        " for this field descriptor with sum factorization on"
+                        " tensor product elements");
+   }
+}
+
+template<int DIM,
+         int MQ1,
+         typename Shared,
+         typename input_fop_ts,
+         std::size_t n_inputs,
+         typename output_fop_t>
+MFEM_HOST_DEVICE void assemble_element_mat_sumfact(
+   const DeviceTensor<5, real_t> &Ae,
+   const DeviceTensor<6, const real_t> &qpdc,
+   const int e,
+   const DeviceTensor<1, const real_t> &itod,
+   const input_fop_ts &inputs,
+   const output_fop_t &output,
+   const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
+   const DofToQuadMap &output_dtq,
+   const int q1d,
+   const int num_trial_dof_1d,
+   Shared &smem)
+{
+   static constexpr int MQN = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
+   // Slab must hold full (test_vdim, test_op_dim, nq) fhat
+   static constexpr int FHAT_SLAB_MAX = MQN * 4;
+
+   static constexpr bool grad_out = is_gradient_fop_v<output_fop_t>;
+   static constexpr bool ident_out = is_identity_fop_v<output_fop_t>;
+
+   // qpdc shape: (nq, total_trial_op_dim, trial_vdim, test_op_dim, test_vdim, ne)
+   const int test_vdim = qpdc.GetShape()[4];
+   const int test_op_dim = qpdc.GetShape()[3];
+   const int trial_vdim = qpdc.GetShape()[2];
+   const int num_test_dof = Ae.GetShape()[0];
+   const int nq = qpdc.GetShape()[0];
+   const int size_on_qp = output.size_on_qp;
+
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+   MFEM_VERIFY(test_op_dim <= DIM,
+               "DerivativeAssemble: test_op_dim exceeds spatial DIM");
+   MFEM_VERIFY(test_op_dim * nq <= FHAT_SLAB_MAX,
+               "DerivativeAssemble: fhat slab exceeds capacity");
+#endif
+
+   MFEM_SHARED real_t fhat_storage[FHAT_SLAB_MAX];
+
+   const auto &inputs_ref = inputs;
+
+   // Iterate quadrature points using the thread-block mapping
+   const auto foreach_qp = [&](auto &&body)
+   {
+      if constexpr (DIM == 2)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         MFEM_FOREACH_THREAD(qy, y, q1d) { body(qx, qy, 0); }
+      }
+      else
+      {
+         for (int qz = 0; qz < q1d; qz++)
+         {
+            MFEM_FOREACH_THREAD(qy, y, q1d)
+            MFEM_FOREACH_THREAD(qx, x, q1d) { body(qx, qy, qz); }
+         }
+      }
+   };
+
+   const auto zero_slab = [&](const int n_comp)
+   {
+      foreach_qp([&](const int qx, const int qy, const int qz)
+      {
+         const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+         for (int k = 0; k < n_comp; k++) { fhat_storage[k * nq + q] = 0.0; }
+      });
+      MFEM_SYNC_THREAD;
+   };
+
+   const auto accumulate_tv = [&](const int Jx,
+                                  const int Jy,
+                                  const int Jz,
+                                  const int j,
+                                  const int tv,
+                                  const int tod_only = -1)
+   {
+      int m_offset = 0;
+      for_constexpr<n_inputs>([&](auto inp)
+      {
+         using fop_t = std::decay_t<decltype(get<inp>(inputs_ref))>;
+
+         const int trial_op_dim = static_cast<int>(itod(static_cast<int>(inp)));
+         if (trial_op_dim == 0) { return; }
+
+         const auto &B = input_dtq_maps[inp].B;
+         const auto &G = input_dtq_maps[inp].G;
+
+         if constexpr (is_value_fop<fop_t>::value)
+         {
+            foreach_qp([&](const int qx, const int qy, const int qz)
+            {
+               const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+               const real_t w =
+                  trial_basis_weight_value<DIM>(B, qx, qy, qz, Jx, Jy, Jz);
+               for (int m = 0; m < trial_op_dim; m++)
+               {
+                  for (int k = 0; k < test_op_dim; k++)
+                  {
+                     if (tod_only >= 0 && k != tod_only) { continue; }
+                     const real_t f = qpdc(q, m + m_offset, j, k, tv, e);
+                     if constexpr (grad_out && !ident_out)
+                     {
+                        fhat_storage[k * nq + q] += f * w;
+                     }
+                     else
+                     {
+                        fhat_storage[q] += f * w;
+                     }
+                  }
+               }
+            });
+         }
+         else if constexpr (is_gradient_fop<fop_t>::value)
+         {
+            foreach_qp([&](const int qx, const int qy, const int qz)
+            {
+               const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+               for (int m = 0; m < trial_op_dim; m++)
+               {
+                  const real_t w = trial_basis_weight_gradient<DIM>(
+                                      B, G, m, qx, qy, qz, Jx, Jy, Jz);
+                  for (int k = 0; k < test_op_dim; k++)
+                  {
+                     if (tod_only >= 0 && k != tod_only) { continue; }
+                     const real_t f = qpdc(q, m + m_offset, j, k, tv, e);
+                     if constexpr (grad_out && !ident_out)
+                     {
+                        fhat_storage[k * nq + q] += f * w;
+                     }
+                     else
+                     {
+                        fhat_storage[q] += f * w;
+                     }
+                  }
+               }
+            });
+         }
+         else
+         {
+            MFEM_ABORT_KERNEL("sum factorized sparse matrix assemble routine "
+                              "not implemented for field operator");
+         }
+         MFEM_SYNC_THREAD;
+         m_offset += trial_op_dim;
+      });
+   };
+
+   for (int Jz = 0; Jz < ((DIM == 2) ? 1 : num_trial_dof_1d); Jz++)
+   {
+      for (int Jy = 0; Jy < num_trial_dof_1d; Jy++)
+      {
+         for (int Jx = 0; Jx < num_trial_dof_1d; Jx++)
+         {
+            const int J = tensor_idx<DIM>(Jx, Jy, Jz, num_trial_dof_1d);
+            for (int j = 0; j < trial_vdim; j++)
+            {
+               auto bvtfhat =
+                  Reshape(&Ae(0, 0, J, j, e), num_test_dof, test_vdim);
+               const int fhat_size = test_vdim * test_op_dim * nq;
+
+               if (fhat_size <= FHAT_SLAB_MAX)
+               {
+                  auto fhat =
+                     Reshape(&fhat_storage[0], test_vdim, test_op_dim, nq);
+                  for (int tv = 0; tv < test_vdim; tv++)
+                  {
+                     for (int tod = 0; tod < test_op_dim; tod++)
+                     {
+                        foreach_qp([&](const int qx, const int qy, const int qz)
+                        {
+                           const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+                           fhat(tv, tod, q) = 0.0;
+                        });
+                     }
+                  }
+                  MFEM_SYNC_THREAD;
+
+                  int m_offset = 0;
+                  for_constexpr<n_inputs>([&](auto inp)
+                  {
+                     using fop_t = std::decay_t<decltype(get<inp>(inputs_ref))>;
+
+                     const int trial_op_dim =
+                        static_cast<int>(itod(static_cast<int>(inp)));
+                     if (trial_op_dim == 0) { return; }
+
+                     const auto &B = input_dtq_maps[inp].B;
+                     const auto &G = input_dtq_maps[inp].G;
+
+                     if constexpr (is_value_fop<fop_t>::value)
+                     {
+                        foreach_qp([&](const int qx, const int qy, const int qz)
+                        {
+                           const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+                           const real_t w = trial_basis_weight_value<DIM>(
+                                               B, qx, qy, qz, Jx, Jy, Jz);
+                           for (int m = 0; m < trial_op_dim; m++)
+                           {
+                              for (int i = 0; i < test_vdim; i++)
+                              {
+                                 for (int k = 0; k < test_op_dim; k++)
+                                 {
+                                    const real_t f =
+                                       qpdc(q, m + m_offset, j, k, i, e);
+                                    fhat(i, k, q) += f * w;
+                                 }
+                              }
+                           }
+                        });
+                     }
+                     else if constexpr (is_gradient_fop<fop_t>::value)
+                     {
+                        foreach_qp([&](const int qx, const int qy, const int qz)
+                        {
+                           const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
+                           for (int m = 0; m < trial_op_dim; m++)
+                           {
+                              const real_t w = trial_basis_weight_gradient<DIM>(
+                                                  B, G, m, qx, qy, qz, Jx, Jy, Jz);
+                              for (int i = 0; i < test_vdim; i++)
+                              {
+                                 for (int k = 0; k < test_op_dim; k++)
+                                 {
+                                    const real_t f =
+                                       qpdc(q, m + m_offset, j, k, i, e);
+                                    fhat(i, k, q) += f * w;
+                                 }
+                              }
+                           }
+                        });
+                     }
+                     else
+                     {
+                        MFEM_ABORT_KERNEL(
+                           "sum factorized sparse matrix assemble routine "
+                           "not implemented for field operator");
+                     }
+                     MFEM_SYNC_THREAD;
+                     m_offset += trial_op_dim;
+                  });
+                  map_quadrature_data_to_fields<DIM, MQ1>(
+                     bvtfhat, fhat, output, output_dtq, smem);
+               }
+               else if constexpr (ident_out)
+               {
+                  for (int sq = 0; sq < size_on_qp; sq++)
+                  {
+                     const int tv = sq / test_op_dim;
+                     const int tod = sq % test_op_dim;
+                     zero_slab(1);
+                     accumulate_tv(Jx, Jy, Jz, j, tv, tod);
+                     auto f_slab = Reshape(&fhat_storage[0], 1, 1, nq);
+                     map_quadrature_data_to_fields<DIM, MQ1>(
+                        bvtfhat, f_slab, output, output_dtq, smem, sq);
+                  }
+               }
+               else if constexpr (grad_out)
+               {
+                  for (int tv = 0; tv < test_vdim; tv++)
+                  {
+                     zero_slab(test_op_dim);
+                     accumulate_tv(Jx, Jy, Jz, j, tv);
+                     auto f_slab =
+                        Reshape(&fhat_storage[0], 1, test_op_dim, nq);
+                     map_quadrature_data_to_fields<DIM, MQ1>(
+                        bvtfhat, f_slab, output, output_dtq, smem, tv);
+                  }
+               }
+               else
+               {
+                  for (int tv = 0; tv < test_vdim; tv++)
+                  {
+                     zero_slab(1);
+                     accumulate_tv(Jx, Jy, Jz, j, tv);
+                     auto f_slab = Reshape(&fhat_storage[0], 1, 1, nq);
+                     map_quadrature_data_to_fields<DIM, MQ1>(
+                        bvtfhat, f_slab, output, output_dtq, smem, tv);
+                  }
+               }
+            }
+         }
+      }
+   }
+}
+
+} // namespace detail
+
+// ────────────────────────────────────────────────────────────────────────────
+// Assemble sparse Jacobian from cached quadrature derivatives (tensor 2D/3D)
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeAssemble
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+
+   const IntegratorContext ctx;
+   const Vector &qp_cache;
+   inputs_t inputs;
+   outputs_t outputs;
+   const bool use_sum_factorization;
+   const std::vector<const DofToQuad *> dtqs;
+   const std::array<DofToQuadMap, n_inputs> input_dtq_maps;
+   const std::array<DofToQuadMap, n_outputs> output_dtq_maps;
+   const std::array<bool, n_inputs> input_is_dependent;
+   const size_t trial_field_uf;
+   const size_t test_field_uf;
+   const ParFiniteElementSpace *test_fes;
+   const ParFiniteElementSpace *trial_fes;
+   const int test_vdim;
+   const int test_op_dim;
+   const int num_test_dof;
+   const int trial_vdim;
+   const int trial_op_dim;
+   const int num_trial_dof;
+   const int dim, ne, nq, q1d;
+   const int num_trial_dof_1d;
+   const int total_trial_op_dim;
+   mutable Vector inputs_trial_op_dim;
+   mutable Vector Ae_mem;
+
+public:
+   DerivativeAssemble() = delete;
+
+   DerivativeAssemble(IntegratorContext ctx_in,
+                      qfunc_t /*qfunc*/,
+                      inputs_t inputs_in,
+                      outputs_t outputs_in,
+                      const Vector &qp_cache_in):
+      ctx(ctx_in), qp_cache(qp_cache_in), inputs(inputs_in),
+      outputs(outputs_in), use_sum_factorization(
+         [&]
+   {
+      const Element::Type etype =
+         Element::TypeFromGeometry(ctx_in.mesh.GetTypicalElementGeometry());
+      return (etype == Element::QUADRILATERAL || etype == Element::HEXAHEDRON);
+   }()),
+   dtqs(
+      [&]
+   {
+      const DofToQuad::Mode dtq_mode = use_sum_factorization
+      ? DofToQuad::Mode::TENSOR
+      : DofToQuad::Mode::FULL;
+      std::vector<const DofToQuad *> maps;
+      maps.reserve(ctx_in.unionfds.size());
+      for (const auto &field : ctx_in.unionfds)
+      {
+         maps.emplace_back(
+            GetDofToQuad<Entity::Element>(field, ctx_in.ir, dtq_mode));
+      }
+      return maps;
+   }()),
+   input_dtq_maps(create_dtq_maps<Entity::Element>(
+                     inputs,
+                     dtqs,
+                     create_union_field_map_for_dtq(ctx_in, inputs),
+                     ctx_in.unionfds,
+                     ctx_in.ir)),
+   output_dtq_maps(create_dtq_maps<Entity::Element>(
+                      outputs,
+                      dtqs,
+                      create_union_field_map_for_dtq(ctx_in, outputs),
+                      ctx_in.unionfds,
+                      ctx_in.ir)),
+   input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
+   trial_field_uf(find_union_field_index(ctx_in, derivative_id)),
+   test_field_uf(
+      find_union_field_index(ctx_in, get<0>(outputs).GetFieldId())),
+   test_fes(
+      [&]
+   {
+      const auto *fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[test_field_uf].data);
+      MFEM_ASSERT(fes != nullptr && *fes != nullptr,
+                  "LocalQFBackend: test space is not a ParFiniteElementSpace");
+      return *fes;
+   }()),
+   trial_fes(
+      [&]
+   {
+      const auto *fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[trial_field_uf].data);
+      MFEM_ASSERT(fes != nullptr && *fes != nullptr,
+                  "LocalQFBackend: trial space is not a ParFiniteElementSpace");
+      return *fes;
+   }()),
+   test_vdim(get<0>(outputs).vdim),
+   test_op_dim(get<0>(outputs).size_on_qp / test_vdim),
+   num_test_dof(test_fes->GetFE(0)->GetDof()),
+   trial_vdim(compute_trial_vdim(inputs, derivative_id)), trial_op_dim(
+      [&]
+   {
+      int top = 0;
+      for_constexpr<n_inputs>([&](auto i)
+      {
+         if (get<i>(inputs).GetFieldId() == derivative_id)
+         {
+            top = get<i>(inputs).size_on_qp / get<i>(inputs).vdim;
+         }
+      });
+      return top;
+   }()),
+   num_trial_dof(trial_fes->GetFE(0)->GetDof()),
+   dim(ctx_in.mesh.Dimension()), ne(ctx_in.nentities),
+   nq(ctx_in.ir.GetNPoints()), q1d(tensor_1d_size(nq, dim)),
+   num_trial_dof_1d(tensor_1d_size(num_trial_dof, dim)), total_trial_op_dim(
+      [&]
+   {
+      const auto in_qp_sizes =
+      get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs>{});
+      return compute_total_trial_op_dim(
+         inputs, input_is_dependent, in_qp_sizes);
+   }()),
+   inputs_trial_op_dim(), Ae_mem()
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+      MFEM_ASSERT(trial_field_uf != SIZE_MAX,
+                  "DerivativeAssemble: trial field not found in unionfds");
+      MFEM_ASSERT(test_field_uf != SIZE_MAX,
+                  "DerivativeAssemble: test field not found in unionfds");
+      MFEM_ASSERT(trial_vdim > 0,
+                  "LocalQFBackend: could not determine trial vdim");
+      MFEM_ASSERT(total_trial_op_dim > 0,
+                  "LocalQFBackend: no dependent inputs found");
+
+      inputs_trial_op_dim.UseDevice(true);
+      inputs_trial_op_dim.SetSize(n_inputs);
+      auto inputs_trial_op_dim_host = inputs_trial_op_dim.HostWrite();
+      for_constexpr<n_inputs>([&](auto i)
+      {
+         inputs_trial_op_dim_host[i] =
+            input_is_dependent[i]
+            ? get<i>(inputs).size_on_qp / get<i>(inputs).vdim
+            : 0;
+      });
+
+      const int elem_mat_size =
+         num_test_dof * test_vdim * num_trial_dof * trial_vdim;
+      Ae_mem.SetSize(elem_mat_size * ne, Device::GetDeviceMemoryType());
+      Ae_mem.UseDevice(true);
+      Ae_mem = 0.0;
+   }
+
+   void operator()(SparseMatrix *&A) const
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      if (!(use_sum_factorization && (dim == 2 || dim == 3)))
+      {
+         MFEM_ABORT("DerivativeAssemble optimized path is implemented "
+                    "for tensor-product 2D/3D elements only");
+      }
+
+      DerivativeAssembleHO::Run(dim,
+                                q1d,
+                                ctx,
+                                qp_cache,
+                                Ae_mem,
+                                inputs,
+                                outputs,
+                                input_dtq_maps,
+                                output_dtq_maps[0],
+                                inputs_trial_op_dim,
+                                test_vdim,
+                                test_op_dim,
+                                num_test_dof,
+                                num_trial_dof,
+                                num_trial_dof_1d,
+                                trial_vdim,
+                                total_trial_op_dim,
+                                nq,
+                                ne,
+                                q1d,
+                                dim);
+
+      A = new SparseMatrix;
+      A->OverrideSize(test_fes->GetVSize(), trial_fes->GetVSize());
+
+      const auto *test_restr = dynamic_cast<const ElementRestriction *>(
+                                  test_fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC));
+      const auto *trial_restr = dynamic_cast<const ElementRestriction *>(
+                                   trial_fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC));
+      MFEM_VERIFY(test_restr != nullptr && trial_restr != nullptr,
+                  "DerivativeAssemble SparseMatrix assembly requires "
+                  "H1/conforming ElementRestriction spaces");
+
+      test_restr->FillSparseMatrix(Ae_mem, *A, *trial_restr);
+   }
+
+   template<typename backend_t = LocalQFHOBackend<3>, int T_Q1D = 0>
+   static void derivative_assemble_callback(
+      const IntegratorContext &ctx,
+      const Vector &qp_cache,
+      Vector &Ae_mem,
+      const inputs_t &inputs,
+      const outputs_t &outputs,
+      const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
+      const DofToQuadMap &output_dtq,
+      const Vector &inputs_trial_op_dim,
+      const int test_vdim,
+      const int test_op_dim,
+      const int num_test_dof,
+      const int num_trial_dof,
+      const int num_trial_dof_1d,
+      const int trial_vdim,
+      const int total_trial_op_dim,
+      const int nq,
+      const int ne,
+      const int q1d,
+      const int dim)
+   {
+      static constexpr int DIM = backend_t::DIM;
+      static constexpr int MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+      static constexpr int MNQ = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
+
+      MFEM_VERIFY(dim == DIM,
+                  "DerivativeAssemble: mesh dim does not match backend");
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+      MFEM_VERIFY(q1d <= MQ1, "q1d exceeds backend MQ1 limit");
+      MFEM_VERIFY(nq <= MNQ,
+                  "DerivativeAssemble: nq exceeds backend quadrature capacity");
+      MFEM_VERIFY(test_op_dim <= DIM,
+                  "DerivativeAssemble: test_op_dim exceeds spatial DIM");
+      if (ctx.attr.Size() == 0) { return; }
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      const auto qpdc = Reshape(qp_cache.Read(),
+                                nq,
+                                total_trial_op_dim,
+                                trial_vdim,
+                                test_op_dim,
+                                test_vdim,
+                                ne);
+      const auto itod = Reshape(inputs_trial_op_dim.Read(), n_inputs);
+
+      auto Ae = Reshape(Ae_mem.ReadWrite(),
+                        num_test_dof,
+                        test_vdim,
+                        num_trial_dof,
+                        trial_vdim,
+                        ne);
+
+      dfem::forall(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         static constexpr int DIM = backend_t::DIM;
+         static constexpr int MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+
+         MFEM_SHARED typename backend_t::Shared s;
+
+         detail::assemble_element_mat_sumfact<DIM, MQ1>(Ae,
+                                                        qpdc,
+                                                        e,
+                                                        itod,
+                                                        inputs,
+                                                        get<0>(outputs),
+                                                        input_dtq_maps,
+                                                        output_dtq,
+                                                        q1d,
+                                                        num_trial_dof_1d,
+                                                        s);
+      },
+      ne,
+      backend_t::thread_blocks(q1d),
+      0,
+      nullptr);
+   }
+
+   using AssembleKernelType =
+      decltype(&DerivativeAssemble::derivative_assemble_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleHO,
+                                     AssembleKernelType,
+                                     (int, int));
+};
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline
+typename DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
+AssembleKernelType
+DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeAssembleHO::Kernel()
+{
+   static_assert(DIM == 2 || DIM == 3);
+   using assemble_t =
+      DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return assemble_t::template derivative_assemble_callback<
+             LocalQFHOBackend<DIM, Q1D>>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline
+typename DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
+AssembleKernelType
+DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeAssembleHO::Fallback(int dim, int q1d)
+{
+   using assemble_t =
+      DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeAssembleHO = typename assemble_t::DerivativeAssembleHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeAssembleHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeAssembleHO, 3, 8>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_assemble_diagonal.hpp

@@ -0,0 +1,540 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include <array>
+
+namespace mfem::future::LocalQFImpl
+{
+
+// Assemble diagonal of cached Jacobian (square trial == test, tensor 2D/3D)
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeAssembleDiagonal
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+
+   const IntegratorContext ctx;
+   const Vector &qp_cache;
+   inputs_t inputs;
+   outputs_t outputs;
+   const bool use_sum_factorization;
+   const std::vector<const DofToQuad *> dtqs;
+   const std::array<DofToQuadMap, n_inputs> input_dtq_maps;
+   const std::array<DofToQuadMap, n_outputs> output_dtq_maps;
+   const std::array<bool, n_inputs> input_is_dependent;
+   const size_t trial_field_uf;
+   const size_t test_field_uf;
+   const bool is_square;
+   const int test_vdim;
+   const std::array<int, n_outputs> out_vdim;
+   const std::array<int, n_outputs> out_op_dim;
+   const std::array<int, n_outputs> out_offsets;
+   const int output_size_on_qp;
+   const int num_test_dof;
+   const int num_test_dof_1d;
+   const int trial_vdim;
+   const int total_trial_op_dim;
+   const int num_trial_dof_1d;
+   const int residual_size_on_qp;
+   const int dim, ne, nq, q1d;
+   const std::array<int, n_inputs> inputs_trial_op_dim;
+   mutable Vector Ye_mem;
+
+public:
+   DerivativeAssembleDiagonal() = delete;
+
+   DerivativeAssembleDiagonal(IntegratorContext ctx_in,
+                              qfunc_t /*qfunc*/,
+                              inputs_t inputs_in,
+                              outputs_t outputs_in,
+                              const Vector &qp_cache_in):
+      ctx(ctx_in), qp_cache(qp_cache_in), inputs(inputs_in),
+      outputs(outputs_in), use_sum_factorization(
+         [&]
+   {
+      const Element::Type etype =
+         Element::TypeFromGeometry(ctx_in.mesh.GetTypicalElementGeometry());
+      return (etype == Element::QUADRILATERAL || etype == Element::HEXAHEDRON);
+   }()),
+   dtqs(
+      [&]
+   {
+      const DofToQuad::Mode dtq_mode = use_sum_factorization
+      ? DofToQuad::Mode::TENSOR
+      : DofToQuad::Mode::FULL;
+      std::vector<const DofToQuad *> maps;
+      maps.reserve(ctx_in.unionfds.size());
+      for (const auto &field : ctx_in.unionfds)
+      {
+         maps.emplace_back(
+            GetDofToQuad<Entity::Element>(field, ctx_in.ir, dtq_mode));
+      }
+      return maps;
+   }()),
+   input_dtq_maps(create_dtq_maps<Entity::Element>(
+                     inputs,
+                     dtqs,
+                     create_union_field_map_for_dtq(ctx_in, inputs),
+                     ctx_in.unionfds,
+                     ctx_in.ir)),
+   output_dtq_maps(create_dtq_maps<Entity::Element>(
+                      outputs,
+                      dtqs,
+                      create_union_field_map_for_dtq(ctx_in, outputs),
+                      ctx_in.unionfds,
+                      ctx_in.ir)),
+   input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
+   trial_field_uf(find_union_field_index(ctx_in, derivative_id)),
+   test_field_uf(
+      find_union_field_index(ctx_in, get<0>(outputs).GetFieldId())),
+   is_square(
+      [&]
+   {
+      const auto *test_fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[test_field_uf].data);
+      const auto *trial_fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[trial_field_uf].data);
+      return test_fes && trial_fes && *test_fes && *trial_fes &&
+      (*test_fes == *trial_fes);
+   }()),
+   test_vdim(get<0>(outputs).vdim),
+   out_vdim(get_vdim(outputs_in)),
+   out_op_dim(compute_out_op_dim(outputs_in)),
+   out_offsets(compute_out_offsets(out_vdim, out_op_dim)),
+   output_size_on_qp(
+      [&]
+   {
+      int s = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      { s += get<o>(outputs_in).size_on_qp; });
+      return s;
+   }()), num_test_dof(
+      [&]
+   {
+      const auto *test_fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[test_field_uf].data);
+      MFEM_ASSERT(test_fes != nullptr && *test_fes != nullptr,
+                  "LocalQFBackend: test space is not a ParFiniteElementSpace");
+      return (*test_fes)->GetFE(0)->GetDof();
+   }()),
+   num_test_dof_1d(tensor_1d_size(num_test_dof, ctx_in.mesh.Dimension())),
+   trial_vdim(compute_trial_vdim(inputs, derivative_id)), total_trial_op_dim(
+      [&]
+   {
+      const auto input_size_on_qp =
+      get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs>{});
+      return compute_total_trial_op_dim(
+         inputs, input_is_dependent, input_size_on_qp);
+   }()),
+   num_trial_dof_1d(
+      [&]
+   {
+      const auto *trial_fes = std::get_if<const ParFiniteElementSpace *>(
+         &ctx_in.unionfds[trial_field_uf].data);
+      MFEM_ASSERT(trial_fes != nullptr && *trial_fes != nullptr,
+                  "LocalQFBackend: trial space is not a ParFiniteElementSpace");
+      const int num_trial_dof = (*trial_fes)->GetFE(0)->GetDof();
+      return tensor_1d_size(num_trial_dof, ctx_in.mesh.Dimension());
+   }()),
+   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
+   dim(ctx_in.mesh.Dimension()), ne(ctx_in.nentities),
+   nq(ctx_in.ir.GetNPoints()), q1d(tensor_1d_size(nq, dim)),
+   inputs_trial_op_dim(
+      [&]
+   {
+      std::array<int, n_inputs> itod{};
+      for_constexpr<n_inputs>([&](auto i)
+      {
+         itod[i] = input_is_dependent[i]
+                   ? get<i>(inputs).size_on_qp / get<i>(inputs).vdim
+                   : 0;
+      });
+      return itod;
+   }()),
+   Ye_mem()
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+      MFEM_ASSERT(
+         trial_field_uf != SIZE_MAX,
+         "DerivativeAssembleDiagonal: trial field not found in unionfds");
+      MFEM_ASSERT(
+         test_field_uf != SIZE_MAX,
+         "DerivativeAssembleDiagonal: test field not found in unionfds");
+      MFEM_ASSERT(trial_vdim > 0,
+                  "LocalQFBackend: could not determine trial vdim");
+      MFEM_ASSERT(total_trial_op_dim > 0,
+                  "LocalQFBackend: no dependent inputs found");
+      for_constexpr<n_outputs>([&](auto o)
+      {
+         MFEM_CONTRACT_VAR(o);
+         MFEM_ASSERT(out_vdim[o] == test_vdim,
+                     "DerivativeAssembleDiagonal: all outputs must share the "
+                     "test field vdim");
+      });
+
+      if (is_square)
+      {
+         Ye_mem.SetSize(num_test_dof * test_vdim * ne);
+         Ye_mem.UseDevice(true);
+      }
+   }
+
+   template<typename Backend>
+   void run_kernels() const
+   {
+      Backend::Run(dim,
+                   q1d,
+                   ctx,
+                   qp_cache,
+                   Ye_mem,
+                   inputs,
+                   outputs,
+                   output_dtq_maps,
+                   input_dtq_maps,
+                   test_vdim,
+                   out_op_dim,
+                   out_offsets,
+                   output_size_on_qp,
+                   num_test_dof,
+                   num_test_dof_1d,
+                   trial_vdim,
+                   total_trial_op_dim,
+                   residual_size_on_qp,
+                   inputs_trial_op_dim,
+                   nq,
+                   ne,
+                   q1d,
+                   dim);
+   }
+
+   void operator()(Vector &diag_e) const
+   {
+      if (!is_square) { return; }
+      if (ctx.attr.Size() == 0) { return; }
+
+      if (!(use_sum_factorization && (dim == 2 || dim == 3)))
+      {
+         MFEM_ABORT("DerivativeAssembleDiagonal optimized path is implemented "
+                    "for tensor-product 2D/3D elements only");
+      }
+      MFEM_VERIFY(num_test_dof_1d == num_trial_dof_1d,
+                  "DerivativeAssembleDiagonal requires matching tensor dofs");
+      MFEM_VERIFY(num_test_dof_1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+      MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+      Ye_mem = 0.0;
+
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<DerivativeAssembleDiagonalLO>();
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<DerivativeAssembleDiagonalHO>();
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+
+      diag_e += Ye_mem;
+   }
+
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void derivative_assemble_diagonal_callback(
+      const IntegratorContext &ctx,
+      const Vector &qp_cache,
+      Vector &Ye_mem,
+      const inputs_t &inputs,
+      const outputs_t &outputs,
+      const std::array<DofToQuadMap, n_outputs> &output_dtq_maps,
+      const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
+      const int test_vdim,
+      const std::array<int, n_outputs> &out_op_dim,
+      const std::array<int, n_outputs> &out_offsets,
+      const int output_size_on_qp,
+      const int num_test_dof,
+      const int num_test_dof_1d,
+      const int trial_vdim,
+      const int total_trial_op_dim,
+      const int residual_size_on_qp,
+      const std::array<int, n_inputs> &inputs_trial_op_dim,
+      const int nq,
+      const int ne,
+      const int q1d,
+      const int dim)
+   {
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+      if (ctx.attr.Size() == 0) { return; }
+
+      static constexpr bool B2D = backend_t::DIM == 2;
+      static constexpr int MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      auto cache_tensor = DeviceTensor<3, const real_t>(
+                             qp_cache.Read(), nq, residual_size_on_qp, ne);
+      const int num_dofs_per_elem = num_test_dof * test_vdim;
+      auto Ye = Reshape(Ye_mem.ReadWrite(), num_dofs_per_elem, ne);
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // The cache is written with the quadrature index fastest, then the
+         // trial op index, then the (test vdim, test op) rows of all outputs
+         // stacked via out_offsets.
+         auto qpdc = Reshape(&cache_tensor(0, 0, e),
+                             nq,
+                             total_trial_op_dim,
+                             trial_vdim,
+                             output_size_on_qp);
+
+         // Backend-owned shared scratch for the sum-factorized contraction.
+         MFEM_SHARED typename backend_t::Shared s_diag;
+         const int nz_dof = B2D ? 1 : num_test_dof_1d;
+
+         for (int vd = 0; vd < test_vdim; vd++)
+         {
+            auto Y = Reshape(&Ye(vd * num_test_dof, e),
+                             num_test_dof_1d,
+                             num_test_dof_1d,
+                             nz_dof);
+
+            MFEM_FOREACH_THREAD(dz_t, z, nz_dof)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dy_t, y, num_test_dof_1d)
+               {
+                  MFEM_FOREACH_THREAD_DIRECT(dx_t, x, num_test_dof_1d)
+                  { Y(dx_t, dy_t, dz_t) = 0.0; }
+               }
+            }
+            MFEM_SYNC_THREAD;
+
+            // Accumulate every (output o, test op k, dependent input s,
+            // trial op m) block of the cached Jacobian into the diagonal via
+            // the backend driver.
+            for_constexpr<n_outputs>([&](auto o)
+            {
+               using test_fop_t = std::decay_t<decltype(get<o>(outputs))>;
+               const auto &out_dtq = output_dtq_maps[o];
+               const int test_op_dim = out_op_dim[static_cast<int>(o)];
+
+               // Test-basis factor along a spatial axis
+               const auto eval_test =
+                  [&](const int k, const int axis, const int q, const int d)
+               {
+                  const auto &B = out_dtq.B;
+                  const auto &G = out_dtq.G;
+                  if constexpr (is_value_fop<test_fop_t>::value)
+                  {
+                     return (k == 0) ? B(q, 0, d) : 0.0;
+                  }
+                  else if constexpr (is_gradient_fop<test_fop_t>::value)
+                  {
+                     return (k == axis) ? G(q, 0, d) : B(q, 0, d);
+                  }
+                  else
+                  {
+                     return 0.0;
+                  }
+               };
+
+               for (int k = 0; k < test_op_dim; k++)
+               {
+                  const int row =
+                     out_offsets[static_cast<int>(o)] + vd * test_op_dim + k;
+                  int m_offset = 0;
+                  for_constexpr<n_inputs>([&](auto s)
+                  {
+                     using fop_t = std::decay_t<decltype(get<s>(inputs))>;
+                     const int trial_op_dim =
+                        inputs_trial_op_dim[static_cast<int>(s)];
+                     if (trial_op_dim == 0) { return; }
+
+                     const auto &in_dtq = input_dtq_maps[s];
+                     const auto eval_input =
+                        [&](const int m, const int axis, const int q,
+                            const int d)
+                     {
+                        if constexpr (is_value_fop<fop_t>::value)
+                        {
+                           return (m == 0) ? in_dtq.B(q, 0, d) : 0.0;
+                        }
+                        else if constexpr (is_gradient_fop<fop_t>::value)
+                        {
+                           return (m == axis) ? in_dtq.G(q, 0, d)
+                                  : in_dtq.B(q, 0, d);
+                        }
+                        else
+                        {
+                           return 0.0;
+                        }
+                     };
+
+                     for (int m = 0; m < trial_op_dim; m++)
+                     {
+                        const int col = m_offset + m;
+                        backend_t::DiagContract(
+                           s_diag,
+                           num_test_dof_1d,
+                           q1d,
+                           nz_dof,
+                           [&](int axis, int q, int d)
+                        { return eval_test(k, axis, q, d); },
+                        [&](int axis, int q, int d)
+                        { return eval_input(m, axis, q, d); },
+                        [&](int q) { return qpdc(q, col, vd, row); },
+                        [&](int dx, int dy, int dz, real_t u)
+                        { Y(dx, dy, dz) += u; });
+                     }
+                     m_offset += trial_op_dim;
+                  });
+               }
+            });
+         }
+      },
+      ne,
+      backend_t::thread_blocks(std::max(q1d, num_test_dof_1d)),
+      0,
+      nullptr);
+   }
+
+   using DiagonalKernelType =
+      decltype(&DerivativeAssembleDiagonal::
+               derivative_assemble_diagonal_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalLO,
+                                     DiagonalKernelType,
+                                     (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalHO,
+                                     DiagonalKernelType,
+                                     (int, int) );
+};
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeAssembleDiagonal<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::DiagonalKernelType
+       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeAssembleDiagonalLO::Kernel()
+{
+   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
+   using diag_t =
+      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return diag_t::template derivative_assemble_diagonal_callback<
+             LocalQFLOBackend<DIM, Q1D>>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeAssembleDiagonal<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::DiagonalKernelType
+       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeAssembleDiagonalLO::Fallback(int dim, int q1d)
+{
+   using diag_t =
+      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeAssembleDiagonalLO =
+      typename diag_t::DerivativeAssembleDiagonalLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeAssembleDiagonal<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::DiagonalKernelType
+       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeAssembleDiagonalHO::Kernel()
+{
+   using diag_t =
+      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return diag_t::template derivative_assemble_diagonal_callback<
+             LocalQFHOBackend<DIM>,
+             Q1D>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeAssembleDiagonal<derivative_id,
+       qfunc_t,
+       inputs_t,
+       outputs_t>::DiagonalKernelType
+       DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
+       DerivativeAssembleDiagonalHO::Fallback(int dim, int q1d)
+{
+   using diag_t =
+      DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeAssembleDiagonalHO =
+      typename diag_t::DerivativeAssembleDiagonalHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/derivative_setup.hpp

@@ -0,0 +1,600 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+
+#include "kernels.hpp"
+#include "util.hpp"
+
+#include <array>
+
+namespace mfem::future::LocalQFImpl
+{
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+class DerivativeSetup
+{
+   static constexpr auto inout_tuple =
+   merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
+   static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
+   static constexpr size_t nfields =
+      count_unique_field_ids(filtered_inout_tuple);
+
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using args_tuple_t = decay_tuple<qf_param_ts>;
+
+   static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
+                 "LocalQF: q-function arity must match inputs + outputs");
+
+   qfunc_t qfunc;
+   const inputs_t inputs;
+   const outputs_t outputs;
+   const IntegratorContext ctx;
+   Vector &qp_cache;
+   const std::vector<const DofToQuad *> dtqs;
+   // inputs: dtq, idx, B, G, d1d, q1d, vdim
+   const std::array<DofToQuadMap, n_inputs> input_dtq;
+   const std::array<size_t, n_inputs> input_idx;
+   const std::array<const real_t *, n_inputs> input_B, input_G;
+   const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
+   // Jacobian cache metadata
+   const std::array<bool, n_inputs> input_is_dependent;
+   const std::array<int, n_inputs> input_size_on_qp;
+   const std::array<int, n_outputs> out_vdim;
+   const std::array<int, n_outputs> out_op_dim;
+   const std::array<int, n_outputs> out_offsets;
+   const int output_size_on_qp;
+   const int trial_vdim;
+   const int total_trial_op_dim;
+   const int residual_size_on_qp;
+   // other constants
+   const int dim, ne, nq, q1d;
+
+public:
+   //////////////////////////////////////////////////////////////////
+   DerivativeSetup() = delete;
+
+   DerivativeSetup(IntegratorContext ctx,
+                   qfunc_t qfunc,
+                   inputs_t inputs,
+                   outputs_t outputs,
+                   Vector &qp_cache):
+      qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
+      qp_cache(qp_cache), dtqs(make_dtqs(ctx)),
+      input_dtq(create_dtq_maps<Entity::Element>(
+                   inputs,
+                   dtqs,
+                   create_union_field_map_for_dtq(ctx, inputs),
+                   ctx.unionfds,
+                   ctx.ir)),
+      input_idx(create_input_vector_map(ctx, inputs)),
+      input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
+      input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
+      input_vdim(get_vdim(inputs)),
+      input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
+      input_size_on_qp(
+         get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
+                           out_vdim(get_vdim(outputs)), out_op_dim(compute_out_op_dim(outputs)),
+                           out_offsets(compute_out_offsets(out_vdim, out_op_dim)), output_size_on_qp(
+                              [&]
+   {
+      int s = 0;
+      for_constexpr<n_outputs>([&](auto o)
+      { s += get<o>(outputs).size_on_qp; });
+      return s;
+   }()),
+   trial_vdim(compute_trial_vdim(inputs, derivative_id)),
+   total_trial_op_dim(compute_total_trial_op_dim(
+                         inputs, input_is_dependent, input_size_on_qp)),
+   residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
+   dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
+   q1d(tensor_1d_size(nq, dim))
+   {
+      MFEM_ASSERT(ctx.unionfds.size() == nfields,
+                  "LocalQFBackend: unionfds size mismatch");
+
+      qp_cache.SetSize(ne * nq * residual_size_on_qp);
+      qp_cache.UseDevice(true);
+   }
+
+   //////////////////////////////////////////////////////////////////
+   void operator()(const std::vector<Vector *> &xe)
+   {
+      if (ctx.attr.Size() == 0) { return; }
+
+      // Quadrature index is fastest-varying so that adjacent threads (one per
+      // quadrature point) touch adjacent addresses.
+      auto cache_tensor = DeviceTensor<3, real_t>(
+                             qp_cache.ReadWrite(), nq, residual_size_on_qp, ne);
+
+      if (q1d <= LocalQFLOBackendMQ1())
+      {
+         run_kernels<DerivativeSetupLO>(xe, cache_tensor);
+      }
+      else if (q1d <= LocalQFHOBackendMQ1())
+      {
+         run_kernels<DerivativeSetupHO>(xe, cache_tensor);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
+      }
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename Backend>
+   void run_kernels(const std::vector<Vector *> &xe,
+                    DeviceTensor<3, real_t> &cache_tensor)
+   {
+      Backend::Run(dim,
+                   q1d,
+                   ctx,
+                   qfunc,
+                   // inputs
+                   input_idx,
+                   input_B,
+                   input_G,
+                   input_vdim,
+                   input_d1d,
+                   input_q1d,
+                   input_size_on_qp,
+                   input_is_dependent,
+                   // outputs / cache metadata
+                   out_vdim,
+                   out_op_dim,
+                   out_offsets,
+                   trial_vdim,
+                   total_trial_op_dim,
+                   residual_size_on_qp,
+                   // vectors
+                   xe,
+                   cache_tensor,
+                   // fallback arguments
+                   dim,
+                   q1d);
+   }
+
+   //////////////////////////////////////////////////////////////////
+   /// Zeroes the q-function *output* slots of an argument tuple.
+   ///
+   /// The argument tuple is built once per quadrature point and reused for
+   /// every trial seed. The q-function writes through its output parameters,
+   /// so those slots have to be restored before each call to give every seed
+   /// the same starting state a freshly value-initialized tuple would.
+   static MFEM_HOST_DEVICE inline void reset_output_args(args_tuple_t &args)
+   {
+      for_constexpr<n_outputs>([&](auto oc)
+      {
+         constexpr size_t ao = n_inputs + oc.value;
+         auto &oarg = get<ao>(args);
+         oarg = std::remove_reference_t<decltype(oarg)> {};
+      });
+   }
+
+   //////////////////////////////////////////////////////////////////
+   template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
+   static void
+   derivative_setup_callback(const IntegratorContext &ctx,
+                             qfunc_t &qfunc,
+                             // inputs: idx, B, G, vdim, d1d, q1d
+                             const std::array<size_t, n_inputs> &in_idx,
+                             const std::array<const real_t *, n_inputs> in_B,
+                             const std::array<const real_t *, n_inputs> in_G,
+                             const std::array<int, n_inputs> &in_vdim,
+                             const std::array<int, n_inputs> &in_d1d,
+                             const std::array<int, n_inputs> &in_q1d,
+                             const std::array<int, n_inputs> &in_size_on_qp,
+                             const std::array<bool, n_inputs> &input_dep,
+                             // outputs / cache metadata
+                             const std::array<int, n_outputs> &out_vdim,
+                             const std::array<int, n_outputs> &out_op_dim,
+                             const std::array<int, n_outputs> &out_offsets,
+                             const int trial_vdim,
+                             const int total_trial_op_dim,
+                             const int residual_size_on_qp,
+                             const std::vector<Vector *> &xe,
+                             DeviceTensor<3, real_t> &cache_tensor,
+                             // fallback arguments
+                             const int dim,
+                             const int q1d)
+   {
+      MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
+      if (ctx.attr.Size() == 0) { return; }
+
+      static constexpr auto B2D = backend_t::DIM == 2;
+      static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
+      static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
+
+      const int ne = ctx.nentities;
+      MFEM_CONTRACT_VAR(residual_size_on_qp);
+
+      constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
+
+      // --------------------------------------------------
+      // INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
+      // --------------------------------------------------
+      std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
+      for_constexpr<n_inputs>([&](auto ic)
+      {
+         constexpr size_t i = ic.value;
+         const size_t k = in_idx[i];
+         const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
+         using FOP = tuple_element_t<i, inputs_t>;
+         if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
+         }
+         else if constexpr (is_identity_fop_v<FOP>)
+         {
+            MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
+            in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
+         }
+         else if constexpr (is_weight_fop_v<FOP>)
+         {
+            MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
+                        "tensor-product IR expected");
+            in_XE[i] = Reshape(
+                          ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      });
+
+      const auto d_attr = ctx.attr.Read();
+      const bool has_attr = ctx.attr.Size() > 0;
+      const auto d_elem_attr = ctx.elem_attr->Read();
+
+      dfem::forall<MTPB>(
+         [=] MFEM_HOST_DEVICE(const int e, void *)
+      {
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+
+         // -----------------------------------------------
+         // Inputs argument registers + shared memory
+         // -----------------------------------------------
+         input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
+         MFEM_SHARED typename backend_t::Shared smem;
+
+         // -----------------------------------------------
+         // Load primal inputs (rargs) once for this element
+         // -----------------------------------------------
+         for_constexpr<n_inputs>([&](auto ic)
+         {
+            constexpr size_t i = ic.value;
+            const auto &XE = in_XE[i];
+            const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
+            const real_t *B = in_B[i], *G = in_G[i];
+            auto &rarg = get<i>(rargs);
+            using XE_t = decltype(XE);
+            using rarg_t = decltype(rarg);
+            using FOP = tuple_element_t<i, inputs_t>;
+            if constexpr (is_value_fop<FOP>::value)
+            {
+               backend_t::template LoadValue<rarg_t, XE_t>(
+                  smem, e, d, q, Q1D, B, XE, rarg);
+            }
+            else if constexpr (is_gradient_fop_v<FOP>)
+            {
+               constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
+               using FieldParamT =
+                  typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+               backend_t::template LoadGradient<RNK, rarg_t, XE_t, FieldParamT>(
+                  smem, e, d, q, q1d, B, G, XE, rarg);
+            }
+            else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
+            {
+               // qp values are read directly from in_XE / IR
+            }
+            else
+            {
+               static_assert(false, "Unsupported");
+            }
+         });
+         MFEM_SYNC_THREAD;
+
+         // -----------------------------------------------
+         // Build the primal arguments once per quadrature point, then, for
+         // each trial seed (j, dependent input s, m), differentiate the
+         // q-function with a unit tangent and store the result row in the
+         // cache. Nothing in the primal pull depends on the seed, so the
+         // thread loop is the outermost one here. The seed loops only touch
+         // per-thread state, hence no barrier inside them.
+         // Warning: no 'DIRECT' on the 'Z' direction,
+         // as one backend may need to iterate over it.
+         // -----------------------------------------------
+         MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+               {
+                  const int q = qx + q1d * (qy + q1d * qz);
+#ifdef MFEM_USE_ENZYME
+                  args_tuple_t primal_args {};
+                  for_constexpr<n_inputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value;
+                     auto &parg = get<i>(primal_args);
+                     const auto &XE = in_XE[i];
+                     using FOP = tuple_element_t<i, inputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
+                     }
+                     else if constexpr (is_weight_fop_v<FOP>)
+                     {
+                        parg = XE(qx, qy, qz, 0, 0);
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        parg = backend_t::template qp_pull<ARG>(
+                           get<i>(rargs), qx, qy, qz);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+
+                  for (int j = 0; j < trial_vdim; j++)
+                  {
+                     int m_offset = 0;
+                     for_constexpr<n_inputs>([&](auto sc)
+                     {
+                        constexpr size_t s = sc.value;
+                        if (!input_dep[s]) { return; }
+
+                        const int vdim_s = in_vdim[s];
+                        const int op_dim_s = in_size_on_qp[s] / vdim_s;
+
+                        for (int m = 0; m < op_dim_s; m++)
+                        {
+                           const int col_m = m + m_offset;
+
+                           // Enzyme writes through the output slots of the
+                           // primal tuple, so they are reset per seed.
+                           reset_output_args(primal_args);
+
+                           args_tuple_t shadow_args {};
+                           qf_set_value_at(get<s>(shadow_args), j, m, 1.0);
+
+                           call_enzyme_fwddiff(qfunc, primal_args, shadow_args);
+
+                           for_constexpr<n_outputs>([&](auto oc)
+                           {
+                              constexpr size_t o = oc.value, ao = n_inputs + o;
+                              const auto &tangent = get<ao>(shadow_args);
+                              const int tv = out_vdim[o], to = out_op_dim[o];
+                              for (int i = 0; i < tv; i++)
+                              {
+                                 for (int k = 0; k < to; k++)
+                                 {
+                                    const int row = out_offsets[o] + i * to + k;
+                                    const int cache_idx =
+                                       row * trial_vdim * total_trial_op_dim +
+                                       j * total_trial_op_dim + col_m;
+                                    cache_tensor(q, cache_idx, e) =
+                                       qf_value_at(tangent, i, k);
+                                 }
+                              }
+                           });
+                        }
+                        m_offset += op_dim_s;
+                     });
+                  }
+#else  // MFEM_USE_ENZYME
+                  args_tuple_t qargs {};
+                  for_constexpr<n_inputs>([&](auto ic)
+                  {
+                     constexpr size_t i = ic.value;
+                     auto &qarg = get<i>(qargs);
+                     const auto &XE = in_XE[i];
+                     using FOP = tuple_element_t<i, inputs_t>;
+                     using ARG =
+                        typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
+                     if constexpr (is_identity_fop_v<FOP>)
+                     {
+                        using DT =
+                           typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
+                        if constexpr (qf_param_uses_dual_v<DT>)
+                        {
+                           qarg = backend_t::template identity_qp_pull_dual<DT>(
+                                     false, XE, XE, qx, qy, qz, e);
+                        }
+                        else
+                        {
+                           qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
+                        }
+                     }
+                     else if constexpr (is_weight_fop_v<FOP>)
+                     {
+                        qarg = XE(qx, qy, qz, 0, 0);
+                     }
+                     else if constexpr (is_value_fop_v<FOP> ||
+                                        is_gradient_fop_v<FOP>)
+                     {
+                        qarg = backend_t::template qp_pull<ARG>(
+                           get<i>(rargs), qx, qy, qz);
+                     }
+                     else
+                     {
+                        static_assert(false, "Unsupported");
+                     }
+                  });
+
+                  for (int j = 0; j < trial_vdim; j++)
+                  {
+                     int m_offset = 0;
+                     for_constexpr<n_inputs>([&](auto sc)
+                     {
+                        constexpr size_t s = sc.value;
+                        if (!input_dep[s]) { return; }
+
+                        const int vdim_s = in_vdim[s];
+                        const int op_dim_s = in_size_on_qp[s] / vdim_s;
+
+                        for (int m = 0; m < op_dim_s; m++)
+                        {
+                           const int col_m = m + m_offset;
+
+                           // The q-function writes through the output slots,
+                           // so they are reset per seed.
+                           reset_output_args(qargs);
+
+                           qf_set_gradient_at(get<s>(qargs), j, m, 1.0);
+
+                           call_qfunc_no_move(qfunc, qargs);
+
+                           for_constexpr<n_outputs>([&](auto oc)
+                           {
+                              constexpr size_t o = oc.value, ao = n_inputs + o;
+                              const auto &tangent = get<ao>(qargs);
+                              const int tv = out_vdim[o], to = out_op_dim[o];
+                              for (int i = 0; i < tv; i++)
+                              {
+                                 for (int k = 0; k < to; k++)
+                                 {
+                                    const int row = out_offsets[o] + i * to + k;
+                                    const int cache_idx =
+                                       row * trial_vdim * total_trial_op_dim +
+                                       j * total_trial_op_dim + col_m;
+                                    cache_tensor(q, cache_idx, e) =
+                                       qf_gradient_at(tangent, i, k);
+                                 }
+                              }
+                           });
+
+                           // Clear the seed so the next direction starts from
+                           // the pristine (zero-tangent) primal state.
+                           qf_set_gradient_at(get<s>(qargs), j, m, 0.0);
+                        }
+                        m_offset += op_dim_s;
+                     });
+                  }
+#endif // MFEM_USE_ENZYME
+               }
+            }
+         }
+      },
+      ne,
+      backend_t::thread_blocks(
+         compute_kernel_thread_1d<inputs_t>(q1d, in_d1d)),
+      0,
+      nullptr);
+   }
+
+   using SetupKernelType =
+      decltype(&DerivativeSetup::derivative_setup_callback<>);
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupLO,
+                                     SetupKernelType,
+                                     (int, int) );
+   MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupHO,
+                                     SetupKernelType,
+                                     (int, int) );
+};
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+SetupKernelType
+DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeSetupLO::Kernel()
+{
+   static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
+   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return setup_t::template derivative_setup_callback<
+             LocalQFLOBackend<DIM, Q1D>>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+SetupKernelType
+DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeSetupLO::Fallback(int dim, int q1d)
+{
+   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeSetupLO = typename setup_t::DerivativeSetupLO;
+   if (dim == 2)
+   {
+      return DispatchLOKernelByQ1D<DerivativeSetupLO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchLOKernelByQ1D<DerivativeSetupLO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+template<int DIM, int Q1D>
+inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+SetupKernelType
+DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeSetupHO::Kernel()
+{
+   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   return setup_t::template derivative_setup_callback<LocalQFHOBackend<DIM>,
+                                                      Q1D>;
+}
+
+template<int derivative_id,
+         typename qfunc_t,
+         typename inputs_t,
+         typename outputs_t>
+inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+SetupKernelType
+DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
+DerivativeSetupHO::Fallback(int dim, int q1d)
+{
+   using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
+   using DerivativeSetupHO = typename setup_t::DerivativeSetupHO;
+   if (dim == 2)
+   {
+      return DispatchHOKernelByQ1D<DerivativeSetupHO, 2>(q1d);
+   }
+   else if (dim == 3)
+   {
+      return DispatchHOKernelByQ1D<DerivativeSetupHO, 3>(q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+      return nullptr;
+   }
+}
+
+} // namespace mfem::future::LocalQFImpl

fem/dfem/backends/local_qf/fwddiff_transformer.hpp

@@ -0,0 +1,274 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../util.hpp"
+
+#ifdef MFEM_USE_ENZYME
+
+namespace mfem::future
+{
+
+template <typename T> struct function_traits;
+
+template <typename R, typename C, typename... Args>
+struct function_traits<R (C::*)(Args...) const>
+{
+   using primal_return_type = R;
+   using args_tuple = std::tuple<Args...>;
+   static constexpr size_t arity = sizeof...(Args);
+};
+
+// Component count and writable counterpart of a per-point argument, which
+// is either a tensor or a plain scalar.
+template <typename Arg> struct qp_traits
+{
+   static_assert(std::is_arithmetic_v<Arg>,
+                 "per-point arguments must be tensors or scalars");
+   using view_type = Arg;
+   static constexpr int components = 1;
+};
+
+template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
+{
+   using view_type = tensor<std::remove_const_t<T>, Sizes...>;
+   static constexpr int components = (Sizes * ... * 1);
+};
+
+// Generic FwdDiff: computes the full gradient of a pointwise qfunction at a
+// single quadrature point.
+//
+// active_input is the index of the argument to differentiate with respect
+// to; active_output is the index of the (scalar) output argument whose
+// derivative is taken.
+//
+// operator()(args...) takes the qfunction's arguments, except that the
+// active output position receives the *gradient*, shaped like the active
+// input (d(output)/d(input component)).
+//
+// Per input component d, one enzyme fwddiff call with the one-hot seed e_d
+// in the input tangent yields gradient entry d, i.e. grad_components enzyme
+// calls per point. The output tangent is not pre-zeroed, so the qfunction
+// must fully write its output (tangent stores overwrite).
+//
+// operator() is MFEM_HOST_DEVICE, stateless and allocation-free, so it is
+// callable inside a GPU kernel: seed, primal scratch and tangent all live
+// on the stack.
+template <typename Func, size_t active_input, size_t active_output>
+struct FwdDiff
+{
+   using traits = function_traits<decltype(&Func::operator())>;
+   using args_tuple = typename traits::args_tuple;
+   static constexpr size_t arity = traits::arity;
+
+   static_assert(std::is_void_v<typename traits::primal_return_type>,
+                 "FwdDiff only supports primal functions with void return type");
+   static_assert(active_input < arity && active_output < arity,
+                 "active argument indices must be within the function arity");
+   static_assert(active_input != active_output,
+                 "active input and output must be different arguments");
+
+   using input_type =
+      std::decay_t<std::tuple_element_t<active_input, args_tuple>>;
+   using output_type =
+      std::decay_t<std::tuple_element_t<active_output, args_tuple>>;
+
+   using grad_type = typename qp_traits<input_type>::view_type;
+   using output_view = typename qp_traits<output_type>::view_type;
+
+   static constexpr int grad_components = qp_traits<input_type>::components;
+
+   static_assert(qp_traits<output_type>::components == 1,
+                 "gradient output requires a scalar output");
+
+   // Signature of the differentiated qfunction: the primal arguments, with
+   // the Active output slot receiving the (writable) gradient instead.
+   // Exposed through create_function_signature below so that
+   // DifferentiableOperator can deduce the parameter types, which it cannot
+   // do from the variadic operator().
+   template <size_t I>
+   using qf_arg_t = std::conditional_t<I == active_output, grad_type &,
+         std::tuple_element_t<I, args_tuple>>;
+
+   template <size_t... Is>
+   static FunctionSignature<void(qf_arg_t<Is>...)>
+   signature_impl(std::index_sequence<Is...>);
+
+   using signature =
+      decltype(signature_impl(std::make_index_sequence<arity> {}));
+
+   // d-th scalar of a per-point argument in flat row-major order, regardless
+   // of rank, built on the native operator[] (tensor has no flat-index
+   // accessor; flatten() returns a copy, so it cannot be written through).
+   MFEM_HOST_DEVICE static double &component(double &t, int) { return t; }
+
+   template <typename T, int n0, int... n>
+   MFEM_HOST_DEVICE static T &component(tensor<T, n0, n...> &t, int d)
+   {
+      if constexpr (sizeof...(n) == 0)
+      {
+         return t[d];
+      }
+      else
+      {
+         constexpr int stride = (n * ... * 1);
+         return component(t[d / stride], d % stride);
+      }
+   }
+
+   // Plain function with the qfunction's exact (reference) signature, so it
+   // can be handed to Enzyme as a function pointer; references are pointers
+   // to Enzyme, so primal arguments and shadows are passed by address below.
+   template <size_t... Is>
+   MFEM_HOST_DEVICE static void
+   static_call(std::tuple_element_t<Is, args_tuple>... args)
+   {
+      Func{}(args...);
+   }
+
+   template <size_t... Is>
+   static constexpr auto fn_ptr(std::index_sequence<Is...>)
+   {
+      return &static_call<Is...>;
+   }
+   static constexpr auto fn = fn_ptr(std::make_index_sequence<arity> {});
+
+   // Writable, zero-initialized scratch with the shape of argument I, used
+   // as its enzyme shadow.
+   template <size_t I>
+   using shadow_t = typename qp_traits<
+                    std::decay_t<std::tuple_element_t<I, args_tuple>>>::view_type;
+
+   template <size_t... Is>
+   MFEM_HOST_DEVICE static auto make_shadows(std::index_sequence<Is...>)
+   {
+      return mfem::future::make_tuple(shadow_t<Is> {}...);
+   }
+
+   template <typename Shadows, size_t... Is>
+   MFEM_HOST_DEVICE static auto make_shadow_ptrs(Shadows &shadows,
+                                                 std::index_sequence<Is...>)
+   {
+      return mfem::future::make_tuple(&mfem::future::get<int(Is)>(shadows)...);
+   }
+
+   // The caller's argument pointers, except the active output slot, which
+   // points to scalar scratch: the caller's slot holds the gradient, while
+   // the primal function writes its scalar output there.
+   template <size_t I, typename Ptrs>
+   MFEM_HOST_DEVICE static auto primal_ptr(Ptrs &ptrs, output_view &primal)
+   {
+      if constexpr (I == active_output) { return &primal; }
+      else { return mfem::future::get<int(I)>(ptrs); }
+   }
+
+   template <typename Ptrs, size_t... Is>
+   MFEM_HOST_DEVICE static auto make_primal_ptrs(Ptrs &ptrs,
+                                                 output_view &primal,
+                                                 std::index_sequence<Is...>)
+   {
+      return mfem::future::make_tuple(primal_ptr<Is>(ptrs, primal)...);
+   }
+
+   // Single flat enzyme call. The activity markers must appear directly in
+   // the __enzyme_fwddiff argument list — Enzyme cannot trace markers that
+   // were forwarded through function parameters (e.g. at -O0, where nothing
+   // is inlined). Every argument is therefore enzyme_dup'd in one sticky
+   // group; Const arguments simply carry a zero tangent, which is equivalent
+   // to marking them enzyme_const.
+   //
+   // always_inline is load-bearing: when FwdDiff is itself differentiated
+   // (second derivatives, forward-over-forward), Enzyme only recognizes this
+   // nested __enzyme_fwddiff call if it sits at most one call level below
+   // the function handed to the outer __enzyme_fwddiff. Without inlining
+   // (-O0) it sits two levels down (wrapper -> operator() -> call_enzyme)
+   // and the outer pass treats it as a regular call: the activity marker
+   // ints then receive undef shadows, which misaligns the argument pairing
+   // (observed as "cannot compute with global variable that doesn't have
+   // marked shadow global" at compile time or null-shadow segfaults at
+   // runtime). The always-inliner runs even at -O0, hoisting this call into
+   // operator() where the nested handling applies.
+   template <typename PrimalPtrs, typename ShadowPtrs, size_t... Is>
+   __attribute__((always_inline))
+   MFEM_HOST_DEVICE static void call_enzyme(PrimalPtrs &primal_ptrs,
+                                            ShadowPtrs &shadow_ptrs,
+                                            std::index_sequence<Is...>)
+   {
+      __enzyme_fwddiff<void>(fn, enzyme_dup,
+                             mfem::future::get<int(Is)>(primal_ptrs)...,
+                             enzyme_interleave,
+                             mfem::future::get<int(Is)>(shadow_ptrs)...,
+                             enzyme_runtime_activity);
+   }
+
+   template <typename... Args>
+   MFEM_HOST_DEVICE void operator()(Args &&...args) const
+   {
+      static_assert(sizeof...(Args) == arity, "Wrong number of arguments");
+      auto ptrs = mfem::future::make_tuple(&args...);
+
+      auto &grad = *mfem::future::get<int(active_output)>(ptrs);
+      static_assert(std::is_same_v<std::decay_t<decltype(grad)>, grad_type>,
+                    "gradient argument must be shaped like the Active input "
+                    "(with writable scalars)");
+
+      constexpr auto seq = std::make_index_sequence<arity> {};
+
+      output_view primal{};
+      auto primal_ptrs = make_primal_ptrs(ptrs, primal, seq);
+
+      auto shadows = make_shadows(seq);
+      auto shadow_ptrs = make_shadow_ptrs(shadows, seq);
+      auto &seed = mfem::future::get<int(active_input)>(shadows);
+      auto &tangent = mfem::future::get<int(active_output)>(shadows);
+
+      // One enzyme call per input component d: seed e_d in the input tangent
+      // and read gradient entry d off the output tangent.
+      for (int d = 0; d < grad_components; d++)
+      {
+         component(seed, d) = 1.0;
+         call_enzyme(primal_ptrs, shadow_ptrs, seq);
+         component(grad, d) = component(tangent, 0);
+         component(seed, d) = 0.0;
+      }
+   }
+
+   static void print() { print_impl(std::make_index_sequence<arity> {}); }
+
+   template <size_t... Is> static void print_impl(std::index_sequence<Is...>)
+   {
+      mfem::out << "for d in [0, " << grad_components
+                << "): __enzyme_fwddiff<void>(fptr, enzyme_dup";
+      ((mfem::out << ", "
+        << get_type_name<std::tuple_element_t<Is, args_tuple>>()),
+       ...);
+      mfem::out << ", enzyme_interleave";
+      (([&]
+      {
+         if constexpr (Is == active_input) { mfem::out << ", e_d seed"; }
+         else if constexpr (Is == active_output) { mfem::out << ", tangent out"; }
+         else { mfem::out << ", zero tangent"; }
+      }()),
+      ...);
+      mfem::out << ")\n";
+   }
+};
+
+template <typename Func, size_t active_input, size_t active_output>
+struct create_function_signature<FwdDiff<Func, active_input, active_output>>
+{
+   using type =
+      typename FwdDiff<Func, active_input, active_output>::signature;
+};
+
+} // namespace mfem::future
+
+#endif // MFEM_USE_ENZYME

fem/dfem/backends/local_qf/kernels.cpp

@@ -0,0 +1,83 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// Explicit instantiation of the local Q-function backend device code
+
+#include "../../../../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "kernels.hpp"
+
+namespace mfem::future
+{
+
+// ────────────────────────────────────────────────────────────────────────────
+// Low-order backends instantiations for (DIM, Q1D)
+// ────────────────────────────────────────────────────────────────────────────
+template struct lo_ker_backend<2, 2>;
+template struct lo_ker_backend<2, 3>;
+template struct lo_ker_backend<2, 4>;
+template struct lo_ker_backend<2, 5>;
+template struct lo_ker_backend<2, 6>;
+template struct lo_ker_backend<2, 7>;
+template struct lo_ker_backend<2, 8>;
+
+template struct lo_ker_backend<3, 2>;
+template struct lo_ker_backend<3, 3>;
+template struct lo_ker_backend<3, 4>;
+template struct lo_ker_backend<3, 5>;
+template struct lo_ker_backend<3, 6>;
+template struct lo_ker_backend<3, 7>;
+template struct lo_ker_backend<3, 8>;
+
+template struct LocalQFLOBackend<2, 2>;
+template struct LocalQFLOBackend<2, 3>;
+template struct LocalQFLOBackend<2, 4>;
+template struct LocalQFLOBackend<2, 5>;
+template struct LocalQFLOBackend<2, 6>;
+template struct LocalQFLOBackend<2, 7>;
+template struct LocalQFLOBackend<2, 8>;
+
+template struct LocalQFLOBackend<3, 2>;
+template struct LocalQFLOBackend<3, 3>;
+template struct LocalQFLOBackend<3, 4>;
+template struct LocalQFLOBackend<3, 5>;
+template struct LocalQFLOBackend<3, 6>;
+template struct LocalQFLOBackend<3, 7>;
+template struct LocalQFLOBackend<3, 8>;
+
+// ────────────────────────────────────────────────────────────────────────────
+// High-order backends instantiations for (DIM, Q1D)
+// ────────────────────────────────────────────────────────────────────────────
+template struct ho_ker_backend<2, 8>;
+template struct ho_ker_backend<2, 10>;
+template struct ho_ker_backend<2, 12>;
+template struct ho_ker_backend<2, 16>;
+
+template struct ho_ker_backend<3, 8>;
+template struct ho_ker_backend<3, 10>;
+template struct ho_ker_backend<3, 12>;
+template struct ho_ker_backend<3, 16>;
+
+template struct LocalQFHOBackend<2, 8>;
+template struct LocalQFHOBackend<2, 10>;
+template struct LocalQFHOBackend<2, 12>;
+template struct LocalQFHOBackend<2, 16>;
+
+template struct LocalQFHOBackend<3, 8>;
+template struct LocalQFHOBackend<3, 10>;
+template struct LocalQFHOBackend<3, 12>;
+template struct LocalQFHOBackend<3, 16>;
+
+} // namespace mfem::future
+
+#endif // MFEM_USE_MPI

fem/dfem/backends/local_qf/kernels.hpp

@@ -0,0 +1,77 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "kernels_lo.hpp" // IWYU pragma: export
+#include "kernels_ho.hpp" // IWYU pragma: export
+
+namespace mfem::future
+{
+
+// ────────────────────────────────────────────────────────────────────────────
+// Low-order backends instantiations for (DIM, Q1D)
+// ────────────────────────────────────────────────────────────────────────────
+extern template struct lo_ker_backend<2, 2>;
+extern template struct lo_ker_backend<2, 3>;
+extern template struct lo_ker_backend<2, 4>;
+extern template struct lo_ker_backend<2, 5>;
+extern template struct lo_ker_backend<2, 6>;
+extern template struct lo_ker_backend<2, 7>;
+extern template struct lo_ker_backend<2, 8>;
+
+extern template struct lo_ker_backend<3, 2>;
+extern template struct lo_ker_backend<3, 3>;
+extern template struct lo_ker_backend<3, 4>;
+extern template struct lo_ker_backend<3, 5>;
+extern template struct lo_ker_backend<3, 6>;
+extern template struct lo_ker_backend<3, 7>;
+extern template struct lo_ker_backend<3, 8>;
+
+extern template struct LocalQFLOBackend<2, 2>;
+extern template struct LocalQFLOBackend<2, 3>;
+extern template struct LocalQFLOBackend<2, 4>;
+extern template struct LocalQFLOBackend<2, 5>;
+extern template struct LocalQFLOBackend<2, 6>;
+extern template struct LocalQFLOBackend<2, 7>;
+extern template struct LocalQFLOBackend<2, 8>;
+
+extern template struct LocalQFLOBackend<3, 2>;
+extern template struct LocalQFLOBackend<3, 3>;
+extern template struct LocalQFLOBackend<3, 4>;
+extern template struct LocalQFLOBackend<3, 5>;
+extern template struct LocalQFLOBackend<3, 6>;
+extern template struct LocalQFLOBackend<3, 7>;
+extern template struct LocalQFLOBackend<3, 8>;
+
+// ────────────────────────────────────────────────────────────────────────────
+// High-order backends instantiations for (DIM, Q1D)
+// ────────────────────────────────────────────────────────────────────────────
+extern template struct ho_ker_backend<2, 8>;
+extern template struct ho_ker_backend<2, 10>;
+extern template struct ho_ker_backend<2, 12>;
+extern template struct ho_ker_backend<2, 16>;
+
+extern template struct ho_ker_backend<3, 8>;
+extern template struct ho_ker_backend<3, 10>;
+extern template struct ho_ker_backend<3, 12>;
+extern template struct ho_ker_backend<3, 16>;
+
+extern template struct LocalQFHOBackend<2, 8>;
+extern template struct LocalQFHOBackend<2, 10>;
+extern template struct LocalQFHOBackend<2, 12>;
+extern template struct LocalQFHOBackend<2, 16>;
+
+extern template struct LocalQFHOBackend<3, 8>;
+extern template struct LocalQFHOBackend<3, 10>;
+extern template struct LocalQFHOBackend<3, 12>;
+extern template struct LocalQFHOBackend<3, 16>;
+
+} // namespace mfem::future

fem/dfem/backends/local_qf/kernels_ho.hpp

@@ -0,0 +1,803 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../../kernels.hpp"
+namespace ker = mfem::kernels::internal;
+
+#include "../../util.hpp" // for ThreadBlocks
+#include "util.hpp"
+
+namespace mfem::future
+{
+
+// ────────────────────────────────────────────────────────────────────────────
+inline constexpr int LocalQFHOBackendMQ1() { return 16; }
+
+// ────────────────────────────────────────────────────────────────────────────
+/// Register type for one HO q-function parameter
+template<typename KerOps, typename T, int rank = qf_param_shape<T>::rank>
+struct ho_qreg;
+
+template<typename KerOps, typename T>
+struct ho_qreg<KerOps, T, 0>
+{
+   using type = typename KerOps::template val_reg_t<1>;
+};
+
+template<typename KerOps, typename T>
+struct ho_qreg<KerOps, T, 1>
+{
+   static constexpr int e0 = qf_param_shape<T>::extents[0];
+   using type = typename KerOps::template val_reg_t<e0>;
+};
+
+template<typename KerOps, typename T>
+struct ho_qreg<KerOps, T, 2>
+{
+   static constexpr int VDIM = qf_param_shape<T>::extents[0];
+   static constexpr int SDIM = qf_param_shape<T>::extents[1];
+   using type = typename KerOps::template del_reg_t<VDIM, SDIM>;
+};
+
+template<typename KerOps, typename T>
+using ho_qreg_t = typename ho_qreg<KerOps, T>::type;
+
+// ────────────────────────────────────────────────────────────────────────────
+namespace hok
+{
+
+/// Load one quadrature-point value
+template<int DIM, typename T, typename Reg>
+MFEM_HOST_DEVICE inline auto load_at(Reg &reg, int qx, int qy, int qz)
+{
+   static_assert(DIM == 2 || DIM == 3);
+   constexpr int RNK = qf_param_shape<T>::rank;
+   if constexpr (DIM == 2)
+   {
+      MFEM_CONTRACT_VAR(qz);
+      if constexpr (RNK == 0) { return T{ reg(0, qy, qx) }; }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         T t{};
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qy, qx); }
+         return t;
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         constexpr int e1 = qf_param_shape<T>::extents[1];
+         T t;
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qy, qx); }
+         }
+         return t;
+      }
+   }
+   else
+   {
+      if constexpr (RNK == 0) { return T{ reg(0, qz, qy, qx) }; }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         T t{};
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qz, qy, qx); }
+         return t;
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         constexpr int e1 = qf_param_shape<T>::extents[1];
+         T t;
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qz, qy, qx); }
+         }
+         return t;
+      }
+   }
+}
+
+template<bool tangent, typename U>
+MFEM_HOST_DEVICE inline auto qp_store(const U &v)
+{
+   if constexpr (tangent) { return qf_store_gradient(v); }
+   else
+   {
+      return qf_store_value(v);
+   }
+}
+
+// Store primal value or dual tangent at one quadrature point
+template<int DIM, typename T, typename Reg, bool tangent>
+MFEM_HOST_DEVICE inline void
+store_at(Reg &reg, int qx, int qy, int qz, const T &out)
+{
+   static_assert(DIM == 2 || DIM == 3);
+   constexpr int RNK = qf_param_shape<T>::rank;
+   if constexpr (DIM == 2)
+   {
+      MFEM_CONTRACT_VAR(qz);
+      if constexpr (RNK == 0) { reg(0, qy, qx) = qp_store<tangent>(out); }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd)
+         {
+            reg(dd, qy, qx) = qp_store<tangent>(out(dd));
+         }
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         constexpr int e1 = qf_param_shape<T>::extents[1];
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j)
+            {
+               reg(i, j, qy, qx) = qp_store<tangent>(out(i, j));
+            }
+         }
+      }
+   }
+   else
+   {
+      if constexpr (RNK == 0) { reg(0, qz, qy, qx) = qp_store<tangent>(out); }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd)
+         {
+            reg(dd, qz, qy, qx) = qp_store<tangent>(out(dd));
+         }
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         constexpr int e1 = qf_param_shape<T>::extents[1];
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j)
+            {
+               reg(i, j, qz, qy, qx) = qp_store<tangent>(out(i, j));
+            }
+         }
+      }
+   }
+}
+
+// Pull primal/tangent pair into a dual q-function argument
+template<int DIM, typename T, typename Reg>
+MFEM_HOST_DEVICE inline auto
+pull_directional(Reg &preg, Reg &sreg, int qx, int qy, int qz, bool dependent)
+{
+   if constexpr (!qf_param_uses_dual_v<T>)
+   {
+      return load_at<DIM, T>(preg, qx, qy, qz);
+   }
+   else
+   {
+      if (!dependent) { return load_at<DIM, T>(preg, qx, qy, qz); }
+      constexpr int RNK = qf_param_shape<T>::rank;
+      if constexpr (DIM == 2)
+      {
+         MFEM_CONTRACT_VAR(qz);
+         if constexpr (RNK == 0)
+         {
+            return T{ preg(0, qy, qx), sreg(0, qy, qx) };
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            T t{};
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               t(dd) = { preg(dd, qy, qx), sreg(dd, qy, qx) };
+            }
+            return t;
+         }
+         else
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            constexpr int e1 = qf_param_shape<T>::extents[1];
+            T t;
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  t(i, j) = { preg(i, j, qy, qx), sreg(i, j, qy, qx) };
+               }
+            }
+            return t;
+         }
+      }
+      else
+      {
+         if constexpr (RNK == 0)
+         {
+            return T{ preg(0, qz, qy, qx), sreg(0, qz, qy, qx) };
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            T t{};
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               t(dd) = { preg(dd, qz, qy, qx), sreg(dd, qz, qy, qx) };
+            }
+            return t;
+         }
+         else
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            constexpr int e1 = qf_param_shape<T>::extents[1];
+            T t;
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  t(i, j) = { preg(i, j, qz, qy, qx), sreg(i, j, qz, qy, qx) };
+               }
+            }
+            return t;
+         }
+      }
+   }
+}
+
+} // namespace hok
+
+// ────────────────────────────────────────────────────────────────────────────
+/// HO tensor-product kernels
+template<int T_DIM, int MQ1>
+struct ho_ker_backend
+{
+   static constexpr int DIM = T_DIM;
+   static_assert(DIM == 2 || DIM == 3);
+
+   template<int VDIM>
+   using val_reg_t = std::conditional_t<(DIM == 2),
+         ker::v_regs2d_t<VDIM, MQ1>,
+         ker::v_regs3d_t<VDIM, MQ1>>;
+
+   template<int VDIM, int SDIM>
+   using del_reg_t = std::conditional_t<(DIM == 2),
+         ker::vd_regs2d_t<VDIM, SDIM, MQ1>,
+         ker::vd_regs3d_t<VDIM, SDIM, MQ1>>;
+
+   struct Shared
+   {
+      real_t M[MQ1][MQ1], B[MQ1][MQ1], G[MQ1][MQ1];
+   };
+
+   template<typename XE_t, typename Dofs>
+   static MFEM_HOST_DEVICE void
+   load_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
+   {
+      if constexpr (DIM == 2) { ker::LoadDofs2d(e, d, XE, dofs); }
+      else
+      {
+         ker::LoadDofs3d(e, d, XE, dofs);
+      }
+   }
+
+   template<int VDIM, int SDIM, typename XE_t, typename Dofs>
+   static MFEM_HOST_DEVICE void
+   load_grad_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
+   {
+      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
+      load_dofs(e, d, XE, dofs);
+   }
+
+   template<typename Smem, typename Dofs, typename ArgReg>
+   static MFEM_HOST_DEVICE void
+   eval_value(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
+   {
+      if constexpr (DIM == 2) { ker::Eval2d(d, q, s.M, s.B, dofs, rarg); }
+      else
+      {
+         ker::Eval3d(d, q, s.M, s.B, dofs, rarg);
+      }
+   }
+
+   template<int VDIM, int SDIM, typename Smem, typename Dofs, typename ArgReg>
+   static MFEM_HOST_DEVICE void
+   grad(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
+   {
+      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
+      if constexpr (DIM == 2) { ker::Grad2d(d, q, s.M, s.B, s.G, dofs, rarg); }
+      else
+      {
+         ker::Grad3d(d, q, s.M, s.B, s.G, dofs, rarg);
+      }
+   }
+
+   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
+   static MFEM_HOST_DEVICE void write_value(const int d,
+                                            const int q,
+                                            const int e,
+                                            Smem &s,
+                                            ArgReg &rarg,
+                                            Dofs &dofs,
+                                            YE_t &YE)
+   {
+      if constexpr (DIM == 2)
+      {
+         ker::EvalTranspose2d(d, q, s.M, s.B, rarg, dofs);
+         ker::WriteDofs2d(e, d, dofs, YE);
+      }
+      else
+      {
+         ker::EvalTranspose3d(d, q, s.M, s.B, rarg, dofs);
+         ker::WriteDofs3d(e, d, dofs, YE);
+      }
+   }
+
+   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
+   static MFEM_HOST_DEVICE void write_gradient_2d(const int d,
+                                                  const int q,
+                                                  const int e,
+                                                  Smem &s,
+                                                  ArgReg &rarg,
+                                                  Dofs &dofs,
+                                                  YE_t &YE)
+   {
+      ker::GradTranspose2d(d, q, s.M, s.B, s.G, rarg, dofs);
+      ker::WriteDofs2d(e, d, dofs, YE);
+   }
+
+   template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
+   static MFEM_HOST_DEVICE void write_gradient_3d(const int d,
+                                                  const int q,
+                                                  const int e,
+                                                  Smem &s,
+                                                  ArgReg &rarg,
+                                                  Dofs &dofs,
+                                                  YE_t &YE)
+   {
+      ker::GradTranspose3d(d, q, s.M, s.B, s.G, rarg, dofs);
+      ker::WriteDofs3d(e, d, dofs, YE);
+   }
+
+   template<int VDIM,
+            int SDIM,
+            typename Smem,
+            typename Dofs,
+            typename ArgReg,
+            typename YE_t>
+   static MFEM_HOST_DEVICE void write_gradient(const int d,
+                                               const int q,
+                                               const int e,
+                                               Smem &s,
+                                               ArgReg &rarg,
+                                               Dofs &dofs,
+                                               YE_t &YE)
+   {
+      static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
+      if constexpr (DIM == 2) { write_gradient_2d(d, q, e, s, rarg, dofs, YE); }
+      else
+      {
+         write_gradient_3d(d, q, e, s, rarg, dofs, YE);
+      }
+   }
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int T_DIM, int T_Q1D = LocalQFHOBackendMQ1()>
+struct LocalQFHOBackend
+{
+   // ─────────────────────────────────────────────────────
+   static constexpr int DIM = T_DIM, MQ1 = T_Q1D, Q1D = T_Q1D;
+   static_assert(DIM == 2 || DIM == 3);
+
+   // ─────────────────────────────────────────────────────
+   static inline ThreadBlocks thread_blocks(const int q1d)
+   {
+      MFEM_ASSERT(q1d <= Q1D, "q1d must be <= " << Q1D);
+      return { q1d, q1d, 1 };
+   }
+
+   // ─────────────────────────────────────────────────────
+   static inline constexpr int MAX_THREADS_PER_BLOCK() { return Q1D * Q1D; }
+
+   // ─────────────────────────────────────────────────────
+   using backend_t = ho_ker_backend<DIM, Q1D>;
+
+   // ─────────────────────────────────────────────────────
+   using Shared = typename backend_t::Shared;
+
+   // ─────────────────────────────────────────────────────
+   template<typename WT, typename WI, typename Cache, typename AddY>
+   static MFEM_HOST_DEVICE inline void DiagContract(Shared &s,
+                                                    const int num_dof_1d,
+                                                    const int q1d,
+                                                    const int nz_dof,
+                                                    WT wt,
+                                                    WI wi,
+                                                    Cache cache,
+                                                    AddY add_y)
+   {
+      MFEM_CONTRACT_VAR(nz_dof);
+      const int nqz = (DIM == 3) ? q1d : 1;
+      const int ndz = (DIM == 3) ? num_dof_1d : 1;
+
+      ker::s_regs3d_t<MQ1> rz, ry;
+      auto &smem = s.M;
+
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+         {
+            for (int dz = 0; dz < ndz; dz++)
+            {
+               real_t u = 0.0;
+               for (int qz = 0; qz < nqz; qz++)
+               {
+                  const int q = qx + (qy + qz * q1d) * q1d;
+                  const real_t wz =
+                     (DIM == 3) ? (wt(2, qz, dz) * wi(2, qz, dz)) : real_t(1);
+                  u += wz * cache(q);
+               }
+               rz[dz][qy][qx] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int dz = 0; dz < ndz; dz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+            { smem[qy][qx] = rz[dz][qy][qx]; }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+            {
+               real_t u = 0.0;
+               for (int qy = 0; qy < q1d; qy++)
+               {
+                  u += wt(1, qy, dy) * wi(1, qy, dy) * smem[qy][qx];
+               }
+               ry[dz][dy][qx] = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+
+      for (int dz = 0; dz < ndz; dz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
+            { smem[dy][qx] = ry[dz][dy][qx]; }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, num_dof_1d)
+            {
+               real_t u = 0.0;
+               for (int qx = 0; qx < q1d; qx++)
+               {
+                  u += wt(0, qx, dx) * wi(0, qx, dx) * smem[dy][qx];
+               }
+               add_y(dx, dy, dz, u);
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   using QReg = ho_qreg_t<backend_t, T>;
+
+   // ─────────────────────────────────────────────────────
+   template<typename ArgRegT, typename XE_T>
+   static inline MFEM_HOST_DEVICE void LoadValue(Shared &s,
+                                                 const int e,
+                                                 const int d,
+                                                 const int q,
+                                                 const int,
+                                                 const real_t *B,
+                                                 const XE_T &XE,
+                                                 ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      std::remove_reference_t<ArgRegT> dofs;
+      backend_t::load_dofs(e, d, XE, dofs);
+      backend_t::eval_value(d, q, s, dofs, rarg);
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<int RNK,
+            typename ArgRegT,
+            typename XE_T,
+            typename FieldParamT = ArgRegT>
+   static inline MFEM_HOST_DEVICE void LoadGradient(Shared &s,
+                                                    const int e,
+                                                    const int d,
+                                                    const int q,
+                                                    const int,
+                                                    const real_t *B,
+                                                    const real_t *G,
+                                                    const XE_T &XE,
+                                                    ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      ker::LoadMatrix(d, q, G, s.G);
+      static_assert(RNK == 1 || RNK == 2);
+      static constexpr int VDIM =
+         (RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
+      static constexpr int SDIM = (RNK == 1)
+                                  ? qf_param_shape<FieldParamT>::extents[0]
+                                  : qf_param_shape<FieldParamT>::extents[1];
+      if constexpr (SDIM == DIM)
+      {
+         typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
+         if constexpr (RNK == 1) { backend_t::load_dofs(e, d, XE, dofs); }
+         else
+         {
+            backend_t::template load_grad_dofs<VDIM, SDIM>(e, d, XE, dofs);
+         }
+         backend_t::template grad<VDIM, SDIM>(d, q, s, dofs, rarg);
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline auto
+   qp_pull(QReg<T> &reg, int qx, int qy, int qz)
+   { return hok::load_at<DIM, T>(reg, qx, qy, qz); }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline auto qp_pull_directional(
+      QReg<T> &preg, QReg<T> &sreg, int qx, int qy, int qz, bool dependent)
+   { return hok::pull_directional<DIM, T>(preg, sreg, qx, qy, qz, dependent); }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename XE_T>
+   static MFEM_HOST_DEVICE inline DT identity_qp_pull_dual(bool dependent,
+                                                           const XE_T &XP,
+                                                           const XE_T &XD,
+                                                           int qx,
+                                                           int qy,
+                                                           int qz,
+                                                           int e)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (RNK == 0)
+      {
+         DT t{};
+         t.value = XP(0, qx, qy, qz, e);
+         t.gradient = dependent ? XD(0, qx, qy, qz, e) : 0.0;
+         return t;
+      }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<DT>::extents[0];
+         DT t{};
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd)
+         {
+            t(dd).value = XP(dd, qx, qy, qz, e);
+            t(dd).gradient = dependent ? XD(dd, qx, qy, qz, e) : 0.0;
+         }
+         return t;
+      }
+      else if constexpr (RNK == 2)
+      {
+         constexpr int e0 = qf_param_shape<DT>::extents[0];
+         constexpr int e1 = qf_param_shape<DT>::extents[1];
+         DT t{};
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j)
+            {
+               t(i, j).value = XP(i + e0 * j, qx, qy, qz, e);
+               t(i, j).gradient =
+                  dependent ? XD(i + e0 * j, qx, qy, qz, e) : 0.0;
+            }
+         }
+         return t;
+      }
+      else
+      {
+         static_assert(false, "Unsupported");
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline void
+   qp_push(QReg<T> &reg, int qx, int qy, int qz, const T &out)
+   { hok::store_at<DIM, T, decltype(reg), false>(reg, qx, qy, qz, out); }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline void
+   qp_push_tangent(QReg<T> &reg, int qx, int qy, int qz, const T &out)
+   {
+      hok::store_at<DIM, T, decltype(reg), qf_param_uses_dual_v<T>>(
+                                                                    reg, qx, qy, qz, out);
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename YE_T>
+   static MFEM_HOST_DEVICE inline void identity_qp_write_value(
+      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (qf_param_uses_dual_v<DT>)
+      {
+         if constexpr (RNK == 0)
+         {
+            YE(0, qx, qy, qz, e) = qf_store_value(qout);
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               YE(dd, qx, qy, qz, e) = qf_store_value(qout(dd));
+            }
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            constexpr int e1 = qf_param_shape<DT>::extents[1];
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_value(qout(i, j));
+               }
+            }
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename YE_T>
+   static MFEM_HOST_DEVICE inline void identity_qp_write_tangent(
+      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (qf_param_uses_dual_v<DT>)
+      {
+         if constexpr (RNK == 0)
+         {
+            YE(0, qx, qy, qz, e) = qf_store_gradient(qout);
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               YE(dd, qx, qy, qz, e) = qf_store_gradient(qout(dd));
+            }
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            constexpr int e1 = qf_param_shape<DT>::extents[1];
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_gradient(qout(i, j));
+               }
+            }
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename ArgRegT, typename YE_T>
+   static inline MFEM_HOST_DEVICE void WriteValue(Shared &s,
+                                                  const int e,
+                                                  const int d,
+                                                  const int q,
+                                                  const int,
+                                                  const real_t *B,
+                                                  YE_T &YE,
+                                                  ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      std::remove_reference_t<ArgRegT> dofs;
+      backend_t::write_value(d, q, e, s, rarg, dofs, YE);
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<int RNK,
+            typename ArgRegT,
+            typename YE_T,
+            typename FieldParamT = ArgRegT>
+   static inline MFEM_HOST_DEVICE void WriteGradient(Shared &s,
+                                                     const int e,
+                                                     const int d,
+                                                     const int q,
+                                                     const int,
+                                                     const real_t *B,
+                                                     const real_t *G,
+                                                     YE_T &YE,
+                                                     ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      ker::LoadMatrix(d, q, G, s.G);
+      static_assert(RNK == 1 || RNK == 2);
+      static constexpr int VDIM =
+         (RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
+      static constexpr int SDIM = (RNK == 1)
+                                  ? qf_param_shape<FieldParamT>::extents[0]
+                                  : qf_param_shape<FieldParamT>::extents[1];
+      if constexpr (SDIM == DIM)
+      {
+         typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
+         backend_t::template write_gradient<VDIM, SDIM>(
+            d, q, e, s, rarg, dofs, YE);
+      }
+   }
+};
+
+/// @brief Dispatch to a compile-time HO kernel with MQ1 >= runtime @a q1d.
+template <typename HOKernelTable, int DIM, int MQ1 = LocalQFHOBackendMQ1()>
+inline typename HOKernelTable::KernelSignature
+DispatchHOKernelByQ1D(int q1d)
+{
+   MFEM_VERIFY(q1d >= 2 && q1d <= MQ1,
+               "Unsupported HO quadrature order: " << q1d);
+   return HOKernelTable::template Kernel<DIM, MQ1>();
+}
+
+} // namespace mfem::future

fem/dfem/backends/local_qf/kernels_lo.hpp

@@ -0,0 +1,813 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../../kernels.hpp"
+namespace ker = mfem::kernels::internal;
+
+#include "../../util.hpp" // for ThreadBlocks
+#include "../util.hpp"    // for as_tensor
+#include "util.hpp"
+
+namespace mfem::future
+{
+
+// ────────────────────────────────────────────────────────────────────────────
+inline constexpr int LocalQFLOBackendMQ1() { return 8; }
+
+// ────────────────────────────────────────────────────────────────────────────
+/// Register type for one LO q-function parameter
+template<typename KerOps, typename T, int rank = qf_param_shape<T>::rank>
+struct lo_qreg;
+
+template<typename KerOps, typename T>
+struct lo_qreg<KerOps, T, 0>
+{
+   using type = typename KerOps::template qreg_t<1>;
+};
+
+template<typename KerOps, typename T>
+struct lo_qreg<KerOps, T, 1>
+{
+   static constexpr int e0 = qf_param_shape<T>::extents[0];
+   using type = typename KerOps::template qreg_t<e0>;
+};
+
+template<typename KerOps, typename T>
+struct lo_qreg<KerOps, T, 2>
+{
+   static constexpr int e0 = qf_param_shape<T>::extents[0];
+   static constexpr int e1 = qf_param_shape<T>::extents[1];
+   using type = typename KerOps::template qreg_vd_t<e0, e1>;
+};
+
+template<typename KerOps, typename T>
+using lo_qreg_t = typename lo_qreg<KerOps, T>::type;
+
+// ────────────────────────────────────────────────────────────────────────────
+namespace lok
+{
+
+template<int DIM, typename Reg>
+MFEM_HOST_DEVICE inline auto &at(Reg &reg, int qx, int qy, int qz)
+{
+   if constexpr (DIM == 2)
+   {
+      MFEM_CONTRACT_VAR(qz);
+      return reg[qy][qx];
+   }
+   else
+   {
+      return reg[qz][qy][qx];
+   }
+}
+
+template<int DIM, typename T, typename Reg>
+MFEM_HOST_DEVICE inline auto load_at(Reg &reg, int qx, int qy, int qz)
+{
+   constexpr int RNK = qf_param_shape<T>::rank;
+   auto &qp = at<DIM>(reg, qx, qy, qz);
+   if constexpr (RNK == 0) { return T{ qp[0] }; }
+   else if constexpr (RNK == 1)
+   {
+      constexpr int e0 = qf_param_shape<T>::extents[0];
+      T t{};
+      MFEM_UNROLL(e0)
+      for (int dd = 0; dd < e0; ++dd) { t(dd) = qp[dd]; }
+      return t;
+   }
+   else
+   {
+      constexpr int e0 = qf_param_shape<T>::extents[0];
+      constexpr int e1 = qf_param_shape<T>::extents[1];
+      T t;
+      MFEM_UNROLL(e0)
+      for (int i = 0; i < e0; ++i)
+      {
+         MFEM_UNROLL(e1)
+         for (int j = 0; j < e1; ++j) { t(i, j) = qp[i][j]; }
+      }
+      return t;
+   }
+}
+
+template<bool tangent, typename U>
+MFEM_HOST_DEVICE inline auto qp_store(const U &v)
+{
+   if constexpr (tangent) { return qf_store_gradient(v); }
+   else
+   {
+      return qf_store_value(v);
+   }
+}
+
+// Store primal value or dual tangent at one quadrature point
+template<int DIM, typename T, typename Reg, bool tangent>
+MFEM_HOST_DEVICE inline void
+store_at(Reg &reg, int qx, int qy, int qz, const T &out)
+{
+   constexpr int RNK = qf_param_shape<T>::rank;
+   auto &qp = at<DIM>(reg, qx, qy, qz);
+   if constexpr (RNK == 0) { qp[0] = qp_store<tangent>(out); }
+   else if constexpr (RNK == 1)
+   {
+      constexpr int e0 = qf_param_shape<T>::extents[0];
+      MFEM_UNROLL(e0)
+      for (int dd = 0; dd < e0; ++dd) { qp[dd] = qp_store<tangent>(out(dd)); }
+   }
+   else
+   {
+      constexpr int e0 = qf_param_shape<T>::extents[0];
+      constexpr int e1 = qf_param_shape<T>::extents[1];
+      MFEM_UNROLL(e0)
+      for (int i = 0; i < e0; ++i)
+      {
+         MFEM_UNROLL(e1)
+         for (int j = 0; j < e1; ++j)
+         {
+            qp[i][j] = qp_store<tangent>(out(i, j));
+         }
+      }
+   }
+}
+
+// Pull primal/tangent pair into a dual q-function argument
+template<int DIM, typename T, typename Reg>
+MFEM_HOST_DEVICE inline auto
+pull_directional(Reg &preg, Reg &sreg, int qx, int qy, int qz, bool dependent)
+{
+   if constexpr (!qf_param_uses_dual_v<T>)
+   {
+      return load_at<DIM, T>(preg, qx, qy, qz);
+   }
+   else
+   {
+      if (!dependent) { return load_at<DIM, T>(preg, qx, qy, qz); }
+      constexpr int RNK = qf_param_shape<T>::rank;
+      auto &pqp = at<DIM>(preg, qx, qy, qz);
+      auto &sqp = at<DIM>(sreg, qx, qy, qz);
+      if constexpr (RNK == 0) { return T{ pqp[0], sqp[0] }; }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         T t{};
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd) { t(dd) = { pqp[dd], sqp[dd] }; }
+         return t;
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<T>::extents[0];
+         constexpr int e1 = qf_param_shape<T>::extents[1];
+         T t;
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j) { t(i, j) = { pqp[i][j], sqp[i][j] }; }
+         }
+         return t;
+      }
+   }
+}
+
+} // namespace lok
+
+// ────────────────────────────────────────────────────────────────────────────
+/// LO tensor-product kernels
+template<int T_DIM, int MQ1>
+struct lo_ker_backend
+{
+   static constexpr int DIM = T_DIM;
+   static_assert(DIM == 2 || DIM == 3);
+
+   template<int VDIM>
+   using qreg_t = std::conditional_t<(DIM == 2),
+         ker::regs2d_t<VDIM, MQ1>,
+         ker::regs3d_t<VDIM, MQ1>>;
+
+   template<int VDIM, int SDIM>
+   using qreg_vd_t = std::conditional_t<(DIM == 2),
+         ker::regs2d_vd_t<VDIM, SDIM, MQ1>,
+         ker::regs3d_vd_t<VDIM, SDIM, MQ1>>;
+
+   struct Shared2d
+   {
+      real_t M[2][MQ1][MQ1][DIM];
+      real_t B[MQ1][MQ1], G[MQ1][MQ1];
+   };
+
+   struct Shared3d
+   {
+      real_t M[2][MQ1][MQ1][MQ1][DIM];
+      real_t B[MQ1][MQ1], G[MQ1][MQ1];
+   };
+
+   using Shared = std::conditional_t<(DIM == 2), Shared2d, Shared3d>;
+
+   template<typename FieldParamT, typename ArgRegT, typename XE_T>
+   static MFEM_HOST_DEVICE void load_value(Shared &s,
+                                           const int e,
+                                           const int d,
+                                           const int q,
+                                           const real_t *B,
+                                           const XE_T &XE,
+                                           ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      using field_t = std::remove_cv_t<std::remove_reference_t<FieldParamT>>;
+      constexpr int RNK = qf_param_shape<field_t>::rank;
+      constexpr int VDIM = []()
+      {
+         if constexpr (RNK == 0) { return 1; }
+         else { return qf_param_shape<field_t>::extents[RNK - 1]; }
+      }();
+      if constexpr (DIM == 2)
+      {
+         ker::LoadDofs2d<VDIM, DIM, MQ1>(e, d, XE, s.M[0]);
+         ker::Eval2d(d, q, s.B, s.M[0], s.M[1], rarg);
+      }
+      else
+      {
+         ker::LoadDofs3d<VDIM, DIM, MQ1>(e, d, XE, s.M[0]);
+         ker::Eval3d(d, q, s.B, s.M[0], s.M[1], rarg);
+      }
+   }
+
+   template<int RNK,
+            typename ArgRegT,
+            typename XE_T,
+            typename FieldParamT = ArgRegT>
+   static MFEM_HOST_DEVICE void load_gradient(Shared &s,
+                                              const int e,
+                                              const int d,
+                                              const int q,
+                                              const real_t *B,
+                                              const real_t *G,
+                                              const XE_T &XE,
+                                              ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      ker::LoadMatrix(d, q, G, s.G);
+      static_assert(RNK == 1 || RNK == 2);
+      if constexpr (RNK == 1)
+      {
+         static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[0];
+         if constexpr (SDIM == DIM)
+         {
+            if constexpr (DIM == 2)
+            {
+               ker::LoadDofs2d(e, d, 0, XE, s.M[0]);
+               ker::Grad2d(d, q, s.B, s.G, s.M[0], s.M[1], rarg);
+            }
+            else
+            {
+               ker::LoadDofs3d(e, d, 0, XE, s.M[0]);
+               ker::Grad3d(d, q, s.B, s.G, s.M[0], s.M[1], rarg);
+            }
+         }
+      }
+      if constexpr (RNK == 2)
+      {
+         static constexpr int VDIM = qf_param_shape<FieldParamT>::extents[0];
+         static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[1];
+         if constexpr (SDIM == DIM)
+         {
+            for (int c = 0; c < VDIM; ++c)
+            {
+               if constexpr (DIM == 2)
+               {
+                  ker::LoadDofs2d(e, d, c, XE, s.M[0]);
+                  ker::VectorGrad2d(d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
+               }
+               else
+               {
+                  ker::LoadDofs3d(e, d, c, XE, s.M[0]);
+                  ker::VectorGrad3d(d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
+               }
+            }
+         }
+      }
+   }
+
+   template<typename ArgRegT, typename YE_T>
+   static MFEM_HOST_DEVICE void write_value(Shared &s,
+                                            const int e,
+                                            const int d,
+                                            const int q,
+                                            const real_t *B,
+                                            const YE_T &YE,
+                                            ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      if constexpr (DIM == 2)
+      {
+         ker::EvalTranspose2d(d, q, s.B, rarg, s.M[1], s.M[0]);
+         ker::WriteEvalDofs2d(d, 0, e, rarg, YE);
+      }
+      else
+      {
+         ker::EvalTranspose3d(d, q, s.B, rarg, s.M[1], s.M[0]);
+         ker::WriteEvalDofs3d(d, 0, e, rarg, YE);
+      }
+   }
+
+   template<int RNK,
+            typename ArgRegT,
+            typename YE_T,
+            typename FieldParamT = ArgRegT>
+   static MFEM_HOST_DEVICE void write_gradient(Shared &s,
+                                               const int e,
+                                               const int d,
+                                               const int q,
+                                               const real_t *B,
+                                               const real_t *G,
+                                               YE_T &YE,
+                                               ArgRegT &rarg)
+   {
+      ker::LoadMatrix(d, q, B, s.B);
+      ker::LoadMatrix(d, q, G, s.G);
+      static_assert(RNK == 1 || RNK == 2);
+      if constexpr (RNK == 1)
+      {
+         static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[0];
+         if constexpr (SDIM == DIM)
+         {
+            if constexpr (DIM == 2)
+            {
+               ker::GradTranspose2d(d, q, s.B, s.G, rarg, s.M[1], s.M[0]);
+               ker::WriteGradDofs2d(d, 0, e, rarg, YE);
+            }
+            else
+            {
+               ker::GradTranspose3d(d, q, s.B, s.G, rarg, s.M[1], s.M[0]);
+               ker::WriteGradDofs3d(d, 0, e, rarg, YE);
+            }
+         }
+      }
+      else if constexpr (RNK == 2)
+      {
+         static constexpr int VDIM = qf_param_shape<FieldParamT>::extents[0];
+         static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[1];
+         if constexpr (SDIM == DIM)
+         {
+            for (int c = 0; c < VDIM; ++c)
+            {
+               if constexpr (DIM == 2)
+               {
+                  ker::VectorGradTranspose2d(
+                     d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
+                  ker::WriteGradDofs2d(d, c, e, rarg, YE);
+               }
+               else
+               {
+                  ker::VectorGradTranspose3d(
+                     d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
+                  ker::WriteGradDofs3d(d, c, e, rarg, YE);
+               }
+            }
+         }
+      }
+      else
+      {
+         static_assert(false, "Unsupported");
+      }
+   }
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int T_DIM, int T_Q1D = LocalQFLOBackendMQ1()>
+struct LocalQFLOBackend
+{
+   // ─────────────────────────────────────────────────────
+   static constexpr int DIM = T_DIM, MQ1 = T_Q1D, Q1D = T_Q1D;
+   static_assert(DIM == 2 || DIM == 3);
+
+   // ─────────────────────────────────────────────────────
+   static inline ThreadBlocks thread_blocks(const int q1d)
+   {
+      MFEM_ASSERT(q1d <= Q1D, "q1d must be <= " << Q1D);
+      return { q1d, q1d, (DIM == 2) ? 1 : q1d };
+   }
+
+   // ─────────────────────────────────────────────────────
+   static inline constexpr int MAX_THREADS_PER_BLOCK()
+   { return Q1D * Q1D * ((DIM == 2) ? 1 : Q1D); }
+
+   // ─────────────────────────────────────────────────────
+   using backend_t = lo_ker_backend<DIM, Q1D>;
+
+   // ─────────────────────────────────────────────────────
+   using Shared = typename backend_t::Shared;
+
+   // ─────────────────────────────────────────────────────
+   template<typename WT, typename WI, typename Cache, typename AddY>
+   static MFEM_HOST_DEVICE inline void DiagContract(Shared &s,
+                                                    const int num_dof_1d,
+                                                    const int q1d,
+                                                    const int nz_dof,
+                                                    WT wt,
+                                                    WI wi,
+                                                    Cache cache,
+                                                    AddY add_y)
+   {
+      MFEM_CONTRACT_VAR(nz_dof);
+      real_t *base = reinterpret_cast<real_t *>(&s.M[0]);
+      auto s0 = reinterpret_cast<real_t(*)[Q1D][Q1D]>(base);
+
+      if constexpr (DIM == 3)
+      {
+         auto s1 =
+            reinterpret_cast<real_t(*)[Q1D][Q1D]>(base + Q1D * Q1D * Q1D);
+
+         // reduce qz → dz : s0[dz][qy][qx]
+         MFEM_FOREACH_THREAD(dz, z, num_dof_1d)
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t u = 0.0;
+            for (int qz = 0; qz < q1d; qz++)
+            {
+               const int q = qx + (qy + qz * q1d) * q1d;
+               u += wt(2, qz, dz) * wi(2, qz, dz) * cache(q);
+            }
+            s0[dz][qy][qx] = u;
+         }
+         MFEM_SYNC_THREAD;
+
+         // reduce qy → dy : s1[dz][dy][qx]
+         MFEM_FOREACH_THREAD(dz, z, num_dof_1d)
+         MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t u = 0.0;
+            for (int qy = 0; qy < q1d; qy++)
+            {
+               u += wt(1, qy, dy) * wi(1, qy, dy) * s0[dz][qy][qx];
+            }
+            s1[dz][dy][qx] = u;
+         }
+         MFEM_SYNC_THREAD;
+
+         // reduce qx → dx : Y(dx,dy,dz)
+         MFEM_FOREACH_THREAD(dz, z, num_dof_1d)
+         MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
+         MFEM_FOREACH_THREAD(dx, x, num_dof_1d)
+         {
+            real_t u = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               u += wt(0, qx, dx) * wi(0, qx, dx) * s1[dz][dy][qx];
+            }
+            add_y(dx, dy, dz, u);
+         }
+         MFEM_SYNC_THREAD;
+      }
+      else
+      {
+         // reduce qy → dy : s0[0][dy][qx]
+         MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t u = 0.0;
+            for (int qy = 0; qy < q1d; qy++)
+            {
+               const int q = qx + qy * q1d;
+               u += wt(1, qy, dy) * wi(1, qy, dy) * cache(q);
+            }
+            s0[0][dy][qx] = u;
+         }
+         MFEM_SYNC_THREAD;
+
+         // reduce qx → dx : Y(dx,dy,0)
+         MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
+         MFEM_FOREACH_THREAD(dx, x, num_dof_1d)
+         {
+            real_t u = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               u += wt(0, qx, dx) * wi(0, qx, dx) * s0[0][dy][qx];
+            }
+            add_y(dx, dy, 0, u);
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   using QReg = lo_qreg_t<backend_t, T>;
+
+   // ─────────────────────────────────────────────────────
+   template<typename ArgRegT, typename XE_T>
+   static inline MFEM_HOST_DEVICE void LoadValue(Shared &s,
+                                                 const int e,
+                                                 const int d,
+                                                 const int q,
+                                                 const int,
+                                                 const real_t *B,
+                                                 const XE_T &XE,
+                                                 ArgRegT &rarg)
+   {
+      backend_t::template load_value<ArgRegT>(s, e, d, q, B, XE, rarg);
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<int RNK,
+            typename ArgRegT,
+            typename XE_T,
+            typename FieldParamT = ArgRegT>
+   static inline MFEM_HOST_DEVICE void LoadGradient(Shared &s,
+                                                    const int e,
+                                                    const int d,
+                                                    const int q,
+                                                    const int,
+                                                    const real_t *B,
+                                                    const real_t *G,
+                                                    const XE_T &XE,
+                                                    ArgRegT &rarg)
+   {
+      backend_t::template load_gradient<RNK, ArgRegT, XE_T, FieldParamT>(
+         s, e, d, q, B, G, XE, rarg);
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline auto
+   qp_pull(QReg<T> &reg, int qx, int qy, int qz)
+   {
+      if constexpr (qf_param_uses_dual_v<T>)
+      {
+         return lok::load_at<DIM, T>(reg, qx, qy, qz);
+      }
+      else
+      {
+         constexpr int RNK = qf_param_shape<T>::rank;
+         if constexpr (RNK == 0)
+         {
+            return as_tensor<real_t>(&lok::at<DIM>(reg, qx, qy, qz)[0]);
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            return as_tensor<real_t, e0>(&lok::at<DIM>(reg, qx, qy, qz)[0]);
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            constexpr int e1 = qf_param_shape<T>::extents[1];
+            return as_tensor<real_t, e0, e1>(
+                      &lok::at<DIM>(reg, qx, qy, qz)[0][0]);
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline auto qp_pull_directional(
+      QReg<T> &preg, QReg<T> &sreg, int qx, int qy, int qz, bool dependent)
+   { return lok::pull_directional<DIM, T>(preg, sreg, qx, qy, qz, dependent); }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename XE_T>
+   static MFEM_HOST_DEVICE inline DT identity_qp_pull_dual(bool dependent,
+                                                           const XE_T &XP,
+                                                           const XE_T &XD,
+                                                           int qx,
+                                                           int qy,
+                                                           int qz,
+                                                           int e)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (RNK == 0)
+      {
+         DT t{};
+         t.value = XP(0, qx, qy, qz, e);
+         t.gradient = dependent ? XD(0, qx, qy, qz, e) : 0.0;
+         return t;
+      }
+      else if constexpr (RNK == 1)
+      {
+         constexpr int e0 = qf_param_shape<DT>::extents[0];
+         DT t{};
+         MFEM_UNROLL(e0)
+         for (int dd = 0; dd < e0; ++dd)
+         {
+            t(dd).value = XP(dd, qx, qy, qz, e);
+            t(dd).gradient = dependent ? XD(dd, qx, qy, qz, e) : 0.0;
+         }
+         return t;
+      }
+      else if constexpr (RNK == 2)
+      {
+         constexpr int e0 = qf_param_shape<DT>::extents[0];
+         constexpr int e1 = qf_param_shape<DT>::extents[1];
+         DT t{};
+         MFEM_UNROLL(e0)
+         for (int i = 0; i < e0; ++i)
+         {
+            MFEM_UNROLL(e1)
+            for (int j = 0; j < e1; ++j)
+            {
+               t(i, j).value = XP(i + e0 * j, qx, qy, qz, e);
+               t(i, j).gradient =
+                  dependent ? XD(i + e0 * j, qx, qy, qz, e) : 0.0;
+            }
+         }
+         return t;
+      }
+      else
+      {
+         static_assert(false, "Unsupported");
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline void
+   qp_push(QReg<T> &reg, int qx, int qy, int qz, const T &out)
+   {
+      if constexpr (qf_param_uses_dual_v<T>)
+      {
+         lok::store_at<DIM, T, decltype(reg), false>(reg, qx, qy, qz, out);
+      }
+      else
+      {
+         constexpr int RNK = qf_param_shape<T>::rank;
+         if constexpr (RNK == 0)
+         {
+            as_tensor<real_t>(&lok::at<DIM>(reg, qx, qy, qz)[0]) = out;
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            as_tensor<real_t, e0>(&lok::at<DIM>(reg, qx, qy, qz)[0]) = out;
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<T>::extents[0];
+            constexpr int e1 = qf_param_shape<T>::extents[1];
+            as_tensor<real_t, e0, e1>(&lok::at<DIM>(reg, qx, qy, qz)[0][0]) =
+               out;
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename T>
+   static MFEM_HOST_DEVICE inline void
+   qp_push_tangent(QReg<T> &reg, int qx, int qy, int qz, const T &out)
+   {
+      if constexpr (!qf_param_uses_dual_v<T>)
+      {
+         qp_push<T>(reg, qx, qy, qz, out);
+      }
+      else
+      {
+         lok::store_at<DIM, T, decltype(reg), true>(reg, qx, qy, qz, out);
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename YE_T>
+   static MFEM_HOST_DEVICE inline void identity_qp_write_value(
+      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (qf_param_uses_dual_v<DT>)
+      {
+         if constexpr (RNK == 0)
+         {
+            YE(0, qx, qy, qz, e) = qf_store_value(qout);
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               YE(dd, qx, qy, qz, e) = qf_store_value(qout(dd));
+            }
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            constexpr int e1 = qf_param_shape<DT>::extents[1];
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_value(qout(i, j));
+               }
+            }
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename DT, typename YE_T>
+   static MFEM_HOST_DEVICE inline void identity_qp_write_tangent(
+      YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
+   {
+      constexpr int RNK = qf_param_shape<DT>::rank;
+      if constexpr (qf_param_uses_dual_v<DT>)
+      {
+         if constexpr (RNK == 0)
+         {
+            YE(0, qx, qy, qz, e) = qf_store_gradient(qout);
+         }
+         else if constexpr (RNK == 1)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            MFEM_UNROLL(e0)
+            for (int dd = 0; dd < e0; ++dd)
+            {
+               YE(dd, qx, qy, qz, e) = qf_store_gradient(qout(dd));
+            }
+         }
+         else if constexpr (RNK == 2)
+         {
+            constexpr int e0 = qf_param_shape<DT>::extents[0];
+            constexpr int e1 = qf_param_shape<DT>::extents[1];
+            MFEM_UNROLL(e0)
+            for (int i = 0; i < e0; ++i)
+            {
+               MFEM_UNROLL(e1)
+               for (int j = 0; j < e1; ++j)
+               {
+                  YE(i + e0 * j, qx, qy, qz, e) = qf_store_gradient(qout(i, j));
+               }
+            }
+         }
+         else
+         {
+            static_assert(false, "Unsupported");
+         }
+      }
+   }
+
+   // ─────────────────────────────────────────────────────
+   template<typename ArgRegT, typename YE_T>
+   static inline MFEM_HOST_DEVICE void WriteValue(Shared &s,
+                                                  const int e,
+                                                  const int d,
+                                                  const int q,
+                                                  const int,
+                                                  const real_t *B,
+                                                  const YE_T &YE,
+                                                  ArgRegT &rarg)
+   { backend_t::write_value(s, e, d, q, B, YE, rarg); }
+
+   // ─────────────────────────────────────────────────────
+   template<int RNK,
+            typename ArgRegT,
+            typename YE_T,
+            typename FieldParamT = ArgRegT>
+   static inline MFEM_HOST_DEVICE void WriteGradient(Shared &s,
+                                                     const int e,
+                                                     const int d,
+                                                     const int q,
+                                                     const int,
+                                                     const real_t *B,
+                                                     const real_t *G,
+                                                     YE_T &YE,
+                                                     ArgRegT &rarg)
+   {
+      backend_t::template write_gradient<RNK, ArgRegT, YE_T, FieldParamT>(
+         s, e, d, q, B, G, YE, rarg);
+   }
+};
+
+/// @brief Dispatch to a compile-time LO kernel matching runtime @a q1d.
+template <typename LOKernelTable, int DIM, int MQ1 = LocalQFLOBackendMQ1()>
+inline typename LOKernelTable::KernelSignature
+DispatchLOKernelByQ1D(int q1d)
+{
+   MFEM_VERIFY(q1d >= 2 && q1d <= MQ1,
+               "Unsupported LO quadrature order: " << q1d);
+   return LOKernelTable::template Kernel<DIM, MQ1>();
+}
+
+} // namespace mfem::future

fem/dfem/backends/local_qf/prelude.hpp

@@ -0,0 +1,242 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../integrator_ctx.hpp"
+#include "action.hpp"
+#include "derivative_action.hpp"
+#include "derivative_setup.hpp"
+#include "derivative_apply.hpp"
+#include "derivative_assemble.hpp"
+#include "derivative_assemble_diagonal.hpp"
+#include "derivative_apply_transpose.hpp"
+
+namespace mfem::future
+{
+
+struct LocalQFBackend
+{
+   /**
+    * @brief Make an action for a local Q-function backend.
+    *
+    * @param ctx The integrator context.
+    * @param args The arguments to the action.
+    * @return The action.
+    */
+   template<typename qfunc_t, typename inputs_t, typename outputs_t>
+   static auto MakeAction(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      return LocalQFImpl::Action<qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs);
+   }
+
+   /**
+    * @brief Make a derivative action for a local Q-function backend.
+    *
+    * @tparam derivative_id The id of the derivative.
+    * @param ctx The integrator context.
+    * @param args The arguments to the derivative action.
+    * @return The derivative action.
+    */
+   template<int id, typename qfunc_t, typename inputs_t, typename outputs_t>
+   static auto MakeDerivativeAction(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs)
+   {
+      return LocalQFImpl::DerivativeAction<id, qfunc_t, inputs_t, outputs_t>(
+                ctx, qfunc, inputs, outputs);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   static auto MakeDerivativeSetup(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeSetup<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
+                                                          outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   static auto MakeDerivativeApply(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeApply<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
+                                                          outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   static auto MakeDerivativeApplyTranspose(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeApplyTranspose<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
+                                                          outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   static auto MakeDerivativeAssemble(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeAssemble<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
+                                                          outputs, qp_cache);
+   }
+
+   template<
+      int derivative_id,
+      typename qfunc_t,
+      typename inputs_t,
+      typename outputs_t>
+   static auto MakeDerivativeAssembleDiagonal(
+      const IntegratorContext &ctx,
+      const qfunc_t &qfunc,
+      inputs_t inputs,
+      outputs_t outputs,
+      const Vector &qp_cache)
+   {
+      return LocalQFImpl::DerivativeAssembleDiagonal<
+             derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
+                                                          outputs, qp_cache);
+   }
+};
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, typename QT, typename IT, typename OT>
+inline void AddAction()
+{
+   using ker = LocalQFImpl::Action<QT, IT, OT>;
+   if constexpr (Q1D <= 8)
+   {
+      ker::ActionLO::template Specialization<DIM, Q1D>::Add();
+   }
+   else
+   {
+      ker::ActionHO::template Specialization<DIM, Q1D>::Add();
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
+inline void AddDerivativeAction()
+{
+   using ker = LocalQFImpl::DerivativeAction<DID, QT, IT, OT>;
+   if constexpr (Q1D <= 8)
+   {
+      ker::DerivativeActionLO::template Specialization<DIM, Q1D>::Add();
+   }
+   else
+   {
+      ker::DerivativeActionHO::template Specialization<DIM, Q1D>::Add();
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
+inline void AddDerivativeSetup()
+{
+   using ker = LocalQFImpl::DerivativeSetup<DID, QT, IT, OT>;
+   if constexpr (Q1D <= 8)
+   {
+      ker::DerivativeSetupLO::template Specialization<DIM, Q1D>::Add();
+   }
+   else
+   {
+      ker::DerivativeSetupHO::template Specialization<DIM, Q1D>::Add();
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
+inline void AddDerivativeApply()
+{
+   using ker = LocalQFImpl::DerivativeApply<DID, QT, IT, OT>;
+   if constexpr (Q1D <= 8)
+   {
+      ker::DerivativeApplyLO::template Specialization<DIM, Q1D>::Add();
+   }
+   else
+   {
+      ker::DerivativeApplyHO::template Specialization<DIM, Q1D>::Add();
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
+inline void AddDerivativeApplyTranspose()
+{
+   using ker = LocalQFImpl::DerivativeApplyTranspose<DID, QT, IT, OT>;
+   if constexpr (Q1D <= 8)
+   {
+      ker::DerivativeApplyTransposeLO::template Specialization<DIM, Q1D>::Add();
+   }
+   else
+   {
+      ker::DerivativeApplyTransposeHO::template Specialization<DIM, Q1D>::Add();
+   }
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+template<int DIM, int Q1D, typename QT, typename IT, typename OT,
+         typename derivative_ids_t = std::index_sequence<>>
+inline void AddLocalSpecializations()
+{
+   AddAction<DIM, Q1D, QT, IT, OT>();
+
+   for_constexpr([&](auto i)
+   {
+      using derivative_id = decltype(i);
+      AddDerivativeAction<DIM, Q1D, derivative_id::value, QT, IT, OT>();
+      AddDerivativeSetup<DIM, Q1D, derivative_id::value, QT, IT, OT>();
+      AddDerivativeApply<DIM, Q1D, derivative_id::value, QT, IT, OT>();
+      AddDerivativeApplyTranspose<DIM, Q1D, derivative_id::value, QT, IT, OT>();
+   }, derivative_ids_t{});
+}
+
+}

fem/dfem/backends/local_qf/revdiff_transformer.hpp

@@ -0,0 +1,274 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../util.hpp"
+
+#ifdef MFEM_USE_ENZYME
+
+namespace mfem::future
+{
+
+template <typename T> struct function_traits;
+
+template <typename R, typename C, typename... Args>
+struct function_traits<R (C::*)(Args...) const>
+{
+   using primal_return_type = R;
+   using args_tuple = tuple<Args...>;
+   static constexpr size_t arity = sizeof...(Args);
+};
+
+// Component count and writable counterpart of a per-point argument, which
+// is either a tensor or a plain scalar.
+template <typename Arg> struct qp_traits
+{
+   static_assert(std::is_arithmetic_v<Arg>,
+                 "per-point arguments must be tensors or scalars");
+   using view_type = Arg;
+   static constexpr int components = 1;
+};
+
+template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
+{
+   using view_type = tensor<std::remove_const_t<T>, Sizes...>;
+   static constexpr int components = (Sizes * ... * 1);
+};
+
+template <typename... T1s, typename... T2s>
+constexpr tuple<T1s..., T2s...> concat_tuples(tuple<T1s...>, tuple<T2s...>);
+
+// RevDiff: computes the full gradient of a pointwise qfunction at a single
+// quadrature point using one Enzyme reverse-mode (autodiff) call.
+//
+// operator()(args...) takes all qfunction input primals followed by one
+// writable gradient output per Active input (same shape as that input).
+// The qfunction's own output is not passed; Enzyme writes it to stack
+// scratch (enzyme_dupnoneed).
+//
+// A single __enzyme_autodiff call with the output adjoint seeded to 1
+// yields all gradient blocks simultaneously — O(1) calls per point
+// regardless of input size, vs O(ncomp) for forward mode.
+//
+// operator() is MFEM_HOST_DEVICE, stateless and allocation-free.
+template <typename Func, typename InputActivityTuple,
+          typename OutputActivityTuple>
+struct RevDiff
+{
+   using traits = function_traits<decltype(&Func::operator())>;
+   using args_tuple = typename traits::args_tuple;
+   using activity =
+      decltype(concat_tuples(InputActivityTuple{}, OutputActivityTuple{}));
+   static constexpr size_t arity = traits::arity;
+   static constexpr size_t num_inputs = tuple_size<InputActivityTuple>::value;
+
+   static_assert(std::is_void_v<typename traits::primal_return_type>,
+                 "RevDiff only supports primal functions with void return type");
+   static_assert(tuple_size<activity>::value == arity,
+                 "Number of input and output activity tags must match function "
+                 "arity");
+
+   template <size_t I>
+   static constexpr bool is_active =
+      std::is_same_v<tuple_element_t<I, activity>, Active>;
+
+   // Index of the single Active tag in [Lo, Hi), or arity if not exactly one.
+   template <size_t Lo, size_t Hi, size_t... Is>
+   static constexpr size_t find_single_active(std::index_sequence<Is...>)
+   {
+      size_t idx = arity, count = 0;
+      (((Is >= Lo && Is < Hi && is_active<Is>) ? (idx = Is, ++count) : size_t{0}),
+       ...);
+      return count == 1 ? idx : arity;
+   }
+
+   // Number of Active inputs and their argument indices, in ascending order.
+   // A qfunction may have several Active inputs at once: e.g. a field's value
+   // u and its gradient dudx both feed the output and both must be
+   // differentiated (the chain-rule contraction with the value/gradient shape
+   // functions then happens at the FE-operator level). We produce one gradient
+   // block, d(output)/d(input), per Active input — each computed with the other
+   // Active inputs frozen, so they come out as isolated partials.
+   template <size_t... Is>
+   static constexpr size_t count_active_inputs(std::index_sequence<Is...>)
+   {
+      return ((Is < num_inputs && is_active<Is> ? size_t{1} : size_t{0}) + ...);
+   }
+   static constexpr size_t num_active_inputs =
+      count_active_inputs(std::make_index_sequence<arity> {});
+
+   template <size_t... Is>
+   static constexpr std::array<size_t, num_active_inputs>
+   collect_active_inputs(std::index_sequence<Is...>)
+   {
+      std::array<size_t, num_active_inputs> idx{};
+      size_t j = 0;
+      (((Is < num_inputs && is_active<Is>) ? (idx[j++] = Is) : size_t{0}), ...);
+      return idx;
+   }
+   static constexpr auto active_inputs =
+   collect_active_inputs(std::make_index_sequence<arity> {});
+
+   // Slot index of argument I in the active_inputs array (compile-time).
+   template <size_t I>
+   static constexpr size_t slot_of = []() constexpr -> size_t
+   {
+      for (size_t s = 0; s < num_active_inputs; s++)
+         if (active_inputs[s] == I) { return s; }
+      return num_active_inputs;
+   }();
+
+   static constexpr size_t active_output =
+      find_single_active<num_inputs, arity>(std::make_index_sequence<arity> {});
+   static_assert(active_output < arity,
+                 "gradient mode requires exactly one Active output");
+   static_assert(num_active_inputs >= 1,
+                 "gradient mode requires at least one Active input");
+   static_assert(tuple_size<OutputActivityTuple>::value == 1,
+                 "gradient mode requires exactly one (scalar) output");
+
+   using output_type =
+      std::decay_t<tuple_element_t<active_output, args_tuple>>;
+
+   using output_view = typename qp_traits<output_type>::view_type;
+
+   static_assert(qp_traits<output_type>::components == 1,
+                 "gradient output requires a scalar output");
+
+   template <size_t I>
+   using primal_arg_t = tuple_element_t<I, args_tuple>;
+
+   template <size_t S>
+   using grad_arg_t =
+      typename qp_traits<std::decay_t<tuple_element_t<active_inputs[S], args_tuple>>>::view_type
+      &;
+
+   template <size_t... Is, size_t... Ss>
+   static FunctionSignature<void(primal_arg_t<Is>..., grad_arg_t<Ss>...)>
+   signature_impl(std::index_sequence<Is...>, std::index_sequence<Ss...>);
+
+   using signature = decltype(signature_impl(std::make_index_sequence<num_inputs> {},
+                                             std::make_index_sequence<num_active_inputs> {}));
+
+   // Plain function with the qfunction's exact (reference) signature so it can
+   // be handed to Enzyme as a function pointer.
+   template <size_t... Is>
+   MFEM_HOST_DEVICE static __attribute__((always_inline)) void
+   static_call(tuple_element_t<Is, args_tuple>... args)
+   {
+      Func{}(args...);
+   }
+
+   template <size_t... Is>
+   static constexpr auto fn_ptr(std::index_sequence<Is...>)
+   {
+      return &static_call<Is...>;
+   }
+   static constexpr auto fn = fn_ptr(std::make_index_sequence<arity> {});
+
+   // Recursive builder of the per-argument reverse-mode enzyme call.
+   template <size_t I = 0, typename AllPtrs, typename... Built>
+   MFEM_HOST_DEVICE static __attribute__((always_inline)) void
+   call_enzyme_rev(AllPtrs &ptrs, output_view &scratch, output_view &adjoint,
+                   Built... built)
+   {
+      if constexpr (I == arity)
+      {
+         __enzyme_autodiff<void>(fn, built...);
+      }
+      else if constexpr (I == active_output)
+      {
+         // Output: primal written to scratch (unused), adjoint seeded to 1.
+         call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
+                                enzyme_dupnoneed, &scratch, &adjoint);
+      }
+      else if constexpr (is_active<I>)
+      {
+         // Active input: gradient accumulates into its grad-output slot.
+         call_enzyme_rev<I + 1>(
+            ptrs, scratch, adjoint, built..., enzyme_dup,
+            mfem::future::get<int(I)>(ptrs),
+            mfem::future::get<int(num_inputs + slot_of<I>)>(ptrs));
+      }
+      else
+      {
+         // Const input: primal only, no shadow.
+         call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
+                                enzyme_const, mfem::future::get<int(I)>(ptrs));
+      }
+   }
+
+   // Zero all gradient outputs before the enzyme call (Enzyme accumulates).
+   template <typename AllPtrs, size_t... Ss>
+   MFEM_HOST_DEVICE static
+   __attribute__((always_inline)) void zero_grads(
+      AllPtrs &ptrs,
+      std::index_sequence<Ss...>)
+   {
+      ((*mfem::future::get<int(num_inputs + Ss)>(ptrs) =
+           std::decay_t<decltype(*mfem::future::get<int(num_inputs + Ss)>(ptrs))> {}),
+       ...);
+   }
+
+   // Called once per quadrature point. Arguments are, in order:
+   //   * the primal value of every qfunction input (active and const), then
+   //   * one gradient output per Active input (ascending index order), each
+   //     shaped like its Active input.
+   // The qfunction's own output slot is not passed; Enzyme writes it to stack
+   // scratch (enzyme_dupnoneed). A single __enzyme_autodiff call yields all
+   // gradient blocks simultaneously.
+   template <typename... Args>
+   MFEM_HOST_DEVICE __attribute__((always_inline)) void operator()(
+      Args &&...args) const
+   {
+      static_assert(sizeof...(Args) == num_inputs + num_active_inputs,
+                    "expected one primal per input plus one gradient output per "
+                    "Active input");
+      auto ptrs = mfem::future::make_tuple(&args...);
+      zero_grads(ptrs, std::make_index_sequence<num_active_inputs> {});
+      output_view out_scratch{};
+      output_view out_adjoint{1.0}; // seed: d(output)/d(output) = 1
+      call_enzyme_rev(ptrs, out_scratch, out_adjoint);
+   }
+
+   static __attribute__((always_inline)) void print() { print_impl(std::make_index_sequence<arity> {}); }
+
+   template <size_t... Is> static __attribute__((always_inline)) void print_impl(
+      std::index_sequence<Is...>)
+   {
+      mfem::out << "__enzyme_autodiff<void>(fptr";
+      (([&]
+      {
+         auto name = get_type_name<tuple_element_t<Is, args_tuple>>();
+         if constexpr (Is == active_output)
+            mfem::out << ", enzyme_dupnoneed, " << name << ", adjoint=1";
+         else if constexpr (is_active<Is>)
+            mfem::out << ", enzyme_dup, " << name << ", grad out";
+         else
+         {
+            mfem::out << ", enzyme_const, " << name;
+         }
+      }()),
+      ...);
+      mfem::out << ")\n";
+   }
+};
+
+template <typename Func, typename InputActivityTuple, typename OutputActivityTuple>
+struct create_function_signature<RevDiff<Func, InputActivityTuple, OutputActivityTuple>>
+{
+   using type = typename
+                RevDiff<Func, InputActivityTuple, OutputActivityTuple>::signature;
+};
+
+} // namespace mfem::future
+
+#endif // MFEM_USE_ENZYME

fem/dfem/backends/local_qf/util.hpp

@@ -0,0 +1,622 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+// Compile-time quadrature argument metadata for local q-functions
+
+#include "../../../../linalg/tensor.hpp"
+
+#include "../../integrator_ctx.hpp"
+#include "../../util.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <type_traits>
+
+namespace mfem::future
+{
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename T>
+MFEM_HOST_DEVICE auto qf_store_value(const T &v)
+{
+   if constexpr (is_dual_number<T>::value) { return v.value; }
+   else { return v; }
+}
+
+template <typename T>
+MFEM_HOST_DEVICE auto qf_store_gradient(const T &v)
+{
+   if constexpr (is_dual_number<T>::value) { return v.gradient; }
+   else { return v; }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// True when quadrature-point values of `T` carry dual-number derivatives
+template <typename T>
+struct qf_param_uses_dual : std::false_type {};
+
+template <typename S, int... Is>
+struct qf_param_uses_dual<tensor<S, Is...>> : is_dual_number<S> {};
+
+template <typename V, typename G>
+struct qf_param_uses_dual<dual<V, G>> : std::true_type {};
+
+template <typename T>
+constexpr bool qf_param_uses_dual_v = qf_param_uses_dual<T>::value;
+
+///////////////////////////////////////////////////////////////////////////////
+/// Static shape for one decayed q-function parameter type
+template <typename T>
+struct qf_param_shape
+{
+   static constexpr int rank = 0;
+   static constexpr std::array<int, 0> extents {};
+};
+
+template <typename scalar_t, int... Is>
+struct qf_param_shape<tensor<scalar_t, Is...>>
+{
+   static constexpr int rank = sizeof...(Is);
+   static constexpr std::array<int, sizeof...(Is)> extents {{Is...}};
+};
+
+template <typename scalar_t>
+struct qf_param_shape<tensor<scalar_t>>
+{
+   static constexpr int rank = 0;
+   static constexpr std::array<int, 0> extents {};
+};
+
+template <>
+struct qf_param_shape<real_t>
+{
+   static constexpr int rank = 0;
+   static constexpr std::array<int, 0> extents {};
+};
+
+///////////////////////////////////////////////////////////////////////////////
+/// Type used in quadrature registers for parameter
+template <typename T>
+struct qf_reg_t { using type = T; };
+
+template <>
+struct qf_reg_t<real_t> { using type = tensor<real_t>; };
+
+///////////////////////////////////////////////////////////////////////////////
+/// Per-parameter tensor info for slot `I` in the decayed q-function parameter tuple
+template <typename qfunc_t, std::size_t I>
+struct qf_param_slot
+{
+   using qf_signature = typename get_function_signature<qfunc_t>::type;
+   using qf_param_ts = typename qf_signature::parameter_ts;
+   using qf_raw_param_t = typename tuple_element<I, qf_param_ts>::type;
+   using qf_decay_param_t =
+      std::remove_cv_t<std::remove_reference_t<qf_raw_param_t>>;
+   using qf_reg_param_t = typename qf_reg_t<qf_decay_param_t>::type;
+
+   static constexpr auto extents = qf_param_shape<qf_decay_param_t>::extents;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+/// Builds a register-bank tuple covering the q-function parameter slots
+/// `[K0, N)`. `K` is the recursion cursor and starts at `K0`; the resulting
+/// tuple is indexed from 0, so a bank starting at `K0 > 0` has its slot indices
+/// rebased by `-K0` relative to the q-function parameter list.
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1,
+   std::size_t K0, std::size_t K, std::size_t N, typename... Acc>
+struct build_args_reg_tuple_impl;
+
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1,
+   std::size_t K0, std::size_t N, typename... Acc>
+struct build_args_reg_tuple_impl<backend_t, qfunc_t, inputs_t, outputs_t, MQ1, K0, N, N, Acc...>
+{
+   using type = tuple<Acc...>;
+   static_assert(sizeof...(Acc) == N - K0);
+   static_assert(sizeof...(Acc) <= 9);
+};
+
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1,
+   std::size_t K0, std::size_t K, std::size_t N, typename... Acc>
+struct build_args_reg_tuple_impl
+{
+   using qf_reg_param_t = typename qf_param_slot<qfunc_t, K>::qf_reg_param_t;
+   using R = typename backend_t::template QReg<qf_reg_param_t>;
+   using type = typename build_args_reg_tuple_impl<backend_t, qfunc_t, inputs_t,
+         outputs_t, MQ1, K0, K + 1, N, Acc..., R>::type;
+};
+
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1>
+using args_reg_t = typename build_args_reg_tuple_impl<backend_t, qfunc_t,
+      inputs_t, outputs_t, MQ1, 0, 0,
+      tuple_size<inputs_t>::value + tuple_size<outputs_t>::value>::type;
+
+/// Register bank covering q-function inputs only (same types as first
+/// `n_inputs` slots of args_reg_t). Used where shadow / tangent paths never
+/// touch output parameter registers.
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1>
+using input_args_reg_t = typename build_args_reg_tuple_impl<backend_t, qfunc_t,
+      inputs_t, outputs_t, MQ1, 0, 0,
+      tuple_size<inputs_t>::value>::type;
+
+/// Register bank covering q-function outputs only (same types as the slots
+/// from `n_inputs` onward in args_reg_t). Used where the primal / trial inputs
+/// live in a separate bank and only the test registers are integrated.
+/// Slot `o` of this bank is q-function parameter `n_inputs + o`.
+template <
+   typename backend_t,
+   typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1>
+using output_args_reg_t = typename build_args_reg_tuple_impl<backend_t, qfunc_t,
+      inputs_t, outputs_t, MQ1,
+      tuple_size<inputs_t>::value,
+      tuple_size<inputs_t>::value,
+      tuple_size<inputs_t>::value + tuple_size<outputs_t>::value>::type;
+
+template <typename ARG>
+MFEM_HOST_DEVICE inline void qf_set_flat_value(ARG &a, int c, real_t v)
+{
+   if constexpr (std::is_same_v<ARG, real_t>) { MFEM_CONTRACT_VAR(c); a = v; }
+   else if constexpr (is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(c);
+      a.value = v;
+   }
+   else
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      constexpr bool D = qf_param_uses_dual_v<ARG>;
+      if constexpr (RNK == 0)
+      {
+         if constexpr (D) { a(0).value = v; }
+         else { a(0) = v; }
+      }
+      else if constexpr (RNK == 1)
+      {
+         if constexpr (D) { a(c).value = v; }
+         else { a(c) = v; }
+      }
+      else
+      {
+         constexpr int e0 = qf_param_shape<ARG>::extents[0];
+         if constexpr (D) { a(c % e0, c / e0).value = v; }
+         else { a(c % e0, c / e0) = v; }
+      }
+   }
+}
+
+template <typename ARG>
+MFEM_HOST_DEVICE inline void qf_set_flat_gradient(ARG &a, int c, real_t v)
+{
+   if constexpr (is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(c);
+      a.gradient = v;
+   }
+   else if constexpr (qf_param_uses_dual_v<ARG>)
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      if constexpr (RNK == 0) { a(0).gradient = v; }
+      else if constexpr (RNK == 1) { a(c).gradient = v; }
+      else
+      {
+         constexpr int e0 = qf_param_shape<ARG>::extents[0];
+         a(c % e0, c / e0).gradient = v;
+      }
+   }
+   else
+   {
+      // Non-dual argument (e.g. Weight): never an active trial direction.
+      MFEM_CONTRACT_VAR(a);
+      MFEM_CONTRACT_VAR(c);
+      MFEM_CONTRACT_VAR(v);
+   }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// Two-index component access for a q-function argument.
+///
+/// `i` indexes vdim, `k` indexes the operator dimension, matching the
+/// column-major packing `c = i + extents[0]*k` of the flat accessors above.
+/// Prefer these wherever the caller already has both indices: the flat form
+/// would have to undo the packing with an integer division, which is expensive
+/// on device and pointless when `(i, k)` are right there.
+///
+/// The runtime extents of the callers agree with the static extents of `ARG`:
+/// for rank 2, `vdim == extents[0]` and `op_dim == extents[1]`; for rank 1 one
+/// of the two is 1 and the corresponding index is always 0, so `a(i + k)`
+/// selects the right component; for rank 0 both are 1.
+template <typename ARG>
+MFEM_HOST_DEVICE inline real_t qf_value_at(const ARG &a, int i, int k)
+{
+   if constexpr (std::is_same_v<ARG, real_t> || is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      return qf_store_value(a);
+   }
+   else
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      if constexpr (RNK == 0)
+      {
+         MFEM_CONTRACT_VAR(i);
+         MFEM_CONTRACT_VAR(k);
+         return qf_store_value(a(0));
+      }
+      else if constexpr (RNK == 1) { return qf_store_value(a(i + k)); }
+      else { return qf_store_value(a(i, k)); }
+   }
+}
+
+template <typename ARG>
+MFEM_HOST_DEVICE inline real_t qf_gradient_at(const ARG &a, int i, int k)
+{
+   if constexpr (is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      return a.gradient;
+   }
+   else if constexpr (qf_param_uses_dual_v<ARG>)
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      if constexpr (RNK == 0)
+      {
+         MFEM_CONTRACT_VAR(i);
+         MFEM_CONTRACT_VAR(k);
+         return a(0).gradient;
+      }
+      else if constexpr (RNK == 1) { return a(i + k).gradient; }
+      else { return a(i, k).gradient; }
+   }
+   else
+   {
+      // Non-dual argument carries no tangent: its derivative contribution is 0.
+      MFEM_CONTRACT_VAR(a);
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      return real_t(0);
+   }
+}
+
+template <typename ARG>
+MFEM_HOST_DEVICE inline void qf_set_value_at(ARG &a, int i, int k, real_t v)
+{
+   if constexpr (std::is_same_v<ARG, real_t>)
+   {
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      a = v;
+   }
+   else if constexpr (is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      a.value = v;
+   }
+   else
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      constexpr bool D = qf_param_uses_dual_v<ARG>;
+      if constexpr (RNK == 0)
+      {
+         MFEM_CONTRACT_VAR(i);
+         MFEM_CONTRACT_VAR(k);
+         if constexpr (D) { a(0).value = v; }
+         else { a(0) = v; }
+      }
+      else if constexpr (RNK == 1)
+      {
+         if constexpr (D) { a(i + k).value = v; }
+         else { a(i + k) = v; }
+      }
+      else
+      {
+         if constexpr (D) { a(i, k).value = v; }
+         else { a(i, k) = v; }
+      }
+   }
+}
+
+template <typename ARG>
+MFEM_HOST_DEVICE inline void qf_set_gradient_at(ARG &a, int i, int k, real_t v)
+{
+   if constexpr (is_dual_number<ARG>::value)
+   {
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      a.gradient = v;
+   }
+   else if constexpr (qf_param_uses_dual_v<ARG>)
+   {
+      constexpr int RNK = qf_param_shape<ARG>::rank;
+      if constexpr (RNK == 0)
+      {
+         MFEM_CONTRACT_VAR(i);
+         MFEM_CONTRACT_VAR(k);
+         a(0).gradient = v;
+      }
+      else if constexpr (RNK == 1) { a(i + k).gradient = v; }
+      else { a(i, k).gradient = v; }
+   }
+   else
+   {
+      // Non-dual argument (e.g. Weight): never an active trial direction.
+      MFEM_CONTRACT_VAR(a);
+      MFEM_CONTRACT_VAR(i);
+      MFEM_CONTRACT_VAR(k);
+      MFEM_CONTRACT_VAR(v);
+   }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// Maps each FOP slot to unionfds indices — used with dtqs / create_dtq_maps
+template<typename C, typename T>
+const auto create_union_field_map_for_dtq(C& ctx, T& io)
+{
+   using FE = Entity::Element;
+   return create_descriptors_to_fields_map<FE>(ctx.unionfds, io);
+}
+
+/// **`xe[i]`** slot per input FOP — indices into **`ctx.infds`** (`SIZE_MAX` for Weight).
+template<typename C, typename T>
+const auto create_input_vector_map(C& ctx, T& io)
+{
+   using FE = Entity::Element;
+   return create_descriptors_to_fields_map<FE>(ctx.infds, io);
+}
+
+/// **`ye[i]`** slot per output FOP — indices into **`ctx.outfds`**.
+template<typename C, typename T>
+const auto create_output_vector_map(C& ctx, T& io)
+{
+   using FE = Entity::Element;
+   return create_descriptors_to_fields_map<FE>(ctx.outfds, io);
+}
+
+template<typename C>
+const auto make_dtqs(C& ctx)
+{
+   std::vector<const DofToQuad*> dtq_vec;
+   dtq_vec.reserve(ctx.unionfds.size());
+   constexpr auto dtq_mode = DofToQuad::Mode::TENSOR;
+   for (const auto &field: ctx.unionfds)
+   {
+      auto dtq = GetDofToQuad<Entity::Element>(field, ctx.ir, dtq_mode);
+      dtq_vec.emplace_back(dtq);
+   }
+   return dtq_vec;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+template<typename Tuple>
+constexpr auto get_vdim(const Tuple& fields)
+{
+   return future::apply([](const auto&... f)
+   {
+      return std::array<int, sizeof...(f)> {f.vdim...};
+   }, fields);
+}
+
+template<typename Tuple>
+constexpr auto get_B(const Tuple& fields)
+{
+   return future::apply([](const auto&... f)
+   {
+      return std::array<const real_t*, sizeof...(f)> {f.B...};
+   }, fields);
+}
+
+template<typename Tuple>
+constexpr auto get_G(const Tuple& fields)
+{
+   return future::apply([](const auto&... f)
+   {
+      return std::array<const real_t*, sizeof...(f)> {f.G...};
+   }, fields);
+}
+
+template<typename Tuple>
+constexpr auto get_D1D(const Tuple& fields)
+{
+   return future::apply([](const auto&... f)
+   {
+      return std::array<int, sizeof...(f)> {f.B.GetShape()[2]...};
+   }, fields);
+}
+
+template<typename Tuple>
+constexpr auto get_Q1D(const Tuple& fields)
+{
+   return future::apply([](const auto&... f)
+   {
+      return std::array<int, sizeof...(f)> {f.B.GetShape()[0]...};
+   }, fields);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+/// Per-output FOP layout metadata (shared by derivative setup / apply kernels).
+
+template<typename outputs_t>
+constexpr auto compute_out_qp_size(const outputs_t &outs)
+{
+   constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   std::array<int, n_outputs> sizes{};
+   for_constexpr<n_outputs>([&](auto o) { sizes[o] = get<o>(outs).size_on_qp; });
+   return sizes;
+}
+
+template<typename outputs_t>
+constexpr auto compute_out_op_dim(const outputs_t &outs)
+{
+   constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
+   std::array<int, n_outputs> op{};
+   for_constexpr<n_outputs>([&](auto o)
+   {
+      op[o] = get<o>(outs).size_on_qp / get<o>(outs).vdim;
+   });
+   return op;
+}
+
+template<std::size_t N>
+constexpr std::array<int, N> compute_out_offsets(
+   const std::array<int, N> &vdim,
+   const std::array<int, N> &op_dim)
+{
+   std::array<int, N> offsets{};
+   offsets[0] = 0;
+   for (std::size_t o = 1; o < N; o++)
+   {
+      offsets[o] = offsets[o - 1] + vdim[o - 1] * op_dim[o - 1];
+   }
+   return offsets;
+}
+
+template<std::size_t N>
+constexpr std::array<int, N> compute_out_flat_offsets(
+   const std::array<int, N> &vdim,
+   const std::array<int, N> &op_dim,
+   const int num_qp)
+{
+   std::array<int, N> offsets{};
+   offsets[0] = 0;
+   for (std::size_t o = 1; o < N; o++)
+   {
+      offsets[o] = offsets[o - 1] + vdim[o - 1] * op_dim[o - 1] * num_qp;
+   }
+   return offsets;
+}
+
+template<typename inputs_t>
+const auto compute_input_is_dependent(const inputs_t &ins, int deriv_id)
+{
+   auto dependency_map = make_dependency_map(ins);
+   auto it = dependency_map.find(deriv_id);
+   MFEM_ASSERT(it != dependency_map.end(),
+               "Derivative ID not found in dependency map");
+   return it->second;
+}
+
+template<typename inputs_t>
+constexpr int compute_trial_vdim(const inputs_t &ins, int deriv_id)
+{
+   constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   int v = 1;
+   for_constexpr<n_inputs>([&](auto i)
+   {
+      if (get<i>(ins).GetFieldId() == deriv_id) { v = get<i>(ins).vdim; }
+   });
+   return v;
+}
+
+template<typename inputs_t>
+constexpr int compute_total_trial_op_dim(
+   const inputs_t &ins,
+   const std::array<bool, tuple_size<inputs_t>::value> &dep,
+   const std::array<int, tuple_size<inputs_t>::value> &size_on_qp)
+{
+   constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
+   int total = 0;
+   for_constexpr<n_inputs>([&](auto i)
+   {
+      if (dep[i]) { total += size_on_qp[i] / get<i>(ins).vdim; }
+   });
+   return total;
+}
+
+inline size_t find_union_field_index(const IntegratorContext &ctx, int field_id)
+{
+   for (size_t uf = 0; uf < ctx.unionfds.size(); uf++)
+   {
+      if (static_cast<int>(ctx.unionfds[uf].id) == field_id) { return uf; }
+   }
+   return SIZE_MAX;
+}
+
+inline size_t find_infd_index(const IntegratorContext &ctx, int field_id)
+{
+   for (size_t i = 0; i < ctx.infds.size(); i++)
+   {
+      if (static_cast<int>(ctx.infds[i].id) == field_id) { return i; }
+   }
+   return SIZE_MAX;
+}
+
+template<typename entity_t = Entity::Element>
+inline int compute_element_dof_sz(
+   const FieldDescriptor &fd,
+   int num_entities,
+   ElementDofOrdering ordering)
+{
+   auto R = get_restriction<entity_t>(fd, ordering);
+   MFEM_ASSERT(R != nullptr, "LocalQF: missing element restriction");
+   return num_entities ? (R->Height() / num_entities) : 0;
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+// Number of threads per 1D direction to launch the kernel with
+template <typename inputs_t, typename outputs_t,
+          std::size_t N_in, std::size_t N_out>
+inline int compute_kernel_thread_1d(
+   const int q1d,
+   const std::array<int, N_in> &in_d1d,
+   const std::array<int, N_out> &out_d1d)
+{
+   int t1d = q1d;
+   for_constexpr<N_in>([&](auto ic)
+   {
+      using FOP = tuple_element_t<ic.value, inputs_t>;
+      if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+      {
+         t1d = std::max(t1d, in_d1d[ic.value]);
+      }
+   });
+   for_constexpr<N_out>([&](auto ic)
+   {
+      using FOP = tuple_element_t<ic.value, outputs_t>;
+      if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+      {
+         t1d = std::max(t1d, out_d1d[ic.value]);
+      }
+   });
+   return t1d;
+}
+
+// Inputs-only variant: used by kernels whose outputs are written at qp
+template <typename inputs_t, std::size_t N_in>
+inline int compute_kernel_thread_1d(
+   const int q1d,
+   const std::array<int, N_in> &in_d1d)
+{
+   int t1d = q1d;
+   for_constexpr<N_in>([&](auto ic)
+   {
+      using FOP = tuple_element_t<ic.value, inputs_t>;
+      if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
+      {
+         t1d = std::max(t1d, in_d1d[ic.value]);
+      }
+   });
+   return t1d;
+}
+
+} // namespace mfem::future

fem/dfem/backends/scratch_bank.hpp

@@ -0,0 +1,277 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+#include "mfem.hpp"
+#include "../tuple.hpp"
+#include <memory>
+#include <type_traits>
+#include <vector>
+
+namespace mfem::future
+{
+// Scratch storage and q-function shadow helpers for dFEM backends. The bank
+// supports two scratch kinds:
+// - quadrature-point scratch: real_t buffers sized as NQ * components_per_qp,
+// - global scratch: one tuple of qfunction-local temporaries, independent of
+//   NQ, used for values such as flags, scalars, or small Vector workspaces.
+template <typename... GlobalScratchTypes>
+struct ScratchBank
+{
+
+   //=================================
+   ///<--- Global scratch utilities.
+   //=================================
+
+   using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
+
+   template <typename T>
+   static T MakeGlobalScratchShadow(const T &)
+   {
+      return T {};
+   }
+
+   static Vector MakeGlobalScratchShadow(const Vector &primal)
+   {
+      Vector shadow(primal.Size());
+      shadow.UseDevice(true);
+      shadow = 0.0;
+      return shadow;
+   }
+
+   template <typename Tuple, size_t... Is>
+   static auto MakeGlobalScratchShadowTuple(const Tuple &primal,
+                                            std::index_sequence<Is...>)
+   {
+      return make_tuple(MakeGlobalScratchShadow(get<Is>(primal))...);
+   }
+
+   template <typename Tuple>
+   static auto MakeGlobalScratchShadowTuple(const Tuple &primal)
+   {
+      return MakeGlobalScratchShadowTuple(
+                primal, std::make_index_sequence<tuple_size<Tuple>::value> {});
+   }
+
+
+   //===========================
+   ///<--- Scratch objects
+   //===========================
+
+   mutable GlobalScratchTuple global;
+
+   int nq = 0;
+   std::vector<int> components;
+   std::vector<int> sizes;
+   std::vector<std::shared_ptr<Vector>> owned;
+   std::vector<real_t *> ptrs;
+
+
+   //===========================
+   ///<--- Setter methods
+   //===========================
+
+   void SetScratch(const int nq_,
+                   std::initializer_list<int> components_per_qp = {1})
+   {
+      SetScratch(nq_, std::vector<int>(components_per_qp));
+   }
+
+   void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
+   {
+      nq = nq_;
+      components.clear();
+      sizes.clear();
+      owned.clear();
+      ptrs.clear();
+      for (int component_count : components_per_qp)
+      {
+         AddScratch(component_count);
+      }
+   }
+
+   void AddScratch(const int components_per_qp = 1)
+   {
+      MFEM_VERIFY(nq > 0, "SetScratch must be called before AddScratch");
+      MFEM_VERIFY(components_per_qp > 0,
+                  "scratch components per quadrature point must be positive");
+      owned.push_back(std::make_shared<Vector>());
+      Vector &scratch = *owned.back();
+      const int size = components_per_qp * nq;
+      scratch.SetSize(size);
+      scratch.UseDevice(true);
+      scratch = 0.0;
+      components.push_back(components_per_qp);
+      sizes.push_back(scratch.Size());
+      ptrs.push_back(scratch.ReadWrite());
+   }
+
+   void SetGlobalScratch(const GlobalScratchTuple &global_)
+   {
+      global = global_;
+   }
+
+
+   //===========================
+   ///<--- Getter methods
+   //===========================
+
+   real_t *GetScratchPointer(const int i) const { return ptrs[i]; }
+   real_t *operator[](const int i) const { return ptrs[i]; }
+
+   Vector &GetScratchVector(const int i) const { return *owned[i]; }
+
+   template <int I>
+   auto &GetGlobalScratch() const
+   {
+      return get<I>(global);
+   }
+
+
+   //===========================
+   ///<--- Utils methods
+   //===========================
+
+   void CloneScratchLayoutTo(ScratchBank &shadow) const
+   {
+      shadow.SetScratch(nq, components);
+      shadow.SetGlobalScratch(MakeGlobalScratchShadowTuple(global));
+   }
+
+   int Size() const { return static_cast<int>(ptrs.size()); }
+};
+
+// Shared base for Q-functions that use ScratchBank and need a matching scratch
+// shadow for forward differentiation.
+template <typename... GlobalScratchTypes>
+struct QFWithScratch
+{
+   using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
+
+   int nq = 0;
+   ScratchBank<GlobalScratchTypes...> scratch;
+
+   void SetScratch(const int nq_,
+                   std::initializer_list<int> components_per_qp = {1})
+   {
+      nq = nq_;
+      scratch.SetScratch(nq, components_per_qp);
+   }
+
+   void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
+   {
+      nq = nq_;
+      scratch.SetScratch(nq, components_per_qp);
+   }
+
+   void SetScratch(const int nq_, const int num_scratch_elem,
+                   const int components_per_qp = 1)
+   {
+      nq = nq_;
+      scratch.SetScratch(nq,
+                         std::vector<int>(num_scratch_elem, components_per_qp));
+   }
+
+   void SetGlobalScratch(const GlobalScratchTuple &global_scratch_)
+   {
+      scratch.SetGlobalScratch(global_scratch_);
+   }
+
+   Vector &GetScratchVector(const int i) const
+   {
+      return scratch.GetScratchVector(i);
+   }
+
+   real_t *GetScratchPointer(const int i) const
+   {
+      return scratch.GetScratchPointer(i);
+   }
+
+   template <int I>
+   auto &GetGlobalScratch() const
+   {
+      return scratch.template GetGlobalScratch<I>();
+   }
+
+   void CloneScratchLayoutTo(QFWithScratch &shadow) const
+   {
+      shadow.nq = nq;
+      scratch.CloneScratchLayoutTo(shadow.scratch);
+   }
+
+   QFWithScratch CreateShadow() const
+   {
+      QFWithScratch shadow;
+      CloneScratchLayoutTo(shadow);
+      return shadow;
+   }
+};
+
+using QFWithScratchType = QFWithScratch<>;
+using QFWithGlobalScratchType = QFWithScratch<bool, real_t, Vector>;
+
+namespace detail
+{
+
+template <typename T>
+struct qfunc_uses_scratch
+{
+private:
+   template <typename... GlobalScratchTypes>
+   static std::true_type Test(const QFWithScratch<GlobalScratchTypes...> *);
+
+   static std::false_type Test(...);
+
+public:
+   static constexpr bool value = decltype(Test(
+                                             static_cast<std::remove_cv_t<std::remove_reference_t<T>> *>(nullptr)))::value;
+};
+
+template <typename T>
+inline constexpr bool qfunc_uses_scratch_v =
+   qfunc_uses_scratch<T>::value;
+
+struct unused_qfunc_shadow { };
+
+template <typename qfunc_t, bool uses_scratch>
+struct qfunc_shadow_type
+{
+   using type = unused_qfunc_shadow;
+};
+
+template <typename qfunc_t>
+struct qfunc_shadow_type<qfunc_t, true>
+{
+   using type = decltype(std::declval<const qfunc_t &>().CreateShadow());
+};
+
+template <typename qfunc_t>
+using qfunc_shadow_t = typename qfunc_shadow_type<qfunc_t,
+      qfunc_uses_scratch_v<qfunc_t>>::type;
+
+// Create a persistent q-function shadow if the q-function uses scratch, otherwise return an empty struct.
+template <typename qfunc_t>
+inline qfunc_shadow_t<qfunc_t> MakeQFunctionShadowStorage(
+   const qfunc_t &qfunc)
+{
+   if constexpr (qfunc_uses_scratch_v<qfunc_t>)
+   {
+      return qfunc.CreateShadow();
+   }
+   else
+   {
+      MFEM_CONTRACT_VAR(qfunc);
+      return {};
+   }
+}
+
+} // namespace detail
+
+}

fem/dfem/doperator.cpp

@@ -11,92 +11,387 @@
 
 #include "doperator.hpp"
 
+#include <algorithm>
+
 #ifdef MFEM_USE_MPI
 
 using namespace mfem;
 using namespace mfem::future;
 
-void DifferentiableOperator::SetParameters(std::vector<Vector *> p) const
+namespace
 {
-   MFEM_ASSERT(parameters.size() == p.size(),
-               "number of parameters doesn't match descriptors");
-   for (size_t i = 0; i < parameters.size(); i++)
+int GetTotalTrueVSize(const std::vector<FieldDescriptor> &fds)
+{
+   int size = 0;
+   for (const auto &fd : fds)
    {
-      p[i]->Read();
-      parameters_l[i] = *p[i];
+      size += mfem::future::GetTrueVSize(fd);
    }
+   return size;
+}
+
+using DerivativeActionMap =
+   std::map<size_t, std::vector<derivative_action_t>>;
+using DerivativeSetupMap =
+   std::map<size_t, std::vector<derivative_setup_t>>;
+using DerivativeFieldMap =
+   std::map<size_t, std::vector<FieldDescriptor>>;
+using SparseAssemblyMap =
+   std::map<size_t,
+   std::vector<assemble_derivative_sparsematrix_callback_t>>;
+using HypreAssemblyMap =
+    std::map<size_t,
+             std::vector<assemble_derivative_hypreparmatrix_callback_t>>;
+using DiagonalAssemblyMap =
+    std::map<size_t, std::vector<assemble_diagonal_callback_t>>;
+using SecondDerivativeActionMap =
+    std::map<second_derivative_key_t, std::vector<derivative_action_t>>;
+using SecondDerivativeSetupMap =
+    std::map<second_derivative_key_t, std::vector<derivative_setup_t>>;
+using SecondDerivativeFieldMap =
+    std::map<second_derivative_key_t, std::vector<FieldDescriptor>>;
+using SecondSparseAssemblyMap =
+    std::map<second_derivative_key_t,
+             std::vector<assemble_derivative_sparsematrix_callback_t>>;
+using SecondHypreAssemblyMap =
+    std::map<second_derivative_key_t,
+             std::vector<assemble_derivative_hypreparmatrix_callback_t>>;
+using SecondDiagonalAssemblyMap =
+    std::map<second_derivative_key_t, std::vector<assemble_diagonal_callback_t>>;
+
+template <typename map_t>
+const typename map_t::mapped_type &FindOrDefault(
+    const map_t &map, const typename map_t::key_type &id,
+    const typename map_t::mapped_type &fallback)
+{
+   const auto it = map.find(id);
+   return it == map.end() ? fallback : it->second;
+}
+
+template <typename map_t>
+typename map_t::mapped_type FindOrEmpty(
+   const map_t &map, const typename map_t::key_type &id)
+{
+   const auto it = map.find(id);
+   return it == map.end() ? typename map_t::mapped_type{} : it->second;
+}
+
+const std::vector<derivative_action_t> &SelectActionCallbacks(
+   const std::vector<derivative_action_t> &direct_actions,
+   const DerivativeActionMap &cached_actions,
+   size_t derivative_id,
+   bool use_cached_setup)
+{
+   if (use_cached_setup)
+   {
+      const auto it_apply = cached_actions.find(derivative_id);
+      if (it_apply != cached_actions.end() && !it_apply->second.empty())
+      {
+         return it_apply->second;
+      }
+   }
+
+   return direct_actions;
+}
+
+struct DerivativeCallbackSet
+{
+   const DerivativeActionMap &actions;
+   const DerivativeActionMap &cached_actions;
+   const DerivativeActionMap &transpose_actions;
+   const DerivativeFieldMap &outfds;
+   const SparseAssemblyMap &assemble_sparse;
+   const HypreAssemblyMap &assemble_hypre;
+   const DiagonalAssemblyMap &assemble_diagonal;
+   const DerivativeSetupMap &setup;
+   const char *missing_action_message;
+};
+
+struct SecondDerivativeCallbackSet
+{
+   const SecondDerivativeActionMap &actions;
+   const SecondDerivativeActionMap &cached_actions;
+   const SecondDerivativeActionMap &transpose_actions;
+   const SecondDerivativeFieldMap &outfds;
+   const SecondSparseAssemblyMap &assemble_sparse;
+   const SecondHypreAssemblyMap &assemble_hypre;
+   const SecondDiagonalAssemblyMap &assemble_diagonal;
+   const SecondDerivativeSetupMap &setup;
+   const char *missing_action_message;
+};
+
+template <typename vector_t>
+std::shared_ptr<DerivativeOperator> MakeStatefulDerivativeOperator(
+   size_t derivative_id,
+   const vector_t &x,
+   const std::vector<FieldDescriptor> &infds,
+   const std::vector<FieldDescriptor> &default_outfds,
+   const DerivativeCallbackSet &callbacks,
+   bool use_cached_setup,
+   bool lvector_mode,
+   bool functional_gradient = false)
+{
+   const auto it_action = callbacks.actions.find(derivative_id);
+   MFEM_ASSERT(it_action != callbacks.actions.end(),
+               callbacks.missing_action_message << derivative_id);
+
+   const size_t dfidx = FindIdx(derivative_id, infds);
+   const auto &doutfds =
+      FindOrDefault(callbacks.outfds, derivative_id, default_outfds);
+   const auto &mult_callbacks =
+      SelectActionCallbacks(it_action->second, callbacks.cached_actions,
+                            derivative_id, use_cached_setup);
+
+   return std::make_shared<DerivativeOperator>(
+             GetTotalTrueVSize(doutfds),
+             GetTrueVSize(infds[dfidx]),
+             mult_callbacks,
+             FindOrEmpty(callbacks.transpose_actions, derivative_id),
+             infds[dfidx],
+             x,
+             infds,
+             doutfds,
+             FindOrEmpty(callbacks.assemble_sparse, derivative_id),
+             FindOrEmpty(callbacks.assemble_hypre, derivative_id),
+             FindOrEmpty(callbacks.assemble_diagonal, derivative_id),
+             FindOrEmpty(callbacks.setup, derivative_id),
+             lvector_mode,
+             functional_gradient);
+}
+
+const std::vector<derivative_action_t> &SelectSecondDerivativeActionCallbacks(
+    const std::vector<derivative_action_t> &direct_actions,
+    const SecondDerivativeActionMap &cached_actions,
+    second_derivative_key_t derivative_key,
+    bool use_cached_setup)
+{
+   if (use_cached_setup)
+   {
+      const auto it_apply = cached_actions.find(derivative_key);
+      if (it_apply != cached_actions.end() && !it_apply->second.empty())
+      {
+         return it_apply->second;
+      }
+   }
+
+   return direct_actions;
+}
+
+template <typename vector_t>
+std::shared_ptr<DerivativeOperator> MakeStatefulSecondDerivativeOperator(
+    size_t gradient_id,
+    size_t direction_id,
+    const vector_t &x,
+    const std::vector<FieldDescriptor> &infds,
+    const std::vector<FieldDescriptor> &default_outfds,
+    const SecondDerivativeCallbackSet &callbacks,
+    bool use_cached_setup,
+    bool lvector_mode)
+{
+   const second_derivative_key_t derivative_key{gradient_id, direction_id};
+   const auto it_action = callbacks.actions.find(derivative_key);
+   MFEM_ASSERT(it_action != callbacks.actions.end(),
+               callbacks.missing_action_message << "(" << gradient_id << ", "
+                                                << direction_id << ")");
+
+   const size_t dfidx = FindIdx(direction_id, infds);
+   const auto &doutfds =
+       FindOrDefault(callbacks.outfds, derivative_key, default_outfds);
+   const auto &mult_callbacks =
+       SelectSecondDerivativeActionCallbacks(
+           it_action->second, callbacks.cached_actions, derivative_key,
+           use_cached_setup);
+
+   return std::make_shared<DerivativeOperator>(
+       GetTotalTrueVSize(doutfds),
+       GetTrueVSize(infds[dfidx]),
+       mult_callbacks,
+       FindOrEmpty(callbacks.transpose_actions, derivative_key),
+       infds[dfidx],
+       x,
+       infds,
+       doutfds,
+       FindOrEmpty(callbacks.assemble_sparse, derivative_key),
+       FindOrEmpty(callbacks.assemble_hypre, derivative_key),
+       FindOrEmpty(callbacks.assemble_diagonal, derivative_key),
+       FindOrEmpty(callbacks.setup, derivative_key),
+       lvector_mode);
+}
 }
 
 DifferentiableOperator::DifferentiableOperator(
-   const std::vector<FieldDescriptor> &solutions,
-   const std::vector<FieldDescriptor> &parameters,
+   const std::vector<FieldDescriptor> &infds,
+   const std::vector<FieldDescriptor> &outfds,
    const ParMesh &mesh) :
+   Operator(),
    mesh(mesh),
-   solutions(solutions),
-   parameters(parameters)
+   infds(infds),
+   outfds(outfds)
 {
-   fields.resize(solutions.size() + parameters.size());
-   fields_e.resize(fields.size());
-   solutions_l.resize(solutions.size());
-   parameters_l.resize(parameters.size());
+   unionfds.clear();
+   unionfds.insert(unionfds.end(), infds.begin(), infds.end());
+   unionfds.insert(unionfds.end(), outfds.begin(), outfds.end());
+   std::sort(unionfds.begin(), unionfds.end());
+   auto last = std::unique(unionfds.begin(), unionfds.end());
+   unionfds.erase(last, unionfds.end());
 
-   for (size_t i = 0; i < solutions.size(); i++)
+   infields_l.resize(infds.size());
+   for (size_t i = 0; i < infds.size(); i++)
    {
-      fields[i] = solutions[i];
+      infields_l[i] = new Vector(GetVSize(infds[i]));
    }
 
-   for (size_t i = 0; i < parameters.size(); i++)
-   {
-      fields[i + solutions.size()] = parameters[i];
-   }
+   infields_e.resize(infds.size());
 }
 
-
-void FDJacobian::Mult(const Vector &v, Vector &y) const
+void DifferentiableOperator::SetMultLevel(MultLevel level)
 {
-   // See [1] for choice of eps.
-   //
-   // [1] Woodward, C.S., Gardner, D.J. and Evans, K.J., 2015. On the use of
-   // finite difference matrix-vector products in Newton-Krylov solvers for
-   // implicit climate dynamics with spectral elements. Procedia Computer
-   // Science, 51, pp.2036-2045.
-   real_t eps;
-   if (fixed_eps > 0.0)
-   {
-      eps = fixed_eps;
-   }
-   else
-   {
-      const real_t vnorm_local = v.Norml2();
-      real_t vnorm;
-      MPI_Allreduce(&vnorm_local, &vnorm, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
-                    MPI_COMM_WORLD);
-      eps = lambda * (lambda + xnorm / vnorm);
-   }
+   mult_level = level;
+}
 
-   // x + eps * v
-   {
-      const auto d_v = v.Read();
-      const auto d_x = x.Read();
-      auto d_xpev = xpev.Write();
-      mfem::forall(x.Size(), [=] MFEM_HOST_DEVICE (int i)
-      {
-         d_xpev[i] = d_x[i] + eps * d_v[i];
-      });
-   }
+void DifferentiableOperator::Mult(const Vector &x, Vector &y) const
+{
+   MFEM_ASSERT(!action_callbacks.empty(),
+               "no integrators have been set");
 
-   // y = f(x + eps * v)
-   op.Mult(xpev, y);
+   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&x),
+               "x needs to be a BlockVector");
 
-   // y = (f(x + eps * v) - f(x)) / eps
+   MFEM_ASSERT(dynamic_cast<const BlockVector*>(&y),
+               "y needs to be a BlockVector");
+
+   const auto &bx = static_cast<const BlockVector &>(x);
+   auto &by = static_cast<BlockVector &>(y);
+
+   Mult(bx, by);
+}
+
+void DifferentiableOperator::DisableTensorProductStructure(bool disable)
+{
+   use_tensor_product_structure = !disable;
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
+   size_t derivative_id, const Vector &x)
+{
+   return MakeStatefulDerivativeOperator(
+             derivative_id, x, infds, outfds,
    {
-      const auto d_f = f.Read();
-      auto d_y = y.ReadWrite();
-      mfem::forall(f.Size(), [=] MFEM_HOST_DEVICE (int i)
-      {
-         d_y[i] = (d_y[i] - d_f[i]) / eps;
-      });
-   }
+      derivative_action_callbacks,
+      derivative_apply_callbacks,
+      daction_transpose_callbacks,
+      derivative_outfds,
+      assemble_derivative_sparsematrix_callbacks,
+      assemble_derivative_hypreparmatrix_callbacks,
+      assemble_diagonal_callbacks,
+      derivative_setup_callbacks,
+      "no derivative action has been found for ID "
+   },
+   true,
+   mult_level == MultLevel::LVECTOR,
+   IsFunctionalDerivative(derivative_id));
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
+   size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
+{
+   return MakeStatefulDerivativeOperator(
+             derivative_id, x, infds, outfds,
+   {
+      derivative_action_callbacks,
+      derivative_apply_callbacks,
+      daction_transpose_callbacks,
+      derivative_outfds,
+      assemble_derivative_sparsematrix_callbacks,
+      assemble_derivative_hypreparmatrix_callbacks,
+      assemble_diagonal_callbacks,
+      derivative_setup_callbacks,
+      "no derivative action has been found for ID "
+   },
+   use_cached_setup,
+   mult_level == MultLevel::LVECTOR,
+   IsFunctionalDerivative(derivative_id));
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
+   size_t derivative_id)
+{
+   MFEM_ASSERT(has_functional_integrator,
+               "stateless GetDerivative is available only for functionals");
+
+   const auto it_action = derivative_action_callbacks.find(derivative_id);
+   MFEM_ASSERT(it_action != derivative_action_callbacks.end(),
+               "no derivative action has been found for ID " << derivative_id);
+
+   const size_t dfidx = FindIdx(derivative_id, infds);
+   const auto &doutfds =
+      FindOrDefault(derivative_outfds, derivative_id, outfds);
+
+   return std::make_shared<DerivativeOperator>(
+             GetTotalTrueVSize(doutfds),
+             GetTrueVSize(infds[dfidx]),
+             it_action->second,
+             infds,
+             doutfds);
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
+   size_t derivative_id, const Vector &x)
+{
+   return GetSecondDerivative(derivative_id, derivative_id, x);
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
+   size_t gradient_id, size_t direction_id, const Vector &x)
+{
+   MFEM_ASSERT(has_functional_integrator,
+               "second derivatives are available only for functionals");
+
+   return MakeStatefulSecondDerivativeOperator(
+             gradient_id, direction_id, x, infds, outfds,
+   {
+      second_derivative_action_callbacks,
+      second_derivative_apply_callbacks,
+      second_daction_transpose_callbacks,
+      second_derivative_outfds,
+      assemble_second_derivative_sparsematrix_callbacks,
+      assemble_second_derivative_hypreparmatrix_callbacks,
+      assemble_second_derivative_diagonal_callbacks,
+      second_derivative_setup_callbacks,
+      "no second derivative action has been found for ID "
+   },
+   false,
+   mult_level == MultLevel::LVECTOR);
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
+   size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
+{
+   return GetSecondDerivative(derivative_id, derivative_id, x, use_cached_setup);
+}
+
+std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
+   size_t gradient_id, size_t direction_id, const MultiVector &x,
+   const bool use_cached_setup)
+{
+   MFEM_ASSERT(has_functional_integrator,
+               "second derivatives are available only for functionals");
+
+   return MakeStatefulSecondDerivativeOperator(
+             gradient_id, direction_id, x, infds, outfds,
+   {
+      second_derivative_action_callbacks,
+      second_derivative_apply_callbacks,
+      second_daction_transpose_callbacks,
+      second_derivative_outfds,
+      assemble_second_derivative_sparsematrix_callbacks,
+      assemble_second_derivative_hypreparmatrix_callbacks,
+      assemble_second_derivative_diagonal_callbacks,
+      second_derivative_setup_callbacks,
+      "no second derivative action has been found for ID "
+   },
+   use_cached_setup,
+   mult_level == MultLevel::LVECTOR);
 }
 
 #endif // MFEM_USE_MPI

fem/dfem/fielddescriptor.hpp

@@ -0,0 +1,64 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../fespace.hpp"
+#include "../qspace.hpp"
+#include "parameterspace.hpp"
+
+namespace mfem::future
+{
+
+/// @brief FieldDescriptor struct
+///
+/// This struct is used to store information about a field.
+struct FieldDescriptor
+{
+   using data_variant_t =
+      std::variant<const FiniteElementSpace *,
+      const ParFiniteElementSpace *,
+      const VectorQuadratureSpace *,
+      const ParameterSpace *>;
+
+   /// Field ID
+   std::size_t id;
+
+   /// Field variant
+   data_variant_t data;
+
+   /// Default constructor
+   FieldDescriptor() :
+      id(SIZE_MAX), data(data_variant_t{}) {}
+
+   /// Constructor
+   template <typename T>
+   FieldDescriptor(std::size_t field_id, const T* v) :
+      id(field_id), data(v) {}
+
+   bool operator==(const FieldDescriptor& other) const
+   {
+      return id == other.id;
+   }
+
+   bool operator<(const FieldDescriptor& other) const
+   {
+      return id < other.id;
+   }
+
+   friend void swap(FieldDescriptor& a, FieldDescriptor& b)
+   {
+      using std::swap;
+      swap(a.id, b.id);
+      swap(a.data, b.data);
+   }
+};
+
+}

fem/dfem/fieldoperator.hpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 #pragma once
 
+#include <ostream>
 #include <type_traits>
 
 namespace mfem::future
@@ -74,6 +75,15 @@ struct is_identity_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_identity_fop<Identity<FIELD_ID>> : std::true_type {};
 
+template <typename T>
+constexpr bool is_identity_fop_v = is_identity_fop<T>::value;
+
+template <int FIELD_ID>
+inline std::ostream& operator<<(std::ostream& out, Identity<FIELD_ID>)
+{
+   return out << "Identity<" << FIELD_ID << ">";
+}
+
 /// @brief Weight FieldOperator.
 ///
 /// This FieldOperator is used to signal that this field contains the quadrature
@@ -90,6 +100,14 @@ struct is_weight_fop : std::false_type {};
 template <>
 struct is_weight_fop<Weight> : std::true_type {};
 
+template <typename T>
+constexpr bool is_weight_fop_v = is_weight_fop<T>::value;
+
+inline std::ostream& operator<<(std::ostream& out, Weight)
+{
+   return out << "Weight";
+}
+
 /// @brief Value FieldOperator.
 ///
 /// This FieldOperator is used to signal that the field contains the
@@ -101,11 +119,20 @@ public:
    constexpr Value() : FieldOperator<FIELD_ID>() {};
 };
 
-template< typename T >
+template <typename T>
 struct is_value_fop : std::false_type {};
 
+template <int T>
+struct is_value_fop<Value<T>> : std::true_type {};
+
+template <typename T>
+constexpr bool is_value_fop_v = is_value_fop<T>::value;
+
 template <int FIELD_ID>
-struct is_value_fop<Value<FIELD_ID>> : std::true_type {};
+inline std::ostream& operator<<(std::ostream& out, Value<FIELD_ID>)
+{
+   return out << "Value<" << FIELD_ID << ">";
+}
 
 /// @brief Gradient FieldOperator.
 ///
@@ -124,6 +151,15 @@ struct is_gradient_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_gradient_fop<Gradient<FIELD_ID>> : std::true_type {};
 
+template <typename T>
+constexpr bool is_gradient_fop_v = is_gradient_fop<T>::value;
+
+template <int FIELD_ID>
+inline std::ostream& operator<<(std::ostream& out, Gradient<FIELD_ID>)
+{
+   return out << "Gradient<" << FIELD_ID << ">";
+}
+
 /// @brief Sum FieldOperator.
 ///
 /// This FieldOperator is commonly used to signal that an output of a quadrature
@@ -141,4 +177,39 @@ struct is_sum_fop : std::false_type {};
 template <int FIELD_ID>
 struct is_sum_fop<Sum<FIELD_ID>> : std::true_type {};
 
+template <typename T>
+constexpr bool is_sum_fop_v = is_sum_fop<T>::value;
+
+template <int FIELD_ID>
+inline std::ostream& operator<<(std::ostream& out, Sum<FIELD_ID>)
+{
+   return out << "Sum<" << FIELD_ID << ">";
+}
+
+/// @brief FunctionalValue FieldOperator.
+///
+/// This FieldOperator is commonly used to signal that an output of a quadrature
+/// function should be summed.
+template <int FIELD_ID = -1>
+class FunctionalValue : public FieldOperator<FIELD_ID>
+{
+public:
+   constexpr FunctionalValue() : FieldOperator<FIELD_ID>() {};
+};
+
+template< typename T >
+struct is_functionalvalue_fop : std::false_type {};
+
+template <int FIELD_ID>
+struct is_functionalvalue_fop<FunctionalValue<FIELD_ID>> : std::true_type {};
+
+template <typename T>
+constexpr bool is_functionalvalue_fop_v = is_functionalvalue_fop<T>::value;
+
+template <int FIELD_ID>
+inline std::ostream& operator<<(std::ostream& out, FunctionalValue<FIELD_ID>)
+{
+   return out << "FunctionalValue<" << FIELD_ID << ">";
+}
+
 } // namespace mfem::future

fem/dfem/integrate.hpp

@@ -1,536 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "util.hpp"
-
-namespace mfem::future
-{
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields_impl(
-   DeviceTensor<2, real_t> &y,
-   const DeviceTensor<3, real_t> &f,
-   const output_t &output,
-   const DofToQuadMap &dtq)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   // assuming the quadrature point residual has to "play nice with
-   // the test function"
-   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [num_qp, cdim, num_dof] = B.GetShape();
-      const int vdim = output.vdim > 0 ? output.vdim : cdim ;
-      for (int dof = 0; dof < num_dof; dof++)
-      {
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            real_t acc = 0.0;
-            for (int qp = 0; qp < num_qp; qp++)
-            {
-               acc += B(qp, 0, dof) * f(vd, 0, qp);
-            }
-            y(dof, vd) += acc;
-         }
-      }
-   }
-   else if constexpr (
-      is_gradient_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [num_qp, dim, num_dof] = G.GetShape();
-      const int vdim = output.vdim;
-      for (int dof = 0; dof < num_dof; dof++)
-      {
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            real_t acc = 0.0;
-            for (int d = 0; d < dim; d++)
-            {
-               for (int qp = 0; qp < num_qp; qp++)
-               {
-                  acc += G(qp, d, dof) * f(vd, d, qp);
-               }
-            }
-            y(dof, vd) += acc;
-         }
-      }
-   }
-   else if constexpr (is_sum_fop<std::decay_t<output_t>>::value)
-   {
-      // This is the "integral over all quadrature points type" applying
-      // B = 1 s.t. B^T * C \in R^1.
-      const auto [num_qp, unused, unused1] = B.GetShape();
-      auto cc = Reshape(&f(0, 0, 0), num_qp);
-      for (int i = 0; i < num_qp; i++)
-      {
-         y(0, 0) += cc(i);
-      }
-   }
-   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [num_qp, unused, num_dof] = B.GetShape();
-      const auto vdim = output.vdim;
-      auto cc = Reshape(&f(0, 0, 0), num_qp * vdim);
-      auto yy = Reshape(&y(0, 0), num_qp * vdim);
-      for (int i = 0; i < num_qp * vdim; i++)
-      {
-         yy(i) = cc(i);
-      }
-   }
-   else
-   {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        " for this field descriptor");
-   }
-}
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields_tensor_impl_1d(
-   DeviceTensor<2, real_t> &y,
-   const DeviceTensor<3, real_t> &f,
-   const output_t &output,
-   const DofToQuadMap &dtq,
-   std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, vdim);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dx, x, d1d)
-         {
-            real_t acc = 0.0;
-            for (int qx = 0; qx < q1d; qx++)
-            {
-               acc += fqp(vd, 0, qx) * B(qx, 0, dx);
-            }
-            yd(dx, vd) = acc;
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = G.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, vdim);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dx, x, d1d)
-         {
-            real_t acc = 0.0;
-            for (int qx = 0; qx < q1d; qx++)
-            {
-               acc += fqp(vd, 0, qx) * G(qx, 0, dx);
-            }
-            yd(dx, vd) = acc;
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
-      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
-
-      for (int sq = 0; sq < output.size_on_qp; sq++)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            yqp(sq, qx) = fqp(sq, qx);
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else
-   {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        "for this field descriptor with sum factorization on"
-                        " tensor product elements");
-   }
-}
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields_tensor_impl_2d(
-   DeviceTensor<2, real_t> &y,
-   const DeviceTensor<3, real_t> &f,
-   const output_t &output,
-   const DofToQuadMap &dtq,
-   std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
-
-      auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               real_t acc = 0.0;
-               for (int qx = 0; qx < q1d; qx++)
-               {
-                  acc += fqp(vd, 0, qx, qy) * B(qx, 0, dx);
-               }
-               s0(qy, dx) = acc;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               real_t acc = 0.0;
-               for (int qy = 0; qy < q1d; qy++)
-               {
-                  acc += s0(qy, dx) * B(qy, 0, dy);
-               }
-               yd(dx, dy, vd) += acc;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = G.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
-
-      auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
-      auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               real_t uv[2] = {0.0, 0.0};
-               for (int qx = 0; qx < q1d; qx++)
-               {
-                  uv[0] += fqp(vd, 0, qx, qy) * G(qx, 0, dx);
-                  uv[1] += fqp(vd, 1, qx, qy) * B(qx, 0, dx);
-               }
-               s0(qy, dx) = uv[0];
-               s1(qy, dx) = uv[1];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               real_t uv[2] = {0.0, 0.0};
-               for (int qy = 0; qy < q1d; qy++)
-               {
-                  uv[0] += s0(qy, dx) * B(qy, 0, dy);
-                  uv[1] += s1(qy, dx) * G(qy, 0, dy);
-               }
-               yd(dx, dy, vd) += uv[0] + uv[1];
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-
-      // // TODO: Check if this is the right fix for all cases
-      // auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
-      // auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
-      // for (int sq = 0; sq < output.size_on_qp; sq++)
-      // {
-      //    MFEM_FOREACH_THREAD(qx, x, q1d)
-      //    {
-      //       yqp(sq, qx) = fqp(sq, qx);
-      //    }
-      //    MFEM_SYNC_THREAD;
-      // }
-
-      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d);
-      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d);
-
-      for (int sq = 0; sq < output.size_on_qp; sq++)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               yqp(sq, qx, qy) = fqp(sq, qx, qy);
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else
-   {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        " for this field descriptor with sum factorization on"
-                        " tensor product elements");
-   }
-}
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields_tensor_impl_3d(
-   DeviceTensor<2, real_t> &y,
-   const DeviceTensor<3, real_t> &f,
-   const output_t &output,
-   const DofToQuadMap &dtq,
-   std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
-
-      auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
-      auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d, d1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  real_t acc = 0.0;
-                  for (int qx = 0; qx < q1d; qx++)
-                  {
-                     acc += fqp(vd, 0, qx, qy, qz) * B(qx, 0, dx);
-                  }
-                  s0(qz, qy, dx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  real_t acc = 0.0;
-                  for (int qy = 0; qy < q1d; qy++)
-                  {
-                     acc += s0(qz, qy, dx) * B(qy, 0, dy);
-                  }
-                  s1(qz, dy, dx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
-            {
-               MFEM_FOREACH_THREAD(dz, z, d1d)
-               {
-                  real_t acc = 0.0;
-                  for (int qz = 0; qz < q1d; qz++)
-                  {
-                     acc += s1(qz, dy, dx) * B(qz, 0, dz);
-                  }
-                  yd(dx, dy, dz, vd) += acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = G.GetShape();
-      const int vdim = output.vdim;
-      const int test_dim = output.size_on_qp / vdim;
-      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
-      auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
-
-      auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
-      auto s1 = Reshape(&scratch_mem[1](0), q1d, q1d, d1d);
-      auto s2 = Reshape(&scratch_mem[2](0), q1d, q1d, d1d);
-      auto s3 = Reshape(&scratch_mem[3](0), q1d, d1d, d1d);
-      auto s4 = Reshape(&scratch_mem[4](0), q1d, d1d, d1d);
-      auto s5 = Reshape(&scratch_mem[5](0), q1d, d1d, d1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int qx = 0; qx < q1d; qx++)
-                  {
-                     uvw[0] += fqp(vd, 0, qx, qy, qz) * G(qx, 0, dx);
-                     uvw[1] += fqp(vd, 1, qx, qy, qz) * B(qx, 0, dx);
-                     uvw[2] += fqp(vd, 2, qx, qy, qz) * B(qx, 0, dx);
-                  }
-                  s0(qz, qy, dx) = uvw[0];
-                  s1(qz, qy, dx) = uvw[1];
-                  s2(qz, qy, dx) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int qy = 0; qy < q1d; qy++)
-                  {
-                     uvw[0] += s0(qz, qy, dx) * B(qy, 0, dy);
-                     uvw[1] += s1(qz, qy, dx) * G(qy, 0, dy);
-                     uvw[2] += s2(qz, qy, dx) * B(qy, 0, dy);
-                  }
-                  s3(qz, dy, dx) = uvw[0];
-                  s4(qz, dy, dx) = uvw[1];
-                  s5(qz, dy, dx) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int qz = 0; qz < q1d; qz++)
-                  {
-                     uvw[0] += s3(qz, dy, dx) * B(qz, 0, dz);
-                     uvw[1] += s4(qz, dy, dx) * B(qz, 0, dz);
-                     uvw[2] += s5(qz, dy, dx) * G(qz, 0, dz);
-                  }
-                  yd(dx, dy, dz, vd) += uvw[0] + uvw[1] + uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d, q1d);
-      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
-
-      for (int sq = 0; sq < output.size_on_qp; sq++)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
-               {
-                  yqp(sq, qx, qy, qz) = fqp(sq, qx, qy, qz);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else
-   {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        " for this field descriptor with sum factorization on"
-                        " tensor product elements");
-   }
-}
-
-template <typename output_t>
-MFEM_HOST_DEVICE
-void map_quadrature_data_to_fields(
-   DeviceTensor<2, real_t> &y,
-   const DeviceTensor<3, real_t> &f,
-   const output_t &output,
-   const DofToQuadMap &dtq,
-   std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 1)
-      {
-         map_quadrature_data_to_fields_tensor_impl_1d(y, f, output, dtq, scratch_mem);
-      }
-      else if (dimension == 2)
-      {
-         map_quadrature_data_to_fields_tensor_impl_2d(y, f, output, dtq, scratch_mem);
-      }
-      else if (dimension == 3)
-      {
-         map_quadrature_data_to_fields_tensor_impl_3d(y, f, output, dtq, scratch_mem);
-      }
-      else { MFEM_ABORT_KERNEL("dimension not supported"); }
-   }
-   else
-   {
-      map_quadrature_data_to_fields_impl(y, f, output, dtq);
-   }
-}
-
-} // namespace mfem::future

fem/dfem/integrator_ctx.hpp

@@ -0,0 +1,37 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include <typeindex>
+#include <unordered_map>
+#include <vector>
+
+#include "../../general/array.hpp"
+#include "fielddescriptor.hpp"
+
+namespace mfem::future
+{
+
+struct IntegratorContext
+{
+   const ParMesh &mesh;
+   const Array<int> *elem_attr;
+   Array<int> attr;
+   const int nentities;
+   const std::vector<FieldDescriptor> &infds;
+   const std::vector<FieldDescriptor> &outfds;
+   const std::vector<FieldDescriptor> &unionfds;
+   const IntegrationRule &ir;
+   std::unordered_map<std::type_index, std::vector<int>> &in_qlayouts;
+   std::unordered_map<std::type_index, std::vector<int>> &out_qlayouts;
+};
+
+}

fem/dfem/interpolate.hpp

@@ -1,678 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "util.hpp"
-
-namespace mfem::future
-{
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE inline
-void map_field_to_quadrature_data_tensor_product_3d(
-   DeviceTensor<2> &field_qp,
-   const DofToQuadMap &dtq,
-   const DeviceTensor<1> &field_e,
-   const field_operator_t &input,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
-   {
-      auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d, q1d);
-      auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
-      auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t acc = 0.0;
-                  for (int dx = 0; dx < d1d; dx++)
-                  {
-                     acc += B(qx, 0, dx) * field(dx, dy, dz, vd);
-                  }
-                  s0(dz, dy, qx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            {
-               MFEM_FOREACH_THREAD(qy, y, q1d)
-               {
-                  real_t acc = 0.0;
-                  for (int dy = 0; dy < d1d; dy++)
-                  {
-                     acc += s0(dz, dy, qx) * B(qy, 0, dy);
-                  }
-                  s1(dz, qy, qx) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t acc = 0.0;
-                  for (int dz = 0; dz < d1d; dz++)
-                  {
-                     acc += s1(dz, qy, qx) * B(qz, 0, dz);
-                  }
-                  fqp(vd, qx, qy, qz) = acc;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (
-      is_gradient_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const int dim = input.dim;
-      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d, q1d);
-
-      auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
-      auto s1 = Reshape(&scratch_mem[1](0), d1d, d1d, q1d);
-      auto s2 = Reshape(&scratch_mem[2](0), d1d, q1d, q1d);
-      auto s3 = Reshape(&scratch_mem[3](0), d1d, q1d, q1d);
-      auto s4 = Reshape(&scratch_mem[4](0), d1d, q1d, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t uv[2] = {0.0, 0.0};
-                  for (int dx = 0; dx < d1d; dx++)
-                  {
-                     const real_t f = field(dx, dy, dz, vd);
-                     uv[0] += f * B(qx, 0, dx);
-                     uv[1] += f * G(qx, 0, dx);
-                  }
-                  s0(dz, dy, qx) = uv[0];
-                  s1(dz, dy, qx) = uv[1];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(dz, z, d1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int dy = 0; dy < d1d; dy++)
-                  {
-                     const real_t s0i = s0(dz, dy, qx);
-                     uvw[0] += s1(dz, dy, qx) * B(qy, 0, dy);
-                     uvw[1] += s0i * G(qy, 0, dy);
-                     uvw[2] += s0i * B(qy, 0, dy);
-                  }
-                  s2(dz, qy, qx) = uvw[0];
-                  s3(dz, qy, qx) = uvw[1];
-                  s4(dz, qy, qx) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qz, z, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
-               {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
-                  for (int dz = 0; dz < d1d; dz++)
-                  {
-                     uvw[0] += s2(dz, qy, qx) * B(qz, 0, dz);
-                     uvw[1] += s3(dz, qy, qx) * B(qz, 0, dz);
-                     uvw[2] += s4(dz, qy, qx) * G(qz, 0, dz);
-                  }
-                  fqp(vd, 0, qx, qy, qz) = uvw[0];
-                  fqp(vd, 1, qx, qy, qz) = uvw[1];
-                  fqp(vd, 2, qx, qy, qz) = uvw[2];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   // TODO: Create separate function for clarity
-   else if constexpr (
-      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
-   {
-      const int num_qp = integration_weights.GetShape()[0];
-      // TODO: eeek
-      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
-      auto w = Reshape(&integration_weights[0], q1d, q1d, q1d);
-      auto f = Reshape(&field_qp[0], q1d, q1d, q1d);
-      MFEM_FOREACH_THREAD(qx, x, q1d)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(qz, z, q1d)
-            {
-               f(qx, qy, qz) = w(qx, qy, qz);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const int q1d = B.GetShape()[0];
-      auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d * q1d);
-      field_qp = field;
-   }
-   else
-   {
-      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
-                    "can't map field to quadrature data");
-   }
-}
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE inline
-void map_field_to_quadrature_data_tensor_product_2d(
-   DeviceTensor<2> &field_qp,
-   const DofToQuadMap &dtq,
-   const DeviceTensor<1> &field_e,
-   const field_operator_t &input,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
-   {
-      auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d);
-      auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            {
-               real_t acc = 0.0;
-               for (int dx = 0; dx < d1d; dx++)
-               {
-                  acc += B(qx, 0, dx) * field(dx, dy, vd);
-               }
-               s0(dy, qx) = acc;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
-            {
-               real_t acc = 0.0;
-               for (int dy = 0; dy < d1d; dy++)
-               {
-                  acc += s0(dy, qx) * B(qy, 0, dy);
-               }
-               fqp(vd, qx, qy) = acc;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   else if constexpr (
-      is_gradient_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const int dim = input.dim;
-      const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d);
-
-      auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
-      auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(dy, y, d1d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            {
-               real_t uv[2] = {0.0, 0.0};
-               for (int dx = 0; dx < d1d; dx++)
-               {
-                  const real_t f = field(dx, dy, vd);
-                  uv[0] += f * B(qx, 0, dx);
-                  uv[1] += f * G(qx, 0, dx);
-               }
-               s0(dy, qx) = uv[0];
-               s1(dy, qx) = uv[1];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            {
-               real_t uv[2] = {0.0, 0.0};
-               for (int dy = 0; dy < d1d; dy++)
-               {
-                  const real_t s0i = s0(dy, qx);
-                  uv[0] += s1(dy, qx) * B(qy, 0, dy);
-                  uv[1] += s0i * G(qy, 0, dy);
-               }
-               fqp(vd, 0, qx, qy) = uv[0];
-               fqp(vd, 1, qx, qy) = uv[1];
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   // TODO: Create separate function for clarity
-   else if constexpr (
-      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
-   {
-      const int num_qp = integration_weights.GetShape()[0];
-      // TODO: eeek
-      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
-      auto w = Reshape(&integration_weights[0], q1d, q1d);
-      auto f = Reshape(&field_qp[0], q1d, q1d);
-      MFEM_FOREACH_THREAD(qx, x, q1d)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            f(qx, qy) = w(qx, qy);
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const int q1d = B.GetShape()[0];
-      auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d);
-      field_qp = field;
-   }
-   else
-   {
-      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
-                    "can't map field to quadrature data");
-   }
-}
-
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE inline
-void map_field_to_quadrature_data_tensor_product_1d(
-   DeviceTensor<2> &field_qp,
-   const DofToQuadMap &dtq,
-   const DeviceTensor<1> &field_e,
-   const field_operator_t &input,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-
-   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
-   {
-      auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e[0], d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            real_t acc = 0.0;
-            for (int dx = 0; dx < d1d; dx++)
-            {
-               acc += B(qx, 0, dx) * field(dx, vd);
-            }
-            fqp(vd, qx) = acc;
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (
-      is_gradient_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const auto [q1d, unused, d1d] = B.GetShape();
-      const int vdim = input.vdim;
-      const int dim = input.dim;
-      const auto field = Reshape(&field_e[0], d1d, vdim);
-      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            real_t acc = 0.0;
-            for (int dx = 0; dx < d1d; dx++)
-            {
-               acc += G(qx, 0, dx) * field(dx, vd);
-            }
-            fqp(vd, 0, qx) = acc;
-         }
-         MFEM_SYNC_THREAD;
-      }
-   }
-   // TODO: Create separate function for clarity
-   else if constexpr (
-      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
-   {
-      const int num_qp = integration_weights.GetShape()[0];
-      // TODO: eeek
-      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
-      auto w = Reshape(&integration_weights[0], q1d);
-      auto f = Reshape(&field_qp[0], q1d);
-      MFEM_FOREACH_THREAD(qx, x, q1d)
-      {
-         f(qx) = w(qx);
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
-   {
-      const int q1d = B.GetShape()[0];
-      auto field = Reshape(&field_e[0], input.size_on_qp, q1d);
-      field_qp = field;
-   }
-   else
-   {
-      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
-                    "can't map field to quadrature data");
-   }
-}
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE
-void map_field_to_quadrature_data(
-   DeviceTensor<2> field_qp,
-   const DofToQuadMap &dtq,
-   const DeviceTensor<1> &field_e,
-   const field_operator_t &input,
-   const DeviceTensor<1, const real_t> &integration_weights)
-{
-   [[maybe_unused]] auto B = dtq.B;
-   [[maybe_unused]] auto G = dtq.G;
-   if constexpr (is_value_fop<field_operator_t>::value)
-   {
-      auto [num_qp, dim, num_dof] = B.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e(0), num_dof, vdim);
-
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         for (int qp = 0; qp < num_qp; qp++)
-         {
-            real_t acc = 0.0;
-            for (int dof = 0; dof < num_dof; dof++)
-            {
-               acc += B(qp, 0, dof) * field(dof, vd);
-            }
-            field_qp(vd, qp) = acc;
-         }
-      }
-   }
-   else if constexpr (is_gradient_fop<field_operator_t>::value)
-   {
-      const auto [num_qp, dim, num_dof] = G.GetShape();
-      const int vdim = input.vdim;
-      const auto field = Reshape(&field_e(0), num_dof, vdim);
-
-      auto f = Reshape(&field_qp[0], vdim, dim, num_qp);
-      for (int vd = 0; vd < vdim; vd++)
-      {
-         for (int qp = 0; qp < num_qp; qp++)
-         {
-            for (int d = 0; d < dim; d++)
-            {
-               real_t acc = 0.0;
-               for (int dof = 0; dof < num_dof; dof++)
-               {
-                  acc += G(qp, d, dof) * field(dof, vd);
-               }
-               f(vd, d, qp) = acc;
-            }
-         }
-      }
-   }
-   else if constexpr (std::is_same_v<field_operator_t, Weight>)
-   {
-      const int num_qp = integration_weights.GetShape()[0];
-      auto f = Reshape(&field_qp[0], num_qp);
-      for (int qp = 0; qp < num_qp; qp++)
-      {
-         f(qp) = integration_weights(qp);
-      }
-   }
-   else if constexpr (is_identity_fop<field_operator_t>::value)
-   {
-      auto [num_qp, unused, num_dof] = B.GetShape();
-      const int size_on_qp = input.size_on_qp;
-      const auto field = Reshape(&field_e[0], size_on_qp * num_qp);
-      auto f = Reshape(&field_qp[0], size_on_qp * num_qp);
-      for (int i = 0; i < size_on_qp * num_qp; i++)
-      {
-         f(i) = field(i);
-      }
-   }
-   else
-   {
-      static_assert(dfem::always_false<field_operator_t>,
-                    "can't map field to quadrature data");
-
-   }
-}
-
-template <typename field_operator_ts, size_t num_inputs, size_t num_fields>
-MFEM_HOST_DEVICE inline
-void map_fields_to_quadrature_data(
-   std::array<DeviceTensor<2>, num_inputs> &fields_qp,
-   const std::array<DeviceTensor<1>, num_fields> &fields_e,
-   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
-   const std::array<size_t, num_inputs> &input_to_field,
-   const field_operator_ts &fops,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const int &dimension,
-   const bool &use_sum_factorization = false)
-{
-   // When the input_to_field map returns -1, this means the requested input
-   // is the integration weight. Weights don't have a user defined field
-   // attached to them and we create a dummy field which is not accessed
-   // inside the functions it is passed to.
-   const auto dummy_field_weight = DeviceTensor<1>(nullptr, 0);
-   for_constexpr<num_inputs>([&](auto i)
-   {
-      const DeviceTensor<1> &field_e =
-         (input_to_field[i] == SIZE_MAX) ? dummy_field_weight :
-         fields_e[input_to_field[i]];
-
-      if (use_sum_factorization)
-      {
-         if (dimension == 1)
-         {
-            map_field_to_quadrature_data_tensor_product_1d(
-               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
-               integration_weights, scratch_mem);
-         }
-         else if (dimension == 2)
-         {
-            map_field_to_quadrature_data_tensor_product_2d(
-               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
-               integration_weights, scratch_mem);
-         }
-         else if (dimension == 3)
-         {
-            map_field_to_quadrature_data_tensor_product_3d(
-               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
-               integration_weights, scratch_mem);
-         }
-         else
-         {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-            MFEM_ABORT("unsupported dimension");
-#endif
-         }
-      }
-      else
-      {
-         map_field_to_quadrature_data(
-            fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
-            integration_weights);
-      }
-   });
-}
-
-template <typename field_operator_t>
-MFEM_HOST_DEVICE
-void map_field_to_quadrature_data_conditional(
-   DeviceTensor<2> &field_qp,
-   const DeviceTensor<1> &field_e,
-   const DofToQuadMap &dtqmap,
-   field_operator_t &fop,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const bool &condition,
-   const int &dimension,
-   const bool &use_sum_factorization = false)
-{
-   if (condition)
-   {
-      if (use_sum_factorization)
-      {
-         if (dimension == 1)
-         {
-            map_field_to_quadrature_data_tensor_product_1d(
-               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
-
-         }
-         else if (dimension == 2)
-         {
-            map_field_to_quadrature_data_tensor_product_2d(
-               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
-         }
-         else if (dimension == 3)
-         {
-            map_field_to_quadrature_data_tensor_product_3d(
-               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
-         }
-      }
-      else
-      {
-         map_field_to_quadrature_data(
-            field_qp, dtqmap, field_e, fop, integration_weights);
-      }
-   }
-}
-
-template <size_t num_fields, size_t num_inputs, typename field_operator_ts>
-MFEM_HOST_DEVICE
-void map_fields_to_quadrature_data_conditional(
-   std::array<DeviceTensor<2>, num_inputs> &fields_qp,
-   const std::array<DeviceTensor<1, const real_t>, num_fields> &fields_e,
-   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
-   field_operator_ts fops,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const std::array<bool, num_inputs> &conditions,
-   const bool &use_sum_factorization = false)
-{
-   for_constexpr<num_inputs>([&](auto i)
-   {
-      map_field_to_quadrature_data_conditional(
-         fields_qp[i], fields_e[i], dtqmaps[i], get<i>(fops), integration_weights,
-         scratch_mem, conditions[i], use_sum_factorization);
-   });
-}
-
-template <size_t num_inputs, typename field_operator_ts>
-MFEM_HOST_DEVICE
-void map_direction_to_quadrature_data_conditional(
-   std::array<DeviceTensor<2>, num_inputs> &directions_qp,
-   const DeviceTensor<1> &direction_e,
-   const std::array<DofToQuadMap, num_inputs> &dtqmaps,
-   field_operator_ts fops,
-   const DeviceTensor<1, const real_t> &integration_weights,
-   const std::array<DeviceTensor<1>, 6> &scratch_mem,
-   const std::array<bool, num_inputs> &conditions,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   for_constexpr<num_inputs>([&](auto i)
-   {
-      if (conditions[i])
-      {
-         if (use_sum_factorization)
-         {
-            if (dimension == 1)
-            {
-               map_field_to_quadrature_data_tensor_product_1d(
-                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
-                  integration_weights, scratch_mem);
-            }
-            else if (dimension == 2)
-            {
-               map_field_to_quadrature_data_tensor_product_2d(
-                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
-                  integration_weights, scratch_mem);
-            }
-            else if (dimension == 3)
-            {
-               map_field_to_quadrature_data_tensor_product_3d(
-                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
-                  integration_weights, scratch_mem);
-            }
-         }
-         else
-         {
-            map_field_to_quadrature_data(
-               directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
-               integration_weights);
-         }
-      }
-   });
-}
-
-}

fem/dfem/parameterspace.hpp

@@ -20,7 +20,7 @@ namespace mfem::future
 class ParameterSpace
 {
 public:
-   ParameterSpace(int vdim = 1) : vdim(vdim) {}
+   ParameterSpace(int vdim = 1) : vdim(vdim) { dtq.FE = nullptr; dtq.IntRule = nullptr; }
 
    /// @brief Get vector dimension at each point
    ///
@@ -43,7 +43,7 @@ public:
    /// Get spatial dimension
    ///
    /// returns always 1.
-   int Dimension() const
+   constexpr int Dimension() const
    {
       return 1;
    }
@@ -65,7 +65,7 @@ public:
    /// It should not be used by a user.
    ///
    /// returns identity by default that is lazy evaluated.
-   virtual const Operator* GetElementRestriction(ElementDofOrdering o) const
+   virtual const Operator* GetElementRestriction(ElementDofOrdering) const
    {
       if (!elem_restr)
       {
@@ -74,11 +74,14 @@ public:
       return elem_restr.get();
    }
 
+   virtual const Operator* GetB() const = 0;
+
+   virtual const Operator* GetBt() const = 0;
+
 protected:
    int vdim;
    DofToQuad dtq;
-   mutable std::unique_ptr<Operator> prolongation;
-   mutable std::unique_ptr<Operator> elem_restr;
+   mutable std::unique_ptr<Operator> prolongation, elem_restr, B, Bt;
 };
 
 /// @brief Uniform parameter space

fem/dfem/qfunction_apply.hpp

@@ -1,619 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "util.hpp"
-#include "qfunction_transform.hpp"
-
-namespace mfem::future
-{
-
-/// @brief Call a qfunction with the given parameters.
-///
-/// @param qfunc the qfunction to call.
-/// @param input_shmem the input shared memory.
-/// @param residual_shmem the residual shared memory.
-/// @param rs_qp the size of the residual.
-/// @param num_qp the number of quadrature points.
-/// @param q1d the number of quadrature points in 1D.
-/// @param dimension the spatial dimension.
-/// @param use_sum_factorization whether to use sum factorization.
-/// @tparam qf_param_ts the tuple type of the qfunction parameters.
-template <
-   typename qf_param_ts,
-   typename qfunc_t,
-   std::size_t num_fields>
-MFEM_HOST_DEVICE inline
-void call_qfunction(
-   qfunc_t &qfunc,
-   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
-   DeviceTensor<2> &residual_shmem,
-   const int &rs_qp,
-   const int &num_qp,
-   const int &q1d,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 1)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
-         {
-            auto qf_args = decay_tuple<qf_param_ts> {};
-            auto r = Reshape(&residual_shmem(0, q), rs_qp);
-            apply_kernel(r, qfunc, qf_args, input_shmem, q);
-         }
-      }
-      else if (dimension == 2)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               const int q = qx + q1d * qy;
-               auto qf_args = decay_tuple<qf_param_ts> {};
-               auto r = Reshape(&residual_shmem(0, q), rs_qp);
-               apply_kernel(r, qfunc, qf_args, input_shmem, q);
-            }
-         }
-      }
-      else if (dimension == 3)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-                  auto qf_args = decay_tuple<qf_param_ts> {};
-                  auto r = Reshape(&residual_shmem(0, q), rs_qp);
-                  apply_kernel(r, qfunc, qf_args, input_shmem, q);
-               }
-            }
-         }
-      }
-      else
-      {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-         MFEM_ABORT("unsupported dimension for sum factorization");
-#endif
-      }
-      MFEM_SYNC_THREAD;
-   }
-   else
-   {
-      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
-      {
-         auto qf_args = decay_tuple<qf_param_ts> {};
-         auto r = Reshape(&residual_shmem(0, q), rs_qp);
-         apply_kernel(r, qfunc, qf_args, input_shmem, q);
-      }
-   }
-}
-
-/// @brief Call a qfunction with the given parameters and
-/// compute it's derivative action.
-///
-/// @param qfunc the qfunction to call.
-/// @param input_shmem the input shared memory.
-/// @param shadow_shmem the shadow shared memory.
-/// @param residual_shmem the residual shared memory.
-/// @param das_qp the size of the derivative action.
-/// @param num_qp the number of quadrature points.
-/// @param q1d the number of quadrature points in 1D.
-/// @param dimension the spatial dimension.
-/// @param use_sum_factorization whether to use sum factorization.
-/// @tparam qf_param_ts the tuple type of the qfunction parameters.
-template <
-   typename qf_param_ts,
-   typename qfunc_t,
-   std::size_t num_fields>
-MFEM_HOST_DEVICE inline
-void call_qfunction_derivative_action(
-   qfunc_t &qfunc,
-   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
-   DeviceTensor<2> &residual_shmem,
-   const int &das_qp,
-   const int &num_qp,
-   const int &q1d,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 1)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
-         {
-            auto r = Reshape(&residual_shmem(0, q), das_qp);
-            auto qf_args = decay_tuple<qf_param_ts> {};
-#ifdef MFEM_USE_ENZYME
-            auto qf_shadow_args = decay_tuple<qf_param_ts> {};
-            apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
-                                        shadow_shmem, q);
-#else
-            apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
-#endif
-         }
-      }
-      else if (dimension == 2)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               const int q = qx + q1d * qy;
-               auto r = Reshape(&residual_shmem(0, q), das_qp);
-               auto qf_args = decay_tuple<qf_param_ts> {};
-#ifdef MFEM_USE_ENZYME
-               auto qf_shadow_args = decay_tuple<qf_param_ts> {};
-               apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
-                                           shadow_shmem, q);
-#else
-               apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
-#endif
-            }
-         }
-      }
-      else if (dimension == 3)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-                  auto r = Reshape(&residual_shmem(0, q), das_qp);
-                  auto qf_args = decay_tuple<qf_param_ts> {};
-#ifdef MFEM_USE_ENZYME
-                  auto qf_shadow_args = decay_tuple<qf_param_ts> {};
-                  apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
-                                              shadow_shmem, q);
-#else
-                  apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
-#endif
-               }
-            }
-         }
-      }
-      else
-      {
-         MFEM_ABORT_KERNEL("unsupported dimension");
-      }
-   }
-   else
-   {
-      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
-      {
-         auto r = Reshape(&residual_shmem(0, q), das_qp);
-         auto qf_args = decay_tuple<qf_param_ts> {};
-#ifdef MFEM_USE_ENZYME
-         auto qf_shadow_args = decay_tuple<qf_param_ts> {};
-         apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
-                                     shadow_shmem, q);
-#else
-         apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
-#endif
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-namespace detail
-{
-template <
-   typename qf_param_ts,
-   typename qfunc_t,
-   std::size_t num_fields>
-MFEM_HOST_DEVICE inline
-void call_qfunction_derivative(
-   qfunc_t &qfunc,
-   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
-   DeviceTensor<2> &residual_shmem,
-   DeviceTensor<5> &qpdc,
-   const DeviceTensor<1, const real_t> &itod,
-   const int &das_qp,
-   const int &q)
-{
-   const int test_vdim = qpdc.GetShape()[0];
-   const int test_op_dim = qpdc.GetShape()[1];
-   const int trial_vdim = qpdc.GetShape()[2];
-   const int num_qp = qpdc.GetShape()[4];
-   const size_t num_inputs = itod.GetShape()[0];
-
-   for (int j = 0; j < trial_vdim; j++)
-   {
-      int m_offset = 0;
-      for (size_t s = 0; s < num_inputs; s++)
-      {
-         const int trial_op_dim = static_cast<int>(itod(s));
-         if (trial_op_dim == 0)
-         {
-            continue;
-         }
-
-         auto d_qp = Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
-         for (int m = 0; m < trial_op_dim; m++)
-         {
-            d_qp(j, m, q) = 1.0;
-
-            auto r = Reshape(&residual_shmem(0, q), das_qp);
-            auto qf_args = decay_tuple<qf_param_ts> {};
-#ifdef MFEM_USE_ENZYME
-            auto qf_shadow_args = decay_tuple<qf_param_ts> {};
-            apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
-                                        shadow_shmem, q);
-#else
-            apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
-#endif
-            d_qp(j, m, q) = 0.0;
-
-            auto f = Reshape(&r(0), test_vdim, test_op_dim);
-            for (int i = 0; i < test_vdim; i++)
-            {
-               for (int k = 0; k < test_op_dim; k++)
-               {
-                  qpdc(i, k, j, m + m_offset, q) = f(i, k);
-               }
-            }
-         }
-         m_offset += trial_op_dim;
-      }
-   }
-}
-}
-
-/// @brief Call a qfunction with the given parameters and
-/// compute it's derivative represented by the Jacobian on
-/// each quadrature point.
-///
-/// @param qfunc the qfunction to call.
-/// @param input_shmem the input shared memory.
-/// @param shadow_shmem the shadow shared memory.
-/// @param residual_shmem the residual shared memory.
-/// @param qpdc the quadrature point data cache holding the resulting
-/// Jacobians on each quadrature point.
-/// @param itod inputs trial operator dimension.
-/// If input is dependent the value corresponds to the spatial dimension, otherwise
-/// a zero indicates non-dependence on the variable.
-/// @param das_qp the size of the derivative action.
-/// @param q1d the number of quadrature points in 1D.
-/// @param dimension the spatial dimension.
-/// @param use_sum_factorization whether to use sum factorization.
-/// @tparam qf_param_ts the tuple type of the qfunction parameters.
-template <
-   typename qf_param_ts,
-   typename qfunc_t,
-   std::size_t num_fields>
-MFEM_HOST_DEVICE inline
-void call_qfunction_derivative(
-   qfunc_t &qfunc,
-   const std::array<DeviceTensor<2>, num_fields> &input_shmem,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
-   DeviceTensor<2> &residual_shmem,
-   DeviceTensor<5> &qpdc,
-   const DeviceTensor<1, const real_t> &itod,
-   const int &das_qp,
-   const int &q1d,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 1)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
-         {
-            detail::call_qfunction_derivative<qf_param_ts>(
-               qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
-         }
-      }
-      else if (dimension == 2)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               const int q = qx + q1d * qy;
-               detail::call_qfunction_derivative<qf_param_ts>(
-                  qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
-            }
-         }
-      }
-      else if (dimension == 3)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-                  detail::call_qfunction_derivative<qf_param_ts>(
-                     qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
-               }
-            }
-         }
-      }
-      else
-      {
-         MFEM_ABORT_KERNEL("unsupported dimension");
-      }
-   }
-   else
-   {
-      const int num_qp = qpdc.GetShape()[4];
-      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
-      {
-         detail::call_qfunction_derivative<qf_param_ts>(
-            qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-namespace detail
-{
-
-/// @brief Apply the quadrature point data cache (qpdc) to a vector
-/// (usually a direction) on quadrature point q.
-///
-/// The qpdc consists of compatible data to be used for integration with a test
-/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
-/// function including integration weights and necessesary transformations.
-///
-/// @param fhat the qpdc applied to a vector in shadow_memory.
-/// @param shadow_shmem the shadow shared memory.
-/// @param qpdc the quadrature point data cache holding the resulting
-/// Jacobians on each quadrature point.
-/// @param itod inputs trial operator dimension.
-/// If input is dependent the value corresponds to the spatial dimension, otherwise
-/// a zero indicates non-dependence on the variable.
-/// @param q the current quadrature point index.
-template <size_t num_fields>
-MFEM_HOST_DEVICE inline
-void apply_qpdc(
-   DeviceTensor<3> &fhat,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
-   const DeviceTensor<5, const real_t> &qpdc,
-   const DeviceTensor<1, const real_t> &itod,
-   const int &q)
-{
-   const int test_vdim = qpdc.GetShape()[0];
-   const int test_op_dim = qpdc.GetShape()[1];
-   const int trial_vdim = qpdc.GetShape()[2];
-   const int num_qp = qpdc.GetShape()[4];
-   const size_t num_inputs = itod.GetShape()[0];
-
-   for (int i = 0; i < test_vdim; i++)
-   {
-      for (int k = 0; k < test_op_dim; k++)
-      {
-         real_t sum = 0.0;
-         int m_offset = 0;
-         for (size_t s = 0; s < num_inputs; s++)
-         {
-            const int trial_op_dim = static_cast<int>(itod(s));
-            if (trial_op_dim == 0)
-            {
-               continue;
-            }
-            const auto d_qp =
-               Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
-            for (int j = 0; j < trial_vdim; j++)
-            {
-               for (int m = 0; m < trial_op_dim; m++)
-               {
-                  sum += qpdc(i, k, j, m + m_offset, q) * d_qp(j, m, q);
-               }
-            }
-            m_offset += trial_op_dim;
-         }
-         fhat(i, k, q) = sum;
-      }
-   }
-}
-}
-
-/// @brief Apply the quadrature point data cache (qpdc) to a vector
-/// (usually a direction).
-///
-/// The qpdc consists of compatible data to be used for integration with a test
-/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
-/// function including integration weights and necessesary transformations.
-///
-/// @param fhat the qpdc applied to a vector in shadow_memory.
-/// @param shadow_shmem the shadow shared memory.
-/// @param qpdc the quadrature point data cache holding the resulting
-/// Jacobians on each quadrature point.
-/// @param itod inputs trial operator dimension.
-/// If input is dependent the value corresponds to the spatial dimension, otherwise
-/// a zero indicates non-dependence on the variable.
-/// @param q1d number of quadrature points in 1D.
-/// @param dimension spatial dimension.
-/// @param use_sum_factorization whether to use sum factorization.
-template <size_t num_fields>
-MFEM_HOST_DEVICE inline
-void apply_qpdc(
-   DeviceTensor<3> &fhat,
-   const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
-   const DeviceTensor<5, const real_t> &qpdc,
-   const DeviceTensor<1, const real_t> &itod,
-   const int &q1d,
-   const int &dimension,
-   const bool &use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 1)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
-         {
-            detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
-         }
-      }
-      else if (dimension == 2)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               const int q = qx + q1d * qy;
-               detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
-            }
-         }
-      }
-      else if (dimension == 3)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
-               {
-                  const int q = qx + q1d * (qy + q1d * qz);
-                  detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
-               }
-            }
-         }
-      }
-      else
-      {
-         MFEM_ABORT_KERNEL("unsupported dimension");
-      }
-   }
-   else
-   {
-      const int num_qp = qpdc.GetShape()[4];
-      MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
-      {
-         detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
-      }
-   }
-}
-
-template <typename qfunc_t, typename args_ts, size_t num_args>
-MFEM_HOST_DEVICE inline
-void apply_kernel(
-   DeviceTensor<1, real_t> &f_qp,
-   const qfunc_t &qfunc,
-   args_ts &args,
-   const std::array<DeviceTensor<2>, num_args> &u,
-   int qp)
-{
-   process_qf_args(u, args, qp);
-   process_qf_result(f_qp, get<0>(apply(qfunc, args)));
-}
-
-template <typename qfunc_t, typename arg_ts, size_t num_args>
-MFEM_HOST_DEVICE inline
-void apply_kernel_native_dual(
-   DeviceTensor<1, real_t> &f_qp,
-   const qfunc_t &qfunc,
-   arg_ts &args,
-   const std::array<DeviceTensor<2>, num_args> &u,
-   const std::array<DeviceTensor<2>, num_args> &v,
-   const int &qp_idx)
-{
-   process_qf_args(u, v, args, qp_idx);
-   auto r = get<0>(apply(qfunc, args));
-   process_derivative_from_native_dual(f_qp, r);
-}
-
-#ifdef MFEM_USE_ENZYME
-
-template <typename func_t, typename... arg_ts>
-MFEM_HOST_DEVICE inline
-auto qfunction_wrapper(const func_t &f, arg_ts &&...args)
-{
-   return f(args...);
-}
-
-// Version for active function arguments only
-//
-// This is an Enzyme regression and can be removed in later versions.
-template <typename qfunc_t, typename arg_ts, std::size_t... Is,
-          typename inactive_arg_ts>
-MFEM_HOST_DEVICE inline
-auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
-                                  arg_ts &&shadow_args,
-                                  std::index_sequence<Is...>,
-                                  inactive_arg_ts &&inactive_args,
-                                  std::index_sequence<>)
-{
-   using qf_return_t = typename create_function_signature<
-                       decltype(&qfunc_t::operator())>::type::return_t;
-   return __enzyme_fwddiff<qf_return_t>(
-             qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...>, enzyme_const,
-             (void *)&qfunc, enzyme_dup, &get<Is>(args)..., enzyme_interleave,
-             &get<Is>(shadow_args)...);
-}
-
-// Interleave function arguments for enzyme
-template <typename qfunc_t, typename arg_ts, std::size_t... Is,
-          typename inactive_arg_ts, std::size_t... Js>
-MFEM_HOST_DEVICE inline
-auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
-                                  arg_ts &&shadow_args,
-                                  std::index_sequence<Is...>,
-                                  inactive_arg_ts &&inactive_args,
-                                  std::index_sequence<Js...>)
-{
-   using qf_return_t = typename create_function_signature<
-                       decltype(&qfunc_t::operator())>::type::return_t;
-   return __enzyme_fwddiff<qf_return_t>(
-             qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...,
-             decltype(get<Js>(inactive_args))...>,
-             enzyme_const, (void *)&qfunc, enzyme_dup, &get<Is>(args)...,
-             enzyme_const, &get<Js>(inactive_args)..., enzyme_interleave,
-             &get<Is>(shadow_args)...);
-}
-
-template <typename qfunc_t, typename arg_ts, typename inactive_arg_ts>
-MFEM_HOST_DEVICE inline
-auto fwddiff_apply_enzyme(qfunc_t &qfunc, arg_ts &&args,
-                          arg_ts &&shadow_args,
-                          inactive_arg_ts &&inactive_args)
-{
-   auto arg_indices = std::make_index_sequence<
-                      tuple_size<std::remove_reference_t<arg_ts>>::value> {};
-
-   auto inactive_arg_indices = std::make_index_sequence<
-                               tuple_size<std::remove_reference_t<inactive_arg_ts>>::value> {};
-
-   return fwddiff_apply_enzyme_indexed(qfunc, args, shadow_args, arg_indices,
-                                       inactive_args, inactive_arg_indices);
-}
-
-template <typename qfunc_t, typename arg_ts, size_t num_args>
-MFEM_HOST_DEVICE inline
-void apply_kernel_fwddiff_enzyme(
-   DeviceTensor<1, real_t> &f_qp,
-   qfunc_t &qfunc,
-   arg_ts &args,
-   arg_ts &shadow_args,
-   const std::array<DeviceTensor<2>, num_args> &u,
-   const std::array<DeviceTensor<2>, num_args> &v,
-   int qp_idx)
-{
-   process_qf_args(u, args, qp_idx);
-   process_qf_args(v, shadow_args, qp_idx);
-   process_qf_result(f_qp,
-                     get<0>(fwddiff_apply_enzyme(qfunc, args, shadow_args, tuple<> {})));
-}
-#endif // MFEM_USE_ENZYME
-
-} // namespace mfem::future

fem/dfem/qfunction_transform.hpp

@@ -1,346 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "util.hpp"
-#include "../../linalg/tensor.hpp"
-
-namespace mfem::future
-{
-
-template <typename T0, typename T1, typename T2>
-MFEM_HOST_DEVICE
-void process_qf_arg(const T0 &, const T1 &, T2 &)
-{
-   static_assert(dfem::always_false<T0, T1, T2>,
-                 "process_qf_arg not implemented for arg type");
-}
-
-template <typename T>
-MFEM_HOST_DEVICE
-void process_qf_arg(
-   const DeviceTensor<1, T> &u,
-   const DeviceTensor<1, T> &v,
-   T &arg)
-{
-   arg = u(0);
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   tensor<dual<T, T>, n, m> &arg)
-{
-   for (int i = 0; i < m; i++)
-   {
-      for (int j = 0; j < n; j++)
-      {
-         arg(j, i).value = u((i * n) + j);
-      }
-   }
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   dual<T, T> &arg)
-{
-   arg.value = u(0);
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   const DeviceTensor<1> &v,
-   dual<T, T> &arg)
-{
-   arg.value = u(0);
-   arg.gradient = v(0);
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   const DeviceTensor<1> &v,
-   tensor<dual<T, T>, n> &arg)
-{
-   for (int i = 0; i < n; i++)
-   {
-      arg(i).value = u(i);
-      arg(i).gradient = v(i);
-   }
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   const DeviceTensor<1> &v,
-   tensor<dual<T, T>, n, m> &arg)
-{
-   for (int i = 0; i < m; i++)
-   {
-      for (int j = 0; j < n; j++)
-      {
-         arg(j, i).value = u((i * n) + j);
-         arg(j, i).gradient = v((i * n) + j);
-      }
-   }
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const tensor<dual<T, T>, n> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      r(i) = x(i).value;
-   }
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const tensor<dual<T, T>, n, m> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      for (size_t j = 0; j < m; j++)
-      {
-         r(i + n * j) = x(i, j).value;
-      }
-   }
-}
-
-template <typename arg_type>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<2> &u,
-   const DeviceTensor<2> &v,
-   arg_type &arg,
-   const int &qp)
-{
-   const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
-   const auto v_qp = Reshape(&v(0, qp), v.GetShape()[0]);
-   process_qf_arg(u_qp, v_qp, arg);
-}
-
-template <size_t num_fields, typename qf_args>
-MFEM_HOST_DEVICE inline
-void process_qf_args(
-   const std::array<DeviceTensor<2>, num_fields> &u,
-   const std::array<DeviceTensor<2>, num_fields> &v,
-   qf_args &args,
-   const int &qp)
-{
-   for_constexpr<tuple_size<qf_args>::value>([&](auto i)
-   {
-      process_qf_arg(u[i], v[i], get<i>(args), qp);
-   });
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_derivative_from_native_dual(
-   DeviceTensor<1, T> &r,
-   const tensor<dual<T, T>, n, m> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      for (size_t j = 0; j < m; j++)
-      {
-         r(i + n * j) = x(i, j).gradient;
-      }
-   }
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE inline
-void process_derivative_from_native_dual(
-   DeviceTensor<1, T> &r,
-   const tensor<dual<T, T>, n> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      r(i) = x(i).gradient;
-   }
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_derivative_from_native_dual(
-   DeviceTensor<1, T> &r,
-   const dual<T, T> &x)
-{
-   r(0) = x.gradient;
-}
-
-template <typename T0, typename T1>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(const T0 &, T1 &)
-{
-   static_assert(dfem::always_false<T0, T1>,
-                 "process_qf_arg not implemented for arg type");
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1, T> &u,
-   T &arg)
-{
-   arg = u(0);
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1, T> &u,
-   tensor<T> &arg)
-{
-   arg(0) = u(0);
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   tensor<T, n> &arg)
-{
-   for (int i = 0; i < n; i++)
-   {
-      arg(i) = u(i);
-   }
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1> &u,
-   tensor<T, n, m> &arg)
-{
-   for (int i = 0; i < m; i++)
-   {
-      for (int j = 0; j < n; j++)
-      {
-         arg(j, i) = u((i * n) + j);
-      }
-   }
-}
-
-template <typename arg_type>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(const DeviceTensor<2> &u, arg_type &arg, int qp)
-{
-   const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
-   process_qf_arg(u_qp, arg);
-}
-
-template <size_t num_fields, typename qf_args>
-MFEM_HOST_DEVICE inline
-void process_qf_args(
-   const std::array<DeviceTensor<2>, num_fields> &u,
-   qf_args &args,
-   const int &qp)
-{
-   for_constexpr<tuple_size<qf_args>::value>([&](auto i)
-   {
-      process_qf_arg(u[i], get<i>(args), qp);
-   });
-}
-
-template <typename T0, typename T1>
-MFEM_HOST_DEVICE inline
-Vector process_qf_result(T0, T1)
-{
-   static_assert(dfem::always_false<T0, T1>,
-                 "process_qf_result not implemented for result type");
-   return Vector{};
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const T &x)
-{
-   r(0) = x;
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1> &r,
-   const dual<T, T> &x)
-{
-   r(0) = x.value;
-}
-
-template <typename T>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const tensor<T> &x)
-{
-   r(0) = x(0);
-}
-
-template <typename T, int n>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const tensor<T, n> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      r(i) = x(i);
-   }
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_result(
-   DeviceTensor<1, T> &r,
-   const tensor<T, n, m> &x)
-{
-   for (size_t i = 0; i < n; i++)
-   {
-      for (size_t j = 0; j < m; j++)
-      {
-         r(i + n * j) = x(i, j);
-      }
-   }
-}
-
-template <typename T, int n, int m>
-MFEM_HOST_DEVICE inline
-void process_qf_arg(
-   const DeviceTensor<1, T> &u,
-   const DeviceTensor<1, T> &v,
-   tensor<T, n, m> &arg)
-{
-   for (int i = 0; i < m; i++)
-   {
-      for (int j = 0; j < n; j++)
-      {
-         arg(j, i) = u((i * n) + j);
-      }
-   }
-}
-
-} // namespace mfem::future

fem/dfem/tuple.hpp

@@ -16,6 +16,24 @@
 #include <type_traits>
 #include <tuple>
 
+// Define a portable unreachable macro
+#if defined(__GNUC__) || defined(__clang__)
+#if defined(__CUDACC_VER_MAJOR__)
+#if __CUDACC_VER_MAJOR__ <= 11 && __CUDACC_VER_MINOR__ < 3
+// nvcc didn't add __builtin_unreachable() until cuda 11.3
+#define MFEM_UNREACHABLE()
+#else
+// nvcc >= 11.3
+#define MFEM_UNREACHABLE() __builtin_unreachable()
+#endif
+#else
+// host-only version
+#define MFEM_UNREACHABLE() __builtin_unreachable()
+#endif
+#elif defined(_MSC_VER)
+#define MFEM_UNREACHABLE() __assume(0)
+#endif
+
 namespace mfem::future
 {
 
@@ -23,31 +41,10 @@ namespace mfem::future
 template <typename... T>
 struct tuple;
 
-// Implementation detail: storage using multiple inheritance from tuple_leaf,
-// which lets the tuple be defined for an arbitrary number of elements.
-// Structured bindings come from the std::tuple_size / std::tuple_element / get
-// specializations at the bottom of this file, not from the layout.
+// Implementation detail: storage using multiple inheritance from tuple_leaf
+// to support structured bindings
 namespace detail
 {
-/**
- * @brief Trait that is true when @a U is a single argument that is (a reference
- * to) @a Self
- *
- * A variadic constructor taking @c "U&&..." is a better match than the copy
- * constructor for a non-const lvalue of its own type; this is used to constrain
- * it out of those overload sets.
- */
-template <typename Self, typename... U>
-struct is_self_arg : std::false_type {};
-
-/// @overload
-template <typename Self, typename U>
-struct is_self_arg<Self, U> : std::is_same<Self, std::decay_t<U>> {};
-
-/// SFINAE guard enabling a constructor for every @a U except @a Self itself
-template <typename Self, typename... U>
-using disable_if_self_t = std::enable_if_t<!is_self_arg<Self, U...>::value>;
-
 /**
  * @brief A single tuple element storage
  * @tparam I The index of this element in the tuple
@@ -62,7 +59,7 @@ struct tuple_leaf
    MFEM_HOST_DEVICE constexpr tuple_leaf() = default;
 
    /// Construct from value
-   template <typename U, typename = disable_if_self_t<tuple_leaf, U>>
+   template <typename U>
    MFEM_HOST_DEVICE constexpr explicit tuple_leaf(U&& v) :
       value(std::forward<U>(v)) {}
 };
@@ -72,9 +69,8 @@ struct tuple_leaf
  * @tparam Indices Index sequence for tuple elements
  * @tparam T The types stored in the tuple
  *
- * This uses multiple inheritance from tuple_leaf base classes so that a single
- * definition covers any number of elements, while keeping the storage layout
- * (and the trivial copyability that device kernels rely on) of a plain struct.
+ * This uses multiple inheritance from tuple_leaf base classes to enable
+ * structured bindings while maintaining efficient storage.
  */
 template <typename Indices, typename... T>
 struct tuple_impl;
@@ -89,61 +85,19 @@ struct tuple_impl<std::index_sequence<I...>, T...> : tuple_leaf<I, T>...
    /**
     * @brief Construct from values
     * @param args The values to store in the tuple
-    *
-    * @note the arguments are perfectly forwarded, so that constructing a tuple
-    * from lvalues costs exactly one copy per element (taking them by value
-    * would add a copy plus a move).
     */
-   template <typename... U, typename = disable_if_self_t<tuple_impl, U...>>
    MFEM_HOST_DEVICE
-   constexpr explicit tuple_impl(U&&... args)
-      : tuple_leaf<I, T>(std::forward<U>(args))... {}
+   constexpr explicit tuple_impl(T... args)
+      : tuple_leaf<I, T>(std::forward<T>(args))... {}
    };
-
-/**
- * @brief Element-wise constructibility check, only instantiated once the
- * argument list is known to have the right length
- * @tparam Viable whether the arity and self-argument checks have passed
- * @tparam Tuple the @p tuple being constructed
- * @tparam U the constructor argument types
- */
-template <bool Viable, typename Tuple, typename... U>
-struct is_constructible_from : std::false_type {};
-
-/// @overload
-template <typename... T, typename... U>
-struct is_constructible_from<true, tuple<T...>, U...>
-: std::bool_constant<(std::is_constructible_v<T, U&&> && ...)> {};
-
-/**
- * @brief Trait that is true when @a Tuple can be constructed element-wise from
- * the argument list @a U
- * @tparam Tuple the @p tuple being constructed
- * @tparam U the constructor argument types
- */
-template <typename Tuple, typename... U>
-struct is_elementwise_constructible : std::false_type {};
-
-/// @overload
-template <typename... T, typename... U>
-struct is_elementwise_constructible<tuple<T...>, U...>
-   : is_constructible_from<sizeof...(U) == sizeof...(T) && sizeof...(U) != 0 &&
-  !is_self_arg<tuple<T...>, U...>::value, tuple<T...>, U...> {};
-
-/// SFINAE guard for the element-wise constructor of @p tuple
-template <typename Tuple, typename... U>
-using enable_elementwise_t =
-   std::enable_if_t<is_elementwise_constructible<Tuple, U...>::value>;
-}  // namespace detail
+}
 
 /**
  * @tparam T the types stored in the tuple
  * @brief This is a class that mimics most of std::tuple's interface,
- * except that it is usable in CUDA kernels and admits some arithmetic operator
- * overloads.
+ * except that it is usable in CUDA kernels and admits some arithmetic operator overloads.
  *
- * See https://en.cppreference.com/w/cpp/utility/tuple for more information
- * about std::tuple.
+ * see https://en.cppreference.com/w/cpp/utility/tuple for more information about std::tuple
  */
 template <typename... T>
 struct tuple : detail::tuple_impl<std::index_sequence_for<T...>, T...>
@@ -157,16 +111,9 @@ struct tuple : detail::tuple_impl<std::index_sequence_for<T...>, T...>
    /**
     * @brief Construct tuple from values
     * @param args The values to store
-    *
-    * @note this constructor is deliberately *not* explicit, so that the
-    * copy-list-initialization forms that worked when @p tuple was an aggregate
-    * (@c "tuple<A,B> t = {a,b};", @c "return {a,b};", passing @c "{a,b}" to a
-    * function) keep working.
     */
-   template <typename... U,
-             typename = detail::enable_elementwise_t<tuple, U...>>
    MFEM_HOST_DEVICE
-   constexpr tuple(U&&... args) : base_type(std::forward<U>(args)...) {}
+   constexpr explicit tuple(T... args) : base_type(std::forward<T>(args)...) {}
 
    /// Copy constructor
    MFEM_HOST_DEVICE
@@ -222,8 +169,10 @@ template <class... Types>
 struct tuple_size;
 
 template <class... Types>
-struct tuple_size<tuple<Types...> >
-: std::integral_constant<std::size_t, sizeof...(Types)> {};
+struct tuple_size<tuple<Types...>> :
+                                std::integral_constant<std::size_t, sizeof...(Types)>
+{
+};
 
 /**
  * @brief a struct used to determine the type at index I of a tuple
@@ -239,8 +188,10 @@ struct tuple_element;
 // recursive case
 /// @overload
 template <size_t I, class Head, class... Tail>
-struct tuple_element<I, tuple<Head, Tail...> >
-   : tuple_element<I - 1, tuple<Tail...>> {};
+struct tuple_element<I, tuple<Head, Tail...>> : tuple_element<I - 1,
+                                              tuple<Tail...>>
+{
+};
 
 // base case
 /// @overload
@@ -250,20 +201,19 @@ struct tuple_element<0, tuple<Head, Tail...>>
    using type = Head;  ///< the type at the specified index
 };
 
-namespace detail
-{
+/// @brief Type alias mirroring std::tuple_element_t for mfem::future::tuple
+template <size_t I, class T>
+using tuple_element_t = typename tuple_element<I, T>::type;
+
 /**
  * @brief Get implementation for tuple_leaf - non-const lvalue reference
  * @tparam I the index of the tuple element
  * @tparam T the type of the tuple element
  * @param leaf the tuple_leaf containing the value
  * @return reference to the value
- *
- * @note @a T is deduced from the (unique) @p tuple_leaf base class of the
- * argument, so callers only have to supply the index @a I.
  */
 template <size_t I, typename T>
-MFEM_HOST_DEVICE constexpr T& get_impl(tuple_leaf<I, T>& leaf)
+MFEM_HOST_DEVICE constexpr T& get_impl(detail::tuple_leaf<I, T>& leaf)
 {
    return leaf.value;
 }
@@ -276,7 +226,8 @@ MFEM_HOST_DEVICE constexpr T& get_impl(tuple_leaf<I, T>& leaf)
  * @return const reference to the value
  */
 template <size_t I, typename T>
-MFEM_HOST_DEVICE constexpr const T& get_impl(const tuple_leaf<I, T>& leaf)
+MFEM_HOST_DEVICE constexpr const T& get_impl(const detail::tuple_leaf<I, T>&
+                                             leaf)
 {
    return leaf.value;
 }
@@ -289,7 +240,7 @@ MFEM_HOST_DEVICE constexpr const T& get_impl(const tuple_leaf<I, T>& leaf)
  * @return rvalue reference to the value
  */
 template <size_t I, typename T>
-MFEM_HOST_DEVICE constexpr T&& get_impl(tuple_leaf<I, T>&& leaf)
+MFEM_HOST_DEVICE constexpr T&& get_impl(detail::tuple_leaf<I, T>&& leaf)
 {
    return static_cast<T&&>(leaf.value);
 }
@@ -302,11 +253,11 @@ MFEM_HOST_DEVICE constexpr T&& get_impl(tuple_leaf<I, T>&& leaf)
  * @return const rvalue reference to the value
  */
 template <size_t I, typename T>
-MFEM_HOST_DEVICE constexpr const T&& get_impl(const tuple_leaf<I, T>&& leaf)
+MFEM_HOST_DEVICE constexpr const T&& get_impl(const detail::tuple_leaf<I, T>&&
+                                              leaf)
 {
    return static_cast<const T&&>(leaf.value);
 }
-}  // namespace detail
 
 /**
  * @tparam I the tuple index to access
@@ -318,7 +269,9 @@ template <size_t I, typename... T>
 MFEM_HOST_DEVICE constexpr auto& get(tuple<T...>& t)
 {
    static_assert(I < sizeof...(T), "Tuple index out of bounds");
-   return detail::get_impl<I>(t);
+   using elem_type = typename tuple_element<I, tuple<T...>>::type;
+   using leaf_type = detail::tuple_leaf<I, elem_type>;
+   return get_impl<I>(static_cast<leaf_type&>(t));
 }
 
 /**
@@ -331,7 +284,9 @@ template <size_t I, typename... T>
 MFEM_HOST_DEVICE constexpr const auto& get(const tuple<T...>& t)
 {
    static_assert(I < sizeof...(T), "Tuple index out of bounds");
-   return detail::get_impl<I>(t);
+   using elem_type = typename tuple_element<I, tuple<T...>>::type;
+   using leaf_type = detail::tuple_leaf<I, elem_type>;
+   return get_impl<I>(static_cast<const leaf_type&>(t));
 }
 
 /**
@@ -344,7 +299,9 @@ template <size_t I, typename... T>
 MFEM_HOST_DEVICE constexpr auto&& get(tuple<T...>&& t)
 {
    static_assert(I < sizeof...(T), "Tuple index out of bounds");
-   return detail::get_impl<I>(std::move(t));
+   using elem_type = typename tuple_element<I, tuple<T...>>::type;
+   using leaf_type = detail::tuple_leaf<I, elem_type>;
+   return get_impl<I>(static_cast<leaf_type&&>(t));
 }
 
 /**
@@ -357,15 +314,15 @@ template <size_t I, typename... T>
 MFEM_HOST_DEVICE constexpr const auto&& get(const tuple<T...>&& t)
 {
    static_assert(I < sizeof...(T), "Tuple index out of bounds");
-   return detail::get_impl<I>(std::move(t));
+   using elem_type = typename tuple_element<I, tuple<T...>>::type;
+   using leaf_type = detail::tuple_leaf<I, elem_type>;
+   return get_impl<I>(static_cast<const leaf_type&&>(t));
 }
 
 /**
- * @brief a function intended to be used for extracting the ith type from a
- * tuple.
+ * @brief a function intended to be used for extracting the ith type from a tuple.
  *
- * @note type<i>(my_tuple) returns a value, whereas get<i>(my_tuple) returns a
- * reference
+ * @note type<i>(my_tuple) returns a value, whereas get<i>(my_tuple) returns a reference
  *
  * @tparam I the index of the tuple to query
  * @tparam T the types stored in the tuple
@@ -379,8 +336,6 @@ MFEM_HOST_DEVICE constexpr auto type(const tuple<T...>& t)
    return get<I>(t);
 }
 
-namespace detail
-{
 /**
  * @brief Helper for applying binary operations element-wise
  *
@@ -402,7 +357,6 @@ MFEM_HOST_DEVICE constexpr auto apply_op_helper(
 {
    return tuple{op(get<I>(x), get<I>(y))...};
 }
-} // namespace detail
 
 /**
  * @tparam S the types stored in the tuple x
@@ -416,8 +370,7 @@ MFEM_HOST_DEVICE constexpr auto operator+(const tuple<S...>& x,
                                           const tuple<T...>& y)
 {
    static_assert(sizeof...(S) == sizeof...(T), "tuples must have same size");
-   return detail::apply_op_helper(x, y,
-   [](const auto& a, const auto& b) { return a + b; },
+   return apply_op_helper(x, y, [](auto a, auto b) { return a + b; },
    std::make_index_sequence<sizeof...(S)> {});
 }
 
@@ -433,8 +386,7 @@ MFEM_HOST_DEVICE constexpr auto operator-(const tuple<S...>& x,
                                           const tuple<T...>& y)
 {
    static_assert(sizeof...(S) == sizeof...(T), "tuples must have same size");
-   return detail::apply_op_helper(x, y,
-   [](const auto& a, const auto& b) { return a - b; },
+   return apply_op_helper(x, y, [](auto a, auto b) { return a - b; },
    std::make_index_sequence<sizeof...(S)> {});
 }
 
@@ -443,16 +395,14 @@ MFEM_HOST_DEVICE constexpr auto operator-(const tuple<S...>& x,
  * @tparam T the types stored in the tuple y
  * @param x a tuple of values
  * @param y a tuple of values
- * @brief return a tuple of values defined by elementwise multiplication of x
- * and y
+ * @brief return a tuple of values defined by elementwise multiplication of x and y
  */
 template <typename... S, typename... T>
 MFEM_HOST_DEVICE constexpr auto operator*(const tuple<S...>& x,
                                           const tuple<T...>& y)
 {
    static_assert(sizeof...(S) == sizeof...(T), "tuples must have same size");
-   return detail::apply_op_helper(x, y,
-   [](const auto& a, const auto& b) { return a * b; },
+   return apply_op_helper(x, y, [](auto a, auto b) { return a * b; },
    std::make_index_sequence<sizeof...(S)> {});
 }
 
@@ -468,13 +418,10 @@ MFEM_HOST_DEVICE constexpr auto operator/(const tuple<S...>& x,
                                           const tuple<T...>& y)
 {
    static_assert(sizeof...(S) == sizeof...(T), "tuples must have same size");
-   return detail::apply_op_helper(x, y,
-   [](const auto& a, const auto& b) { return a / b; },
+   return apply_op_helper(x, y, [](auto a, auto b) { return a / b; },
    std::make_index_sequence<sizeof...(S)> {});
 }
 
-namespace detail
-{
 /**
  * @brief A helper function for the += operator of tuples
  *
@@ -491,7 +438,6 @@ MFEM_HOST_DEVICE constexpr void inplace_add_helper(
 {
    ((get<I>(x) += get<I>(y)), ...);
 }
-} // namespace detail
 
 /**
  * @tparam T the types stored in the tuples x and y
@@ -500,15 +446,12 @@ MFEM_HOST_DEVICE constexpr void inplace_add_helper(
  * @brief add values contained in y, to the tuple x
  */
 template <typename... T>
-MFEM_HOST_DEVICE constexpr tuple<T...>& operator+=(tuple<T...>& x,
-                                                   const tuple<T...>& y)
+MFEM_HOST_DEVICE constexpr auto operator+=(tuple<T...>& x, const tuple<T...>& y)
 {
-   detail::inplace_add_helper(x, y, std::make_index_sequence<sizeof...(T)> {});
+   inplace_add_helper(x, y, std::make_index_sequence<sizeof...(T)> {});
    return x;
 }
 
-namespace detail
-{
 /**
  * @brief A helper function for the -= operator of tuples
  *
@@ -525,7 +468,6 @@ MFEM_HOST_DEVICE constexpr void inplace_sub_helper(
 {
    ((get<I>(x) -= get<I>(y)), ...);
 }
-} // namespace detail
 
 /**
  * @tparam T the types stored in the tuples x and y
@@ -534,15 +476,12 @@ MFEM_HOST_DEVICE constexpr void inplace_sub_helper(
  * @brief subtract values contained in y from the tuple x
  */
 template <typename... T>
-MFEM_HOST_DEVICE constexpr tuple<T...>& operator-=(tuple<T...>& x,
-                                                   const tuple<T...>& y)
+MFEM_HOST_DEVICE constexpr auto operator-=(tuple<T...>& x, const tuple<T...>& y)
 {
-   detail::inplace_sub_helper(x, y, std::make_index_sequence<sizeof...(T)> {});
+   inplace_sub_helper(x, y, std::make_index_sequence<sizeof...(T)> {});
    return x;
 }
 
-namespace detail
-{
 /**
  * @brief A helper function for the unary - operator of tuples
  *
@@ -558,23 +497,18 @@ MFEM_HOST_DEVICE constexpr auto unary_minus_helper(
 {
    return tuple{-get<I>(x)...};
 }
-} // namespace detail
 
 /**
  * @tparam T the types stored in the tuple x
  * @param x a tuple of values
- * @brief return a tuple of values defined by applying the unary minus operator
- * to each element of x
+ * @brief return a tuple of values defined by applying the unary minus operator to each element of x
  */
 template <typename... T>
 MFEM_HOST_DEVICE constexpr auto operator-(const tuple<T...>& x)
 {
-   return detail::unary_minus_helper(
-             x, std::make_index_sequence<sizeof...(T)> {});
+   return unary_minus_helper(x, std::make_index_sequence<sizeof...(T)> {});
 }
 
-namespace detail
-{
 /**
  * @brief A helper function for the * operator of tuples with scalar
  *
@@ -584,15 +518,14 @@ namespace detail
  * @param x tuple of values
  * @return the returned tuple product
  */
-template <typename scalar_t, typename... T, size_t... I>
+template <typename... T, size_t... I>
 MFEM_HOST_DEVICE constexpr auto scalar_mult_helper(
-   scalar_t a,
+   real_t a,
    const tuple<T...>& x,
    std::index_sequence<I...>)
 {
    return tuple{a * get<I>(x)...};
 }
-} // namespace detail
 
 /**
  * @tparam T the types stored in the tuple
@@ -600,11 +533,10 @@ MFEM_HOST_DEVICE constexpr auto scalar_mult_helper(
  * @param x the tuple object
  * @brief multiply each component of x by the value a on the left
  */
-template <typename scalar_t, typename... T>
-MFEM_HOST_DEVICE constexpr auto operator*(scalar_t a, const tuple<T...>& x)
+template <typename... T>
+MFEM_HOST_DEVICE constexpr auto operator*(real_t a, const tuple<T...>& x)
 {
-   return detail::scalar_mult_helper(
-             a, x, std::make_index_sequence<sizeof...(T)> {});
+   return scalar_mult_helper(a, x, std::make_index_sequence<sizeof...(T)> {});
 }
 
 /**
@@ -613,14 +545,12 @@ MFEM_HOST_DEVICE constexpr auto operator*(scalar_t a, const tuple<T...>& x)
  * @param a a scaling factor
  * @brief multiply each component of x by the value a on the right
  */
-template <typename scalar_t, typename... T>
-MFEM_HOST_DEVICE constexpr auto operator*(const tuple<T...>& x, scalar_t a)
+template <typename... T>
+MFEM_HOST_DEVICE constexpr auto operator*(const tuple<T...>& x, real_t a)
 {
    return a * x;
 }
 
-namespace detail
-{
 /**
  * @brief A helper function for the / operator of tuples with scalar denominator
  *
@@ -630,15 +560,14 @@ namespace detail
  * @param a the constant denominator
  * @return the returned tuple ratio
  */
-template <typename scalar_t, typename... T, size_t... I>
+template <typename... T, size_t... I>
 MFEM_HOST_DEVICE constexpr auto scalar_div_helper(
    const tuple<T...>& x,
-   scalar_t a,
+   real_t a,
    std::index_sequence<I...>)
 {
    return tuple{get<I>(x) / a...};
 }
-} // namespace detail
 
 /**
  * @tparam T the types stored in the tuple x
@@ -646,15 +575,12 @@ MFEM_HOST_DEVICE constexpr auto scalar_div_helper(
  * @param a a denominator
  * @brief return a tuple of values defined by elementwise division of x by a
  */
-template <typename scalar_t, typename... T>
-MFEM_HOST_DEVICE constexpr auto operator/(const tuple<T...>& x, scalar_t a)
+template <typename... T>
+MFEM_HOST_DEVICE constexpr auto operator/(const tuple<T...>& x, real_t a)
 {
-   return detail::scalar_div_helper(
-             x, a, std::make_index_sequence<sizeof...(T)> {});
+   return scalar_div_helper(x, a, std::make_index_sequence<sizeof...(T)> {});
 }
 
-namespace detail
-{
 /**
  * @brief A helper function for the / operator with scalar numerator
  *
@@ -664,15 +590,14 @@ namespace detail
  * @param x tuple of values
  * @return the returned tuple ratio
  */
-template <typename scalar_t, typename... T, size_t... I>
+template <typename... T, size_t... I>
 MFEM_HOST_DEVICE constexpr auto scalar_div_inv_helper(
-   scalar_t a,
+   real_t a,
    const tuple<T...>& x,
    std::index_sequence<I...>)
 {
    return tuple{a / get<I>(x)...};
 }
-} // namespace detail
 
 /**
  * @tparam T the types stored in the tuple x
@@ -680,15 +605,12 @@ MFEM_HOST_DEVICE constexpr auto scalar_div_inv_helper(
  * @param x a tuple of denominator values
  * @brief return a tuple of values defined by division of a by the elements of x
  */
-template <typename scalar_t, typename... T>
-MFEM_HOST_DEVICE constexpr auto operator/(scalar_t a, const tuple<T...>& x)
+template <typename... T>
+MFEM_HOST_DEVICE constexpr auto operator/(real_t a, const tuple<T...>& x)
 {
-   return detail::scalar_div_inv_helper(
-             a, x, std::make_index_sequence<sizeof...(T)> {});
+   return scalar_div_inv_helper(a, x, std::make_index_sequence<sizeof...(T)> {});
 }
 
-namespace detail
-{
 /**
  * @tparam T the types stored in the tuple
  * @tparam I a list of indices used to access each element of the tuple
@@ -705,7 +627,6 @@ auto& print_helper(std::ostream& out, const tuple<T...>& t,
    out << "}";
    return out;
 }
-} // namespace detail
 
 /**
  * @tparam T the types stored in the tuple
@@ -716,12 +637,9 @@ auto& print_helper(std::ostream& out, const tuple<T...>& t,
 template <typename... T>
 auto& operator<<(std::ostream& out, const tuple<T...>& t)
 {
-   return detail::print_helper(
-             out, t, std::make_index_sequence<sizeof...(T)> {});
+   return print_helper(out, t, std::make_index_sequence<sizeof...(T)> {});
 }
 
-namespace detail
-{
 /**
  * @brief A helper to apply a lambda to a tuple
  *
@@ -738,27 +656,23 @@ MFEM_HOST_DEVICE auto apply_helper(F&& f, tuple<T...>& args,
 {
    return std::forward<F>(f)(get<I>(args)...);
 }
-} // namespace detail
 
 /**
  * @tparam F a callable type
  * @tparam T the types of arguments to be passed in to f
  * @param f the callable object
  * @param args a tuple of arguments
- * @brief a way of passing an n-tuple to a function that expects n separate
- * arguments
+ * @brief a way of passing an n-tuple to a function that expects n separate arguments
  *
- * For example, foo(bar, baz) is equivalent to apply(foo, mfem::tuple(bar,baz)).
+ *   e.g. foo(bar, baz) is equivalent to apply(foo, mfem::tuple(bar,baz));
  */
 template <typename F, typename... T>
 MFEM_HOST_DEVICE auto apply(F&& f, tuple<T...>& args)
 {
-   return detail::apply_helper(std::forward<F>(f), args,
-                               std::make_index_sequence<sizeof...(T)> {});
+   return apply_helper(std::forward<F>(f), args,
+                       std::make_index_sequence<sizeof...(T)> {});
 }
 
-namespace detail
-{
 /**
  * @overload
  */
@@ -768,23 +682,21 @@ MFEM_HOST_DEVICE auto apply_helper(F&& f, const tuple<T...>& args,
 {
    return std::forward<F>(f)(get<I>(args)...);
 }
-} // namespace detail
 
 /**
  * @tparam F a callable type
  * @tparam T the types of arguments to be passed in to f
  * @param f the callable object
  * @param args a tuple of arguments
- * @brief a way of passing an n-tuple to a function that expects n separate
- * arguments
+ * @brief a way of passing an n-tuple to a function that expects n separate arguments
  *
- * For example, foo(bar, baz) is equivalent to apply(foo, mfem::tuple(bar,baz)).
+ *   e.g. foo(bar, baz) is equivalent to apply(foo, mfem::tuple(bar,baz));
  */
 template <typename F, typename... T>
 MFEM_HOST_DEVICE auto apply(F&& f, const tuple<T...>& args)
 {
-   return detail::apply_helper(std::forward<F>(f), args,
-                               std::make_index_sequence<sizeof...(T)> {});
+   return apply_helper(std::forward<F>(f), args,
+                       std::make_index_sequence<sizeof...(T)> {});
 }
 
 /**
@@ -802,8 +714,7 @@ struct is_tuple<tuple<T...>> : std::true_type
 };
 
 /**
- * @brief Trait for checking if a type if a @p mfem::tuple containing only
- * @p mfem::tuple
+ * @brief Trait for checking if a type if a @p mfem::tuple containing only @p mfem::tuple
  */
 template <typename T>
 struct is_tuple_of_tuples : std::false_type
@@ -811,8 +722,7 @@ struct is_tuple_of_tuples : std::false_type
 };
 
 /**
- * @brief Trait for checking if a type if a @p mfem::tuple containing only
- * @p mfem::tuple
+ * @brief Trait for checking if a type if a @p mfem::tuple containing only @p mfem::tuple
  */
 template <typename... T>
 struct is_tuple_of_tuples<tuple<T...>>
@@ -842,8 +752,8 @@ namespace std
  * @tparam T The types in the mfem::future::tuple
  */
 template <typename... T>
-struct tuple_size<mfem::future::tuple<T...> >
-: integral_constant<size_t, sizeof...(T)> {};
+struct tuple_size<mfem::future::tuple<T...>>
+                                          : integral_constant<size_t, sizeof...(T)> {};
 
 /**
  * @brief Specialization of std::tuple_element for mfem::future::tuple
@@ -856,4 +766,20 @@ struct tuple_element<I, mfem::future::tuple<T...>>
    using type = typename
                 mfem::future::tuple_element<I, mfem::future::tuple<T...>>::type;
 };
-} // namespace std
+
+template <size_t I, typename... T>
+constexpr decltype(auto) get(mfem::future::tuple<T...>& t) noexcept
+{ return mfem::future::get<I>(t); }
+
+template <size_t I, typename... T>
+constexpr decltype(auto) get(const mfem::future::tuple<T...>& t) noexcept
+{ return mfem::future::get<I>(t); }
+
+template <size_t I, typename... T>
+constexpr decltype(auto) get(mfem::future::tuple<T...>&& t) noexcept
+{ return mfem::future::get<I>(std::move(t)); }
+
+template <size_t I, typename... T>
+constexpr decltype(auto) get(const mfem::future::tuple<T...>&& t) noexcept
+{ return mfem::future::get<I>(std::move(t)); }
+}

fem/fe/fe_base.cpp

@@ -1316,14 +1316,13 @@ void VectorFiniteElement::Project_RT(
    }
 }
 
-void VectorFiniteElement::ProjectCurl2D_RT(
+void VectorFiniteElement::ProjectGrad_RT(
    const real_t *nk, const Array<int> &d2n, const FiniteElement &fe,
    ElementTransformation &Trans, DenseMatrix &grad) const
 {
-   // 2D "ProjectCurl_RT"
    if (dim != 2)
    {
-      mfem_error("VectorFiniteElement::ProjectCurl2D_RT works only in 2D!");
+      mfem_error("VectorFiniteElement::ProjectGrad_RT works only in 2D!");
    }
 
    DenseMatrix dshape(fe.GetDof(), fe.GetDim());
@@ -1334,8 +1333,8 @@ void VectorFiniteElement::ProjectCurl2D_RT(
    for (int k = 0; k < dof; k++)
    {
       fe.CalcDShape(Nodes.IntPoint(k), dshape);
-      tk[0] = -nk[d2n[k]*dim+1];
-      tk[1] = nk[d2n[k]*dim];
+      tk[0] = nk[d2n[k]*dim+1];
+      tk[1] = -nk[d2n[k]*dim];
       dshape.Mult(tk, grad_k);
       for (int j = 0; j < grad_k.Size(); j++)
       {
@@ -1382,7 +1381,7 @@ void VectorFiniteElement::ProjectCurl_ND(
    }
 }
 
-void VectorFiniteElement::ProjectCurl3D_RT(
+void VectorFiniteElement::ProjectCurl_RT(
    const real_t *nk, const Array<int> &d2n, const FiniteElement &fe,
    ElementTransformation &Trans, DenseMatrix &curl) const
 {

fem/fe/fe_base.hpp

@@ -957,11 +957,10 @@ protected:
                    const FiniteElement &fe, ElementTransformation &Trans,
                    DenseMatrix &I) const;
 
-   // Input is a scalar representing the Z (out of plane) component, Output is
-   // the X-Y (in-plane) RT curl
-   void ProjectCurl2D_RT(const real_t *nk, const Array<int> &d2n,
-                         const FiniteElement &fe, ElementTransformation &Trans,
-                         DenseMatrix &grad) const;
+   // rotated gradient in 2D
+   void ProjectGrad_RT(const real_t *nk, const Array<int> &d2n,
+                       const FiniteElement &fe, ElementTransformation &Trans,
+                       DenseMatrix &grad) const;
 
    // Compute the curl as a discrete operator from ND FE (fe) to ND FE (this).
    // The natural FE for the range is RT, so this is an approximation.
@@ -969,9 +968,9 @@ protected:
                        const FiniteElement &fe, ElementTransformation &Trans,
                        DenseMatrix &curl) const;
 
-   void ProjectCurl3D_RT(const real_t *nk, const Array<int> &d2n,
-                         const FiniteElement &fe, ElementTransformation &Trans,
-                         DenseMatrix &curl) const;
+   void ProjectCurl_RT(const real_t *nk, const Array<int> &d2n,
+                       const FiniteElement &fe, ElementTransformation &Trans,
+                       DenseMatrix &curl) const;
 
    /** @brief Project a vector coefficient onto the ND basis functions
        @param tk    Edge tangent vectors for this element type
@@ -1447,8 +1446,6 @@ public:
                                 dof2quad_array_open);
    }
 
-   const Poly_1D::Basis &GetOpenBasis1D() const { return obasis1d; }
-
    virtual ~VectorTensorFiniteElement();
 };
 

fem/fe/fe_fixed_order.hpp

@@ -307,12 +307,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (6) */
+       them in the vector shape of dimension Dof (4) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (6 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;
@@ -336,12 +336,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (5) */
+       them in the vector shape of dimension Dof (4) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (5 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;

fem/fe/fe_h1.cpp

@@ -1757,45 +1757,22 @@ H1_BergotPyramidElement::H1_BergotPyramidElement(const int p, const int btype)
       real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       real_t z = ip.z;
 
-      if (std::abs(z - 1.0) < apex_tol)
-      {
-         // Compute the limit of the basis functions as z->1 with x and y on the
-         // line between the center of the base and the apex
-         o = 0;
-         for (int i = 0; i <= p; i++)
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               for (int k = 0; k <= p - maxij; k++)
-                  if (i == 0 && j == 0)
-                  {
-                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
-                  }
-                  else
-                  {
-                     T(o++, m) = 0.;
-                  }
-            }
-      }
-      else
-      {
-         poly1d.CalcLegendre(p, x, shape_x.GetData());
-         poly1d.CalcLegendre(p, y, shape_y.GetData());
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-         o = 0;
-         for (int i = 0; i <= p; i++)
+      o = 0;
+      for (int i = 0; i <= p; i++)
+      {
+         for (int j = 0; j <= p; j++)
          {
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                                z, 1.0, shape_z);
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
 
-               for (int k = 0; k <= p - maxij; k++)
-               {
-                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                              pow(1.0 - ip.z, maxij);
-               }
+            for (int k = 0; k <= p - maxij; k++)
+            {
+               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                           pow(1.0 - ip.z, maxij);
             }
          }
       }
@@ -1816,44 +1793,25 @@ void H1_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    Vector u(dof);
 #endif
 
-   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   const real_t z = ip.z;
+   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the basis functions as z->1 with x and y on the
-      // line between the center of the base and the apex
-      u = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-               if (i == 0 && j == 0)
-               {
-                  u(o) = ((k + 3.) * k + 2.) / 2.;
-               }
-         }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
+   poly1d.CalcLegendre(p, x, shape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData());
+
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z);
+         for (int k = 0; k <= p - maxij; k++)
+            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                     pow(1.0 - ip.z, maxij);
+      }
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
-            for (int k = 0; k <= p - maxij; k++)
-               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                        pow(1.0 - ip.z, maxij);
-         }
-   }
    Ti.Mult(u, shape);
 }
 
@@ -1872,68 +1830,37 @@ void H1_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    Vector dshape_z(order+1);
    Vector dshape_z_dt(order+1);
 #endif
-   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   const real_t z = ip.z;
+   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the gradients of the basis functions as
-      // z->1 with x and y on the line between the center of the base and the
-      // apex
-      du = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
+   poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z, dshape_z, dshape_z_dt);
+
+         for (int k = 0; k <= p - maxij; k++, o++)
          {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               if (i == 0 && j == 0)
-               {
-                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
-               }
-               else if (i == 1 && j == 0)
-               {
-                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-               else if (i == 0 && j == 1)
-               {
-                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-            }
+            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                      pow(1.0 - ip.z, maxij) +
+                      (ip.x * dshape_x(i) * shape_y(j) +
+                       ip.y * shape_x(i) * dshape_y(j)) *
+                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                      maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
          }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+      }
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0,
-                                             shape_z, dshape_z, dshape_z_dt);
-
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                         pow(1.0 - ip.z, maxij) +
-                         (ip.x * dshape_x(i) * shape_y(j) +
-                          ip.y * shape_x(i) * dshape_y(j)) *
-                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
-            }
-         }
-   }
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_h1.hpp

@@ -208,8 +208,6 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
-   static constexpr real_t apex_tol = 1e-8;
-
 public:
    H1_BergotPyramidElement(const int p,
                            const int btype = BasisType::GaussLobatto);

fem/fe/fe_l2.cpp

@@ -1106,16 +1106,9 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
          {
             const real_t wik = op[i] + op[k] + op[p-i-k];
             const real_t w = wik * wjk * op[p-k];
-            if (std::abs(w) < apex_tol)
-            {
-               Nodes.IntPoint(o++).Set3(0.,0.,1.);
-            }
-            else
-            {
-               Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
-                                        op[j] * (op[i] + op[p-i-k]) / w,
-                                        op[k] * op[p-k] / w);
-            }
+            Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
+                                     op[j] * (op[j] + op[p-j-k]) / w,
+                                     op[k] * op[p-k] / w);
          }
       }
 
@@ -1132,45 +1125,22 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
       const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       const real_t z = ip.z;
 
-      if (std::abs(z - 1.0) < apex_tol)
-      {
-         // Compute the limit of the basis functions as z->1 with x and y on the
-         // line between the center of the base and the apex
-         o = 0;
-         for (int i = 0; i <= p; i++)
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               for (int k = 0; k <= p - maxij; k++)
-                  if (i == 0 && j == 0)
-                  {
-                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
-                  }
-                  else
-                  {
-                     T(o++, m) = 0.;
-                  }
-            }
-      }
-      else
-      {
-         poly1d.CalcLegendre(p, x, shape_x.GetData());
-         poly1d.CalcLegendre(p, y, shape_y.GetData());
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-         o = 0;
-         for (int i = 0; i <= p; i++)
+      o = 0;
+      for (int i = 0; i <= p; i++)
+      {
+         for (int j = 0; j <= p; j++)
          {
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                                z, 1.0, shape_z);
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
 
-               for (int k = 0; k <= p - maxij; k++)
-               {
-                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                              pow(1.0 - ip.z, maxij);
-               }
+            for (int k = 0; k <= p - maxij; k++)
+            {
+               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                           pow(1.0 - ip.z, maxij);
             }
          }
       }
@@ -1195,41 +1165,26 @@ void L2_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the basis functions as z->1 with x and y on the
-      // line between the center of the base and the apex
-      u = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-               if (i == 0 && j == 0)
-               {
-                  u(o) = ((k + 3.) * k + 2.) / 2.;
-               }
-         }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
+   poly1d.CalcLegendre(p, x, shape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+   {
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z);
 
-            for (int k = 0; k <= p - maxij; k++)
-               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                        pow(1.0 - ip.z, maxij);
+         for (int k = 0; k <= p - maxij; k++)
+         {
+            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                     pow(1.0 - ip.z, maxij);
          }
+      }
    }
+
    Ti.Mult(u, shape);
 }
 
@@ -1253,64 +1208,35 @@ void L2_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the gradients of the basis functions as
-      // z->1 with x and y on the line between the center of the base and the
-      // apex
-      du = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               if (i == 0 && j == 0)
-               {
-                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
-               }
-               else if (i == 1 && j == 0)
-               {
-                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-               else if (i == 0 && j == 1)
-               {
-                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-            }
-         }
-   }
-   else
-   {
-      Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-      Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+   Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+   Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0,
-                                             shape_z, dshape_z, dshape_z_dt);
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+   {
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z, dshape_z, dshape_z_dt);
 
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                         pow(1.0 - ip.z, maxij) +
-                         (ip.x * dshape_x(i) * shape_y(j) +
-                          ip.y * shape_x(i) * dshape_y(j)) *
-                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
-            }
+         for (int k = 0; k <= p - maxij; k++, o++)
+         {
+            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                      pow(1.0 - ip.z, maxij) +
+                      (ip.x * dshape_x(i) * shape_y(j) +
+                       ip.y * shape_x(i) * dshape_y(j)) *
+                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                      ((maxij > 0) ? (maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                                      pow(1.0 - ip.z, maxij - 1)) : 0.0);
          }
+      }
    }
+
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_l2.hpp

@@ -225,8 +225,6 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
-   static constexpr real_t apex_tol = 1e-8;
-
 public:
    /// Construct the L2_PyramidElement of order @a p and BasisType @a btype
    L2_BergotPyramidElement(const int p,

fem/fe/fe_nd.cpp

@@ -1282,49 +1282,12 @@ ND_SegmentElement::ND_SegmentElement(const int p, const int ob_type)
    }
 }
 
-void ND_SegmentElement::CalcShape(const IntegrationPoint &ip,
-                                  Vector &shape) const
-{
-   if (obasis1d.IsIntegratedType()) { obasis1d.ScaleIntegrated(false); }
-   obasis1d.Eval(ip.x, shape);
-}
-
 void ND_SegmentElement::CalcVShape(const IntegrationPoint &ip,
                                    DenseMatrix &shape) const
 {
    Vector vshape(shape.Data(), dof);
 
-   CalcShape(ip, vshape);
-}
-
-void ND_SegmentElement::ProjectIntegrated(VectorCoefficient &vc,
-                                          ElementTransformation &Trans,
-                                          Vector &dofs) const
-{
-   MFEM_ASSERT(obasis1d.IsIntegratedType(), "Not integrated type");
-   real_t vk[Geometry::MaxDim];
-   Vector xk(vk, vc.GetVDim());
-
-   const real_t *cp = poly1d.ClosedPoints(dof, BasisType::GaussLobatto);
-   const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, dof);
-   IntegrationPoint ip;
-
-   for (int i = 0; i < dof; i++)
-   {
-      const real_t h = cp[i+1] - cp[i];
-      real_t val = 0.0;
-
-      for (int q = 0; q < ir.GetNPoints(); q++)
-      {
-         const IntegrationPoint &ip1d = ir.IntPoint(q);
-         ip.x = cp[i] + h*ip1d.x;
-         Trans.SetIntPoint(&ip);
-         vc.Eval(xk, Trans, ip);
-         val += ip1d.weight*Trans.Jacobian().InnerProduct(tk, vk);
-      }
-
-      dofs(i) = val*h;
-   }
+   obasis1d.Eval(ip.x, vshape);
 }
 
 const real_t ND_WedgeElement::tk[15] =

fem/fe/fe_nd.hpp

@@ -303,7 +303,8 @@ public:
    /** @brief Construct the ND_SegmentElement of order @a p and open
        BasisType @a ob_type */
    ND_SegmentElement(const int p, const int ob_type = BasisType::GaussLegendre);
-   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
+   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override
+   { obasis1d.Eval(ip.x, shape); }
    void CalcVShape(const IntegrationPoint &ip,
                    DenseMatrix &shape) const override;
    void CalcVShape(ElementTransformation &Trans,
@@ -324,10 +325,7 @@ public:
    using FiniteElement::Project;
    void Project(VectorCoefficient &vc,
                 ElementTransformation &Trans, Vector &dofs) const override
-   {
-      if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
-      else { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-   }
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    void ProjectMatrixCoefficient(MatrixCoefficient &mc,
                                  ElementTransformation &T,
                                  Vector &dofs) const override
@@ -340,11 +338,6 @@ public:
                     ElementTransformation &Trans,
                     DenseMatrix &grad) const override
    { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
-
-protected:
-   void ProjectIntegrated(VectorCoefficient &vc,
-                          ElementTransformation &Trans,
-                          Vector &dofs) const;
 };
 
 class ND_WedgeElement : public VectorFiniteElement

fem/fe/fe_pyramid.hpp

@@ -17,12 +17,6 @@
 namespace mfem
 {
 
-struct ScalarPyramid
-{
-   // Default basis type for H1 and L2 pyramids
-   static inline int DefaultType = 1; // Bergot(0) or Fuentes(1)
-};
-
 /** Base class for arbitrary order basis functions on pyramid-shaped elements
 
   This base class provides a common class to store temporary vectors,

fem/fe/fe_rt.hpp

@@ -73,11 +73,16 @@ public:
    void Project(const FiniteElement &fe, ElementTransformation &Trans,
                 DenseMatrix &I) const override
    { Project_RT(nk, dof2nk, fe, Trans, I); }
+   // Gradient + rotation = Curl: H1 -> H(div)
+   void ProjectGrad(const FiniteElement &fe,
+                    ElementTransformation &Trans,
+                    DenseMatrix &grad) const override
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
    // Curl = Gradient + rotation: H1 -> H(div)
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl2D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
 
    void GetFaceMap(const int face_id, Array<int> &face_map) const override;
 
@@ -143,7 +148,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
 
    /// @brief Return the mapping from lexicographically ordered face DOFs to
    /// lexicographically ordered element DOFs corresponding to local face
@@ -205,11 +210,16 @@ public:
    void Project(const FiniteElement &fe, ElementTransformation &Trans,
                 DenseMatrix &I) const override
    { Project_RT(nk, dof2nk, fe, Trans, I); }
+   // Gradient + rotation = Curl: H1 -> H(div)
+   void ProjectGrad(const FiniteElement &fe,
+                    ElementTransformation &Trans,
+                    DenseMatrix &grad) const override
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
    // Curl = Gradient + rotation: H1 -> H(div)
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl2D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 
@@ -264,7 +274,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 class RT_WedgeElement : public VectorFiniteElement
@@ -322,7 +332,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 /** Arbitrary order H(Div) basis functions defined on pyramid-shaped elements
@@ -418,7 +428,7 @@ public:
    virtual void ProjectCurl(const FiniteElement &fe,
                             ElementTransformation &Trans,
                             DenseMatrix &curl) const
-   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
 
    void CalcRawVShape(const IntegrationPoint &ip,
                       DenseMatrix &shape) const;

fem/fe_coll.cpp

@@ -228,19 +228,7 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1_", 3))
    {
-      // Parse pyramid basis type if included in the name
-      const char *pyr = strstr(name, "Pyr");
-      if (pyr == NULL)
-      {
-         // Use default pyramid type elements
-         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
-      }
-      else
-      {
-         // Use specific pyramid type elements
-         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3),
-                                   BasisType::GaussLobatto, atoi(pyr + 3));
-      }
+      fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
    }
    else if (!strncmp(name, "H1Pos_Trace_", 12))
    {
@@ -257,44 +245,26 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1@", 3))
    {
-      // Parse pyramid basis type if included in the name
-      const char *pyr = strstr(name, "Pyr");
-      if (pyr == NULL)
-      {
-         // Use default pyramid type elements
-         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                   BasisType::GetType(name[3]));
-      }
-      else
-      {
-         // Use specific pyramid type elements
-         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                   BasisType::GetType(name[3]),
-                                   atoi(pyr + 3));
-      }
+      fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
+                                BasisType::GetType(name[3]));
    }
-   else if (!strncmp(name, "L2", 2))
+   else if (!strncmp(name, "L2_T", 4))
+      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
+                                atoi(name + 4));
+   else if (!strncmp(name, "L2_", 3))
    {
-      // Parse Map Type
-      const int mtype = strstr(name, "Int") == NULL ?
-                        FiniteElement::VALUE : FiniteElement::INTEGRAL;
-
-      // Parse the base order
-      const int p = atoi(strstr(name, "_P") + 2);
-
-      // Parse the mesh dimension
-      const int dim = atoi(strstr(name, "D") - 1);
-
-      // Parse basis type if specified
-      const char *t = strstr(name, "_T");
-      const int btype = t == NULL ? BasisType::GaussLegendre : atoi(t + 2);
-
-      // Parse the pyramid type if specified
-      const char *pyr = strstr(name, "Pyr");
-      const int ptype = pyr == NULL ? 1 : atoi(pyr + 3);
-
-      // Create collection
-      fec = new L2_FECollection(p, dim, btype, mtype, ptype);
+      fec = new L2_FECollection(atoi(name + 7), atoi(name + 3));
+   }
+   else if (!strncmp(name, "L2Int_T", 7))
+   {
+      fec = new L2_FECollection(atoi(name + 13), atoi(name + 9),
+                                atoi(name + 7), FiniteElement::INTEGRAL);
+   }
+   else if (!strncmp(name, "L2Int_", 6))
+   {
+      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
+                                BasisType::GaussLegendre,
+                                FiniteElement::INTEGRAL);
    }
    else if (!strncmp(name, "RT_Trace_", 9))
    {
@@ -1739,10 +1709,9 @@ const int *RT1_3DFECollection::DofOrderForOrientation(Geometry::Type GeomType,
 
 
 H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
-                                 const int pyr_type)
+                                 const int pyrtype)
    : FiniteElementCollection(p)
    , dim(dim)
-   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 1, "H1_FECollection requires order >= 1.");
    MFEM_VERIFY(dim >= 0 && dim <= 3, "H1_FECollection requires 0 <= dim <= 3.");
@@ -1755,14 +1724,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
    {
       case BasisType::GaussLobatto:
       {
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
-         }
-         else
-         {
-            snprintf(h1_name, 32, "H1_%dD_P%d_Pyr%d", dim, p, pyr_type);
-         }
+         snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
          break;
       }
       case BasisType::Positive:
@@ -1948,11 +1910,11 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
          H1_dof[Geometry::TETRAHEDRON] = (TriDof*pm3)/3;
          H1_dof[Geometry::CUBE] = QuadDof*pm1;
          H1_dof[Geometry::PRISM] = TriDof*pm1;
-         if (pyr_type == 0 || b_type == BasisType::Positive)
+         if (pyrtype == 0 || b_type == BasisType::Positive)
          {
             H1_dof[Geometry::PYRAMID] = pm2*pm1*(2*p-3)/6; // Bergot (JSC)
          }
-         else if (pyr_type == 1)
+         else if (pyrtype == 1)
          {
             H1_dof[Geometry::PYRAMID] = pm1*pm1*pm1; // Fuentes
          }
@@ -1973,15 +1935,13 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
                new H1_TetrahedronElement(p, btype);
             H1_Elements[Geometry::CUBE] = new H1_HexahedronElement(p, btype);
             H1_Elements[Geometry::PRISM] = new H1_WedgeElement(p, btype);
-            if (pyr_type == 0)
+            if (pyrtype == 0)
             {
-               H1_Elements[Geometry::PYRAMID] =
-                  new H1_BergotPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] = new H1_BergotPyramidElement(p, btype);
             }
             else
             {
-               H1_Elements[Geometry::PYRAMID] =
-                  new H1_FuentesPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] = new H1_FuentesPyramidElement(p, btype);
             }
          }
 
@@ -2188,7 +2148,6 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    : FiniteElementCollection(p)
    , dim(dim)
    , m_type(map_type)
-   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 0, "L2_FECollection requires order >= 0.");
 
@@ -2204,25 +2163,10 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    switch (btype)
    {
       case BasisType::GaussLegendre:
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
-         }
-         else
-         {
-            snprintf(d_name, 32, "%s_%dD_P%d_Pyr%d", prefix, dim, p, pyr_type);
-         }
+         snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
          break;
       default:
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
-         }
-         else
-         {
-            snprintf(d_name, 32, "%s_T%d_%dD_P%d_Pyr%d",
-                     prefix, btype, dim, p, pyr_type);
-         }
+         snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
    }
 
    for (int g = 0; g < Geometry::NumGeom; g++)
@@ -2341,13 +2285,11 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
          L2_Elements[Geometry::PRISM] = new L2_WedgeElement(p, btype);
          if (pyr_type == 0)
          {
-            L2_Elements[Geometry::PYRAMID] =
-               new L2_BergotPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] = new L2_BergotPyramidElement(p, btype);
          }
          else
          {
-            L2_Elements[Geometry::PYRAMID] =
-               new L2_FuentesPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] = new L2_FuentesPyramidElement(p, btype);
          }
       }
 

fem/fe_coll.hpp

@@ -100,10 +100,6 @@ public:
       return FiniteElementForGeometry(GeomType);
    }
 
-   /** @brief Returns a collection of the trace elements.
-
-       @note The collection is owned by the caller and is NOT deleted in the
-       destructor. */
    virtual FiniteElementCollection *GetTraceCollection() const;
 
    virtual ~FiniteElementCollection();
@@ -290,7 +286,7 @@ protected:
 class H1_FECollection : public FiniteElementCollection
 {
 protected:
-   int dim, b_type, p_type;
+   int dim, b_type;
    char h1_name[32];
    FiniteElement *H1_Elements[Geometry::NumGeom];
    int H1_dof[Geometry::NumGeom];
@@ -299,7 +295,7 @@ protected:
 public:
    explicit H1_FECollection(const int p, const int dim = 3,
                             const int btype = BasisType::GaussLobatto,
-                            const int pyr_type = ScalarPyramid::DefaultType);
+                            const int pyrtype = 1);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -324,7 +320,7 @@ public:
    const int *GetDofMap(Geometry::Type GeomType, int p) const;
 
    FiniteElementCollection *Clone(int p) const override
-   { return new H1_FECollection(p, dim, b_type, p_type); }
+   { return new H1_FECollection(p, dim, b_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }
@@ -371,7 +367,6 @@ private:
    int dim;
    int b_type; // BasisType
    int m_type; // map type
-   int p_type; // Pyramid type (0 -> Bergot, 1 -> Fuentes)
    char d_name[32];
    ScalarFiniteElement *L2_Elements[Geometry::NumGeom];
    ScalarFiniteElement *Tr_Elements[Geometry::NumGeom];
@@ -384,7 +379,7 @@ public:
    L2_FECollection(const int p, const int dim,
                    const int btype = BasisType::GaussLegendre,
                    const int map_type = FiniteElement::VALUE,
-                   const int pyr_type = ScalarPyramid::DefaultType);
+                   const int pyrtype = 1);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -414,7 +409,7 @@ public:
    int GetBasisType() const { return b_type; }
 
    FiniteElementCollection *Clone(int p) const override
-   { return new L2_FECollection(p, dim, b_type, m_type, p_type); }
+   { return new L2_FECollection(p, dim, b_type, m_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }

fem/fem.hpp

@@ -52,7 +52,7 @@
 #include "bounds.hpp"
 #include "particleset.hpp"
 
-#include "dfem/doperator.hpp"
+// #include "dfem/doperator.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"

fem/fespace.cpp

@@ -22,8 +22,6 @@
 #include <algorithm>
 #include <cmath>
 #include <cstdarg>
-#include <unordered_map>
-#include <unordered_set>
 
 using namespace std;
 
@@ -4529,210 +4527,6 @@ void FiniteElementSpace
    }
 }
 
-void FiniteElementSpace::GetBoundaryLoopEdgeDofs(
-   const Array<int> &boundary_element_indices,
-   Array<int> &boundary_edge_dofs,
-   Array<int> *dof_edges,
-   Array<int> *dof_boundary_elements) const
-{
-   MFEM_VERIFY(mesh->Dimension() >= 2,
-               "GetBoundaryLoopEdgeDofs requires 2D or 3D meshes to find edge objects");
-
-   boundary_edge_dofs.SetSize(0);
-   if (dof_edges) { dof_edges->SetSize(0); }
-   if (dof_boundary_elements) { dof_boundary_elements->SetSize(0); }
-
-   // A DOF that appears in exactly one selected boundary element lies on the
-   // bounding loop; one appearing in two or more is interior to the boundary
-   // region and is dropped. Count occurrences of each DOF (using scratch maps,
-   // exposed only as parallel-indexed Array<int> below) and record, on first
-   // sight, the local edge and boundary element carrying it.
-   //
-   // The count is over GetEdgeDofs, which returns endpoint vertex DOFs as well
-   // as edge-interior DOFs (relevant for collections such as ND_R2D that carry
-   // vertex DOFs). Edge-interior DOFs occur once per edge, so the count mainly
-   // resolves vertex DOFs: a vertex shared by several elements is interior and
-   // dropped, while a genuine loop-corner (open-curve endpoint) vertex is kept.
-   // This is why we count GetEdgeDofs rather than collecting GetEdgeInteriorDofs,
-   // which would omit the endpoint vertex DOFs the method is documented to keep.
-   // The 3D removal criterion (any edge in two or more faces) matches the
-   // parallel version rather than a parity toggle.
-   std::unordered_map<int, int> dof_count, dof_edge, dof_belem;
-   Array<int> edge_dofs, edges, edge_orientations;
-
-   const int dim = mesh->Dimension();
-   for (int i = 0; i < boundary_element_indices.Size(); ++i)
-   {
-      const int boundary_element_idx = boundary_element_indices[i];
-      std::unordered_set<int> boundary_element_dofs;
-
-      if (dim == 3)
-      {
-         // Boundary elements are 2D faces; extract their 1D edges.
-         int face_index, face_orientation;
-         mesh->GetBdrElementFace(boundary_element_idx, &face_index,
-                                 &face_orientation);
-         mesh->GetFaceEdges(face_index, edges, edge_orientations);
-      }
-      else
-      {
-         // Boundary elements are 1D segments, each being a single edge.
-         mesh->GetBdrElementEdges(boundary_element_idx, edges, edge_orientations);
-         MFEM_VERIFY(edges.Size() == 1,
-                     "2D boundary element should have exactly one edge");
-      }
-
-      for (int j = 0; j < edges.Size(); ++j)
-      {
-         GetEdgeDofs(edges[j], edge_dofs);
-         for (int k = 0; k < edge_dofs.Size(); ++k)
-         {
-            const int dof = edge_dofs[k];
-            // Count each DOF once per boundary element and record metadata the
-            // first time it is seen, so H1 DOFs shared by multiple edges of the
-            // same element are not double counted.
-            if (boundary_element_dofs.insert(dof).second &&
-                dof_count[dof]++ == 0)
-            {
-               dof_edge[dof] = edges[j];
-               dof_belem[dof] = boundary_element_idx;
-            }
-         }
-      }
-   }
-
-   // Emit the DOFs seen in exactly one selected boundary element, in a
-   // deterministic (increasing DOF index) order shared by all output arrays.
-   std::vector<int> kept;
-   kept.reserve(dof_count.size());
-   for (const auto &[dof, count] : dof_count)
-   {
-      if (count == 1) { kept.push_back(dof); }
-   }
-   std::sort(kept.begin(), kept.end());
-
-   boundary_edge_dofs.Reserve(static_cast<int>(kept.size()));
-   if (dof_edges) { dof_edges->Reserve(static_cast<int>(kept.size())); }
-   if (dof_boundary_elements)
-   {
-      dof_boundary_elements->Reserve(static_cast<int>(kept.size()));
-   }
-   for (int dof : kept)
-   {
-      boundary_edge_dofs.Append(dof);
-      if (dof_edges) { dof_edges->Append(dof_edge[dof]); }
-      if (dof_boundary_elements) { dof_boundary_elements->Append(dof_belem[dof]); }
-   }
-}
-
-void FiniteElementSpace::GetBoundaryElementsByAttribute(
-   const Array<int> &bdr_attrs,
-   std::vector<Array<int>> &attr_to_elements)
-{
-   // One (initially empty) list of boundary elements per requested attribute,
-   // indexed to match bdr_attrs.
-   attr_to_elements.assign(bdr_attrs.Size(), Array<int>());
-
-   // Map attribute value -> position in bdr_attrs for quick lookup.
-   std::unordered_map<int, int> attr_to_index;
-   for (int i = 0; i < bdr_attrs.Size(); ++i)
-   {
-      attr_to_index[bdr_attrs[i]] = i;
-   }
-
-   // Bucket boundary elements by their attribute.
-   for (int i = 0; i < mesh->GetNBE(); ++i)
-   {
-      int attr = mesh->GetBdrElement(i)->GetAttribute();
-      auto it = attr_to_index.find(attr);
-      if (it != attr_to_index.end())
-      {
-         attr_to_elements[it->second].Append(i);
-      }
-   }
-}
-
-void FiniteElementSpace::GetBoundaryElementsByAttribute(int bdr_attr,
-                                                        Array<int> &boundary_elements)
-{
-   boundary_elements.SetSize(0);
-
-   for (int i = 0; i < mesh->GetNBE(); ++i)
-   {
-      if (mesh->GetBdrElement(i)->GetAttribute() == bdr_attr)
-      {
-         boundary_elements.Append(i);
-      }
-   }
-}
-
-void FiniteElementSpace::ComputeLoopEdgeOrientations(
-   const Array<int> &dof_edges,
-   const Array<int> &dof_boundary_elements,
-   const Vector &loop_normal,
-   Array<int> &dof_orientations) const
-{
-   MFEM_VERIFY(dof_edges.Size() == dof_boundary_elements.Size(),
-               "dof_edges and dof_boundary_elements must be parallel-indexed");
-
-   const int ndof = dof_edges.Size();
-   dof_orientations.SetSize(ndof);
-
-   Array<int> edge_verts, bdr_elem_verts;
-   Vector edge_vec(3), to_edge_vec(3), cross_product(3);
-   for (int i = 0; i < ndof; i++)
-   {
-      const int edge_id = dof_edges[i];
-      const int bdr_elem_idx = dof_boundary_elements[i];
-
-      // Get edge vertices
-      mesh->GetEdgeVertices(edge_id, edge_verts);
-
-      const real_t *v0 = mesh->GetVertex(edge_verts[0]);
-      const real_t *v1 = mesh->GetVertex(edge_verts[1]);
-
-      // Get boundary element vertices
-      mesh->GetBdrElement(bdr_elem_idx)->GetVertices(bdr_elem_verts);
-
-      // Find the third vertex (not part of the edge)
-      int third_vertex = -1;
-      for (int j = 0; j < bdr_elem_verts.Size(); j++)
-      {
-         int v = bdr_elem_verts[j];
-         if (v != edge_verts[0] && v != edge_verts[1])
-         {
-            third_vertex = v;
-            break;
-         }
-      }
-
-      if (third_vertex == -1)
-      {
-         MFEM_ABORT("Boundary element " << bdr_elem_idx << " has only 2 vertices, "
-                    "but 3D boundary elements must have at least 3 vertices");
-      }
-
-      const real_t *v2 = mesh->GetVertex(third_vertex);
-
-      // Edge vector
-      for (int j = 0; j < 3; j++) { edge_vec[j] = v1[j] - v0[j]; }
-
-      // Vector from third vertex to edge (use edge midpoint)
-      for (int j = 0; j < 3; j++)
-      {
-         real_t edge_midpoint = (v0[j] + v1[j]) * 0.5;
-         to_edge_vec[j] = edge_midpoint - v2[j];
-      }
-
-      // Cross product: to_edge × edge
-      to_edge_vec.cross3D(edge_vec, cross_product);
-
-      // Check alignment with loop normal
-      real_t dot_product = cross_product * loop_normal;
-      dof_orientations[i] = (dot_product > 0) ? 1 : -1;
-   }
-}
-
 FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)
 {
    string buff;

fem/fespace.hpp

@@ -22,7 +22,6 @@
 #include "restriction.hpp"
 #include <iostream>
 #include <unordered_map>
-#include <vector>
 
 namespace mfem
 {
@@ -1390,80 +1389,6 @@ public:
    virtual void GetExteriorTrueDofs(Array<int> &exterior_dofs,
                                     int component = -1) const;
 
-   /** @brief Extract the edge degrees of freedom of a boundary "loop".
-
-       Here a "loop" is the set of boundary edges bounding the region covered by
-       @a boundary_element_indices: in 3D the outer edges of a patch of boundary
-       faces, in 2D the boundary segments themselves. An edge that is shared by
-       two (or more) of the selected boundary elements is interior to that region
-       rather than on its bounding loop, so its DOFs are excluded from the result.
-       This exclusion of interior DOFs is the defining feature of the method.
-
-       The three output arrays share a single indexing: for each valid index @a i,
-       @a dof_edges[i] and @a dof_boundary_elements[i] describe the DOF
-       @a boundary_edge_dofs[i].
-
-       @param[in]  boundary_element_indices Boundary element indices spanning a
-                   boundary surface (3D) or curve (2D).
-       @param[out] boundary_edge_dofs Local DOF indices on the boundary loop.
-       @param[out] dof_edges Optional; local edge index carrying each DOF.
-       @param[out] dof_boundary_elements Optional; a boundary element containing
-                   each DOF.
-
-       @note In 3D the edge DOFs are extracted from the 1D edges of the 2D
-       boundary faces; in 2D they come directly from the 1D boundary segments, so
-       @a dof_edges then holds the boundary element (segment) edge indices.
-       @note This method uses GetEdgeDofs internally, which returns both vertex and
-       edge DOFs. Standard Nédélec elements (ND_FECollection) have no vertex DOFs,
-       so only genuine edge DOFs appear. Collections that carry vertex DOFs (e.g.
-       ND_R2D_FECollection) additionally contribute the vertex DOFs at loop
-       endpoints.
-       @note This is the serial version. For parallel meshes, use the parallel
-       version in ParFiniteElementSpace which handles processor boundaries
-       correctly.
-       @note Requires a 2D or 3D mesh to identify edge objects. The method will
-       assert if called on 1D meshes.
-       @note Only supports conforming meshes; non-conforming meshes are not
-       supported. */
-   void GetBoundaryLoopEdgeDofs(const Array<int> &boundary_element_indices,
-                                Array<int> &boundary_edge_dofs,
-                                Array<int> *dof_edges = nullptr,
-                                Array<int> *dof_boundary_elements = nullptr) const;
-
-   /** @brief Get boundary elements grouped by attribute.
-
-       For each attribute in @a bdr_attrs, collect the indices of all boundary
-       elements carrying that attribute. The result is indexed to match
-       @a bdr_attrs: @a attr_to_elements[i] holds the boundary elements with
-       attribute @a bdr_attrs[i]. */
-   void GetBoundaryElementsByAttribute(
-      const Array<int> &bdr_attrs,
-      std::vector<Array<int>> &attr_to_elements);
-
-   /** @brief Get all boundary elements with a specific attribute. */
-   void GetBoundaryElementsByAttribute(int bdr_attr,
-                                       Array<int> &boundary_elements);
-
-   /** @brief Compute edge orientations relative to a boundary loop direction.
-
-       For each boundary-loop DOF described by @a dof_edges and
-       @a dof_boundary_elements (see GetBoundaryLoopEdgeDofs), determine whether
-       the carrying edge is
-       traversed in the direction consistent with @a loop_normal, following the
-       right-hand rule. Intended for 3D meshes.
-
-       @param[in]  dof_edges Local edge index of each DOF (parallel-indexed with
-                   the boundary_edge_dofs output of GetBoundaryLoopEdgeDofs).
-       @param[in]  dof_boundary_elements A boundary element containing each DOF,
-                   using the same indexing as @a dof_edges.
-       @param[in]  loop_normal Normal vector defining the loop orientation.
-       @param[out] dof_orientations Orientation (+1 or -1) for each DOF, using the
-                   same indexing as @a dof_edges. */
-   void ComputeLoopEdgeOrientations(const Array<int> &dof_edges,
-                                    const Array<int> &dof_boundary_elements,
-                                    const Vector &loop_normal,
-                                    Array<int> &dof_orientations) const;
-
    /// Convert a Boolean marker array to a list containing all marked indices.
    static void MarkerToList(const Array<int> &marker, Array<int> &list);
 

fem/gslib.cpp

@@ -76,9 +76,6 @@ namespace gslib
 #ifndef GSLIB_RELEASE_VERSION //gslib v1.0.7
 #define GSLIB_RELEASE_VERSION 10007
 #endif
-static_assert(std::is_same_v<uint,unsigned int>,
-              "GSLIB's integer-type, 'uint', defined in gslib.h, must be the same as 'unsigned int'!");
-
 extern "C" {
    struct hash_data_3
    {
@@ -556,7 +553,7 @@ void obboxsurf_calc_3(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, n);
    lag(I0, work, n, 1, 0);
 
-   for (int ie = 0; (unsigned)ie < nel; ie++,x+=n2,y+=n2,z+=n2)
+   for (int ie = 0; ie < nel; ie++,x+=n2,y+=n2,z+=n2)
    {
       struct gslib::dbl_range ab[3];
       struct gslib::dbl_range tb[3];
@@ -780,7 +777,7 @@ void obboxedge_calc_2(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, nr);
    lag(I0r, work, nr,1, 0);
 
-   for (int ie = 0; (unsigned)ie < nel; ie++,x+=nr,y+=nr)
+   for (int ie = 0; ie < nel; ie++,x+=nr,y+=nr)
    {
       double x0[2], A[4];
       struct gslib::dbl_range ab[2], tb[2];
@@ -892,7 +889,7 @@ void obboxedge_calc_3(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, nr);
    lag(I0r, work, nr, 1, 0);
 
-   for (int ie = 0; (unsigned)ie < nel; ie++,x+=nr,y+=nr,z+=nr)
+   for (int ie = 0; ie < nel; ie++,x+=nr,y+=nr,z+=nr)
    {
       double x0[3], A[9], Ai[9];
       struct gslib::dbl_range ab[3], tb[3];
@@ -4518,7 +4515,7 @@ Mesh* FindPointsGSLIB::GetBoundingBoxMesh(int type)
    int eidx = 0;
    if (myid == save_rank)
    {
-      for (int p = 0; (unsigned)p < gsl_comm->np; p++)
+      for (int p = 0; p < gsl_comm->np; p++)
       {
          if (static_cast<unsigned int>(p) != save_rank)
          {

fem/gslib.hpp

@@ -326,22 +326,22 @@ protected:
    void findptsedge_setup_2(DEV_STRUCT &devs,
                             const double *const elx[2],
                             const unsigned n,
-                            const unsigned int nel,
+                            const uint nel,
                             const unsigned m,
                             const double bbox_rel_size_inc,
-                            const unsigned int local_hash_size,
-                            const unsigned int global_hash_size,
+                            const uint local_hash_size,
+                            const uint global_hash_size,
                             const Vector *aabb_sz_inc);
 
    /// Preprocess 3D surface mesh needed for FindPoints.
    void findptssurf_setup_3(DEV_STRUCT &devs,
                             const double *const elx[3],
                             const unsigned n,
-                            const unsigned int nel,
+                            const uint nel,
                             const unsigned m,
                             const double bbox_rel_size_inc,
-                            const unsigned int local_hash_size,
-                            const unsigned int global_hash_size,
+                            const uint local_hash_size,
+                            const uint global_hash_size,
                             const int rD,
                             const Vector *aabb_sz_inc);
 

fem/integ/bilininteg_convection_pa.cpp

@@ -178,8 +178,6 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    // Assumes tensor-product elements
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
-   MFEM_VERIFY(el.GetMapType() == FiniteElement::VALUE,
-               "Only value map type currently supported");
    ElementTransformation &Trans = *mesh->GetTypicalElementTransformation();
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Trans);
    if (DeviceCanUseCeed())

fem/integ/bilininteg_elasticity_kernels.cpp

@@ -91,15 +91,15 @@ void ElasticityAddMultPA(const int dim, const int nDofs,
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag)
+                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
    switch (dim)
    {
       case 2:
-         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, ir, diag);
+         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, QVec, diag);
          break;
       case 3:
-         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, ir, diag);
+         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, QVec, diag);
          break;
       default:
          MFEM_ABORT("Only dimensions 2 and 3 supported.");

fem/integ/bilininteg_elasticity_kernels.hpp

@@ -38,6 +38,7 @@
 #include "../../linalg/vector.hpp"
 #include "../../linalg/tensor.hpp"
 #include "../quadinterpolator.hpp"
+#include "../bilininteg.hpp"
 #include "../coefficient.hpp"
 #include "../qfunction.hpp"
 
@@ -132,12 +133,12 @@ void ElasticityAssembleEA(const int dim, const int i_block, const int j_block,
 /// @param[in] mu Quadrature function for second Lame param.
 /// @param[in] geom Geometric factors corresponding to fespace.
 /// @param[in] maps DofToQuad maps for one element (assume elements all same).
-/// @param[in] ir Integration rule.
+/// @param QVec Scratch Q-Vector. nQuad x dim x dim x dim x dim x numEls.
 /// @param[out] diag diagonal of A. nDofs x dim x numEls.
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag);
+                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag);
 
 /// Templated implementation of ElasticityAddMultPA.
 template<int dim, int i_block = -1, int j_block = -1>
@@ -279,67 +280,77 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
 template<int dim>
 void ElasticityAssembleDiagonalPA_(const int nDofs,
                                    const CoefficientVector &lambda,
-                                   const CoefficientVector &mu,
-                                   const GeometricFactors &geom,
-                                   const DofToQuad &maps,
-                                   const IntegrationRule &ir,
-                                   Vector &diag)
+                                   const CoefficientVector &mu, const GeometricFactors &geom,
+                                   const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
+   using future::tensor;
+   using future::make_tensor;
    using future::det;
    using future::inv;
-   using future::make_tensor;
-   using future::tensor;
 
    // Assuming all elements are the same
+   const auto &ir = QVec.GetIntRule(0);
    static constexpr int d = dim;
    const int numPoints = ir.GetNPoints();
-   const int numEls = lambda.Size() / numPoints;
-
+   const int numEls = lambda.Size()/numPoints;
    const auto lamDev = Reshape(lambda.Read(), numPoints, numEls);
    const auto muDev = Reshape(mu.Read(), numPoints, numEls);
    const auto J = Reshape(geom.J.Read(), numPoints, d, d, numEls);
+   auto Q = Reshape(QVec.ReadWrite(), numPoints, d,d, d, numEls);
    const real_t *ipWeights = ir.GetWeights().Read();
-   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
-   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
+   mfem::forall_2D(numEls, numPoints,1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(p, x,numPoints)
+      {
+         auto invJ = inv(make_tensor<d, d>(
+         [&](int i, int j) { return J(p, i, j, e); }));
+         const real_t w = ipWeights[p] /det(invJ);
+         for (int n = 0; n < d; n++)
+         {
+            for (int m = 0; m < d; m++)
+            {
+               for (int q = 0; q < d; q++)
+               {
+                  // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                  // this contraction could be made slightly cheaper using Voigt
+                  // notation, but repeated entries are summed for simplicity.
+                  real_t contraction = 0.;
+                  for (int a = 0; a < d; a++)
+                  {
+                     for (int b = 0; b < d; b++)
+                     {
+                        contraction += ((a == q)*invJ(m,b) + (b==q)*invJ(m,a))*((a == q)
+                                                                                *invJ(n, b) + (b==q)*invJ(n,a));
+                     }
+                  }
+                  // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
+                  // contraction = 4*sym(grad(u))sym(grad(v))
+                  Q(p,m,n,q,e) = w*(lamDev(p, e)*invJ(m,q)*invJ(n,q)
+                                    + 0.5*muDev(p, e)*contraction);
+               }
+            }
+         }
+      }
+   });
 
+   // Reduce quadrature function to an E-Vector
+   const auto QRead = Reshape(QVec.Read(), numPoints, d, d, d, numEls);
+   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
+   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
    mfem::forall_2D(numEls, d, nDofs, [=] MFEM_HOST_DEVICE (int e)
    {
-      MFEM_FOREACH_THREAD_DIRECT(i, y, nDofs)
+      MFEM_FOREACH_THREAD(i, y, nDofs)
       {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, d)
+         MFEM_FOREACH_THREAD(q, x, d)
          {
-            real_t sum = 0.0;
-            for (int p = 0; p < numPoints; p++)
+            real_t sum = 0.;
+            for (int n = 0; n < d; n++)
             {
-               const auto invJ = inv(make_tensor<d, d>([&](int r, int c)
+               for (int m = 0; m < d; m++)
                {
-                  return J(p, r, c, e);
-               }));
-               const real_t w = ipWeights[p] / det(invJ);
-
-               for (int n = 0; n < d; n++)
-               {
-                  for (int m = 0; m < d; m++)
+                  for (int p = 0; p < numPoints; p++ )
                   {
-                     // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-                     // this contraction could be made slightly cheaper using Voigt
-                     // notation, but repeated entries are summed for simplicity.
-                     real_t contraction = 0.0;
-                     for (int a = 0; a < d; a++)
-                     {
-                        for (int b = 0; b < d; b++)
-                        {
-                           contraction +=
-                              ((a == q) * invJ(m, b) + (b == q) * invJ(m, a)) *
-                              ((a == q) * invJ(n, b) + (b == q) * invJ(n, a));
-                        }
-                     }
-                     // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
-                     // contraction = 4*sym(grad(u))sym(grad(v))
-                     const real_t Q =
-                        w * (lamDev(p, e) * invJ(m, q) * invJ(n, q)
-                             + 0.5 * muDev(p, e) * contraction);
-                     sum += Q * G(p, m, i) * G(p, n, i);
+                     sum += QRead(p,m,n,q,e)*G(p,m,i)*G(p,n,i);
                   }
                }
             }

fem/integ/bilininteg_elasticity_pa.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 #include "bilininteg_elasticity_kernels.hpp"
 
@@ -58,8 +59,9 @@ void ElasticityIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
 void ElasticityIntegrator::AssembleDiagonalPA(Vector &diag)
 {
+   q_vec->SetVDim(vdim*vdim*vdim*vdim);
    internal::ElasticityAssembleDiagonalPA(vdim, ndofs, *lambda_quad, *mu_quad,
-                                          *geom, *maps, *IntRule, diag);
+                                          *geom, *maps, *q_vec, diag);
 }
 
 void ElasticityIntegrator::AddMultPA(const Vector &x, Vector &y) const

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -147,16 +147,18 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHcurlMassApply2D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int D1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D)
+void PAHcurlMassApply2D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y)
 {
-   MFEM_ASSERT(D1D == TestD1D,
-               "Trial and Test space must have the same number of dofs");
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(bc.Read(), Q1D, D1D);
    auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
@@ -275,16 +277,18 @@ void PAHcurlMassApply2D(const int NE, const bool symmetric,
    }); // end of element loop
 }
 
-void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int D1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D)
+void PAHcurlMassApply3D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
@@ -785,23 +789,6 @@ void PAHcurlL2Setup2D(const int Q1D,
    });
 }
 
-void PAHcurlL2IntSetup2D(const int Q1D, const int NE, const Array<real_t> &w,
-                         Vector &coeff, const Vector &detJ, Vector &op)
-{
-   const int NQ = Q1D*Q1D;
-   auto W = w.Read();
-   auto C = Reshape(coeff.Read(), NQ, NE);
-   auto J = Reshape(detJ.Read(), NQ, NE);
-   auto y = Reshape(op.Write(), NQ, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         y(q,e) = W[q] * C(q,e) / J(q,e);
-      }
-   });
-}
-
 void PAHcurlL2Setup3D(const int NQ,
                       const int coeffDim,
                       const int NE,

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -181,312 +181,228 @@ inline void SmemPAHcurlMassAssembleDiagonal3D(const int d1d,
 }
 
 // PA H(curl) Mass Apply 2D kernel
-void PAHcurlMassApply2D(const int NE, const bool symmetric,
-                        const bool scalar_coeff, const Array<real_t> &bo,
-                        const Array<real_t> &bc, const Array<real_t> &bot,
-                        const Array<real_t> &bct, const Vector &pa_data,
-                        const Vector &x, Vector &y, const int TrialD1D,
-                        const int TestD1D, const int Q1D);
+void PAHcurlMassApply2D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
 
 // PA H(curl) Mass Apply 3D kernel
-void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int TrialD1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D);
+void PAHcurlMassApply3D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
 
 // Shared memory PA H(curl) Mass Apply 3D kernel
-template <int T_D1D = 0, int T_Q1D = 0, int TBATCH = 0, bool ACCUMULATE = true>
-inline void SmemPAHcurlMassApply3D(
-   const int NE, const bool symmetric, [[maybe_unused]] const bool scalar_coeff,
-   const Array<real_t> &bo, const Array<real_t> &bc,
-   [[maybe_unused]] const Array<real_t> &bot,
-   [[maybe_unused]] const Array<real_t> &bct, const Vector &pa_data,
-   const Vector &x, Vector &y, const int d1d = 0,
-   [[maybe_unused]] const int test_d1d = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHcurlMassApply3D(const int d1d,
+                                   const int q1d,
+                                   const int NE,
+                                   const bool symmetric,
+                                   const Array<real_t> &bo,
+                                   const Array<real_t> &bc,
+                                   const Array<real_t> &bot,
+                                   const Array<real_t> &bct,
+                                   const Vector &pa_data,
+                                   const Vector &x,
+                                   Vector &y)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
    MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: d1d > HCURL_MAX_D1D");
    MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
                "Error: q1d > HCURL_MAX_Q1D");
-   MFEM_ASSERT(Q1D >= D1D, "Expected Q1D >= D1D");
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
    const int dataSize = symmetric ? 6 : 9;
 
-   // assume trial space == test space
-   auto Bo = bo.Read();
-   auto Bc = bc.Read();
-   auto op =
-      Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
-   auto X_ = Reshape(x.Read(), 3 * (D1D - 1) * D1D * D1D, NE);
-   auto y_ = y.ReadWrite();
+   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
+   auto X = Reshape(x.Read(), 3*(D1D-1)*D1D*D1D, NE);
+   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
 
-   constexpr int MD_ = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-   constexpr int MQ_ = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-   constexpr int MDQ_ = std::max(MD_, MQ_);
-   constexpr int MB_ = TBATCH ? TBATCH : 1;
-
-   mfem::forall_2D_batch<MDQ_ * MDQ_ * MDQ_ * MB_>(
-      NE, MDQ_ * MDQ_ * MDQ_, 1, MB_, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-      constexpr int nbz = TBATCH ? TBATCH : 1;
-      int tidz = MFEM_THREAD_ID(z);
-#else
-      constexpr int nbz = 1;
-      constexpr int tidz = 0;
-#endif
-
       constexpr int VDIM = 3;
       constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
       constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-      constexpr int MDQ = std::max(MD1D, MQ1D);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
 
-      // nvcc limit work-around: can't have Y_ be captured first in
-      // if constexpr, so capture y_ and construct Y_ locally
-      // only works on GPU
-      auto Y = Reshape(y_, VDIM * (D1D - 1) * D1D * D1D, NE);
+      MFEM_SHARED real_t sBo[MQ1D][MD1D];
+      MFEM_SHARED real_t sBc[MQ1D][MD1D];
 
-      MFEM_SHARED real_t sBo[MDQ * (MD1D - 1)];
-      MFEM_SHARED real_t sBc[MDQ * MD1D];
-      auto BO = Reshape(sBo, Q1D, D1D - 1);
-      auto BC = Reshape(sBc, Q1D, D1D);
+      real_t op9[9];
+      MFEM_SHARED real_t sop[9*MQ1D*MQ1D];
+      MFEM_SHARED real_t mass[MQ1D][MQ1D][3];
 
-      MFEM_SHARED real_t sX[nbz * VDIM * (MD1D - 1) * MD1D * MD1D];
-      MFEM_SHARED real_t sm0[nbz * VDIM * MDQ * MDQ * MDQ];
-      MFEM_SHARED real_t sm1[nbz * VDIM * MDQ * MDQ * MDQ];
+      MFEM_SHARED real_t sX[MD1D][MD1D][MD1D];
 
-      real_t(*X)[nbz][(MD1D - 1) * MD1D * MD1D] =
-         (real_t(*)[nbz][(MD1D - 1) * MD1D * MD1D])(sX);
-      // shapes of buffers always use MQ1D to mitigate shared memory bank
-      // conflicts
-      real_t(*DDQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-      real_t(*DQQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
-      real_t(*QQQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-      real_t(*QQD)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
-      real_t(*QDD)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-
-      // load dofs into smem
-      const int offset = (D1D - 1) * D1D * D1D;
-      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
-         for (int dim = 0; dim < VDIM; ++dim)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
-            X[dim][tidz][ix] = X_(ix + dim * offset, e);
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               for (int i=0; i<dataSize; ++i)
+               {
+                  op9[i] = op(qx,qy,qz,i,e);
+               }
+            }
          }
       }
-      // load basis functions data
+
+      const int tidx = MFEM_THREAD_ID(x);
+      const int tidy = MFEM_THREAD_ID(y);
+      const int tidz = MFEM_THREAD_ID(z);
+
       if (tidz == 0)
       {
-         MFEM_FOREACH_THREAD_DIRECT(ix, x, D1D * Q1D) { sBc[ix] = Bc[ix]; }
-         MFEM_FOREACH_THREAD_DIRECT(ix, x, (D1D - 1) * Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            sBo[ix] = Bo[ix];
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               sBc[q][d] = Bc(q,d);
+               if (d < D1D-1)
+               {
+                  sBo[q][d] = Bo(q,d);
+               }
+            }
          }
       }
+      MFEM_SYNC_THREAD;
 
-      for (int dim0 = 0; dim0 < VDIM; ++dim0)
+      for (int qz=0; qz < Q1D; ++qz)
       {
-         MFEM_SYNC_THREAD;
-         // sum factor to QQQ = Q_{dim0,dim1} B X_{dim1}
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         int osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
          {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
 
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, Q1D, D1Dy, D1Dz,
-                                                 Q1D, Q1D, Q1D)
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
             {
-               real_t u = 0;
-               for (int dx = 0; dx < D1Dx; ++dx)
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
                {
-                  real_t b;
-                  if (dim1 == 0)
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
                   {
-                     b = BO(qx, dx);
+                     sX[dz][dy][dx] = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
                   }
-                  else
-                  {
-                     b = BC(qx, dx);
-                  }
-                  u += X[dim1][tidz][dx + (dy + dz * D1Dy) * D1Dx] * b;
-               }
-               DDQ[dim1][tidz][dz][dy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
-         {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, Q1D, Q1D, D1Dz,
-                                                 Q1D, Q1D, Q1D)
-            {
-               real_t u = 0;
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  real_t b;
-                  if (dim1 == 1)
-                  {
-                     b = BO(qy, dy);
-                  }
-                  else
-                  {
-                     b = BC(qy, dy);
-                  }
-                  u += DDQ[dim1][tidz][dz][dy][qx] * b;
-               }
-               DQQ[dim1][tidz][dz][qy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
-         {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            // const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
-            MFEM_FOREACH_THREAD_DIRECT_3D(qx, qy, qz, x, Q1D, Q1D, Q1D)
-            {
-               real_t u = 0;
-               for (int dz = 0; dz < D1Dz; ++dz)
-               {
-                  real_t b;
-                  if (dim1 == 2)
-                  {
-                     b = BO(qz, dz);
-                  }
-                  else
-                  {
-                     b = BC(qz, dz);
-                  }
-                  u += DQQ[dim1][tidz][dz][qy][qx] * b;
-               }
-               // pa_data is row major
-               int idx;
-               if (symmetric)
-               {
-                  int row;
-                  int col;
-                  if (dim0 > dim1)
-                  {
-                     row = dim1;
-                     col = dim0;
-                  }
-                  else
-                  {
-                     row = dim0;
-                     col = dim1;
-                  }
-                  idx = col + VDIM * row - row * (row + 1) / 2;
-               }
-               else
-               {
-                  idx = dim0 * VDIM + dim1;
-               }
-               QQQ[dim1][tidz][qz][qy][qx] = op(qx, qy, qz, idx, e) * u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         // sum factor back to Y
-         // Assume bot and bct == bo^t and bc^t respectively (i.e. test ==
-         // trial functions), skip loading them again.
-         {
-            const int D1Dz = (dim0 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim0 == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (dim0 == 0) ? D1D - 1 : D1D;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, D1Dz, Q1D, Q1D,
-                                                 Q1D, Q1D, Q1D)
-            {
-               for (int dim1 = 0; dim1 < VDIM; ++dim1)
-               {
-                  real_t u = 0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     real_t b = 0;
-                     if (dim0 == 2)
-                     {
-                        b = BO(qz, dz);
-                     }
-                     else
-                     {
-                        b = BC(qz, dz);
-                     }
-                     u += QQQ[dim1][tidz][qz][qy][qx] * b;
-                  }
-                  QQD[dim1][tidz][qy][qx][dz] = u;
-               }
-            }
-            MFEM_SYNC_THREAD;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, D1Dy, D1Dz, Q1D,
-                                                 Q1D, Q1D, Q1D)
-            {
-               for (int dim1 = 0; dim1 < VDIM; ++dim1)
-               {
-                  real_t u = 0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     real_t b;
-                     if (dim0 == 1)
-                     {
-                        b = BO(qy, dy);
-                     }
-                     else
-                     {
-                        b = BC(qy, dy);
-                     }
-                     u += QQD[dim1][tidz][qy][qx][dz] * b;
-                  }
-                  QDD[dim1][tidz][qx][dz][dy] = u;
                }
             }
             MFEM_SYNC_THREAD;
 
-            MFEM_FOREACH_THREAD_DIRECT_3D(dx, dy, dz, x, D1Dx, D1Dy, D1Dz)
+            if (tidz == qz)
             {
-               int ix = dx + D1Dx * (dy + D1Dy * dz);
-               real_t u = 0;
-               for (int qx = 0; qx < Q1D; ++qx)
+               for (int i=0; i<dataSize; ++i)
                {
-                  real_t b;
-                  if (dim0 == 0)
-                  {
-                     b = BO(qx, dx);
-                  }
-                  else
-                  {
-                     b = BC(qx, dx);
-                  }
-                  for (int dim1 = 0; dim1 < VDIM; ++dim1)
-                  {
-                     u += QDD[dim1][tidz][qx][dz][dy] * b;
-                  }
+                  sop[i + (dataSize*tidx) + (dataSize*Q1D*tidy)] = op9[i];
                }
-               if constexpr (ACCUMULATE)
+
+               MFEM_FOREACH_THREAD(qy,y,Q1D)
                {
-                  Y(ix + dim0 * offset, e) += u;
-               }
-               else
+                  MFEM_FOREACH_THREAD(qx,x,Q1D)
+                  {
+                     real_t u = 0.0;
+
+                     for (int dz = 0; dz < D1Dz; ++dz)
+                     {
+                        const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
+                        for (int dy = 0; dy < D1Dy; ++dy)
+                        {
+                           const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
+                           for (int dx = 0; dx < D1Dx; ++dx)
+                           {
+                              const real_t t = sX[dz][dy][dx];
+                              const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
+                              u += t * wx * wy * wz;
+                           }
+                        }
+                     }
+
+                     mass[qy][qx][c] = u;
+                  } // qx
+               } // qy
+            } // tidz == qz
+
+            osc += D1Dx * D1Dy * D1Dz;
+            MFEM_SYNC_THREAD;
+         } // c
+
+         MFEM_SYNC_THREAD;  // Sync mass[qy][qx][d] and sop
+
+         osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
+         {
+            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
+
+            real_t dxyz = 0.0;
+
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
+            {
+               const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
+
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
                {
-                  Y(ix + dim0 * offset, e) = u;
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
+                  {
+                     for (int qy = 0; qy < Q1D; ++qy)
+                     {
+                        const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
+                        for (int qx = 0; qx < Q1D; ++qx)
+                        {
+                           const int os = (dataSize*qx) + (dataSize*Q1D*qy);
+                           const int id1 = os + ((c == 0) ? 0 : ((c == 1) ? (symmetric ? 1 : 3) :
+                                                                 (symmetric ? 2 : 6))); // O11, O21, O31
+                           const int id2 = os + ((c == 0) ? 1 : ((c == 1) ? (symmetric ? 3 : 4) :
+                                                                 (symmetric ? 4 : 7))); // O12, O22, O32
+                           const int id3 = os + ((c == 0) ? 2 : ((c == 1) ? (symmetric ? 4 : 5) :
+                                                                 (symmetric ? 5 : 8))); // O13, O23, O33
+
+                           const real_t m_c = (sop[id1] * mass[qy][qx][0]) + (sop[id2] * mass[qy][qx][1]) +
+                                              (sop[id3] * mass[qy][qx][2]);
+
+                           const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
+                           dxyz += m_c * wx * wy * wz;
+                        }
+                     }
+                  }
                }
             }
-         }
-      }
+
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
+            {
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
+               {
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
+                  {
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) += dxyz;
+                  }
+               }
+            }
+
+            osc += D1Dx * D1Dy * D1Dz;
+         } // c loop
+      } // qz
    }); // end of element loop
 }
 
@@ -1889,17 +1805,13 @@ inline void SmemPACurlCurlApply3D(const int d1d,
    ForallWrap<3>(true, NE, device_kernel, host_kernel, Q1D, Q1D, Q1D);
 }
 
-// PA H(curl)-L2 value Assemble 2D kernel
+// PA H(curl)-L2 Assemble 2D kernel
 void PAHcurlL2Setup2D(const int Q1D,
                       const int NE,
                       const Array<real_t> &w,
                       Vector &coeff,
                       Vector &op);
 
-// PA H(curl)-L2 integral Assemble 2D kernel
-void PAHcurlL2IntSetup2D(const int Q1D, const int NE, const Array<real_t> &w,
-                         Vector &coeff, const Vector &detJ, Vector &op);
-
 // PA H(curl)-L2 Assemble 3D kernel
 void PAHcurlL2Setup3D(const int NQ,
                       const int coeffDim,

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -62,30 +62,6 @@ void PAHcurlHdivMassApply2D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
-/// H(curl) test, H(div) trial
-inline void
-PAHcurlHdivMassApply2D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false,
-                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
-/// H(div) test, H(curl) trial
-inline void
-PAHdivHcurlMassApply2D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true,
-                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
 // PA H(curl)-H(div) Mass Apply 3D kernel
 void PAHcurlHdivMassApply3D(const int D1D,
                             const int D1Dtest,
@@ -102,30 +78,6 @@ void PAHcurlHdivMassApply3D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
-/// H(curl) test, H(div) trial
-inline void
-PAHcurlHdivMassApply3D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false, false, Bo_,
-                          Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
-/// H(div) test, H(curl) trial
-inline void
-PAHdivHcurlMassApply3D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true, false, Bo_,
-                          Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
 // PA H(curl)-H(div) Curl Apply 3D kernel
 template<int T_D1D = 0, int T_D1D_TEST = 0, int T_Q1D = 0>
 inline void PAHcurlHdivApply3D(const int d1d,
@@ -864,656 +816,8 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
    }); // end of element loop
 }
 
-namespace curlinterp
-{
-constexpr int NBZ3D(int ndof_o, int nquad_o, int mdq)
-{
-   if (ndof_o <= 0 || nquad_o <= 0)
-   {
-      return 1;
-   }
-   int ndof_c = ndof_o + 1;
-   int nquad_c = nquad_o + 1;
-   // z dimension is capped at 64 on nvidia and amd gpus
-   int tmp =
-      std::min((128 + mdq * mdq * (mdq - 1) - 1) / (mdq * mdq * (mdq - 1)), 64);
-   int smem_req =
-      sizeof(mfem::real_t) *
-      ((3 * ndof_c * ndof_c * ndof_o + 2 * 2 * mdq * mdq * mdq) * tmp +
-       ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
-   // assume GPU has at least 48k shared memory
-   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
-}
-}
-
-template <int T_NDOF_O, int T_NQUAD_O>
-void CurlInterpolatorApply3DSmem(const int ne, const int ndof_o,
-                                 const int nquad_o, const Vector &pa,
-                                 const Vector &x_, Vector &y_)
-{
-   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-   constexpr int mnq_o =
-      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
-   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
-   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
-   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
-   int mnq = std::max(ndof_o + 1, nquad_o + 1);
-   auto pa_data = pa.Read();
-   auto x_d = x_.Read();
-   auto y_d = y_.ReadWrite();
-   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
-      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MND_O =
-         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-      constexpr int MNQ_O =
-         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
-      int tidz = MFEM_THREAD_ID(z);
-      // Make mnq a local variable since capturing would result in different
-      // captures between host/device versions, and spuriously fails
-      int mnq = std::max(ndof_o + 1, nquad_o + 1);
-#else
-      constexpr int nbz = 1;
-      constexpr int tidz = 0;
-#endif
-      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
-      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
-      const int NDOF_C = NDOF_O + 1;
-      const int NQUAD_C = NQUAD_O + 1;
-      MFEM_SHARED real_t
-      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
-      auto X_ = Reshape(x_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
-      auto Y = Reshape(y_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
-      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
-      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
-      auto Boo =
-         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
-      MFEM_SHARED real_t X[3][nbz][MND_O * (MND_O + 1) * (MND_O + 1)];
-      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
-      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
-
-      // shapes of buffers always use MNDQ to mitigate shared memory bank
-      // conflicts
-      real_t(*DDQ)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      real_t(*DQQ)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
-      real_t(*QQQ)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      const int offset = NDOF_O * NDOF_C * NDOF_C;
-      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
-      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
-      {
-         for (int dim = 0; dim < 3; ++dim)
-         {
-            X[dim][tidz][ix] = X_(ix + dim * offset, e);
-         }
-      }
-      // load basis functions data
-      if (tidz == 0)
-      {
-         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
-         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
-      }
-      MFEM_SYNC_THREAD;
-
-      // x: Vz Bcc Gco Boo - Vy Bcc Boo Gco
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Bcc(qx, dx);
-         }
-         DDQ[0][tidz][dz][dy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Bcc(qx, dx);
-         }
-         DDQ[1][tidz][dz][dy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[0][tidz][dz][dy][qx] * Gco(qy, dy);
-         }
-         DQQ[0][tidz][dz][qy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_O; ++dy)
-         {
-            u += DDQ[1][tidz][dz][dy][qx] * Boo(qy, dy);
-         }
-         DQQ[1][tidz][dz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_O; ++dz)
-         {
-            u += DQQ[0][tidz][dz][qy][qx] * Boo(qz, dz);
-         }
-         QQQ[0][tidz][qz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[1][tidz][dz][qy][qx] * Gco(qz, dz);
-         }
-         Y(qx + (qy + qz * NQUAD_O) * NQUAD_C, e) =
-            QQQ[0][tidz][qz][qy][qx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // y: Vx Boo Bcc Gco - Vz Gco Bcc Boo
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_O; ++dx)
-         {
-            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
-         }
-         DDQ[0][tidz][dz][dy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Gco(qx, dx);
-         }
-         DDQ[1][tidz][dz][dy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[0][tidz][dz][dy][qx] * Bcc(qy, dy);
-         }
-         DQQ[0][tidz][dz][qy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
-                                           NDOF_O, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[1][tidz][dz][dy][qx] * Bcc(qy, dy);
-         }
-         DQQ[1][tidz][dz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[0][tidz][dz][qy][qx] * Gco(qz, dz);
-         }
-         QQQ[0][tidz][qz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_O; ++dz)
-         {
-            u += DQQ[1][tidz][dz][qy][qx] * Boo(qz, dz);
-         }
-         Y(qx + (qy + qz * NQUAD_C) * NQUAD_O + offsetq, e) =
-            QQQ[0][tidz][qz][qy][qx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // z: Vy Gco Boo Bcc - Vx Boo Gco Bcc
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Gco(qx, dx);
-         }
-         DDQ[0][tidz][dz][dy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_O; ++dx)
-         {
-            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
-         }
-         DDQ[1][tidz][dz][dy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_O; ++dy)
-         {
-            u += DDQ[0][tidz][dz][dy][qx] * Boo(qy, dy);
-         }
-         DQQ[0][tidz][dz][qy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[1][tidz][dz][dy][qx] * Gco(qy, dy);
-         }
-         DQQ[1][tidz][dz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[0][tidz][dz][qy][qx] * Bcc(qz, dz);
-         }
-         QQQ[0][tidz][qz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[1][tidz][dz][qy][qx] * Bcc(qz, dz);
-         }
-         Y(qx + (qy + qz * NQUAD_O) * NQUAD_O + 2 * offsetq, e) =
-            QQQ[0][tidz][qz][qy][qx] - u;
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-template <int T_NDOF_O, int T_NQUAD_O>
-void CurlInterpolatorTApply3DSmem(const int ne, const int ndof_o,
-                                  const int nquad_o, const Vector &pa,
-                                  const Vector &x_, Vector &y_)
-{
-   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-   constexpr int mnq_o =
-      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
-   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
-   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
-   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
-   int mnq = std::max(ndof_o + 1, nquad_o + 1);
-   auto pa_data = pa.Read();
-   auto x_d = x_.Read();
-   auto y_d = y_.ReadWrite();
-   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
-      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MND_O =
-         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-      constexpr int MNQ_O =
-         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
-      int tidz = MFEM_THREAD_ID(z);
-      // Make mnq a local variable since capturing would result in different
-      // captures between host/device versions, and spuriously fails
-      int mnq = std::max(ndof_o + 1, nquad_o + 1);
-#else
-      constexpr int nbz = 1;
-      constexpr int tidz = 0;
-#endif
-      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
-      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
-      const int NDOF_C = NDOF_O + 1;
-      const int NQUAD_C = NQUAD_O + 1;
-      MFEM_SHARED real_t
-      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
-      auto X_ = Reshape(x_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
-      auto Y = Reshape(y_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
-      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
-      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
-      auto Boo =
-         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
-      MFEM_SHARED real_t X[3][nbz][MNQ_O * MNQ_O * (MNQ_O + 1)];
-      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
-      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
-
-      // shapes of buffers always use MNDQ to mitigate shared memory bank
-      // conflicts
-      real_t(*QQD)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      real_t(*QDD)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
-      real_t(*DDD)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      const int offset = NDOF_O * NDOF_C * NDOF_C;
-      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
-      MFEM_FOREACH_THREAD_DIRECT(ix, x, offsetq)
-      {
-         for (int dim = 0; dim < 3; ++dim)
-         {
-            X[dim][tidz][ix] = X_(ix + dim * offsetq, e);
-         }
-      }
-      // load basis functions data
-      if (tidz == 0)
-      {
-         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
-         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
-      }
-      MFEM_SYNC_THREAD;
-
-      // x: Vy Boo Bcc Gco - Vz Boo Gco Bcc
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Gco(qz, dz);
-         }
-         QQD[0][tidz][qy][qx][dz] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_C; ++qz)
-         {
-            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
-         }
-         QQD[1][tidz][qy][qx][dz] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_C; ++qy)
-         {
-            u += QQD[0][tidz][qy][qx][dz] * Bcc(qy, dy);
-         }
-         QDD[0][tidz][qx][dz][dy] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[1][tidz][qy][qx][dz] * Gco(qy, dy);
-         }
-         QDD[1][tidz][qx][dz][dy] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[0][tidz][qx][dz][dy] * Boo(qx, dx);
-         }
-         DDD[0][tidz][dz][dy][dx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[1][tidz][qx][dz][dy] * Boo(qx, dx);
-         }
-         Y(dx + (dy + dz * NDOF_C) * NDOF_O, e) =
-            DDD[0][tidz][dz][dy][dx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // y: Vz Gco Boo Bcc - Vx Bcc Boo Gco
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_C; ++qz)
-         {
-            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
-         }
-         QQD[0][tidz][qy][qx][dz] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Gco(qz, dz);
-         }
-         QQD[1][tidz][qy][qx][dz] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[0][tidz][qy][qx][dz] * Boo(qy, dy);
-         }
-         QDD[0][tidz][qx][dz][dy] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
-                                           NQUAD_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[1][tidz][qy][qx][dz] * Boo(qy, dy);
-         }
-         QDD[1][tidz][qx][dz][dy] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[0][tidz][qx][dz][dy] * Gco(qx, dx);
-         }
-         DDD[0][tidz][dz][dy][dx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_C; ++qx)
-         {
-            u += QDD[1][tidz][qx][dz][dy] * Bcc(qx, dx);
-         }
-         Y(dx + (dy + dz * NDOF_O) * NDOF_C + offset, e) =
-            DDD[0][tidz][dz][dy][dx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // z: Vx Bcc Gco Boo - Vy Gco Bcc Boo
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Boo(qz, dz);
-         }
-         QQD[0][tidz][qy][qx][dz] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Boo(qz, dz);
-         }
-         QQD[1][tidz][qy][qx][dz] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[0][tidz][qy][qx][dz] * Gco(qy, dy);
-         }
-         QDD[0][tidz][qx][dz][dy] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_C; ++qy)
-         {
-            u += QQD[1][tidz][qy][qx][dz] * Bcc(qy, dy);
-         }
-         QDD[1][tidz][qx][dz][dy] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_C; ++qx)
-         {
-            u += QDD[0][tidz][qx][dz][dy] * Bcc(qx, dx);
-         }
-         DDD[0][tidz][dz][dy][dx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[1][tidz][qx][dz][dy] * Gco(qx, dx);
-         }
-         Y(dx + (dy + dz * NDOF_C) * NDOF_C + 2 * offset, e) =
-            DDD[0][tidz][dz][dy][dx] - u;
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
 } // namespace internal
 
-template <int DIM, int NDOF_O, int NQUAD_O>
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyPAKernels::Kernel()
-{
-   if constexpr (DIM == 3)
-   {
-      return internal::CurlInterpolatorApply3DSmem<NDOF_O, NQUAD_O>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
-template <int DIM, int NDOF_O, int NQUAD_O>
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyTPAKernels::Kernel()
-{
-   if constexpr (DIM == 3)
-   {
-      return internal::CurlInterpolatorTApply3DSmem<NDOF_O, NQUAD_O>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
 } // namespace mfem
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_hdiv_kernels.cpp

@@ -294,14 +294,61 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int TestD1D, const int Q1D)
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<real_t> &Bo,
+                     const Array<real_t> &Bc,
+                     const Array<real_t> &Bot,
+                     const Array<real_t> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y)
+{
+   const int id = (D1D << 4) | Q1D;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x22: return SmemPAHdivMassApply2D<2,2>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x33: return SmemPAHdivMassApply2D<3,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x44: return SmemPAHdivMassApply2D<4,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x55: return SmemPAHdivMassApply2D<5,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply2D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x23: return SmemPAHdivMassApply3D<2,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x34: return SmemPAHdivMassApply3D<3,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x45: return SmemPAHdivMassApply3D<4,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x56: return SmemPAHdivMassApply3D<5,6>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x67: return SmemPAHdivMassApply3D<6,7>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x78: return SmemPAHdivMassApply3D<7,8>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply3D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+}
+
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
@@ -421,14 +468,18 @@ void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
    }); // end of element loop
 }
 
-void PAHdivMassApply3D(const int NE, const bool symmetric, const bool,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int TestD1D, const int Q1D)
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
                "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,

fem/integ/bilininteg_hdiv_kernels.hpp

@@ -66,29 +66,58 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
                                   const Vector &op_,
                                   Vector &diag_);
 
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<real_t> &Bo,
+                     const Array<real_t> &Bc,
+                     const Array<real_t> &Bot,
+                     const Array<real_t> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y);
+
 // PA H(div) Mass Apply 2D kernel
-void PAHdivMassApply2D(const int NE, const bool symmetric,
-                       const bool scalar_coeff, const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_, const Vector &op_,
-                       const Vector &x_, Vector &y_, const int D1D,
-                       const int TestD1D, const int Q1D);
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
 
 // PA H(div) Mass Apply 3D kernel
-void PAHdivMassApply3D(const int NE, const bool symmetric,
-                       const bool scalar_coeff, const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_, const Vector &op_,
-                       const Vector &x_, Vector &y_, const int D1D,
-                       const int TestD1D, const int Q1D);
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
 
 // Shared memory PA H(div) Mass Apply 2D kernel
-template <int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHdivMassApply2D(
-   const int NE, const bool symmetric, const bool, const Array<real_t> &Bo_,
-   const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-   const Array<real_t> &Bct_, const Vector &op_, const Vector &x_, Vector &y_,
-   const int d1d = 0, const int = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply2D(const int NE,
+                                  const bool symmetric,
+                                  const Array<real_t> &Bo_,
+                                  const Array<real_t> &Bc_,
+                                  const Array<real_t> &Bot_,
+                                  const Array<real_t> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);
@@ -251,13 +280,18 @@ inline void SmemPAHdivMassApply2D(
 }
 
 // Shared memory PA H(div) Mass Apply 3D kernel
-template <int T_D1D = 0, int T_Q1D = 0>
-inline void
-SmemPAHdivMassApply3D(const int NE, const bool symmetric, const bool,
-                      const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                      const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                      const Vector &op_, const Vector &x_, Vector &y_,
-                      const int d1d = 0, const int = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply3D(const int NE,
+                                  const bool symmetric,
+                                  const Array<real_t> &Bo_,
+                                  const Array<real_t> &Bc_,
+                                  const Array<real_t> &Bot_,
+                                  const Array<real_t> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);

fem/integ/bilininteg_interp_pa.cpp

@@ -14,218 +14,9 @@
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 
-#include "bilininteg_hcurlhdiv_kernels.hpp"
-
 namespace mfem
 {
 
-namespace
-{
-
-void PAHcurlApplyCurl2D(const int c_dofs1D,
-                        const int o_dofs1D,
-                        const int NE,
-                        const Array<real_t> &Bo_,
-                        const Array<real_t> &Gc_,
-                        const Vector &x_,
-                        Vector &y_)
-{
-   auto Bo = Reshape(Bo_.Read(), o_dofs1D, o_dofs1D);
-   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
-   auto X = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), o_dofs1D, o_dofs1D, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int iy = 0; iy < c_dofs1D; ++iy)
-      {
-         for (int ix = 0; ix < o_dofs1D; ++ix)
-         {
-            const real_t xv = X(ix + iy * o_dofs1D, e);
-            for (int oy = 0; oy < o_dofs1D; ++oy)
-            {
-               const real_t gy = Gc(oy, iy);
-               for (int ox = 0; ox < o_dofs1D; ++ox)
-               {
-                  Y(ox, oy, e) -= Bo(ox, ix) * gy * xv;
-               }
-            }
-         }
-      }
-
-      const int y_nd = c_dofs1D * o_dofs1D;
-      for (int iy = 0; iy < o_dofs1D; ++iy)
-      {
-         for (int ix = 0; ix < c_dofs1D; ++ix)
-         {
-            const real_t xv = X(y_nd + ix + iy * c_dofs1D, e);
-            for (int oy = 0; oy < o_dofs1D; ++oy)
-            {
-               const real_t by = Bo(oy, iy);
-               for (int ox = 0; ox < o_dofs1D; ++ox)
-               {
-                  Y(ox, oy, e) += Gc(ox, ix) * by * xv;
-               }
-            }
-         }
-      }
-   });
-}
-
-void PAHcurlApplyCurlTranspose2D(const int c_dofs1D,
-                                 const int o_dofs1D,
-                                 const int NE,
-                                 const Array<real_t> &Bo_,
-                                 const Array<real_t> &Gc_,
-                                 const Vector &x_,
-                                 Vector &y_)
-{
-   auto Bo = Reshape(Bo_.Read(), o_dofs1D, o_dofs1D);
-   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
-   auto X = Reshape(x_.Read(), o_dofs1D, o_dofs1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int dy = 0; dy < c_dofs1D; ++dy)
-      {
-         for (int dx = 0; dx < o_dofs1D; ++dx)
-         {
-            real_t sum = 0.0;
-            for (int oy = 0; oy < o_dofs1D; ++oy)
-            {
-               const real_t gy = Gc(oy, dy);
-               for (int ox = 0; ox < o_dofs1D; ++ox)
-               {
-                  sum -= Bo(ox, dx) * gy * X(ox, oy, e);
-               }
-            }
-            Y(dx + dy * o_dofs1D, e) += sum;
-         }
-      }
-
-      const int y_nd = c_dofs1D * o_dofs1D;
-      for (int dy = 0; dy < o_dofs1D; ++dy)
-      {
-         for (int dx = 0; dx < c_dofs1D; ++dx)
-         {
-            real_t sum = 0.0;
-            for (int oy = 0; oy < o_dofs1D; ++oy)
-            {
-               const real_t by = Bo(oy, dy);
-               for (int ox = 0; ox < o_dofs1D; ++ox)
-               {
-                  sum += Gc(ox, dx) * by * X(ox, oy, e);
-               }
-            }
-            Y(y_nd + dx + dy * c_dofs1D, e) += sum;
-         }
-      }
-   });
-}
-
-void PAHdivApplyCurl2D(const int c_dofs1D,
-                       const int o_dofs1D,
-                       const int NE,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Gc_,
-                       const Vector &x_,
-                       Vector &y_)
-{
-   auto Bc = Reshape(Bc_.Read(), c_dofs1D, c_dofs1D);
-   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
-   auto X = Reshape(x_.Read(), c_dofs1D, c_dofs1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int iy = 0; iy < c_dofs1D; ++iy)
-      {
-         for (int ix = 0; ix < c_dofs1D; ++ix)
-         {
-            const real_t xv = X(ix, iy, e);
-            for (int oy = 0; oy < o_dofs1D; ++oy)
-            {
-               const real_t gy = Gc(oy, iy);
-               for (int ox = 0; ox < c_dofs1D; ++ox)
-               {
-                  Y(ox + oy * c_dofs1D, e) += Bc(ox, ix) * gy * xv;
-               }
-            }
-         }
-      }
-
-      const int y_nd = c_dofs1D * o_dofs1D;
-      for (int iy = 0; iy < c_dofs1D; ++iy)
-      {
-         for (int ix = 0; ix < c_dofs1D; ++ix)
-         {
-            const real_t xv = X(ix, iy, e);
-            for (int oy = 0; oy < c_dofs1D; ++oy)
-            {
-               const real_t by = Bc(oy, iy);
-               for (int ox = 0; ox < o_dofs1D; ++ox)
-               {
-                  Y(y_nd + ox + oy * o_dofs1D, e) -= Gc(ox, ix) * by * xv;
-               }
-            }
-         }
-      }
-   });
-}
-
-void PAHdivApplyCurlTranspose2D(const int c_dofs1D,
-                                const int o_dofs1D,
-                                const int NE,
-                                const Array<real_t> &Bc_,
-                                const Array<real_t> &Gc_,
-                                const Vector &x_,
-                                Vector &y_)
-{
-   auto Bc = Reshape(Bc_.Read(), c_dofs1D, c_dofs1D);
-   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
-   auto X = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), c_dofs1D, c_dofs1D, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int dy = 0; dy < o_dofs1D; ++dy)
-      {
-         for (int dx = 0; dx < c_dofs1D; ++dx)
-         {
-            const real_t xv = X(dx + dy * c_dofs1D, e);
-            for (int iy = 0; iy < c_dofs1D; ++iy)
-            {
-               const real_t gy = Gc(dy, iy);
-               for (int ix = 0; ix < c_dofs1D; ++ix)
-               {
-                  Y(ix, iy, e) += Bc(dx, ix) * gy * xv;
-               }
-            }
-         }
-      }
-
-      const int y_nd = c_dofs1D * o_dofs1D;
-      for (int dy = 0; dy < c_dofs1D; ++dy)
-      {
-         for (int dx = 0; dx < o_dofs1D; ++dx)
-         {
-            const real_t xv = X(y_nd + dx + dy * o_dofs1D, e);
-            for (int iy = 0; iy < c_dofs1D; ++iy)
-            {
-               const real_t by = Bc(dy, iy);
-               for (int ix = 0; ix < c_dofs1D; ++ix)
-               {
-                  Y(ix, iy, e) -= Gc(dx, ix) * by * xv;
-               }
-            }
-         }
-      }
-   });
-}
-
-}
-
 // Apply to x corresponding to DOFs in H^1 (domain) the (topological) gradient
 // to get a dof in H(curl) (range). You can think of the range as the "test" space
 // and the domain as the "trial" space, but there's no integration.
@@ -2159,266 +1950,4 @@ void IdentityInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
    }
 }
 
-void CurlInterpolator::AssemblePA(const FiniteElementSpace &dom_fes,
-                                  const FiniteElementSpace &ran_fes)
-{
-   Mesh *mesh = dom_fes.GetMesh();
-   dim = mesh->Dimension();
-   ne = dom_fes.GetNE();
-   pa_mode_2d = 0;
-   MFEM_VERIFY(ne == ran_fes.GetNE(),
-               "Different meshes for domain and range spaces");
-
-   if (dim == 2)
-   {
-      pa_data.SetSize(0);
-      const FiniteElement *dom_fel = dom_fes.GetTypicalFE();
-      const FiniteElement *ran_fel = ran_fes.GetTypicalFE();
-      const bool hcurl_to_scalar =
-         dynamic_cast<const VectorTensorFiniteElement*>(dom_fel) != NULL &&
-         dom_fel->GetDerivType() == FiniteElement::CURL &&
-         dynamic_cast<const TensorBasisElement*>(ran_fel) != NULL &&
-         ran_fel->GetRangeType() == FiniteElement::SCALAR;
-      const bool scalar_to_hdiv =
-         dynamic_cast<const TensorBasisElement*>(dom_fel) != NULL &&
-         dom_fel->GetRangeType() == FiniteElement::SCALAR &&
-         dynamic_cast<const VectorTensorFiniteElement*>(ran_fel) != NULL &&
-         ran_fel->GetDerivType() == FiniteElement::DIV;
-
-      MFEM_VERIFY(hcurl_to_scalar || scalar_to_hdiv,
-                  "2D CurlInterpolator PA supports H(curl)->scalar and scalar->H(div) only.");
-
-      int closed_basis_type = -1;
-      int open_basis_type = -1;
-      if (hcurl_to_scalar)
-      {
-         const auto *trial_fec = dynamic_cast<const ND_FECollection*>(dom_fes.FEColl());
-         const auto *range_fec = dynamic_cast<const L2_FECollection*>(ran_fes.FEColl());
-         MFEM_VERIFY(trial_fec != NULL, "H(curl) domain must use ND_FECollection.");
-         MFEM_VERIFY(range_fec != NULL, "Scalar range must use L2_FECollection.");
-         MFEM_VERIFY(ran_fel->GetMapType() == FiniteElement::INTEGRAL,
-                     "2D H(curl)->scalar CurlInterpolator PA supports integral-map scalar range spaces only.");
-         closed_basis_type = trial_fec->GetClosedBasisType();
-         open_basis_type = trial_fec->GetOpenBasisType();
-         MFEM_VERIFY(range_fec->GetBasisType() == open_basis_type,
-                     "Domain/range open basis types do not match.");
-         pa_mode_2d = 1;
-      }
-      else
-      {
-         const auto *trial_fec = dynamic_cast<const H1_FECollection*>(dom_fes.FEColl());
-         const auto *range_fec = dynamic_cast<const RT_FECollection*>(ran_fes.FEColl());
-         MFEM_VERIFY(trial_fec != NULL, "Scalar domain must use H1_FECollection.");
-         MFEM_VERIFY(range_fec != NULL, "H(div) range must use RT_FECollection.");
-         closed_basis_type = trial_fec->GetBasisType();
-         open_basis_type = range_fec->GetOpenBasisType();
-         MFEM_VERIFY(range_fec->GetClosedBasisType() == closed_basis_type,
-                     "Domain/range closed basis types do not match.");
-         pa_mode_2d = 2;
-      }
-
-      const int order = hcurl_to_scalar
-                        ? dynamic_cast<const VectorTensorFiniteElement*>(dom_fel)->GetOrder()
-                        : dynamic_cast<const NodalTensorFiniteElement*>(dom_fel)->GetOrder();
-      c_dofs1D = order + 1;
-      o_dofs1D = order;
-
-      closed_dofquad_fe.reset(new H1_SegmentElement(order, closed_basis_type));
-      open_dofquad_fe.reset(new L2_SegmentElement(order - 1, open_basis_type));
-
-      mfem::QuadratureFunctions1D qf1d;
-      mfem::IntegrationRule closed_ir;
-      closed_ir.SetSize(c_dofs1D);
-      qf1d.GaussLobatto(c_dofs1D, &closed_ir);
-
-      mfem::IntegrationRule open_ir;
-      open_ir.SetSize(o_dofs1D);
-      qf1d.GaussLegendre(o_dofs1D, &open_ir);
-
-      maps_C_C = &closed_dofquad_fe->GetDofToQuad(closed_ir, DofToQuad::TENSOR);
-      maps_O_C = &closed_dofquad_fe->GetDofToQuad(open_ir, DofToQuad::TENSOR);
-      maps_O_O = &open_dofquad_fe->GetDofToQuad(open_ir, DofToQuad::TENSOR);
-
-      MFEM_VERIFY(maps_C_C->ndof == c_dofs1D && maps_C_C->nqpt == c_dofs1D, "");
-      MFEM_VERIFY(maps_O_C->ndof == c_dofs1D && maps_O_C->nqpt == o_dofs1D, "");
-      MFEM_VERIFY(maps_O_O->ndof == o_dofs1D && maps_O_O->nqpt == o_dofs1D, "");
-      return;
-   }
-
-   closed_dofquad_fe.reset();
-   open_dofquad_fe.reset();
-   maps_C_C = nullptr;
-   maps_O_C = nullptr;
-   maps_O_O = nullptr;
-
-   const VectorTensorFiniteElement *dom_el =
-      dynamic_cast<const VectorTensorFiniteElement *>(dom_fes.GetTypicalFE());
-   const VectorTensorFiniteElement *ran_el =
-      dynamic_cast<const VectorTensorFiniteElement *>(ran_fes.GetTypicalFE());
-   MFEM_VERIFY(dom_el != NULL, "Only VectorTensorFiniteElement is supported!");
-   MFEM_VERIFY(ran_el != NULL, "Only VectorTensorFiniteElement is supported!");
-   MFEM_VERIFY(dom_el->GetDerivType() == FiniteElement::CURL,
-               "Domain space must be H(curl)");
-   MFEM_VERIFY(ran_el->GetDerivType() == FiniteElement::DIV,
-               "Range space must be H(div)");
-
-   const int dims = dom_el->GetDim();
-   MFEM_VERIFY(dims == 3, "");
-
-   ndof_o = dom_el->GetOrder();
-   int ndof_c = ndof_o + 1;
-   nquad_o = ran_el->GetOrder();
-   int nquad_c = nquad_o + 1;
-
-   // extract the tensor product range dof locations
-   std::vector<real_t> qc(nquad_c);
-   std::vector<real_t> qo(nquad_o);
-   {
-      const IntegrationRule &ran_nodes = ran_el->GetNodes();
-      const Array<int> &quad_map = ran_el->GetDofMap();
-      for (int i = 0; i < nquad_c; ++i)
-      {
-         int idx = UnsignIndex(quad_map[i]);
-         qc[i] = ran_nodes.IntPoint(idx).x;
-      }
-      int offset = ndof_c * ndof_o * ndof_o;
-      for (int i = 0; i < nquad_o; ++i)
-      {
-         int idx = UnsignIndex(quad_map[i + offset]);
-         qo[i] = ran_nodes.IntPoint(idx).x;
-      }
-   }
-
-   // evaluate closed/open 1D basis (and their derivatives) at closed and
-   // open quads
-   // storage order: GCO, BCC, BOO
-   pa_data.SetSize(ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
-   auto ptr = pa_data.HostWrite();
-   auto &cbasis1d = dom_el->GetBasis1D();
-   auto &obasis1d = dom_el->GetOpenBasis1D();
-   Vector b, g;
-   b.SetSize(ndof_c);
-   g.SetSize(ndof_c);
-   for (int j = 0; j < nquad_o; ++j)
-   {
-      cbasis1d.Eval(qo[j], b, g);
-      for (int i = 0; i < ndof_c; ++i)
-      {
-         ptr[j + i * nquad_o] = g[i];
-      }
-   }
-   ptr += nquad_o * ndof_c;
-
-   for (int j = 0; j < nquad_c; ++j)
-   {
-      cbasis1d.Eval(qc[j], b);
-      for (int i = 0; i < ndof_c; ++i)
-      {
-         ptr[j + i * nquad_c] = b[i];
-      }
-   }
-   ptr += ndof_c * nquad_c;
-
-   b.SetSize(ndof_o);
-   for (int j = 0; j < nquad_o; ++j)
-   {
-      obasis1d.Eval(qo[j], b);
-      for (int i = 0; i < ndof_o; ++i)
-      {
-         ptr[j + i * nquad_o] = b[i];
-      }
-   }
-}
-
-CurlInterpolator::Kernels::Kernels()
-{
-   CurlInterpolator::AddSpecialization<3, 1, 1>();
-   CurlInterpolator::AddSpecialization<3, 2, 2>();
-   CurlInterpolator::AddSpecialization<3, 3, 3>();
-   CurlInterpolator::AddSpecialization<3, 4, 4>();
-   CurlInterpolator::AddSpecialization<3, 5, 5>();
-}
-
-CurlInterpolator::CurlInterpolator() { static Kernels kernels{}; }
-
-void CurlInterpolator::AddMultPA(const Vector &x, Vector &y) const
-{
-   if (dim == 2)
-   {
-      MFEM_VERIFY(maps_C_C != nullptr && maps_O_C != nullptr,
-                  "2D CurlInterpolator PA data is not assembled.");
-      if (pa_mode_2d == 1)
-      {
-         MFEM_VERIFY(maps_O_O != nullptr,
-                     "2D CurlInterpolator scalar curl map is not assembled.");
-         PAHcurlApplyCurl2D(c_dofs1D, o_dofs1D, ne, maps_O_O->B, maps_O_C->G,
-                            x, y);
-      }
-      else if (pa_mode_2d == 2)
-      {
-         PAHdivApplyCurl2D(c_dofs1D, o_dofs1D, ne, maps_C_C->B, maps_O_C->G,
-                           x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported 2D CurlInterpolator mode.");
-      }
-      return;
-   }
-
-   ApplyPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x, y);
-}
-
-void CurlInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
-{
-   if (dim == 2)
-   {
-      MFEM_VERIFY(maps_C_C != nullptr && maps_O_C != nullptr,
-                  "2D CurlInterpolator PA data is not assembled.");
-      if (pa_mode_2d == 1)
-      {
-         MFEM_VERIFY(maps_O_O != nullptr,
-                     "2D CurlInterpolator scalar curl map is not assembled.");
-         PAHcurlApplyCurlTranspose2D(c_dofs1D, o_dofs1D, ne, maps_O_O->B,
-                                     maps_O_C->G, x, y);
-      }
-      else if (pa_mode_2d == 2)
-      {
-         PAHdivApplyCurlTranspose2D(c_dofs1D, o_dofs1D, ne, maps_C_C->B,
-                                    maps_O_C->G, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported 2D CurlInterpolator mode.");
-      }
-      return;
-   }
-
-   ApplyTPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x, y);
-}
-
-/// \cond DO_NOT_DOCUMENT
-
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyPAKernels::Fallback(int DIM, int, int)
-{
-   if (DIM == 3)
-   {
-      return internal::CurlInterpolatorApply3DSmem<0, 0>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyTPAKernels::Fallback(int DIM, int, int)
-{
-   if (DIM == 3)
-   {
-      return internal::CurlInterpolatorTApply3DSmem<0, 0>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
 } // namespace mfem

fem/integ/bilininteg_vecdiv_pa.hpp

@@ -1,365 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/array.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../linalg/vector.hpp"
-#include "../bilininteg.hpp"
-#include "../kernels.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-// Shared memory PA Divergence Apply 2D kernel
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApply2D(const int NE,
-                                    const Array<real_t> &b_,
-                                    const Array<real_t> &g_,
-                                    const Array<real_t> &bt_,
-                                    const Vector &q_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
-   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, 2, NE);
-   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, 1, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
-      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
-
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs2d(e, TR_D1D, X, g0);
-      kernels::internal::Grad2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            r0[0][qy][qx] =
-               g1[0][0][qy][qx] * Q(qx, qy, 0, 0, e) +
-               g1[0][1][qy][qx] * Q(qx, qy, 1, 0, e) +
-               g1[1][0][qy][qx] * Q(qx, qy, 0, 1, e) +
-               g1[1][1][qy][qx] * Q(qx, qy, 1, 1, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TE_D1D, Q1D, Bt, sB);
-      kernels::internal::EvalTranspose2d(TE_D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs2d(e, TE_D1D, r1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 2D kernel transpose
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApplyTranspose2D(const int NE,
-                                             const Array<real_t> &bt,
-                                             const Array<real_t> &gt,
-                                             const Array<real_t> &b,
-                                             const Vector &q_,
-                                             const Vector &x_,
-                                             Vector &y_,
-                                             const int tr_d1d = 0,
-                                             const int te_d1d = 0,
-                                             const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
-   const auto X  = Reshape(x_.Read(), TE_D1D, TE_D1D, 1, NE);
-   auto Y  = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, 2, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
-      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
-      kernels::internal::LoadDofs2d(e, TE_D1D, X, r0);
-      kernels::internal::Eval2d(TE_D1D, Q1D, smem, sB, r0, r1);
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            g0[0][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 0, e);
-            g0[0][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 0, e);
-            g0[1][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 1, e);
-            g0[1][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 1, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
-      kernels::internal::GradTranspose2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-      kernels::internal::WriteDofs2d(e, TR_D1D, g1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 3D kernel transpose
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApplyTranspose3D(const int NE,
-                                             const Array<real_t> &bt,
-                                             const Array<real_t> &gt,
-                                             const Array<real_t> &b,
-                                             const Vector &q_,
-                                             const Vector &x_,
-                                             Vector &y_,
-                                             int tr_d1d = 0,
-                                             int te_d1d = 0,
-                                             int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
-   const auto X = Reshape(x_.Read(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
-   auto Y = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
-      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
-      kernels::internal::LoadDofs3d(e, TE_D1D, X, r0);
-      kernels::internal::Eval3d(TE_D1D, Q1D, smem, sB, r0, r1);
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const auto r = r1[0][qz][qy][qx];
-               g0[0][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 0, e);
-               g0[0][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 0, e);
-               g0[0][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 0, e);
-
-               g0[1][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 1, e);
-               g0[1][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 1, e);
-               g0[1][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 1, e);
-
-               g0[2][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 2, e);
-               g0[2][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 2, e);
-               g0[2][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 2, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
-      kernels::internal::GradTranspose3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-      kernels::internal::WriteDofs3d(e, TR_D1D, g1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 3D kernel
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApply3D(const int NE,
-                                    const Array<real_t> &b_,
-                                    const Array<real_t> &g_,
-                                    const Array<real_t> &bt_,
-                                    const Vector &q_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3,3, NE);
-   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
-   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
-      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
-
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs3d(e, TR_D1D, X, g0);
-      kernels::internal::Grad3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               r0[0][qz][qy][qx] =
-                  // c = 0
-                  g1[0][0][qz][qy][qx] * Q(qx, qy, qz, 0, 0, e) +
-                  g1[0][1][qz][qy][qx] * Q(qx, qy, qz, 1, 0, e) +
-                  g1[0][2][qz][qy][qx] * Q(qx, qy, qz, 2, 0, e) +
-                  // c = 1
-                  g1[1][0][qz][qy][qx] * Q(qx, qy, qz, 0, 1, e) +
-                  g1[1][1][qz][qy][qx] * Q(qx, qy, qz, 1, 1, e) +
-                  g1[1][2][qz][qy][qx] * Q(qx, qy, qz, 2, 1, e) +
-                  // c = 2
-                  g1[2][0][qz][qy][qx] * Q(qx, qy, qz, 0, 2, e) +
-                  g1[2][1][qz][qy][qx] * Q(qx, qy, qz, 1, 2, e) +
-                  g1[2][2][qz][qy][qx] * Q(qx, qy, qz, 2, 2, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1, true>(TE_D1D, Q1D, Bt, sB);
-      kernels::internal::EvalTranspose3d(TE_D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs3d(e, TE_D1D, r1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
-VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Kernel()
-{
-   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPADivergenceApply2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADivergenceApply3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Fallback
-(int dim, int tr_d1d, int te_d1d, int q1d)
-{
-   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
-   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPADivergenceApply2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPADivergenceApply3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Kernel()
-{
-   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPADivergenceApplyTranspose2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADivergenceApplyTranspose3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Fallback
-(int dim, int tr_d1d, int te_d1d, int q1d)
-{
-   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
-   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPADivergenceApplyTranspose2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPADivergenceApplyTranspose3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/bilininteg_vecmass_pa.cpp

@@ -22,8 +22,6 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
-   MFEM_VERIFY(el.GetMapType() == FiniteElement::VALUE,
-               "Only value map type supported");
    ElementTransformation &Trans = *mesh->GetTypicalElementTransformation();
    const auto *ir = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, Trans);
 
@@ -207,40 +205,157 @@ void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 
 }
 
+template <const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal2D(const int NE,
+                                           const Array<real_t> &b,
+                                           const Vector &pa_data, Vector &diag,
+                                           const int d1d = 0, const int q1d = 0)
+{
+   constexpr int VDIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const auto B = Reshape(b.Read(), Q1D, D1D);
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, VDIM, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t temp[max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            temp[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               temp[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            real_t temp1 = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               temp1 += B(qx, dx) * B(qx, dx) * temp[qx][dy];
+            }
+            Y(dx, dy, 0, e) = temp1;
+            Y(dx, dy, 1, e) = temp1;
+         }
+      }
+   });
+}
+
+template <const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal3D(const int NE,
+                                           const Array<real_t> &B_,
+                                           const Vector &pa_data, Vector &diag,
+                                           const int d1d = 0, const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const auto B = Reshape(B_.Read(), Q1D, D1D);
+   MFEM_VERIFY(pa_data.Size() == Q1D * Q1D * Q1D * NE, "pa_data size error");
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t temp[max_Q1D][max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               temp[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  temp[qx][qy][dz] +=
+                     B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      real_t temp2[max_Q1D][max_D1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               temp2[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  temp2[qx][dy][dz] +=
+                     B(qy, dy) * B(qy, dy) * temp[qx][qy][dz];
+               }
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               real_t temp3 = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  temp3 += B(qx, dx) * B(qx, dx) * temp2[qx][dy][dz];
+               }
+               Y(dx, dy, dz, 0, e) = temp3;
+               Y(dx, dy, dz, 1, e) = temp3;
+               Y(dx, dy, dz, 2, e) = temp3;
+            }
+         }
+      }
+   });
+}
+
+static void PAVectorMassAssembleDiagonal(const int dim, const int D1D,
+                                         const int Q1D, const int NE,
+                                         const Array<real_t> &B,
+                                         const Vector &pa_data,
+                                         Vector &diag)
+{
+   if (dim == 2)
+   {
+      return PAVectorMassAssembleDiagonal2D(NE, B, pa_data, diag, D1D, Q1D);
+   }
+   else if (dim == 3)
+   {
+      return PAVectorMassAssembleDiagonal3D(NE, B, pa_data, diag, D1D, Q1D);
+   }
+   MFEM_ABORT("Dimension not implemented.");
+}
+
 void VectorMassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   if (DeviceCanUseCeed()) { return ceedOp->GetDiagonal(diag); }
-
-   MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
-   MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
-
-   // Add the VectorMassAssembleDiagonalPA specializations
-   static const auto vector_mass_assemble_diagonal_kernel_specializations =
-      ( // 2D
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 2>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 3>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 4>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 5>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 6>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 7>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 8>::Add(),
-         // 3D
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 2>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 3>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 4>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 5>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 6>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 7>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 8>::Add(),
-         true);
-   MFEM_CONTRACT_VAR(vector_mass_assemble_diagonal_kernel_specializations);
-
-   VectorMassAssembleDiagonalPA::Run(dim, quad1D, // templated arguments
-                                     ne, dofs1D, quad1D,
-                                     maps->B.Read(),
-                                     pa_data.Read(),
-                                     diag.ReadWrite());
-
+   if (DeviceCanUseCeed()) { ceedOp->GetDiagonal(diag); }
+   else
+   {
+      MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
+      MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
+      PAVectorMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
+   }
 }
 
 } // namespace mfem

fem/integ/bilininteg_vecmass_pa.hpp

@@ -176,146 +176,8 @@ void SmemPAVectorMassApply3D(const int NE,
    });
 }
 
-template <int T_Q1D = 0, int T_MDQ = 16>
-static void SmemPAVectorMassAssembleDiagonal2D(const int ne,
-                                               const int d1d,
-                                               const int q1d,
-                                               const real_t *b_r,
-                                               const real_t *d_r,
-                                               real_t *y_rw)
-{
-   constexpr int VDIM = 2;
-
-   const int D1D = d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
-
-   const auto B = Reshape(b_r, Q1D, D1D);
-   const auto D = Reshape(d_r, Q1D, Q1D, ne);
-   auto Y = Reshape(y_rw, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(
-      ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
-
-      MFEM_SHARED real_t sm[MQ1][MQ1];
-
-      MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            real_t u = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               u += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-            sm[qx][dy] = u;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-         {
-            real_t u = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               u += B(qx, dx) * B(qx, dx) * sm[qx][dy];
-            }
-            Y(dx, dy, 0, e) += u;
-            Y(dx, dy, 1, e) += u;
-         }
-      }
-   });
-}
-
-// T_MDQ <= 10 so the Q1D^3 thread block stays within the 1024/block GPU limit
-template <int T_Q1D = 0, int T_MDQ = 10>
-static void SmemPAVectorMassAssembleDiagonal3D(const int ne,
-                                               const int d1d,
-                                               const int q1d,
-                                               const real_t *b_r,
-                                               const real_t *d_r,
-                                               real_t *y_rw)
-{
-   constexpr int VDIM = 3;
-
-   const int D1D = d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
-
-   const auto B = Reshape(b_r, Q1D, D1D);
-   const auto D = Reshape(d_r, Q1D, Q1D, Q1D, ne);
-   auto Y = Reshape(y_rw, D1D, D1D, D1D, VDIM, ne);
-
-   mfem::forall_3D<T_Q1D*T_Q1D*T_Q1D>(
-      ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
-
-      MFEM_SHARED real_t sm[2][MQ1][MQ1][MQ1];
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               real_t u = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-               sm[0][dz][qy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += B(qy, dy) * B(qy, dy) * sm[0][dz][qy][qx];
-               }
-               sm[1][dz][dy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += B(qx, dx) * B(qx, dx) * sm[1][dz][dy][qx];
-               }
-               Y(dx, dy, dz, 0, e) += u;
-               Y(dx, dy, dz, 1, e) += u;
-               Y(dx, dy, dz, 2, e) += u;
-            }
-         }
-      }
-   });
-}
-
 } // namespace internal
 
-// AddMultPA kernels
 template<int DIM, int T_D1D, int T_Q1D>
 VectorMassIntegrator::VectorMassAddMultPAType
 VectorMassIntegrator::VectorMassAddMultPA::Kernel()
@@ -332,7 +194,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Kernel()
 }
 
 inline VectorMassIntegrator::VectorMassAddMultPAType
-VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
+VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int d1d, int q1d)
 {
    if (dim == 2)
    {
@@ -342,37 +204,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
    {
       return internal::SmemPAVectorMassApply3D;
    }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-// DiagonalPA kernels
-template<int DIM, int T_Q1D>
-VectorMassIntegrator::VectorMassAssembleDiagonalPAType
-VectorMassIntegrator::VectorMassAssembleDiagonalPA::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal2D<T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal3D<T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorMassIntegrator::VectorMassAssembleDiagonalPAType
-VectorMassIntegrator::VectorMassAssembleDiagonalPA::Fallback(int dim, int)
-{
-   if (dim == 2)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
+   else { MFEM_ABORT("Unsupported kernel"); }
 }
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_vectorfemass_kernels.hpp

@@ -1,113 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
-#define MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
-
-#include "../../config/config.hpp"
-#include "../bilininteg.hpp"
-
-#include "bilininteg_diffusion_kernels.hpp"
-#include "bilininteg_hcurl_kernels.hpp"
-#include "bilininteg_hdiv_kernels.hpp"
-#include "bilininteg_hcurlhdiv_kernels.hpp"
-
-namespace mfem
-{
-/// \cond DO_NOT_DOCUMENT
-namespace internal
-{
-namespace hcurlmass
-{
-constexpr int NBZ3D(int d1d, int q1d)
-{
-   if (d1d <= 1 || q1d <= 0)
-   {
-      return 1;
-   }
-   // assume q1d >= d1d
-   // z dimension is capped at 64 on nvidia and amd gpus
-   int tmp = std::min((128 + q1d * q1d * q1d - 1) / (q1d * q1d * q1d), 64);
-   int smem_req =
-      sizeof(mfem::real_t) *
-      (3 * ((d1d - 1) * d1d * d1d + 2 * q1d * q1d * q1d) * tmp +
-       q1d * (d1d - 1) + q1d * d1d);
-   // assume GPU has at least 48k shared memory
-   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
-}
-} // namespace hcurlmass
-} // namespace internal
-
-template <FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
-          int DIM, int TrialD1D, int TestD1D, int Q1D>
-VectorFEMassIntegrator::ApplyKernelType
-VectorFEMassIntegrator::ApplyPAKernels::Kernel()
-{
-   constexpr bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
-   constexpr bool trial_div = (TrialType == mfem::FiniteElement::DIV);
-   constexpr bool test_curl = (TestType == mfem::FiniteElement::CURL);
-   constexpr bool test_div = (TestType == mfem::FiniteElement::DIV);
-
-   if constexpr (DIM == 3)
-   {
-      if constexpr (trial_curl && test_curl)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            // assume TrialD1D == TestD1D
-            return internal::SmemPAHcurlMassApply3D<
-                   TrialD1D, Q1D, internal::hcurlmass::NBZ3D(TrialD1D, Q1D)>;
-         }
-         else
-         {
-            return internal::PAHcurlMassApply3D;
-         }
-      }
-      else if constexpr (trial_div && test_div)
-      {
-         // assumes TrialD1D == TestD1D
-         return internal::SmemPAHdivMassApply3D<TrialD1D, Q1D>;
-      }
-      else if constexpr (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply3D;
-      }
-      else if constexpr (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply3D;
-      }
-   }
-   else if constexpr (DIM == 2) // 2D
-   {
-      if constexpr (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply2D;
-      }
-      else if constexpr (trial_div && test_div)
-      {
-         // assumes TrialD1D == TestD1D
-         return internal::SmemPAHdivMassApply2D<TrialD1D, Q1D>;
-      }
-      else if constexpr (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply2D;
-      }
-      else if constexpr (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply2D;
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-/// \endcond DO_NOT_DOCUMENT
-}
-
-#endif