clean up pass

MixedVector gradient partial assembly

.github/workflows/builds-and-tests.yml

@@ -142,6 +142,10 @@ jobs:
 
     continue-on-error: ${{ matrix.enzyme && true || false }}
 
+    # Enable ccache for all jobs except Windows (would need sccache).
+    env:
+      USE_CCACHE: ${{ matrix.os != 'windows-latest' }}
+
     steps:
       # Fix 'No space left on device' errors for Ubuntu builds.
       - name: Run Actions Cleaner
@@ -290,6 +294,52 @@ jobs:
           echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
           echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
 
+      # Restore the compiler cache (ccache). The key embeds the run id, so new
+      # runs save a fresh snapshot; the restore-keys prefix warm-starts from the
+      # most recent prior run (incl. the base branch for PRs).
+      - name: cache ccache
+        if: ${{ env.USE_CCACHE == 'true' }}
+        uses: actions/cache@v5
+        with:
+          path: .ccache
+          key: ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-${{ github.run_id }}
+          restore-keys: |
+            ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-
+
+      # Configure ccache and select how it is injected into the MFEM build:
+      # - make:  set CXX="ccache g++"; for MPI, OMPI_CXX="ccache g++" so mpicxx
+      #          runs ccache around g++ (not ccache around the mpicxx wrapper).
+      # - cmake: set CMAKE_<LANG>_COMPILER_LAUNCHER=ccache.
+      # - enzyme: wrap the brew clang++ via OMPI_CXX.
+      # The chosen options are passed through build-mfem's 'config-options'
+      # input (see the build step below).
+      - name: configure ccache
+        if: ${{ env.USE_CCACHE == 'true' }}
+        run: |
+          command -v ccache >/dev/null 2>&1 || {
+            if [[ "${{ runner.os }}" == "Linux" ]]; then
+              sudo apt-get update && sudo apt-get install -y ccache
+            else
+              brew install ccache
+            fi
+          }
+          echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
+          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
+          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
+          # Ignore header timestamps (restamped by each checkout) so direct mode hits.
+          echo "CCACHE_SLOPPINESS=include_file_mtime,include_file_ctime,time_macros" >> $GITHUB_ENV
+          # Hash absolute paths relative to the workspace.
+          echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
+          if [[ "${{ matrix.enzyme }}" == "true" ]]; then
+            echo "OMPI_CXX=ccache $LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
+          elif [[ "${{ matrix.build-system }}" == "cmake" ]]; then
+            echo 'CCACHE_CONFIG_OPTS=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache' >> $GITHUB_ENV
+          else
+            echo "OMPI_CXX=ccache g++" >> $GITHUB_ENV
+            echo 'CCACHE_CONFIG_OPTS=CXX="ccache g++" MPICXX="mpicxx"' >> $GITHUB_ENV
+          fi
+        shell: bash
+
       # MFEM build and test
       - name: build
         uses: mfem/github-actions/build-mfem@v2.7
@@ -305,9 +355,14 @@ jobs:
           metis-dir: ${{ env.METIS_TOP_DIR }}
           mfem-dir: ${{ env.MFEM_TOP_DIR }}
           precision: ${{ matrix.precision }}
-          config-options: ${{ matrix.config-opts }}
+          config-options: ${{ matrix.config-opts }} ${{ env.CCACHE_CONFIG_OPTS }}
           library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
 
+      - name: ccache stats
+        if: ${{ env.USE_CCACHE == 'true' }}
+        run: ccache -s
+        shell: bash
+
       # Run checks (and only checks) on debug targets
       - name: checks
         if: matrix.build-system == 'make' && matrix.target == 'dbg'

.github/workflows/cleanup-pr-caches.yml

@@ -0,0 +1,42 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+# A closed PR's caches can never be restored again, so delete them to free
+# space against the 10 GB per-repo cache limit.
+name: Cleanup PR caches
+
+on:
+  pull_request:
+    types: [closed]
+
+permissions:
+  actions: write
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Delete caches for the closed PR
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          PR_REF: refs/pull/${{ github.event.pull_request.number }}/merge
+        run: |
+          echo "Deleting caches for $PR_REF"
+          while :; do
+            ids=$(gh cache list --ref "$PR_REF" --limit 100 --json id --jq '.[].id')
+            [ -n "$ids" ] || break
+            echo "$ids" | while read -r id; do
+              [ -n "$id" ] || continue
+              echo "Deleting cache $id"
+              gh cache delete "$id" || echo "  (already gone)"
+            done
+          done

.github/workflows/repo-check.yml

@@ -13,6 +13,7 @@ name: "Checks"
 
 permissions:
   actions: write
+  pull-requests: read
 
 on:
   push:
@@ -29,6 +30,11 @@ concurrency:
 # by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
 # and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
 # github.repository").
+#
+# The logic for the branch-history check is slightly different, since that check
+# also inspects the PR's labels to allow for overriding failures. In this case,
+# we run on all 'pull_request' triggers, but only run for 'push' triggers that
+# do not correspond to any open PRs.
 
 jobs:
   file-headers-check:
@@ -128,10 +134,7 @@ jobs:
 
   branch-history:
     if: |
-      github.ref != 'refs/heads/next' &&
-      github.ref != 'refs/heads/master' &&
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
+      github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
     runs-on: ubuntu-latest
     steps:
       - name: checkout mfem
@@ -139,7 +142,27 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: check for pull request
+        id: check_pr
+        if: github.event_name == 'push'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          pr_exists=$(gh pr list --repo "$GITHUB_REPOSITORY" \
+            --head "$GITHUB_REF_NAME" \
+            --state open \
+            --json number \
+            --jq 'length > 0')
+          echo "pr_exists=$pr_exists" >> "$GITHUB_OUTPUT"
+
       - name: branch-history
+        id: branch_history
+        if: |
+          (github.event_name == 'pull_request' ||
+           github.event_name == 'workflow_dispatch' ||
+           steps.check_pr.outputs.pr_exists == 'false')
+        continue-on-error: ${{ contains(github.event.pull_request.labels.*.name,
+                                        'branch-history-override') }}
         run: |
           # We override origin to make sure we point to the main repo.
           # This is to have consistent test results on PRs from forks.
@@ -147,3 +170,9 @@ jobs:
           git remote add origin https://github.com/mfem/mfem.git
           git checkout -b gh-actions-branch-history
           ./config/githooks/pre-push --history
+
+      - name: report branch-history override
+        if: steps.branch_history.outcome == 'failure'
+        run: |
+          echo "::warning::branch-history check failed, but the" \
+            "'branch-history-override' label is set."

.gitignore

@@ -260,6 +260,7 @@ miniapps/meshing/polar-nc
 miniapps/meshing/mesh-quality
 miniapps/meshing/hpref
 miniapps/meshing/phpref
+miniapps/meshing/pref321
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh

.gitlab/dane-baseline.yml

@@ -102,12 +102,14 @@ report_baseline:
         mkdir -p ${MACHINE_NAME}
         rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
         rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
+        status=0
+        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir} || { status=1; }
         printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
           >> ${rundir}/pipeline.txt
         # We create an autotest-email.html file, because that's how we signal
         # that there was an error / diff (temporary).
-        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
+        if [[ $status -ne 0 ]] || \
+           [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
            [[ -f ${rundir}/${BASELINE_TEST}-${MACHINE_NAME}.diff ]]; then
           cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
         fi

CHANGELOG

@@ -46,8 +46,19 @@ Discretization improvements
 
 - Extend FindPointsGSLIB to support surface meshes.
 
+- Added support for complex-valued mixed bilinear forms via the new classes
+  MixedSesquilinearForm and ParMixedSesquilinearForm, mirroring the existing
+  SesquilinearForm classes. Rectangular complex operators are now also
+  handled correctly by ComplexSparseMatrix::GetSystemMatrix and
+  ComplexHypreParMatrix::GetSystemMatrix, which previously assumed equal
+  trial and test spaces.
+
 Meshing improvements
 --------------------
+- Added support for nonuniform anisotropic mesh refinement on parallel quad/hex
+  meshes with arbitrary spacing in each direction. This enables in particular
+  3:1 refinement in parallel, as demonstrated in the new meshing miniapp pref321.
+
 - Added option to guarantee mesh validity during TMOP-based r-adaptivity, using
   bounds on the determinant of the mesh transformation Jacobian.
 
@@ -68,16 +79,55 @@ Linear and nonlinear solvers
   PRefinement multigrid methods for problems posed on trace spaces (see e.g. the
   DPG miniapps).
 
+- Added new class MultiVector: an array of Vectors of different sizes where each
+  Vector can be allocated independently. Also, added associated methods in class
+  Operator: MultMV, MultTransposeMV, and GetGradientMV, that use MultiVector
+  objects for input and/or output parameters. [PR #5249]
+
 GPU computing
 -------------
+- Improved partial assembly for VectorDivergenceIntegrator with shared-memory
+  kernels, kernel registration, and transpose support.
+
+- Improved partial-assembly diagonal kernels for VectorMassIntegrator (shared-
+  memory specializations) and ElasticityIntegrator (no scratch Q-vector).
+
+- Added PA gradient and diagonal support for VectorConvectionNLFIntegrator
+ (AssembleGradPA, AddMultGradPA, AssembleGradDiagonalPA).
+
 - Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
 
+- Added partial assembly support for MixedScalarWeakGradientIntegrator.
+
+- Added partial assembly support for MixedDotProductIntegrator.
+
+- Added partial assembly support for MixedScalarCrossProductIntegrator.
+
+- Added partial assembly support for MixedScalarWeakCrossProductIntegrator.
+
+- Added partial assembly support for MixedVectorGradientIntegrator for H1->RT.
+
+- Added support for device partial assembly CurlInterpolator.
+  This supports 2D and 3D variants:
+    2D H1 (out-of-plane) to RT (in-plane)
+    2D ND (in-plane) to Integral L2 (out-of-plane)
+    3D ND to RT
+
 - Added NVIDIA cuDSS library interface. Implementation examples have been
   added to ex1 and ex1p. See https://developer.nvidia.com/cudss for more
   details. Supported versions >= 0.6.0.
 
 - Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
 
+- Changed VectorFEMassIntegrator to use kernel specialization dispatch for
+  partial assembly.
+
+- Added support for FiniteElement::MapType::INTEGRAL spaces to
+  QuadratureInterpolator.
+
+- Added support for FiniteElement::MapType::INTEGRAL spaces to
+  MixedScalarCurlIntegrator.
+
 New and updated examples and miniapps
 -------------------------------------
 - The Lorentz miniapp (in miniapps/electromagnetics) has been updated to
@@ -92,6 +142,15 @@ Miscellaneous
   using the new method ApplyDofSigns() in class ParFiniteElementSpace: the
   method will return immediately if no sign flips are needed.
 
+- Added support for saving DataCollection output on the node-local storage,
+  instead of requiring that the filesystem is shared among all the ranks.
+
+API changes
+-----------
+- Removed ProjectGrad from 2D RT elements. Users should use ProjectCurl instead.
+  This also fixes a bug where ProjectCurl was returning the negative curl,
+  identical to ProjectGrad.
+
 
 Version 4.9, released on Dec 11, 2025
 =====================================

CMakeLists.txt

@@ -88,18 +88,9 @@ if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
-# Ginkgo requires C++17:
-if ((MFEM_USE_GINKGO) AND ("${CMAKE_CXX_STANDARD}" LESS "17"))
-  set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to use." FORCE)
-# Google Benchmark, SUNDIALS, STRUMPACK, Tribol, RAJA and Umpire require C++14:
-elseif ((MFEM_USE_BENCHMARK OR
-         MFEM_USE_SUNDIALS OR
-         MFEM_USE_STRUMPACK OR
-         MFEM_USE_TRIBOL OR
-         MFEM_USE_RAJA OR
-         MFEM_USE_UMPIRE) AND
-    ("${CMAKE_CXX_STANDARD}" LESS "14"))
-  set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ standard to use." FORCE)
+# RAJA requires C++20:
+if ((MFEM_USE_UMPIRE OR MFEM_USE_RAJA) AND ("${CMAKE_CXX_STANDARD}" LESS "20"))
+  set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard to use." FORCE)
 endif()
 
 # Include xSDK default CMake file.
@@ -239,6 +230,13 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
+# Shadow warnings for clang only; GCC's -Wshadow flags more.
+if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall -Wshadow")
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall")
+endif()
+
 # Shared build on Windows
 if (WIN32 AND BUILD_SHARED_LIBS)
   # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -962,7 +962,7 @@ function(mfem_export_mk_files)
     set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} -I${dir}")
   endforeach()
   # TODO: MFEM_TPLFLAGS: add other TPL flags, in addition to the -I flags.
-  set(MFEM_INCFLAGS "-I\$(MFEM_INC_DIR) \$(MFEM_TPLFLAGS)")
+  set(MFEM_INCFLAGS "-I\$(MFEM_INC_DIR) -I\$(MFEM_SOURCE_DIR) \$(MFEM_TPLFLAGS)")
   set(MFEM_PICFLAG "")
   if (BUILD_SHARED_LIBS)
     set(MFEM_PICFLAG "${CMAKE_SHARED_LIBRARY_CXX_FLAGS}")
@@ -1098,6 +1098,7 @@ function(mfem_export_mk_files)
     "${INSTALL_LIB_DIR}" "${CMAKE_INSTALL_PREFIX}" MFEM_LIB_DIR)
   mfem_path_to_fullpath(
     "${INSTALL_SHARE_DIR}/mfem/test.mk" "${CMAKE_INSTALL_PREFIX}" MFEM_TEST_MK)
+  set(MFEM_INCFLAGS "-I\$(MFEM_INC_DIR) \$(MFEM_TPLFLAGS)")
   set(MFEM_CONFIG_EXTRA "")
 
   # Create the install-tree version of 'config.mk'

config/defaults.mk

@@ -27,7 +27,10 @@ MPICXX = mpicxx
 
 BASE_FLAGS  = -std=c++17
 OPTIM_FLAGS = -O3 $(BASE_FLAGS)
-DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS)
+
+# The variable WARNING_FLAGS depends on which compiler is used, and is defined
+# later in this file.
+DEBUG_FLAGS = $(strip -g $(addprefix $(XCOMPILER),$(WARNING_FLAGS)) $(BASE_FLAGS))
 
 # Prefixes for passing flags to the compiler and linker when using CXX or MPICXX
 CXX_XCOMPILER =
@@ -46,6 +49,10 @@ SHARED = NO
 #
 # If you set MFEM_USE_ENZYME=YES, must use CUDA_CXX=clang++
 CUDA_CXX = nvcc
+# CUDA compute capability used during compilation, e.g. sm_60. Multiple
+# architectures can be requested as a comma-separated list, e.g. sm_70,sm_80.
+# A single value may also be one of the nvcc special values "all",
+# "all-major", or "native".
 CUDA_ARCH = sm_60
 # Base CUDA install directory, only needed if building with clang+cuda:
 # The default setting is:
@@ -54,11 +61,23 @@ CUDA_ARCH = sm_60
 # 3. Use /usr/local/cuda
 CUDA_DIR = $(or $(CUDA_HOME),$(patsubst %/,%,$(dir \
  $(patsubst %/,%,$(dir $(shell command -v nvcc))))),/usr/local/cuda)
+# Derive nvcc/clang architecture flags from CUDA_ARCH. A comma-separated list
+# expands into one -gencode / --cuda-gpu-arch flag per architecture; otherwise
+# use the -arch / --cuda-gpu-arch shorthand.
+MFEM_COMMA := ,
+CUDA_ARCH_NUMS = $(patsubst sm_%,%,$(subst $(MFEM_COMMA), ,$(CUDA_ARCH)))
+NVCC_ARCH_FLAGS = $(strip $(if $(findstring $(MFEM_COMMA),$(CUDA_ARCH)),\
+ $(foreach arch,$(CUDA_ARCH_NUMS),\
+ -gencode arch=compute_$(arch)$(MFEM_COMMA)code=sm_$(arch)),\
+ -arch=$(CUDA_ARCH)))
+CLANG_ARCH_FLAGS = $(strip $(if $(findstring $(MFEM_COMMA),$(CUDA_ARCH)),\
+ $(foreach arch,$(CUDA_ARCH_NUMS),--cuda-gpu-arch=sm_$(arch)),\
+ --cuda-gpu-arch=$(CUDA_ARCH)))
 # flags for clang+cuda
-CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) --cuda-gpu-arch=$(CUDA_ARCH)
+CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) $(CLANG_ARCH_FLAGS)
 # flags for nvcc
 NVCC_FLAGS = -x=cu --expt-extended-lambda --expt-relaxed-constexpr \
- -arch=$(CUDA_ARCH) -isystem "$(CUDA_DIR)/include"
+ $(NVCC_ARCH_FLAGS) -isystem "$(CUDA_DIR)/include"
 # Prefixes for passing flags to the host compiler and linker when using
 # CUDA_CXX=nvcc
 CUDA_XCOMPILER = -Xcompiler=
@@ -376,7 +395,7 @@ CUDSS_LIBRARY_DIR = $(CUDSS_DIR)/lib
 CUDSS_OPT         = -I$(CUDSS_INCLUDE_DIR)
 CUDSS_LIB         = \
  $(XLINKER)-rpath,$(CUDSS_LIBRARY_DIR) -L$(CUDSS_LIBRARY_DIR) -lcudss
-# The cuDSS communication and threading libraries. 
+# The cuDSS communication and threading libraries.
 MFEM_CUDSS_COMM_LIB = $(abspath $(wildcard $(or $(CUDSS_COMM_LIB),\
    $(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so))))
 MFEM_CUDSS_THREADING_LIB = $(abspath $(wildcard $(or $(CUDSS_THREADING_LIB),\
@@ -659,3 +678,15 @@ VERBOSE = NO
 
 # Optional build tag
 MFEM_BUILD_TAG = $(shell uname -snm)
+
+# Enable -pedantic flag only for gcc or clang. nvcc complains with -pedantic
+# because of line directives.
+PEDANTIC_FLAG = $(if \
+   $(findstring NVIDIA,$(shell $(MFEM_CXX) --version 2>&1)),, \
+   $(if $(or \
+      $(findstring gcc version,$(shell $(MFEM_CXX) -v 2>&1)), \
+      $(findstring clang version,$(shell $(MFEM_CXX) -v 2>&1))),-pedantic,))
+# Enable shadow warnings for clang only; GCC's -Wshadow flags more.
+SHADOW_WARNING_FLAG = $(if $(findstring clang,\
+ $(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
+WARNING_FLAGS = $(PEDANTIC_FLAG) -Wall $(SHADOW_WARNING_FLAG)

config/githooks/README.md

@@ -39,3 +39,8 @@ when a picture was added for documentation.
 If that is the case, make sure the failure is indeed justified, and rerun the
 push command with the `--no-verify` option. This will skip the hooks, allowing
 you to push those changes.
+
+The `branch-history` check is run automatically through GitHub Actions. If a
+branch is known to have a large number of changes that are legitimate, the
+check can be overridden by setting  the label 'branch-history-override' on the
+pull request.

data/tinyzoo-2d.mesh

@@ -0,0 +1,38 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see fem/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+
+dimension
+2
+
+elements
+2
+1 3 0 1 4 3
+1 2 1 2 4
+
+boundary
+5
+1 1 0 1
+1 1 1 2
+1 1 2 4
+1 1 4 3
+1 1 3 0
+
+vertices
+5
+2
+0 0
+1 0
+2 0
+0 1
+1 1

doc/CodeDocumentation.conf.in

@@ -1083,7 +1083,8 @@ EXCLUDE_PATTERNS       =
 # ANamespace::AClass, ANamespace::*Test
 
 EXCLUDE_SYMBOLS        = mfem::internal \
-                         mfem::kernels::internal
+                         mfem::kernels::internal \
+                         mfem::future::detail
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include

doc/CodeDocumentation.dox

@@ -201,6 +201,7 @@ namespace mfem {
  * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
  * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
  * - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
+ * - <a class="el" href="nurbs__surface_8cpp_source.html">NURBS Surface</a>: interpolate a 3D Surface in a NURBS Patch
  *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -245,6 +246,9 @@ namespace mfem {
  * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
  * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
  * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
+ * - <a class="el" href="reflector_8cpp_source.html">Reflector Miniapp</a>: reflect a mesh about a plane
+ * - <a class="el" href="ref321_8cpp_source.html">3:1 Refinement Miniapp</a>: perform 3:1 anisotropic mesh refinements
+ * - <a class="el" href="pref321_8cpp_source.html">3:1 Refinement Miniapp</a>: parallel 3:1 anisotropic mesh refinements
  *
  * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
  */

examples/CMakeLists.txt

@@ -262,3 +262,8 @@ endif()
 if(MFEM_USE_MOONOLITH)
   add_subdirectory(moonolith)
 endif()
+
+# Include the examples/magma directory if MAGMA is enabled.
+if (MFEM_USE_MAGMA)
+  add_subdirectory(magma)
+endif()

examples/magma/CMakeLists.txt

@@ -0,0 +1,27 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+set(MAGMA_EXAMPLES_SRCS)
+list(APPEND MAGMA_EXAMPLES_SRCS
+  batched_mass_bench.cpp
+  )
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add "test_magma" target (build-only by default).
+add_custom_target(test_magma
+  COMMENT "Building MAGMA examples ...")
+
+# Add one executable per cpp file, adding "magma_" as prefix to avoid
+# collisions in a unified build tree.
+set(PFX magma_)
+add_mfem_examples(MAGMA_EXAMPLES_SRCS ${PFX} "" test_magma)

examples/magma/README

@@ -0,0 +1,12 @@
+This directory contains an MFEM example focused on benchmarking MAGMA batched
+element-mass-matrix inverses/solves on GPU backends (CUDA/HIP).
+
+- `batched_mass_bench.cpp`: Assemble packed element mass matrices and compare:
+  - MFEM tripack inverse apply (packed lower-triangular)
+  - MAGMA packed Cholesky factor + solve
+  - MAGMA full (dense) batched Cholesky factor + solve
+
+Build and run from this directory (after building MFEM with MAGMA enabled):
+
+- `make batched_mass_bench`
+- `./batched_mass_bench -m ../../data/beam-hex.mesh -o 3 -r 2 -d hip -reps 100`

examples/magma/batched_mass_bench.cpp

@@ -0,0 +1,798 @@
+//                    MFEM Batched Mass Benchmark
+//
+// Compile with: make batched_mass_bench
+//
+// Sample runs:  batched_mass_bench -m ../../data/beam-hex.mesh -o 3 -r 2 -d hip -reps 100
+//               batched_mass_bench -m ../../data/beam-hex.mesh -o 3 -r 3 -d cuda -reps 50
+//
+// Description:  This example isolates the element mass-matrix
+//               inverse paths used by element matrix kernels. It
+//               assembles packed lower-triangular L2 element mass matrices,
+//               then times repeated mass inverse applications using MFEM's
+//               tripack path and MAGMA (packed and full-matrix variants).
+
+#include "mfem.hpp"
+#include <cmath>
+#include <iomanip>
+#include <iostream>
+#include <limits>
+#include <memory>
+
+using namespace mfem;
+using namespace std;
+
+namespace
+{
+
+#ifdef MFEM_USE_MAGMA
+inline void MagmaSetPointer(float **output, float *input,
+                            magma_int_t inc, magma_int_t row, magma_int_t col,
+                            magma_int_t lda, magma_int_t batch_count,
+                            magma_queue_t queue)
+{
+   magma_sset_pointer(output, input, inc, row, col, lda, batch_count, queue);
+}
+
+inline void MagmaSetPointer(double **output, double *input,
+                            magma_int_t inc, magma_int_t row, magma_int_t col,
+                            magma_int_t lda, magma_int_t batch_count,
+                            magma_queue_t queue)
+{
+   magma_dset_pointer(output, input, inc, row, col, lda, batch_count, queue);
+}
+
+inline magma_int_t MagmaPotrfBatched(magma_uplo_t uplo, magma_int_t n,
+                                     float **dA, magma_int_t ldda,
+                                     magma_int_t *info, magma_int_t batch_count,
+                                     magma_queue_t queue)
+{
+   return magma_spotrf_batched(uplo, n, dA, ldda, info, batch_count, queue);
+}
+
+inline magma_int_t MagmaPotrfBatched(magma_uplo_t uplo, magma_int_t n,
+                                     double **dA, magma_int_t ldda,
+                                     magma_int_t *info, magma_int_t batch_count,
+                                     magma_queue_t queue)
+{
+   return magma_dpotrf_batched(uplo, n, dA, ldda, info, batch_count, queue);
+}
+
+inline magma_int_t MagmaPotrsBatched(magma_uplo_t uplo, magma_int_t n,
+                                     magma_int_t nrhs, float **dA,
+                                     magma_int_t ldda, float **dB,
+                                     magma_int_t lddb, magma_int_t batch_count,
+                                     magma_queue_t queue)
+{
+   return magma_spotrs_batched(uplo, n, nrhs, dA, ldda, dB, lddb,
+                              batch_count, queue);
+}
+
+inline magma_int_t MagmaPotrsBatched(magma_uplo_t uplo, magma_int_t n,
+                                     magma_int_t nrhs, double **dA,
+                                     magma_int_t ldda, double **dB,
+                                     magma_int_t lddb, magma_int_t batch_count,
+                                     magma_queue_t queue)
+{
+   return magma_dpotrs_batched(uplo, n, nrhs, dA, ldda, dB, lddb,
+                              batch_count, queue);
+}
+
+void PrintMagmaFasterCondition(const double eq_fixed_ms,
+                               const double eq_apply_ms,
+                               const double magma_fixed_ms,
+                               const double magma_apply_ms)
+{
+   cout << "MAGMA faster condition (assembly+setup+N applies): ";
+
+   if (magma_fixed_ms <= eq_fixed_ms && magma_apply_ms <= eq_apply_ms)
+   {
+      if (magma_fixed_ms == eq_fixed_ms && magma_apply_ms == eq_apply_ms)
+      {
+         cout << "modeled totals are equal for all positive integer N.\n";
+      }
+      else
+      {
+         cout << "faster for every positive integer N.\n";
+      }
+      return;
+   }
+
+   if (magma_fixed_ms >= eq_fixed_ms && magma_apply_ms >= eq_apply_ms)
+   {
+      cout << "not faster for any positive integer N.\n";
+      return;
+   }
+
+   if (magma_apply_ms > eq_apply_ms)
+   {
+      const double crossover =
+         (eq_fixed_ms - magma_fixed_ms)/(magma_apply_ms - eq_apply_ms);
+      if (crossover <= 1.0)
+      {
+         cout << "only for N < " << crossover
+              << ", so not for any positive integer N.\n";
+      }
+      else
+      {
+         const double last_n =
+            floor(nextafter(crossover, -numeric_limits<double>::infinity()));
+         cout << "faster for N < " << crossover
+              << " applies (positive integer N <= " << (long long)last_n
+              << "); eq-iter is faster above that.\n";
+      }
+   }
+   else
+   {
+      const double crossover =
+         (magma_fixed_ms - eq_fixed_ms)/(eq_apply_ms - magma_apply_ms);
+      const double first_n = floor(crossover) + 1.0;
+      cout << "faster for N > " << crossover
+           << " applies (positive integer N >= " << (long long)first_n
+           << "); eq-iter is faster below that.\n";
+	   }
+	}
+
+real_t **SetMagmaPackedPointerArray(Array<real_t *> &ptrs, real_t *data,
+                                    const int stride,
+                                    const int batch_size,
+                                    const magma_queue_t queue)
+{
+   if (ptrs.Size() != batch_size)
+   {
+      if (ptrs.Size() != 0) { magma_queue_sync(queue); }
+      ptrs.SetSize(batch_size, Device::GetDeviceMemoryType());
+   }
+
+   real_t **d_ptrs = ptrs.Write();
+   MagmaSetPointer(d_ptrs, data, 1, 0, 0, stride, batch_size, queue);
+   return d_ptrs;
+}
+
+void ComputeMagmaFullCholeskyLower(
+   const Vector &full,
+   const int n,
+   Vector &factor,
+   Array<real_t *> &factor_ptrs,
+   const magma_queue_t queue)
+{
+   const int batch_size = full.Size()/(n*n);
+   MFEM_VERIFY(full.Size() == batch_size*n*n, "Invalid full matrix storage.");
+
+   factor = full;
+
+   if (batch_size == 0) { return; }
+
+   real_t *factor_data = factor.ReadWrite();
+   real_t **dA = SetMagmaPackedPointerArray(factor_ptrs, factor_data, n*n,
+                                            batch_size, queue);
+
+   Array<magma_int_t> info_array(batch_size, Device::GetDeviceMemoryType());
+   magma_int_t *d_info = info_array.Write();
+   magma_memset(d_info, 0, batch_size*sizeof(magma_int_t));
+
+   const magma_int_t status =
+      MagmaPotrfBatched(MagmaLower, n, dA, n, d_info, batch_size, queue);
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA full potrf batched failed.");
+
+   magma_queue_sync(queue);
+   const magma_int_t *info = info_array.HostRead();
+   for (int e = 0; e < batch_size; ++e)
+   {
+      MFEM_VERIFY(info[e] == 0, "MAGMA full potrf failed on matrix " << e << '.');
+   }
+}
+
+void SolveMagmaFullCholeskyLowerInPlace(
+   const Vector &full_factor,
+   const int n,
+   const Array<real_t *> &factor_ptrs,
+   Array<real_t *> &rhs_ptrs,
+   Vector &rhs_sol,
+   const magma_queue_t queue)
+{
+   const int batch_size = rhs_sol.Size()/n;
+   MFEM_VERIFY(rhs_sol.Size() == batch_size*n, "Invalid RHS size.");
+   MFEM_VERIFY(factor_ptrs.Size() == batch_size,
+               "Factor pointer array has the wrong size.");
+
+   if (batch_size == 0) { return; }
+
+   real_t *factor_data = const_cast<real_t *>(full_factor.Read());
+   real_t **dA = const_cast<real_t **>(factor_ptrs.Read());
+   (void)factor_data; // Factor data is referenced by dA (for clarity).
+
+   real_t *rhs_data = rhs_sol.ReadWrite();
+   real_t **dB = SetMagmaPackedPointerArray(rhs_ptrs, rhs_data, n, batch_size,
+                                            queue);
+
+   const magma_int_t status =
+      MagmaPotrsBatched(MagmaLower, n, 1, dA, n, dB, n, batch_size, queue);
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA full potrs batched failed.");
+}
+#endif
+
+void FillRHS(Vector &rhs)
+{
+   rhs.UseDevice(true);
+   real_t *x = rhs.HostWrite();
+   for (int i = 0; i < rhs.Size(); ++i)
+   {
+      x[i] = 1.0 + real_t((13*i + 7) % 29)/real_t(29);
+   }
+}
+
+void ComputeLowerPackedResidual(
+   const TriPackLowerMatrix &lower,
+   const Vector &x,
+   const Vector &rhs,
+   double &l2_residual,
+   double &relative_l2_residual,
+   real_t &max_residual,
+   real_t &relative_max_residual)
+{
+   const int n = lower.GetNumRows();
+   const int batch_size = lower.GetNumMatrices();
+   const int packed_size = lower.GetPackedSize();
+   MFEM_VERIFY(x.Size() == batch_size*n, "Solution vector has the wrong size.");
+   MFEM_VERIFY(rhs.Size() == batch_size*n, "Right-hand side has the wrong size.");
+
+   const real_t *A = lower.Data().HostRead();
+   const real_t *X = x.HostRead();
+   const real_t *B = rhs.HostRead();
+
+   long double l2_sum = 0.0;
+   long double rhs_l2_sum = 0.0;
+   real_t max_abs = 0.0;
+   real_t rhs_max_abs = 0.0;
+
+   for (int e = 0; e < batch_size; ++e)
+   {
+      const real_t *Ae = A + e*packed_size;
+      const real_t *Xe = X + e*n;
+      const real_t *Be = B + e*n;
+      for (int i = 0; i < n; ++i)
+      {
+         long double ax = 0.0;
+         for (int j = 0; j < n; ++j)
+         {
+         const real_t aij =
+            (i >= j) ?
+               Ae[TriPackLowerMatrix::LowerIndex(i, j, n)] :
+               Ae[TriPackLowerMatrix::LowerIndex(j, i, n)];
+            ax += (long double)aij * (long double)Xe[j];
+         }
+         const long double residual = ax - (long double)Be[i];
+         l2_sum += residual*residual;
+         rhs_l2_sum += (long double)Be[i]*(long double)Be[i];
+         max_abs = max(max_abs, (real_t)fabs((double)residual));
+         rhs_max_abs = max(rhs_max_abs, fabs(Be[i]));
+      }
+   }
+
+   l2_residual = sqrt((double)l2_sum);
+   const double rhs_l2_norm = sqrt((double)rhs_l2_sum);
+   relative_l2_residual =
+      (rhs_l2_norm > 0.0) ? l2_residual/rhs_l2_norm : l2_residual;
+   max_residual = max_abs;
+   relative_max_residual =
+      (rhs_max_abs > 0.0) ? max_residual/rhs_max_abs : max_residual;
+}
+
+void ApplyLowerInverseInPlace(
+   const TriPackLowerMatrix &lower_inverse,
+   Vector &x,
+   Vector &work)
+{
+   const int n = lower_inverse.GetNumRows();
+   const int batch_size = lower_inverse.GetNumMatrices();
+   const int packed_size = lower_inverse.GetPackedSize();
+   MFEM_VERIFY(x.Size() == batch_size*n, "Input vector has the wrong size.");
+
+   work.SetSize(batch_size*n);
+   work.UseDevice(true);
+
+   const real_t *L = lower_inverse.Data().Read();
+   const real_t *X = x.Read();
+   real_t *T = work.Write();
+
+   mfem::forall(batch_size*n, [=] MFEM_HOST_DEVICE (int idx)
+   {
+      const int i = idx % n;
+      const int e = idx / n;
+      const real_t *Le = L + e*packed_size;
+      const real_t *Xe = X + e*n;
+      real_t sum = 0.0;
+      for (int j = 0; j <= i; ++j)
+      {
+         sum += Le[TriPackLowerMatrix::LowerIndex(i, j, n)]*Xe[j];
+      }
+      T[idx] = sum;
+   });
+
+   const real_t *T_in = work.Read();
+   real_t *Y = x.Write();
+   mfem::forall(batch_size*n, [=] MFEM_HOST_DEVICE (int idx)
+   {
+      const int i = idx % n;
+      const int e = idx / n;
+      const real_t *Le = L + e*packed_size;
+      const real_t *Te = T_in + e*n;
+      real_t sum = 0.0;
+      for (int j = i; j < n; ++j)
+      {
+         sum += Le[TriPackLowerMatrix::LowerIndex(j, i, n)]*Te[j];
+      }
+      Y[idx] = sum;
+   });
+}
+
+double TimeLowerInverseApply(
+   const TriPackLowerMatrix &inverse,
+   const Vector &rhs,
+   const int reps,
+   Vector &x,
+   Vector &work)
+{
+   StopWatch sw;
+
+   // Dry run to remove first-use kernel and workspace allocation costs.
+   x = rhs;
+   ApplyLowerInverseInPlace(inverse, x, work);
+   MFEM_DEVICE_SYNC;
+   sw.Start();
+   for (int r = 0; r < reps; ++r)
+   {
+      x = rhs;
+      ApplyLowerInverseInPlace(inverse, x, work);
+   }
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   return 1000.0*sw.RealTime()/reps;
+}
+
+#ifdef MFEM_USE_MAGMA
+double TimeMagmaSolve(
+   const TriPackLowerMatrix &lower_factor,
+   const Vector &rhs,
+   const int reps,
+   Vector &x,
+   MagmaPackedLowerCholesky &ws)
+{
+   StopWatch sw;
+
+   // Dry run to remove first-use MAGMA and RHS pointer-array setup costs.
+   x = rhs;
+   ws.SolveInPlace(lower_factor, x);
+   MFEM_DEVICE_SYNC;
+   sw.Start();
+   for (int r = 0; r < reps; ++r)
+   {
+      x = rhs;
+      ws.SolveInPlace(lower_factor, x);
+   }
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   return 1000.0*sw.RealTime()/reps;
+}
+
+double TimeMagmaInverseApply(
+   const TriPackLowerMatrix &lower_inverse,
+   const Vector &rhs,
+   const int reps,
+   Vector &x,
+   MagmaPackedLowerInverse &ws)
+{
+   StopWatch sw;
+
+   // Dry run to remove first-use MAGMA and RHS pointer-array setup costs.
+   x = rhs;
+   ws.ApplyInPlace(lower_inverse, x);
+   MFEM_DEVICE_SYNC;
+   sw.Start();
+   for (int r = 0; r < reps; ++r)
+   {
+      x = rhs;
+      ws.ApplyInPlace(lower_inverse, x);
+   }
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   return 1000.0*sw.RealTime()/reps;
+}
+
+double TimeMagmaFullSolve(
+   const Vector &full_factor,
+   const int n,
+   const Array<real_t *> &factor_ptrs,
+   const Vector &rhs,
+   const int reps,
+   Vector &x,
+   const magma_queue_t queue)
+{
+   StopWatch sw;
+   Array<real_t *> rhs_ptrs;
+
+   x = rhs;
+   SolveMagmaFullCholeskyLowerInPlace(full_factor, n, factor_ptrs, rhs_ptrs, x,
+                                      queue);
+   MFEM_DEVICE_SYNC;
+   sw.Start();
+   for (int r = 0; r < reps; ++r)
+   {
+      x = rhs;
+      SolveMagmaFullCholeskyLowerInPlace(full_factor, n, factor_ptrs, rhs_ptrs,
+                                         x, queue);
+   }
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   return 1000.0*sw.RealTime()/reps;
+}
+#endif
+
+} // namespace
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../data/beam-hex.mesh";
+   int order = 3;
+   int ref_levels = 1;
+   int reps = 100;
+   int setup_reps = 10;
+   const char *device_config = "cpu";
+   bool use_magma = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of uniform refinements.");
+   args.AddOption(&reps, "-reps", "--apply-repetitions",
+                  "Number of mass inverse applications to time.");
+   args.AddOption(&setup_reps, "-sreps", "--setup-repetitions",
+                  "Number of setup repetitions to time.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&use_magma, "-magma", "--magma-solve",
+                  "-no-magma", "--no-magma-solve",
+                  "Time the MAGMA packed Cholesky solve when available.");
+   args.ParseCheck();
+
+   MFEM_VERIFY(reps > 0, "The number of apply repetitions must be positive.");
+   MFEM_VERIFY(setup_reps > 0,
+               "The number of setup repetitions must be positive.");
+
+   Device device(device_config);
+   device.Print();
+
+#ifndef MFEM_USE_MAGMA
+   MFEM_VERIFY(!use_magma, "MFEM was built without MAGMA support.");
+#endif
+
+   Mesh mesh(mesh_file, 1, 1);
+   for (int l = 0; l < ref_levels; ++l)
+   {
+      mesh.UniformRefinement();
+   }
+
+   const int dim = mesh.Dimension();
+   L2_FECollection fec(order, dim, BasisType::Positive);
+   FiniteElementSpace fespace(&mesh, &fec);
+
+   MFEM_VERIFY(UsesTensorBasis(fespace),
+               "This example requires a tensor-product finite element space.");
+
+   const int ne = mesh.GetNE();
+   const int elem_dofs = fespace.GetTypicalFE()->GetDof();
+
+   MassIntegrator mass;
+   StopWatch sw;
+
+   // Dry run assembly before timing steady-state work.
+   TriPackLowerMatrix packed_ea;
+   mass.AssembleEATriangular(fespace, packed_ea, false);
+   MFEM_DEVICE_SYNC;
+
+   sw.Clear();
+   sw.Start();
+   mass.AssembleEATriangular(fespace, packed_ea, false);
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   const double assemble_ms = 1000.0*sw.RealTime();
+
+   TriPackLowerMatrix tripack_inverse;
+   // Dry run setup before timing steady-state setup work.
+   tripack::ComputeCholeskyLowerInverse(packed_ea, tripack_inverse);
+   MFEM_DEVICE_SYNC;
+
+   sw.Clear();
+   sw.Start();
+   for (int r = 0; r < setup_reps; ++r)
+   {
+      tripack::ComputeCholeskyLowerInverse(packed_ea, tripack_inverse);
+   }
+   MFEM_DEVICE_SYNC;
+   sw.Stop();
+   const double tripack_inverse_setup_ms = 1000.0*sw.RealTime()/setup_reps;
+
+#ifdef MFEM_USE_MAGMA
+   magma_queue_t magma_queue = nullptr;
+   if (use_magma) { magma_queue = Magma::Queue(); }
+
+   TriPackLowerMatrix magma_factor;
+   std::unique_ptr<MagmaPackedLowerCholesky> magma_chol_ws;
+   double magma_factor_ms = 0.0;
+
+   TriPackLowerMatrix magma_inverse;
+   std::unique_ptr<MagmaPackedLowerInverse> magma_inv_ws;
+   double magma_inverse_ms = 0.0;
+   bool magma_ppinv_enabled = false;
+   Vector magma_full_factor;
+   Array<real_t *> magma_full_factor_ptrs;
+   double magma_full_factor_ms = 0.0;
+
+   if (use_magma)
+   {
+      magma_chol_ws.reset(new MagmaPackedLowerCholesky());
+      magma_chol_ws->SetQueue(magma_queue);
+
+      // Dry run setup before timing steady-state setup work.
+      magma_chol_ws->Factor(packed_ea, magma_factor);
+      MFEM_DEVICE_SYNC;
+
+      sw.Clear();
+      sw.Start();
+      for (int r = 0; r < setup_reps; ++r)
+      {
+         magma_chol_ws->Factor(packed_ea, magma_factor);
+      }
+      MFEM_DEVICE_SYNC;
+      sw.Stop();
+      magma_factor_ms = 1000.0*sw.RealTime()/setup_reps;
+
+      // Benchmark packed inverse (ppinv) only for sizes supported by MAGMA's
+      // current packed-inverse apply kernel.
+      if (elem_dofs <= 64)
+      {
+         magma_ppinv_enabled = true;
+         magma_inv_ws.reset(new MagmaPackedLowerInverse());
+         magma_inv_ws->SetQueue(magma_queue);
+
+         // Dry run setup before timing steady-state setup work.
+         magma_inv_ws->Compute(packed_ea, magma_inverse);
+         MFEM_DEVICE_SYNC;
+
+         sw.Clear();
+         sw.Start();
+         for (int r = 0; r < setup_reps; ++r)
+         {
+            magma_inv_ws->Compute(packed_ea, magma_inverse);
+         }
+         MFEM_DEVICE_SYNC;
+         sw.Stop();
+         magma_inverse_ms = 1000.0*sw.RealTime()/setup_reps;
+      }
+
+      // Full (dense) batched Cholesky factorization for comparison.
+      Vector full_ea(ne*elem_dofs*elem_dofs);
+      full_ea.UseDevice(true);
+      mass.AssembleEA(fespace, full_ea, false);
+
+      magma_full_factor.SetSize(full_ea.Size(), Device::GetDeviceMemoryType());
+      magma_full_factor.UseDevice(true);
+
+      ComputeMagmaFullCholeskyLower(full_ea, elem_dofs, magma_full_factor,
+                                    magma_full_factor_ptrs, magma_queue);
+      MFEM_DEVICE_SYNC;
+
+      sw.Clear();
+      sw.Start();
+      for (int r = 0; r < setup_reps; ++r)
+      {
+         ComputeMagmaFullCholeskyLower(full_ea, elem_dofs, magma_full_factor,
+                                       magma_full_factor_ptrs, magma_queue);
+      }
+      MFEM_DEVICE_SYNC;
+      sw.Stop();
+      magma_full_factor_ms = 1000.0*sw.RealTime()/setup_reps;
+   }
+#endif
+
+   Vector rhs(ne*elem_dofs);
+   FillRHS(rhs);
+
+   Vector tripack_x(rhs.Size()), work;
+   tripack_x.UseDevice(true);
+
+   const double tripack_apply_ms =
+      TimeLowerInverseApply(tripack_inverse, rhs, reps, tripack_x, work);
+
+   double tripack_res_l2 = 0.0, tripack_rel_res_l2 = 0.0;
+   real_t tripack_res_max = 0.0, tripack_rel_res_max = 0.0;
+   ComputeLowerPackedResidual(packed_ea, tripack_x, rhs,
+                              tripack_res_l2, tripack_rel_res_l2,
+                              tripack_res_max, tripack_rel_res_max);
+
+#ifdef MFEM_USE_MAGMA
+   double magma_solve_ms = 0.0;
+   double magma_full_solve_ms = 0.0;
+   double magma_res_l2 = 0.0, magma_rel_res_l2 = 0.0;
+   real_t magma_res_max = 0.0, magma_rel_res_max = 0.0;
+   double magma_full_res_l2 = 0.0, magma_full_rel_res_l2 = 0.0;
+   real_t magma_full_res_max = 0.0, magma_full_rel_res_max = 0.0;
+   Vector magma_x;
+   Vector magma_full_x;
+   if (use_magma)
+   {
+      magma_x.SetSize(rhs.Size());
+      magma_x.UseDevice(true);
+      magma_solve_ms =
+         TimeMagmaSolve(magma_factor, rhs, reps, magma_x, *magma_chol_ws);
+      ComputeLowerPackedResidual(packed_ea, magma_x, rhs,
+                                 magma_res_l2, magma_rel_res_l2,
+                                 magma_res_max, magma_rel_res_max);
+
+      magma_full_x.SetSize(rhs.Size());
+      magma_full_x.UseDevice(true);
+      magma_full_solve_ms =
+         TimeMagmaFullSolve(magma_full_factor, elem_dofs,
+                            magma_full_factor_ptrs, rhs, reps, magma_full_x,
+                            magma_queue);
+      ComputeLowerPackedResidual(packed_ea, magma_full_x, rhs,
+                                 magma_full_res_l2, magma_full_rel_res_l2,
+                                 magma_full_res_max, magma_full_rel_res_max);
+   }
+
+   double magma_ppinv_apply_ms = 0.0;
+   double magma_ppinv_res_l2 = 0.0, magma_ppinv_rel_res_l2 = 0.0;
+   real_t magma_ppinv_res_max = 0.0, magma_ppinv_rel_res_max = 0.0;
+   Vector magma_ppinv_x;
+   if (use_magma && magma_ppinv_enabled)
+   {
+      magma_ppinv_x.SetSize(rhs.Size());
+      magma_ppinv_x.UseDevice(true);
+      magma_ppinv_apply_ms =
+         TimeMagmaInverseApply(magma_inverse, rhs, reps, magma_ppinv_x,
+                               *magma_inv_ws);
+      ComputeLowerPackedResidual(packed_ea, magma_ppinv_x, rhs,
+                                 magma_ppinv_res_l2, magma_ppinv_rel_res_l2,
+                                 magma_ppinv_res_max,
+                                 magma_ppinv_rel_res_max);
+   }
+#endif
+
+   cout << fixed << setprecision(6);
+   cout << "Mass matrix inverse microbenchmark" << '\n';
+   cout << "Mesh: " << mesh_file << '\n';
+   cout << "Dimension: " << dim << '\n';
+   cout << "Elements: " << ne << '\n';
+   cout << "Element dofs: " << elem_dofs << '\n';
+   cout << "Scalar element unknowns: " << ne*elem_dofs << '\n';
+   cout << "Apply repetitions: " << reps << '\n';
+   cout << "Setup repetitions: " << setup_reps << '\n';
+   cout << '\n';
+
+   cout << "Assembly packed EA (ms): " << assemble_ms << '\n';
+   cout << "Setup MFEM tripack inverse (ms): " << tripack_inverse_setup_ms
+        << '\n';
+#ifdef MFEM_USE_MAGMA
+   if (use_magma)
+   {
+      cout << "Setup MAGMA packed Cholesky factor (ms): " << magma_factor_ms
+           << '\n';
+      cout << "Setup MAGMA full Cholesky factor (ms): " << magma_full_factor_ms
+           << '\n';
+      if (magma_ppinv_enabled)
+      {
+         cout << "Setup MAGMA packed inverse (ppinv) (ms): " << magma_inverse_ms
+              << '\n';
+      }
+      else
+      {
+         cout << "Setup MAGMA packed inverse (ppinv) (ms): skipped "
+              << "(requires element dofs <= 64)\n";
+      }
+   }
+#endif
+   cout << '\n';
+
+   cout << "Apply MFEM tripack inverse (ms/apply): " << tripack_apply_ms
+        << '\n';
+#ifdef MFEM_USE_MAGMA
+   if (use_magma)
+   {
+      cout << "Apply MAGMA packed Cholesky solve (ms/apply): "
+           << magma_solve_ms << '\n';
+      cout << "Apply MAGMA full Cholesky solve (ms/apply): "
+           << magma_full_solve_ms << '\n';
+      cout << "MAGMA solve / MFEM tripack inverse apply: "
+           << magma_solve_ms/tripack_apply_ms << '\n';
+      cout << "MAGMA full solve / MFEM tripack inverse apply: "
+           << magma_full_solve_ms/tripack_apply_ms << '\n';
+
+      if (magma_ppinv_enabled)
+      {
+         cout << "Apply MAGMA packed inverse (ppinv) (ms/apply): "
+              << magma_ppinv_apply_ms << '\n';
+         cout << "MAGMA ppinv apply / MFEM tripack inverse apply: "
+              << magma_ppinv_apply_ms/tripack_apply_ms << '\n';
+      }
+      else
+      {
+         cout << "Apply MAGMA packed inverse (ppinv) (ms/apply): skipped "
+              << "(requires element dofs <= 64)\n";
+      }
+
+      const double tripack_fixed_ms = assemble_ms + tripack_inverse_setup_ms;
+      const double magma_fixed_ms = assemble_ms + magma_factor_ms;
+      const double magma_full_fixed_ms = assemble_ms + magma_full_factor_ms;
+      const double tripack_total_ms =
+         tripack_fixed_ms + reps*tripack_apply_ms;
+      const double magma_total_ms = magma_fixed_ms + reps*magma_solve_ms;
+      const double magma_full_total_ms =
+         magma_full_fixed_ms + reps*magma_full_solve_ms;
+      cout << "Total MFEM tripack inverse for current repetitions "
+           << "(assembly+setup+applies, ms): " << tripack_total_ms << '\n';
+      cout << "Total MAGMA packed Cholesky solve for current repetitions "
+           << "(assembly+setup+applies, ms): " << magma_total_ms << '\n';
+      cout << "Total MAGMA full Cholesky solve for current repetitions "
+           << "(assembly+setup+applies, ms): " << magma_full_total_ms << '\n';
+      cout << "Faster approach for current repetitions: "
+           << ((magma_total_ms < tripack_total_ms &&
+                magma_total_ms <= magma_full_total_ms) ? "MAGMA packed" :
+               ((magma_full_total_ms < tripack_total_ms &&
+                 magma_full_total_ms < magma_total_ms) ? "MAGMA full" :
+                ((tripack_total_ms < magma_total_ms &&
+                  tripack_total_ms <= magma_full_total_ms) ? "tripack" :
+                 "tie")))
+           << '\n';
+      PrintMagmaFasterCondition(tripack_fixed_ms, tripack_apply_ms,
+                                magma_fixed_ms, magma_solve_ms);
+      PrintMagmaFasterCondition(tripack_fixed_ms, tripack_apply_ms,
+                                magma_full_fixed_ms, magma_full_solve_ms);
+
+      if (magma_ppinv_enabled)
+      {
+         const double magma_ppinv_fixed_ms =
+            assemble_ms + magma_inverse_ms;
+         const double magma_ppinv_total_ms =
+            magma_ppinv_fixed_ms + reps*magma_ppinv_apply_ms;
+         cout << "Total MAGMA packed inverse (ppinv) for current "
+              << "repetitions (assembly+setup+applies, ms): "
+              << magma_ppinv_total_ms << '\n';
+      }
+   }
+#endif
+   cout << '\n';
+
+   cout << scientific << setprecision(12);
+   cout << "Residual, MFEM tripack inverse, max: "
+        << tripack_res_max << " (relative "
+        << tripack_rel_res_max << "), L2: "
+        << tripack_res_l2 << " (relative "
+        << tripack_rel_res_l2 << ")\n";
+#ifdef MFEM_USE_MAGMA
+   if (use_magma)
+   {
+      cout << "Residual, MAGMA packed Cholesky solve, max: "
+           << magma_res_max << " (relative "
+           << magma_rel_res_max << "), L2: "
+           << magma_res_l2 << " (relative "
+           << magma_rel_res_l2 << ")\n";
+      cout << "Residual, MAGMA full Cholesky solve, max: "
+           << magma_full_res_max << " (relative "
+           << magma_full_rel_res_max << "), L2: "
+           << magma_full_res_l2 << " (relative "
+           << magma_full_rel_res_l2 << ")\n";
+      if (magma_ppinv_enabled)
+      {
+         cout << "Residual, MAGMA packed inverse (ppinv), max: "
+              << magma_ppinv_res_max << " (relative "
+              << magma_ppinv_rel_res_max << "), L2: "
+              << magma_ppinv_res_l2 << " (relative "
+              << magma_ppinv_rel_res_l2 << ")\n";
+      }
+   }
+#endif
+
+   return 0;
+}

examples/magma/makefile

@@ -0,0 +1,55 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+MFEM_INSTALL_DIR ?= ../../mfem
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/magma/,)
+CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
+   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = batched_mass_bench
+EXAMPLES = $(SEQ_EXAMPLES)
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+ifeq ($(MFEM_USE_MAGMA),NO)
+$(EXAMPLES):
+	$(error MFEM is not configured with MAGMA)
+endif
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints

examples/makefile

@@ -68,6 +68,9 @@ endif
 ifeq ($(MFEM_USE_CALIPER),YES)
    SUBDIRS += caliper
 endif
+ifeq ($(MFEM_USE_MAGMA),YES)
+   SUBDIRS += magma
+endif
 
 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
@@ -84,7 +87,8 @@ SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))
 
 # Replace the default implicit rule for *.cpp files
 %: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< -o $@.o
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) $@.o -o $@ $(MFEM_LIBS)
 
 all: $(EXAMPLES) $(SUBDIRS_ALL)
 

fem/CMakeLists.txt

@@ -57,6 +57,8 @@ set(SRCS
   integ/lininteg_domain_grad.cpp
   integ/lininteg_domain_vectorfe.cpp
   integ/nonlininteg_vecconvection_pa.cpp
+  integ/nonlininteg_vecconvection_pa_diag.cpp
+  integ/nonlininteg_vecconvection_pa_grad.cpp
   integ/nonlininteg_vecconvection_mf.cpp
   coefficient.cpp
   complex_fem.cpp
@@ -204,7 +206,11 @@ set(HDRS
   integ/bilininteg_mass_kernels.hpp
   integ/bilininteg_mass_pa_simplices.hpp
   integ/bilininteg_vecdiffusion_pa.hpp
+  integ/bilininteg_vecdiv_pa.hpp
   integ/bilininteg_vecmass_pa.hpp
+  integ/nonlininteg_vecconvection_pa.hpp
+  integ/nonlininteg_vecconvection_pa_diag.hpp
+  integ/nonlininteg_vecconvection_pa_grad.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp

fem/bilinearform.cpp

@@ -1255,6 +1255,31 @@ void BilinearForm::Mult(const Vector &x, Vector &y) const
    }
 }
 
+void BilinearForm::AddMult(const Vector &x, Vector &y, const real_t a) const
+{
+   if (ext)
+   {
+      ext->AddMult(x, y, a);
+   }
+   else
+   {
+      mat->AddMult(x, y, a);
+   }
+}
+
+void BilinearForm::AddMultTranspose(const Vector &x, Vector &y,
+                                    const real_t a) const
+{
+   if (ext)
+   {
+      ext->AddMultTranspose(x, y, a);
+   }
+   else
+   {
+      mat->AddMultTranspose(x, y, a);
+   }
+}
+
 void BilinearForm::MultTranspose(const Vector & x, Vector & y) const
 {
    if (ext)

fem/bilinearform.hpp

@@ -307,8 +307,8 @@ public:
    { mat->Mult(x, y); mat_e->AddMult(x, y); }
 
    /// Add the matrix vector multiple to a vector:  $ y += a M x $
-   void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const override
-   { mat -> AddMult (x, y, a); }
+   void AddMult(const Vector &x, Vector &y,
+                const real_t a = 1.0) const override;
 
    /** @brief Add the original uneliminated matrix vector multiple to a vector.
        The original matrix is $ M + Me $ so we have:
@@ -318,8 +318,7 @@ public:
 
    /// Add the matrix transpose vector multiplication:  $ y += a M^T x $
    void AddMultTranspose(const Vector & x, Vector & y,
-                         const real_t a = 1.0) const override
-   { mat->AddMultTranspose(x, y, a); }
+                         const real_t a = 1.0) const override;
 
    /** @brief Add the original uneliminated matrix transpose vector
        multiple to a vector. The original matrix is $ M + M_e $

fem/bilinearform_ext.cpp

@@ -1997,7 +1997,11 @@ void PADiscreteLinearOperatorExtension::Assemble()
    }
    else
    {
-      mfem_error("A real ElementRestriction is required in this setting!");
+      const L2ElementRestriction* l2_elem_restrict =
+         dynamic_cast<const L2ElementRestriction*>(elem_restrict_test);
+      MFEM_VERIFY(l2_elem_restrict,
+                  "A real ElementRestriction is required in this setting!");
+      test_multiplicity = 1.0;
    }
 
    auto tm = test_multiplicity.ReadWrite();
@@ -2036,7 +2040,13 @@ void PADiscreteLinearOperatorExtension::AddMult(
    }
    else
    {
-      mfem_error("In this setting you need a real ElementRestriction!");
+      const L2ElementRestriction* l2_elem_restrict =
+         dynamic_cast<const L2ElementRestriction*>(elem_restrict_test);
+      MFEM_VERIFY(l2_elem_restrict,
+                  "In this setting you need a real ElementRestriction!");
+      tempY.SetSize(y.Size());
+      l2_elem_restrict->MultTranspose(localTest, tempY);
+      y += tempY;
    }
 }
 

fem/coefficient.hpp

@@ -1055,7 +1055,8 @@ public:
 
 typedef VectorCoefficient DiagonalMatrixCoefficient;
 
-/// Base class for Matrix Coefficients that optionally depend on time and space.
+/** Base class for matrix-valued coefficients that optionally depend on time
+    and space. */
 class MatrixCoefficient
 {
 protected:
@@ -1102,6 +1103,9 @@ public:
    /// the quadrature points. The matrix will be transposed or not according to
    /// the boolean argument @a transpose.
    ///
+   /// The stored entries use the same row/column convention as `Eval()`,
+   /// unless `transpose == true`, in which case `K^T` is stored instead.
+   ///
    /// The @a vdim of the QuadratureFunction should be equal to the height times
    /// the width of the matrix.
    virtual void Project(QuadratureFunction &qf, bool transpose=false);

fem/complex_fem.hpp

@@ -392,6 +392,9 @@ private:
    bool RealInteg();
    bool ImagInteg();
 
+   void BuildComplexOperator(OperatorHandle &A_r, OperatorHandle &A_i,
+                             OperatorHandle &A) const;
+
 public:
    SesquilinearForm(FiniteElementSpace *fes,
                     ComplexOperator::Convention
@@ -505,6 +508,186 @@ public:
    virtual ~SesquilinearForm();
 };
 
+/** Class for a mixed sesquilinear form
+
+    A mixed sesquilinear form is a generalization of a mixed bilinear form to
+    complex-valued fields. Mixed sesquilinear forms are linear in the second
+    argument but the first argument involves a complex conjugate in the sense
+    that:
+
+                a(alpha u, beta v) = conj(alpha) beta a(u, v)
+
+    The @a convention argument in the class's constructor is documented in the
+    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
+
+    When supplying integrators to the MixedSesquilinearForm either the real or
+    imaginary integrator can be NULL. This indicates that the corresponding
+    portion of the complex-valued material coefficient is equal to zero.
+*/
+class MixedSesquilinearForm
+{
+private:
+   ComplexOperator::Convention conv;
+
+   MixedBilinearForm * mblfr;
+   MixedBilinearForm * mblfi;
+
+   /* These methods check if the real/imag parts of the sesqulinear form are not
+      empty */
+   bool RealInteg();
+   bool ImagInteg();
+
+public:
+   MixedSesquilinearForm(
+      FiniteElementSpace * trial_fes,
+      FiniteElementSpace * test_fes,
+      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
+
+   /** @brief Create a MixedSesquilinearForm on the given trial and test
+       FiniteElementSpaces, using the same integrators as the
+       MixedBilinearForms @a bfr and @a bfi.
+
+       The FiniteElementSpace pointers are not owned by the newly constructed
+       object.
+
+       The integrators are copied as pointers and they are not owned by the
+       newly constructed MixedSesquilinearForm. */
+   MixedSesquilinearForm(
+      FiniteElementSpace * trial_fes,
+      FiniteElementSpace * test_fes,
+      MixedBilinearForm * bfr,
+      MixedBilinearForm * bfi,
+      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
+
+   ComplexOperator::Convention GetConvention() const { return conv; }
+   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
+
+   /// Set the desired assembly level.
+   /** Valid choices are:
+
+       - AssemblyLevel::LEGACY (default)
+       - AssemblyLevel::FULL
+       - AssemblyLevel::PARTIAL
+       - AssemblyLevel::ELEMENT
+       - AssemblyLevel::NONE
+
+       This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level)
+   {
+      mblfr->SetAssemblyLevel(assembly_level);
+      mblfi->SetAssemblyLevel(assembly_level);
+   }
+
+   MixedBilinearForm & real() { return *mblfr; }
+   MixedBilinearForm & imag() { return *mblfi; }
+   const MixedBilinearForm & real() const { return *mblfr; }
+   const MixedBilinearForm & imag() const { return *mblfi; }
+
+   /// Adds new Domain Integrator.
+   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                            BilinearFormIntegrator * bfi_imag);
+
+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                            BilinearFormIntegrator * bfi_imag,
+                            Array<int> & elem_marker);
+
+   /// Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
+                              BilinearFormIntegrator * bfi_imag);
+
+   /** @brief Adds new boundary Integrator, restricted to specific boundary
+       attributes.
+
+       Assumes ownership of @a bfi.
+
+       The mfem::array @a bdr_marker is stored internally as a pointer to the given
+       mfem::Array<int> object. */
+   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
+                              BilinearFormIntegrator * bfi_imag,
+                              Array<int> & bdr_marker);
+
+   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
+   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag);
+
+   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
+   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                             BilinearFormIntegrator * bfi_imag);
+
+   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
+       attributes.
+
+       Assumes ownership of @a bfi.
+
+       The mfem::array @a bdr_marker is stored internally as a pointer to the given
+       mfem::Array<int> object. */
+   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                             BilinearFormIntegrator * bfi_imag,
+                             Array<int> & bdr_marker);
+
+   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
+
+       This type of integrator assembles terms over all faces of the mesh using
+       the face FE from the trial space and the two adjacent volume FEs from
+       the test space. */
+   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                               BilinearFormIntegrator * bfi_imag);
+
+   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag);
+
+   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag,
+                                  Array<int> &bdr_marker);
+
+   /// Assemble the local matrix
+   void Assemble(int skip_zeros = 1);
+
+   /// Finalizes the matrix initialization.
+   void Finalize(int skip_zeros = 1);
+
+   /// Updates the internal mixed forms with the new finite element space.
+   virtual void Update();
+
+   /** @brief Return a ComplexSparseMatrix wrapping the local (L-dof) real
+       and imaginary matrices of the form.
+
+       The returned wrapper has to be deleted by the caller, but it does not
+       own the wrapped real and imaginary matrices, which remain owned by
+       this form. */
+   ComplexSparseMatrix *AssembleComplexSparseMatrix();
+
+   /// Return the trial FE space associated with the MixedSesquilinearForm.
+   FiniteElementSpace *TrialFESpace() { return mblfr->TrialFESpace(); }
+
+   /// Read-only access to the associated trial FiniteElementSpace.
+   const FiniteElementSpace *TrialFESpace() const { return mblfr->TrialFESpace(); }
+
+   /// Return the test FE space associated with the MixedSesquilinearForm.
+   FiniteElementSpace *TestFESpace() { return mblfr->TestFESpace(); }
+
+   /// Read-only access to the associated test FiniteElementSpace.
+   const FiniteElementSpace *TestFESpace() const { return mblfr->TestFESpace(); }
+
+
+   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
+                                    const Array<int> & ess_test_tdof_list,
+                                    Vector & x,
+                                    Vector & b,
+                                    OperatorHandle & A,
+                                    Vector & X,
+                                    Vector & B);
+
+   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
+                                    const Array<int> & ess_test_tdof_list,
+                                    OperatorHandle & A);
+
+   virtual ~MixedSesquilinearForm();
+};
+
 #ifdef MFEM_USE_MPI
 
 /// Class for parallel complex-valued grid function - real + imaginary part
@@ -806,6 +989,12 @@ private:
    bool RealInteg();
    bool ImagInteg();
 
+   void SetImaginaryEssentialDiagonalToZero(
+      const Array<int> &ess_tdof_list, OperatorHandle &A);
+
+   void BuildComplexOperator(OperatorHandle &A_r, OperatorHandle &A_i,
+                             OperatorHandle &A) const;
+
 public:
    ParSesquilinearForm(ParFiniteElementSpace *pf,
                        ComplexOperator::Convention
@@ -921,6 +1110,169 @@ public:
    virtual ~ParSesquilinearForm();
 };
 
+/** Class for a parallel mixed sesquilinear form
+
+    A mixed sesquilinear form is a generalization of a mixed bilinear form to
+    complex-valued fields. Mixed sesquilinear forms are linear in the second
+    argument but the first argument involves a complex conjugate in the sense
+    that:
+
+                a(alpha u, beta v) = conj(alpha) beta a(u, v)
+
+    The @a convention argument in the class's constructor is documented in the
+    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
+
+    When supplying integrators to the ParMixedSesquilinearForm either the real
+    or imaginary integrator can be NULL. This indicates that the corresponding
+    portion of the complex-valued material coefficient is equal to zero.
+*/
+class ParMixedSesquilinearForm
+{
+private:
+   ComplexOperator::Convention conv;
+
+   ParMixedBilinearForm * pmblfr;
+   ParMixedBilinearForm * pmblfi;
+
+   /* These methods check if the real/imag parts of the sesqulinear form are
+    not empty */
+   bool RealInteg();
+   bool ImagInteg();
+
+public:
+   ParMixedSesquilinearForm(
+      ParFiniteElementSpace * trial_fes,
+      ParFiniteElementSpace * test_fes,
+      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
+
+   /** @brief Create a ParMixedSesquilinearForm on the given trial and test
+       ParFiniteElementSpaces, using the same integrators as the
+       ParMixedBilinearForms @a pbfr and @a pbfi.
+
+       The ParFiniteElementSpace pointers are not owned by the newly
+       constructed object.
+
+       The integrators are copied as pointers and they are not owned by the
+       newly constructed ParMixedSesquilinearForm. */
+   ParMixedSesquilinearForm(
+      ParFiniteElementSpace * trial_fes,
+      ParFiniteElementSpace * test_fes,
+      ParMixedBilinearForm * pbfr,
+      ParMixedBilinearForm * pbfi,
+      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
+
+   ComplexOperator::Convention GetConvention() const { return conv; }
+   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
+
+   /// Set the desired assembly level.
+   /** Valid choices are:
+
+       - AssemblyLevel::LEGACY (default)
+       - AssemblyLevel::FULL
+       - AssemblyLevel::PARTIAL
+       - AssemblyLevel::ELEMENT
+       - AssemblyLevel::NONE
+
+       This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level)
+   {
+      pmblfr->SetAssemblyLevel(assembly_level);
+      pmblfi->SetAssemblyLevel(assembly_level);
+   }
+
+   ParMixedBilinearForm & real() { return *pmblfr; }
+   ParMixedBilinearForm & imag() { return *pmblfi; }
+   const ParMixedBilinearForm & real() const { return *pmblfr; }
+   const ParMixedBilinearForm & imag() const { return *pmblfi; }
+
+   /// Adds new Domain Integrator.
+   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                            BilinearFormIntegrator * bfi_imag);
+
+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
+                            BilinearFormIntegrator * bfi_imag,
+                            Array<int> & elem_marker);
+
+   /// Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
+                              BilinearFormIntegrator * bfi_imag);
+
+   /** @brief Adds new boundary Integrator, restricted to specific boundary
+       attributes.
+
+       Assumes ownership of @a bfi.
+
+       The mfem::array @a bdr_marker is stored internally as a pointer to the given
+       mfem::Array<int> object. */
+   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
+                              BilinearFormIntegrator * bfi_imag,
+                              Array<int> & bdr_marker);
+
+   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
+   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag);
+
+   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
+   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                             BilinearFormIntegrator * bfi_imag);
+
+   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
+       attributes.
+
+       Assumes ownership of @a bfi.
+
+       The mfem::array @a bdr_marker is stored internally as a pointer to the given
+       mfem::Array<int> object. */
+   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                             BilinearFormIntegrator * bfi_imag,
+                             Array<int> & bdr_marker);
+
+   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
+
+      This type of integrator assembles terms over all faces of the mesh using
+      the face FE from the trial space and the two adjacent volume FEs from
+      the test space. */
+   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                               BilinearFormIntegrator * bfi_imag);
+
+   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag);
+
+   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
+                                  BilinearFormIntegrator * bfi_imag,
+                                  Array<int> &bdr_marker);
+
+   /// Assemble the local matrix
+   void Assemble(int skip_zeros = 1);
+
+   /// Finalizes the matrix initialization.
+   void Finalize(int skip_zeros = 1);
+
+   /// Updates the internal mixed forms with the new finite element space.
+   virtual void Update();
+
+   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
+   /** The returned matrix has to be deleted by the caller. */
+   ComplexHypreParMatrix * ParallelAssemble();
+
+   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
+                                    const Array<int> & ess_test_tdof_list,
+                                    Vector & x,
+                                    Vector & b,
+                                    OperatorHandle & A,
+                                    Vector & X,
+                                    Vector & B);
+
+   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
+                                    const Array<int> & ess_test_tdof_list,
+                                    OperatorHandle & A);
+
+   virtual ~ParMixedSesquilinearForm();
+};
+
 #endif // MFEM_USE_MPI
 
 }

fem/datacollection.cpp

@@ -38,9 +38,24 @@ int DataCollection::create_directory(const std::string &dir_name,
    // create directories recursively
    const char path_delim = '/';
    std::string::size_type pos = 0;
-   int err_flag;
+   int err_flag = 0;
 #ifdef MFEM_USE_MPI
    const ParMesh *pmesh = dynamic_cast<const ParMesh*>(mesh);
+   // In addition to the global root, let the lowest rank on each shared-memory
+   // node create the directory too, so that node-local (non-shared) filesystems
+   // get it on every node rather than only where the global root lives. On a
+   // shared filesystem the extra mkdir() hits EEXIST and is tolerated below.
+   bool node_root = true;
+   if (pmesh)
+   {
+      MPI_Comm node_comm;
+      MPI_Comm_split_type(pmesh->GetComm(), MPI_COMM_TYPE_SHARED, myid,
+                          MPI_INFO_NULL, &node_comm);
+      int node_rank;
+      MPI_Comm_rank(node_comm, &node_rank);
+      node_root = (node_rank == 0);
+      MPI_Comm_free(&node_comm);
+   }
 #endif
 
    do
@@ -52,7 +67,7 @@ int DataCollection::create_directory(const std::string &dir_name,
       err_flag = mkdir(subdir.c_str(), 0777);
       err_flag = (err_flag && (errno != EEXIST)) ? 1 : 0;
 #else
-      if (myid == 0 || pmesh == NULL)
+      if (node_root || pmesh == NULL)
       {
          err_flag = mkdir(subdir.c_str(), 0777);
          err_flag = (err_flag && (errno != EEXIST)) ? 1 : 0;
@@ -64,7 +79,8 @@ int DataCollection::create_directory(const std::string &dir_name,
 #ifdef MFEM_USE_MPI
    if (pmesh)
    {
-      MPI_Bcast(&err_flag, 1, MPI_INT, 0, pmesh->GetComm());
+      MPI_Allreduce(MPI_IN_PLACE, &err_flag, 1, MPI_INT, MPI_MAX,
+                    pmesh->GetComm());
    }
 #endif
 
@@ -809,7 +825,7 @@ ParaViewDataCollectionBase::ParaViewDataCollectionBase(
 
 void ParaViewDataCollectionBase::SetLevelsOfDetail(int levels_of_detail_)
 {
-   levels_of_detail = levels_of_detail_;
+   levels_of_detail = std::max(levels_of_detail_, 1);
 }
 
 void ParaViewDataCollectionBase::SetHighOrderOutput(bool high_order_output_)

fem/dfem/doperator.cpp

@@ -51,4 +51,52 @@ DifferentiableOperator::DifferentiableOperator(
    }
 }
 
+
+void FDJacobian::Mult(const Vector &v, Vector &y) const
+{
+   // See [1] for choice of eps.
+   //
+   // [1] Woodward, C.S., Gardner, D.J. and Evans, K.J., 2015. On the use of
+   // finite difference matrix-vector products in Newton-Krylov solvers for
+   // implicit climate dynamics with spectral elements. Procedia Computer
+   // Science, 51, pp.2036-2045.
+   real_t eps;
+   if (fixed_eps > 0.0)
+   {
+      eps = fixed_eps;
+   }
+   else
+   {
+      const real_t vnorm_local = v.Norml2();
+      real_t vnorm;
+      MPI_Allreduce(&vnorm_local, &vnorm, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                    MPI_COMM_WORLD);
+      eps = lambda * (lambda + xnorm / vnorm);
+   }
+
+   // x + eps * v
+   {
+      const auto d_v = v.Read();
+      const auto d_x = x.Read();
+      auto d_xpev = xpev.Write();
+      mfem::forall(x.Size(), [=] MFEM_HOST_DEVICE (int i)
+      {
+         d_xpev[i] = d_x[i] + eps * d_v[i];
+      });
+   }
+
+   // y = f(x + eps * v)
+   op.Mult(xpev, y);
+
+   // y = (f(x + eps * v) - f(x)) / eps
+   {
+      const auto d_f = f.Read();
+      auto d_y = y.ReadWrite();
+      mfem::forall(f.Size(), [=] MFEM_HOST_DEVICE (int i)
+      {
+         d_y[i] = (d_y[i] - d_f[i]) / eps;
+      });
+   }
+}
+
 #endif // MFEM_USE_MPI

fem/dfem/doperator.hpp

@@ -697,17 +697,18 @@ void DifferentiableOperator::AddIntegrator(
 
    // The explicit captures are necessary to avoid dependency on
    // the specific instance of this class (this pointer).
-   restriction_callback =
-      [=, solutions = this->solutions, parameters = this->parameters]
-      (std::vector<Vector> &sol,
-       const std::vector<Vector> &par,
-       std::vector<Vector> &f)
+   restriction_callback = [element_dof_ordering,
+                           solutions_ = this->solutions,
+                           parameters_ = this->parameters]
+   (std::vector<Vector> &sol,
+    const std::vector<Vector> &par,
+    std::vector<Vector> &f)
    {
-      restriction<entity_t>(solutions, sol, f,
+      restriction<entity_t>(solutions_, sol, f,
                             element_dof_ordering);
-      restriction<entity_t>(parameters, par, f,
+      restriction<entity_t>(parameters_, par, f,
                             element_dof_ordering,
-                            solutions.size());
+                            solutions_.size());
    };
 
    prolongation_transpose = get_prolongation_transpose(
@@ -835,19 +836,19 @@ void DifferentiableOperator::AddIntegrator(
 
          // capture by ref:
          &restriction_cb = this->restriction_callback,
-         &fields_e = this->fields_e,
-         &residual_e = this->residual_e,
-         &output_restriction_transpose = this->output_restriction_transpose
+         &fields_e_ = this->fields_e,
+         &residual_e_ = this->residual_e,
+         &output_restriction_transpose_ = this->output_restriction_transpose
       ]
       (std::vector<Vector> &sol, const std::vector<Vector> &par, Vector &res)
       mutable // mutable: needed to modify 'shmem_cache'
    {
-      restriction_cb(sol, par, fields_e);
+      restriction_cb(sol, par, fields_e_);
 
-      residual_e = 0.0;
-      auto ye = Reshape(residual_e.ReadWrite(), test_vdim, num_test_dof, num_entities);
+      residual_e_ = 0.0;
+      auto ye = Reshape(residual_e_.ReadWrite(), test_vdim, num_test_dof, num_entities);
 
-      auto wrapped_fields_e = wrap_fields(fields_e,
+      auto wrapped_fields_e = wrap_fields(fields_e_,
                                           action_shmem_info.field_sizes,
                                           num_entities);
 
@@ -878,7 +879,7 @@ void DifferentiableOperator::AddIntegrator(
             y, fhat, output_fop, output_dtq_shmem[0],
             scratch_shmem, dimension, use_sum_factorization);
       }, num_entities, thread_blocks, action_shmem_info.total_size, shmem_cache.ReadWrite());
-      output_restriction_transpose(residual_e, res);
+      output_restriction_transpose_(residual_e_, res);
    });
 
    // Without this compile-time check, some valid instantiations of this method
@@ -1193,7 +1194,7 @@ void DifferentiableOperator::AddIntegrator(
 
                // capture by ref:
                &qpdc_mem = derivative_qp_caches_ref,
-               &fields = fields_ref
+               &fields_ = fields_ref
             ](std::vector<Vector> &f_e, SparseMatrix *&A) mutable
          {
             auto wrapped_fields_e = wrap_fields(f_e, shmem_info.field_sizes,
@@ -1241,14 +1242,14 @@ void DifferentiableOperator::AddIntegrator(
             {
                if (input_is_dependent[s])
                {
-                  trial_field = &fields[input_to_field[s]];
+                  trial_field = &fields_[input_to_field[s]];
                }
             }
 
             auto trial_fes = *std::get_if<const ParFiniteElementSpace *>
                              (&trial_field->data);
             auto test_fes = *std::get_if<const ParFiniteElementSpace *>
-                            (&fields[output_to_field[0]].data);
+                            (&fields_[output_to_field[0]].data);
 
             A = new SparseMatrix(test_fes->GetVSize(), trial_fes->GetVSize());
 
@@ -1334,7 +1335,7 @@ void DifferentiableOperator::AddIntegrator(
                input_to_field,
                output_to_field,
                &spmatcb = assemble_derivative_sparsematrix_callbacks_ref,
-               &fields = fields_ref
+               &fields_ = fields_ref
             ](std::vector<Vector> &f_e, HypreParMatrix *&A) mutable
          {
             SparseMatrix *spmat = nullptr;
@@ -1366,14 +1367,14 @@ void DifferentiableOperator::AddIntegrator(
             {
                if (input_is_dependent[s])
                {
-                  trial_field = &fields[input_to_field[s]];
+                  trial_field = &fields_[input_to_field[s]];
                }
             }
 
             auto trial_fes = *std::get_if<const ParFiniteElementSpace *>
                              (&trial_field->data);
             auto test_fes = *std::get_if<const ParFiniteElementSpace *>
-                            (&fields[output_to_field[0]].data);
+                            (&fields_[output_to_field[0]].data);
 
             if (same_test_and_trial)
             {

fem/dfem/util.hpp

@@ -597,7 +597,7 @@ struct ThreadBlocks
    int z = 1;
 };
 
-#if defined(MFEM_USE_CUDA_OR_HIP)
+#if defined(MFEM_USE_CUDA_OR_HIP_LANG)
 template <typename func_t>
 __global__ void forall_kernel_shmem(func_t f, int n)
 {
@@ -617,10 +617,11 @@ void forall(func_t f,
             int num_shmem = 0,
             real_t *shmem = nullptr)
 {
-   if (Device::Allows(Backend::CUDA_MASK) ||
-       Device::Allows(Backend::HIP_MASK))
+   internal::RequireKernelCompilation();
+
+#if defined(MFEM_USE_CUDA_OR_HIP_LANG)
+   if (Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
    {
-#if defined(MFEM_USE_CUDA_OR_HIP)
       // int gridsize = (N + Z - 1) / Z;
       int num_bytes = num_shmem * sizeof(decltype(shmem));
       dim3 block_size(blocks.x, blocks.y, blocks.z);
@@ -631,9 +632,10 @@ void forall(func_t f,
       MFEM_GPU_CHECK(hipGetLastError());
 #endif
       MFEM_DEVICE_SYNC;
-#endif
+      return;
    }
-   else if (Device::Allows(Backend::CPU_MASK))
+#endif
+   if (Device::Allows(Backend::CPU_MASK))
    {
       MFEM_ASSERT(!((bool)num_shmem != (bool)shmem),
                   "Backend::CPU needs a pre-allocated shared memory block");
@@ -671,52 +673,7 @@ public:
                     MPI_COMM_WORLD);
    }
 
-   void Mult(const Vector &v, Vector &y) const override
-   {
-      // See [1] for choice of eps.
-      //
-      // [1] Woodward, C.S., Gardner, D.J. and Evans, K.J., 2015. On the use of
-      // finite difference matrix-vector products in Newton-Krylov solvers for
-      // implicit climate dynamics with spectral elements. Procedia Computer
-      // Science, 51, pp.2036-2045.
-      real_t eps;
-      if (fixed_eps > 0.0)
-      {
-         eps = fixed_eps;
-      }
-      else
-      {
-         const real_t vnorm_local = v.Norml2();
-         real_t vnorm;
-         MPI_Allreduce(&vnorm_local, &vnorm, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
-                       MPI_COMM_WORLD);
-         eps = lambda * (lambda + xnorm / vnorm);
-      }
-
-      // x + eps * v
-      {
-         const auto d_v = v.Read();
-         const auto d_x = x.Read();
-         auto d_xpev = xpev.Write();
-         mfem::forall(x.Size(), [=] MFEM_HOST_DEVICE (int i)
-         {
-            d_xpev[i] = d_x[i] + eps * d_v[i];
-         });
-      }
-
-      // y = f(x + eps * v)
-      op.Mult(xpev, y);
-
-      // y = (f(x + eps * v) - f(x)) / eps
-      {
-         const auto d_f = f.Read();
-         auto d_y = y.ReadWrite();
-         mfem::forall(f.Size(), [=] MFEM_HOST_DEVICE (int i)
-         {
-            d_y[i] = (d_y[i] - d_f[i]) / eps;
-         });
-      }
-   }
+   void Mult(const Vector &v, Vector &y) const override;
 
    virtual MemoryClass GetMemoryClass() const override
    {

fem/fe/fe_base.cpp

@@ -1316,13 +1316,14 @@ void VectorFiniteElement::Project_RT(
    }
 }
 
-void VectorFiniteElement::ProjectGrad_RT(
+void VectorFiniteElement::ProjectCurl2D_RT(
    const real_t *nk, const Array<int> &d2n, const FiniteElement &fe,
    ElementTransformation &Trans, DenseMatrix &grad) const
 {
+   // 2D "ProjectCurl_RT"
    if (dim != 2)
    {
-      mfem_error("VectorFiniteElement::ProjectGrad_RT works only in 2D!");
+      mfem_error("VectorFiniteElement::ProjectCurl2D_RT works only in 2D!");
    }
 
    DenseMatrix dshape(fe.GetDof(), fe.GetDim());
@@ -1333,8 +1334,8 @@ void VectorFiniteElement::ProjectGrad_RT(
    for (int k = 0; k < dof; k++)
    {
       fe.CalcDShape(Nodes.IntPoint(k), dshape);
-      tk[0] = nk[d2n[k]*dim+1];
-      tk[1] = -nk[d2n[k]*dim];
+      tk[0] = -nk[d2n[k]*dim+1];
+      tk[1] = nk[d2n[k]*dim];
       dshape.Mult(tk, grad_k);
       for (int j = 0; j < grad_k.Size(); j++)
       {
@@ -1381,7 +1382,7 @@ void VectorFiniteElement::ProjectCurl_ND(
    }
 }
 
-void VectorFiniteElement::ProjectCurl_RT(
+void VectorFiniteElement::ProjectCurl3D_RT(
    const real_t *nk, const Array<int> &d2n, const FiniteElement &fe,
    ElementTransformation &Trans, DenseMatrix &curl) const
 {

fem/fe/fe_base.hpp

@@ -957,10 +957,11 @@ protected:
                    const FiniteElement &fe, ElementTransformation &Trans,
                    DenseMatrix &I) const;
 
-   // rotated gradient in 2D
-   void ProjectGrad_RT(const real_t *nk, const Array<int> &d2n,
-                       const FiniteElement &fe, ElementTransformation &Trans,
-                       DenseMatrix &grad) const;
+   // Input is a scalar representing the Z (out of plane) component, Output is
+   // the X-Y (in-plane) RT curl
+   void ProjectCurl2D_RT(const real_t *nk, const Array<int> &d2n,
+                         const FiniteElement &fe, ElementTransformation &Trans,
+                         DenseMatrix &grad) const;
 
    // Compute the curl as a discrete operator from ND FE (fe) to ND FE (this).
    // The natural FE for the range is RT, so this is an approximation.
@@ -968,9 +969,9 @@ protected:
                        const FiniteElement &fe, ElementTransformation &Trans,
                        DenseMatrix &curl) const;
 
-   void ProjectCurl_RT(const real_t *nk, const Array<int> &d2n,
-                       const FiniteElement &fe, ElementTransformation &Trans,
-                       DenseMatrix &curl) const;
+   void ProjectCurl3D_RT(const real_t *nk, const Array<int> &d2n,
+                         const FiniteElement &fe, ElementTransformation &Trans,
+                         DenseMatrix &curl) const;
 
    /** @brief Project a vector coefficient onto the ND basis functions
        @param tk    Edge tangent vectors for this element type
@@ -1446,6 +1447,8 @@ public:
                                 dof2quad_array_open);
    }
 
+   const Poly_1D::Basis &GetOpenBasis1D() const { return obasis1d; }
+
    virtual ~VectorTensorFiniteElement();
 };
 

fem/fe/fe_fixed_order.hpp

@@ -307,12 +307,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (4) */
+       them in the vector shape of dimension Dof (6) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (6 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;
@@ -336,12 +336,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (4) */
+       them in the vector shape of dimension Dof (5) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (5 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;

fem/fe/fe_h1.cpp

@@ -1757,22 +1757,45 @@ H1_BergotPyramidElement::H1_BergotPyramidElement(const int p, const int btype)
       real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       real_t z = ip.z;
 
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
-
-      o = 0;
-      for (int i = 0; i <= p; i++)
+      if (std::abs(z - 1.0) < apex_tol)
       {
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
-
-            for (int k = 0; k <= p - maxij; k++)
+         // Compute the limit of the basis functions as z->1 with x and y on the
+         // line between the center of the base and the apex
+         o = 0;
+         for (int i = 0; i <= p; i++)
+            for (int j = 0; j <= p; j++)
             {
-               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                           pow(1.0 - ip.z, maxij);
+               int maxij = std::max(i, j);
+               for (int k = 0; k <= p - maxij; k++)
+                  if (i == 0 && j == 0)
+                  {
+                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
+                  }
+                  else
+                  {
+                     T(o++, m) = 0.;
+                  }
+            }
+      }
+      else
+      {
+         poly1d.CalcLegendre(p, x, shape_x.GetData());
+         poly1d.CalcLegendre(p, y, shape_y.GetData());
+
+         o = 0;
+         for (int i = 0; i <= p; i++)
+         {
+            for (int j = 0; j <= p; j++)
+            {
+               int maxij = std::max(i, j);
+               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                                z, 1.0, shape_z);
+
+               for (int k = 0; k <= p - maxij; k++)
+               {
+                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                              pow(1.0 - ip.z, maxij);
+               }
             }
          }
       }
@@ -1793,25 +1816,44 @@ void H1_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    Vector u(dof);
 #endif
 
-   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   real_t z = ip.z;
+   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   const real_t z = ip.z;
 
-   poly1d.CalcLegendre(p, x, shape_x.GetData());
-   poly1d.CalcLegendre(p, y, shape_y.GetData());
-
-   int o = 0;
-   for (int i = 0; i <= p; i++)
-      for (int j = 0; j <= p; j++)
-      {
-         int maxij = std::max(i, j);
-         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
-                                          shape_z);
-         for (int k = 0; k <= p - maxij; k++)
-            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                     pow(1.0 - ip.z, maxij);
-      }
+   if (std::abs(z - 1.0) < apex_tol)
+   {
+      // Compute the limit of the basis functions as z->1 with x and y on the
+      // line between the center of the base and the apex
+      u = 0.;
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
+         {
+            int maxij = std::max(i, j);
+            for (int k = 0; k <= p - maxij; k++, o++)
+               if (i == 0 && j == 0)
+               {
+                  u(o) = ((k + 3.) * k + 2.) / 2.;
+               }
+         }
+   }
+   else
+   {
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
+         {
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
+            for (int k = 0; k <= p - maxij; k++)
+               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                        pow(1.0 - ip.z, maxij);
+         }
+   }
    Ti.Mult(u, shape);
 }
 
@@ -1830,37 +1872,68 @@ void H1_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    Vector dshape_z(order+1);
    Vector dshape_z_dt(order+1);
 #endif
-   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   real_t z = ip.z;
+   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   const real_t z = ip.z;
 
-   poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-   poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
-
-   int o = 0;
-   for (int i = 0; i <= p; i++)
-      for (int j = 0; j <= p; j++)
-      {
-         int maxij = std::max(i, j);
-         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
-                                          shape_z, dshape_z, dshape_z_dt);
-
-         for (int k = 0; k <= p - maxij; k++, o++)
+   if (std::abs(z - 1.0) < apex_tol)
+   {
+      // Compute the limit of the gradients of the basis functions as
+      // z->1 with x and y on the line between the center of the base and the
+      // apex
+      du = 0.;
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
          {
-            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                      pow(1.0 - ip.z, maxij - 1);
-            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                      pow(1.0 - ip.z, maxij - 1);
-            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                      pow(1.0 - ip.z, maxij) +
-                      (ip.x * dshape_x(i) * shape_y(j) +
-                       ip.y * shape_x(i) * dshape_y(j)) *
-                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                      maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                      pow(1.0 - ip.z, maxij - 1);
+            int maxij = std::max(i, j);
+            for (int k = 0; k <= p - maxij; k++, o++)
+            {
+               if (i == 0 && j == 0)
+               {
+                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
+               }
+               else if (i == 1 && j == 0)
+               {
+                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
+               }
+               else if (i == 0 && j == 1)
+               {
+                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
+               }
+            }
          }
-      }
+   }
+   else
+   {
+      poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
 
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
+         {
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0,
+                                             shape_z, dshape_z, dshape_z_dt);
+
+            for (int k = 0; k <= p - maxij; k++, o++)
+            {
+               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                         pow(1.0 - ip.z, maxij - 1);
+               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                         pow(1.0 - ip.z, maxij - 1);
+               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                         pow(1.0 - ip.z, maxij) +
+                         (ip.x * dshape_x(i) * shape_y(j) +
+                          ip.y * shape_x(i) * dshape_y(j)) *
+                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
+            }
+         }
+   }
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_h1.hpp

@@ -208,6 +208,8 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
+   static constexpr real_t apex_tol = 1e-8;
+
 public:
    H1_BergotPyramidElement(const int p,
                            const int btype = BasisType::GaussLobatto);

fem/fe/fe_l2.cpp

@@ -1106,9 +1106,16 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
          {
             const real_t wik = op[i] + op[k] + op[p-i-k];
             const real_t w = wik * wjk * op[p-k];
-            Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
-                                     op[j] * (op[j] + op[p-j-k]) / w,
-                                     op[k] * op[p-k] / w);
+            if (std::abs(w) < apex_tol)
+            {
+               Nodes.IntPoint(o++).Set3(0.,0.,1.);
+            }
+            else
+            {
+               Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
+                                        op[j] * (op[i] + op[p-i-k]) / w,
+                                        op[k] * op[p-k] / w);
+            }
          }
       }
 
@@ -1125,22 +1132,45 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
       const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       const real_t z = ip.z;
 
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
-
-      o = 0;
-      for (int i = 0; i <= p; i++)
+      if (std::abs(z - 1.0) < apex_tol)
       {
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
-
-            for (int k = 0; k <= p - maxij; k++)
+         // Compute the limit of the basis functions as z->1 with x and y on the
+         // line between the center of the base and the apex
+         o = 0;
+         for (int i = 0; i <= p; i++)
+            for (int j = 0; j <= p; j++)
             {
-               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                           pow(1.0 - ip.z, maxij);
+               int maxij = std::max(i, j);
+               for (int k = 0; k <= p - maxij; k++)
+                  if (i == 0 && j == 0)
+                  {
+                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
+                  }
+                  else
+                  {
+                     T(o++, m) = 0.;
+                  }
+            }
+      }
+      else
+      {
+         poly1d.CalcLegendre(p, x, shape_x.GetData());
+         poly1d.CalcLegendre(p, y, shape_y.GetData());
+
+         o = 0;
+         for (int i = 0; i <= p; i++)
+         {
+            for (int j = 0; j <= p; j++)
+            {
+               int maxij = std::max(i, j);
+               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                                z, 1.0, shape_z);
+
+               for (int k = 0; k <= p - maxij; k++)
+               {
+                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                              pow(1.0 - ip.z, maxij);
+               }
             }
          }
       }
@@ -1165,26 +1195,41 @@ void L2_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   poly1d.CalcLegendre(p, x, shape_x.GetData());
-   poly1d.CalcLegendre(p, y, shape_y.GetData());
-
-   int o = 0;
-   for (int i = 0; i <= p; i++)
+   if (std::abs(z - 1.0) < apex_tol)
    {
-      for (int j = 0; j <= p; j++)
-      {
-         int maxij = std::max(i, j);
-         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
-                                          shape_z);
-
-         for (int k = 0; k <= p - maxij; k++)
+      // Compute the limit of the basis functions as z->1 with x and y on the
+      // line between the center of the base and the apex
+      u = 0.;
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
          {
-            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                     pow(1.0 - ip.z, maxij);
+            int maxij = std::max(i, j);
+            for (int k = 0; k <= p - maxij; k++, o++)
+               if (i == 0 && j == 0)
+               {
+                  u(o) = ((k + 3.) * k + 2.) / 2.;
+               }
          }
-      }
    }
+   else
+   {
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
+         {
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
+
+            for (int k = 0; k <= p - maxij; k++)
+               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                        pow(1.0 - ip.z, maxij);
+         }
+   }
    Ti.Mult(u, shape);
 }
 
@@ -1208,35 +1253,64 @@ void L2_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-   Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
-
-   int o = 0;
-   for (int i = 0; i <= p; i++)
+   if (std::abs(z - 1.0) < apex_tol)
    {
-      for (int j = 0; j <= p; j++)
-      {
-         int maxij = std::max(i, j);
-         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
-                                          shape_z, dshape_z, dshape_z_dt);
-
-         for (int k = 0; k <= p - maxij; k++, o++)
+      // Compute the limit of the gradients of the basis functions as
+      // z->1 with x and y on the line between the center of the base and the
+      // apex
+      du = 0.;
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
          {
-            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                      pow(1.0 - ip.z, maxij - 1);
-            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                      pow(1.0 - ip.z, maxij - 1);
-            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                      pow(1.0 - ip.z, maxij) +
-                      (ip.x * dshape_x(i) * shape_y(j) +
-                       ip.y * shape_x(i) * dshape_y(j)) *
-                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                      ((maxij > 0) ? (maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                                      pow(1.0 - ip.z, maxij - 1)) : 0.0);
+            int maxij = std::max(i, j);
+            for (int k = 0; k <= p - maxij; k++, o++)
+            {
+               if (i == 0 && j == 0)
+               {
+                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
+               }
+               else if (i == 1 && j == 0)
+               {
+                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
+               }
+               else if (i == 0 && j == 1)
+               {
+                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
+               }
+            }
          }
-      }
    }
+   else
+   {
+      Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+      Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
 
+      int o = 0;
+      for (int i = 0; i <= p; i++)
+         for (int j = 0; j <= p; j++)
+         {
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0,
+                                             shape_z, dshape_z, dshape_z_dt);
+
+            for (int k = 0; k <= p - maxij; k++, o++)
+            {
+               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                         pow(1.0 - ip.z, maxij - 1);
+               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                         pow(1.0 - ip.z, maxij - 1);
+               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                         pow(1.0 - ip.z, maxij) +
+                         (ip.x * dshape_x(i) * shape_y(j) +
+                          ip.y * shape_x(i) * dshape_y(j)) *
+                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
+            }
+         }
+   }
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_l2.hpp

@@ -225,6 +225,8 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
+   static constexpr real_t apex_tol = 1e-8;
+
 public:
    /// Construct the L2_PyramidElement of order @a p and BasisType @a btype
    L2_BergotPyramidElement(const int p,

fem/fe/fe_nd.cpp

@@ -1282,12 +1282,49 @@ ND_SegmentElement::ND_SegmentElement(const int p, const int ob_type)
    }
 }
 
+void ND_SegmentElement::CalcShape(const IntegrationPoint &ip,
+                                  Vector &shape) const
+{
+   if (obasis1d.IsIntegratedType()) { obasis1d.ScaleIntegrated(false); }
+   obasis1d.Eval(ip.x, shape);
+}
+
 void ND_SegmentElement::CalcVShape(const IntegrationPoint &ip,
                                    DenseMatrix &shape) const
 {
    Vector vshape(shape.Data(), dof);
 
-   obasis1d.Eval(ip.x, vshape);
+   CalcShape(ip, vshape);
+}
+
+void ND_SegmentElement::ProjectIntegrated(VectorCoefficient &vc,
+                                          ElementTransformation &Trans,
+                                          Vector &dofs) const
+{
+   MFEM_ASSERT(obasis1d.IsIntegratedType(), "Not integrated type");
+   real_t vk[Geometry::MaxDim];
+   Vector xk(vk, vc.GetVDim());
+
+   const real_t *cp = poly1d.ClosedPoints(dof, BasisType::GaussLobatto);
+   const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, dof);
+   IntegrationPoint ip;
+
+   for (int i = 0; i < dof; i++)
+   {
+      const real_t h = cp[i+1] - cp[i];
+      real_t val = 0.0;
+
+      for (int q = 0; q < ir.GetNPoints(); q++)
+      {
+         const IntegrationPoint &ip1d = ir.IntPoint(q);
+         ip.x = cp[i] + h*ip1d.x;
+         Trans.SetIntPoint(&ip);
+         vc.Eval(xk, Trans, ip);
+         val += ip1d.weight*Trans.Jacobian().InnerProduct(tk, vk);
+      }
+
+      dofs(i) = val*h;
+   }
 }
 
 const real_t ND_WedgeElement::tk[15] =

fem/fe/fe_nd.hpp

@@ -303,8 +303,7 @@ public:
    /** @brief Construct the ND_SegmentElement of order @a p and open
        BasisType @a ob_type */
    ND_SegmentElement(const int p, const int ob_type = BasisType::GaussLegendre);
-   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override
-   { obasis1d.Eval(ip.x, shape); }
+   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
    void CalcVShape(const IntegrationPoint &ip,
                    DenseMatrix &shape) const override;
    void CalcVShape(ElementTransformation &Trans,
@@ -325,7 +324,10 @@ public:
    using FiniteElement::Project;
    void Project(VectorCoefficient &vc,
                 ElementTransformation &Trans, Vector &dofs) const override
-   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   {
+      if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
+      else { Project_ND(tk, dof2tk, vc, Trans, dofs); }
+   }
    void ProjectMatrixCoefficient(MatrixCoefficient &mc,
                                  ElementTransformation &T,
                                  Vector &dofs) const override
@@ -338,6 +340,11 @@ public:
                     ElementTransformation &Trans,
                     DenseMatrix &grad) const override
    { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
+
+protected:
+   void ProjectIntegrated(VectorCoefficient &vc,
+                          ElementTransformation &Trans,
+                          Vector &dofs) const;
 };
 
 class ND_WedgeElement : public VectorFiniteElement

fem/fe/fe_pyramid.hpp

@@ -17,6 +17,12 @@
 namespace mfem
 {
 
+struct ScalarPyramid
+{
+   // Default basis type for H1 and L2 pyramids
+   static inline int DefaultType = 1; // Bergot(0) or Fuentes(1)
+};
+
 /** Base class for arbitrary order basis functions on pyramid-shaped elements
 
   This base class provides a common class to store temporary vectors,

fem/fe/fe_rt.hpp

@@ -73,16 +73,11 @@ public:
    void Project(const FiniteElement &fe, ElementTransformation &Trans,
                 DenseMatrix &I) const override
    { Project_RT(nk, dof2nk, fe, Trans, I); }
-   // Gradient + rotation = Curl: H1 -> H(div)
-   void ProjectGrad(const FiniteElement &fe,
-                    ElementTransformation &Trans,
-                    DenseMatrix &grad) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
    // Curl = Gradient + rotation: H1 -> H(div)
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl2D_RT(nk, dof2nk, fe, Trans, curl); }
 
    void GetFaceMap(const int face_id, Array<int> &face_map) const override;
 
@@ -148,7 +143,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 
    /// @brief Return the mapping from lexicographically ordered face DOFs to
    /// lexicographically ordered element DOFs corresponding to local face
@@ -210,16 +205,11 @@ public:
    void Project(const FiniteElement &fe, ElementTransformation &Trans,
                 DenseMatrix &I) const override
    { Project_RT(nk, dof2nk, fe, Trans, I); }
-   // Gradient + rotation = Curl: H1 -> H(div)
-   void ProjectGrad(const FiniteElement &fe,
-                    ElementTransformation &Trans,
-                    DenseMatrix &grad) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
    // Curl = Gradient + rotation: H1 -> H(div)
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl2D_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 
@@ -274,7 +264,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 class RT_WedgeElement : public VectorFiniteElement
@@ -332,7 +322,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 /** Arbitrary order H(Div) basis functions defined on pyramid-shaped elements
@@ -428,7 +418,7 @@ public:
    virtual void ProjectCurl(const FiniteElement &fe,
                             ElementTransformation &Trans,
                             DenseMatrix &curl) const
-   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
 
    void CalcRawVShape(const IntegrationPoint &ip,
                       DenseMatrix &shape) const;

fem/fe_coll.cpp

@@ -228,7 +228,19 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1_", 3))
    {
-      fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
+      // Parse pyramid basis type if included in the name
+      const char *pyr = strstr(name, "Pyr");
+      if (pyr == NULL)
+      {
+         // Use default pyramid type elements
+         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
+      }
+      else
+      {
+         // Use specific pyramid type elements
+         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3),
+                                   BasisType::GaussLobatto, atoi(pyr + 3));
+      }
    }
    else if (!strncmp(name, "H1Pos_Trace_", 12))
    {
@@ -245,26 +257,44 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1@", 3))
    {
-      fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                BasisType::GetType(name[3]));
+      // Parse pyramid basis type if included in the name
+      const char *pyr = strstr(name, "Pyr");
+      if (pyr == NULL)
+      {
+         // Use default pyramid type elements
+         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
+                                   BasisType::GetType(name[3]));
+      }
+      else
+      {
+         // Use specific pyramid type elements
+         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
+                                   BasisType::GetType(name[3]),
+                                   atoi(pyr + 3));
+      }
    }
-   else if (!strncmp(name, "L2_T", 4))
-      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
-                                atoi(name + 4));
-   else if (!strncmp(name, "L2_", 3))
+   else if (!strncmp(name, "L2", 2))
    {
-      fec = new L2_FECollection(atoi(name + 7), atoi(name + 3));
-   }
-   else if (!strncmp(name, "L2Int_T", 7))
-   {
-      fec = new L2_FECollection(atoi(name + 13), atoi(name + 9),
-                                atoi(name + 7), FiniteElement::INTEGRAL);
-   }
-   else if (!strncmp(name, "L2Int_", 6))
-   {
-      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
-                                BasisType::GaussLegendre,
-                                FiniteElement::INTEGRAL);
+      // Parse Map Type
+      const int mtype = strstr(name, "Int") == NULL ?
+                        FiniteElement::VALUE : FiniteElement::INTEGRAL;
+
+      // Parse the base order
+      const int p = atoi(strstr(name, "_P") + 2);
+
+      // Parse the mesh dimension
+      const int dim = atoi(strstr(name, "D") - 1);
+
+      // Parse basis type if specified
+      const char *t = strstr(name, "_T");
+      const int btype = t == NULL ? BasisType::GaussLegendre : atoi(t + 2);
+
+      // Parse the pyramid type if specified
+      const char *pyr = strstr(name, "Pyr");
+      const int ptype = pyr == NULL ? 1 : atoi(pyr + 3);
+
+      // Create collection
+      fec = new L2_FECollection(p, dim, btype, mtype, ptype);
    }
    else if (!strncmp(name, "RT_Trace_", 9))
    {
@@ -1709,9 +1739,10 @@ const int *RT1_3DFECollection::DofOrderForOrientation(Geometry::Type GeomType,
 
 
 H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
-                                 const int pyrtype)
+                                 const int pyr_type)
    : FiniteElementCollection(p)
    , dim(dim)
+   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 1, "H1_FECollection requires order >= 1.");
    MFEM_VERIFY(dim >= 0 && dim <= 3, "H1_FECollection requires 0 <= dim <= 3.");
@@ -1724,7 +1755,14 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
    {
       case BasisType::GaussLobatto:
       {
-         snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
+         if (pyr_type == ScalarPyramid::DefaultType)
+         {
+            snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
+         }
+         else
+         {
+            snprintf(h1_name, 32, "H1_%dD_P%d_Pyr%d", dim, p, pyr_type);
+         }
          break;
       }
       case BasisType::Positive:
@@ -1910,11 +1948,11 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
          H1_dof[Geometry::TETRAHEDRON] = (TriDof*pm3)/3;
          H1_dof[Geometry::CUBE] = QuadDof*pm1;
          H1_dof[Geometry::PRISM] = TriDof*pm1;
-         if (pyrtype == 0 || b_type == BasisType::Positive)
+         if (pyr_type == 0 || b_type == BasisType::Positive)
          {
             H1_dof[Geometry::PYRAMID] = pm2*pm1*(2*p-3)/6; // Bergot (JSC)
          }
-         else if (pyrtype == 1)
+         else if (pyr_type == 1)
          {
             H1_dof[Geometry::PYRAMID] = pm1*pm1*pm1; // Fuentes
          }
@@ -1935,13 +1973,15 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
                new H1_TetrahedronElement(p, btype);
             H1_Elements[Geometry::CUBE] = new H1_HexahedronElement(p, btype);
             H1_Elements[Geometry::PRISM] = new H1_WedgeElement(p, btype);
-            if (pyrtype == 0)
+            if (pyr_type == 0)
             {
-               H1_Elements[Geometry::PYRAMID] = new H1_BergotPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] =
+                  new H1_BergotPyramidElement(p, btype);
             }
             else
             {
-               H1_Elements[Geometry::PYRAMID] = new H1_FuentesPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] =
+                  new H1_FuentesPyramidElement(p, btype);
             }
          }
 
@@ -2148,6 +2188,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    : FiniteElementCollection(p)
    , dim(dim)
    , m_type(map_type)
+   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 0, "L2_FECollection requires order >= 0.");
 
@@ -2163,10 +2204,25 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    switch (btype)
    {
       case BasisType::GaussLegendre:
-         snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
+         if (pyr_type == ScalarPyramid::DefaultType)
+         {
+            snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
+         }
+         else
+         {
+            snprintf(d_name, 32, "%s_%dD_P%d_Pyr%d", prefix, dim, p, pyr_type);
+         }
          break;
       default:
-         snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
+         if (pyr_type == ScalarPyramid::DefaultType)
+         {
+            snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
+         }
+         else
+         {
+            snprintf(d_name, 32, "%s_T%d_%dD_P%d_Pyr%d",
+                     prefix, btype, dim, p, pyr_type);
+         }
    }
 
    for (int g = 0; g < Geometry::NumGeom; g++)
@@ -2285,11 +2341,13 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
          L2_Elements[Geometry::PRISM] = new L2_WedgeElement(p, btype);
          if (pyr_type == 0)
          {
-            L2_Elements[Geometry::PYRAMID] = new L2_BergotPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] =
+               new L2_BergotPyramidElement(p, btype);
          }
          else
          {
-            L2_Elements[Geometry::PYRAMID] = new L2_FuentesPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] =
+               new L2_FuentesPyramidElement(p, btype);
          }
       }
 

fem/fe_coll.hpp

@@ -100,6 +100,10 @@ public:
       return FiniteElementForGeometry(GeomType);
    }
 
+   /** @brief Returns a collection of the trace elements.
+
+       @note The collection is owned by the caller and is NOT deleted in the
+       destructor. */
    virtual FiniteElementCollection *GetTraceCollection() const;
 
    virtual ~FiniteElementCollection();
@@ -286,7 +290,7 @@ protected:
 class H1_FECollection : public FiniteElementCollection
 {
 protected:
-   int dim, b_type;
+   int dim, b_type, p_type;
    char h1_name[32];
    FiniteElement *H1_Elements[Geometry::NumGeom];
    int H1_dof[Geometry::NumGeom];
@@ -295,7 +299,7 @@ protected:
 public:
    explicit H1_FECollection(const int p, const int dim = 3,
                             const int btype = BasisType::GaussLobatto,
-                            const int pyrtype = 1);
+                            const int pyr_type = ScalarPyramid::DefaultType);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -320,7 +324,7 @@ public:
    const int *GetDofMap(Geometry::Type GeomType, int p) const;
 
    FiniteElementCollection *Clone(int p) const override
-   { return new H1_FECollection(p, dim, b_type); }
+   { return new H1_FECollection(p, dim, b_type, p_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }
@@ -367,6 +371,7 @@ private:
    int dim;
    int b_type; // BasisType
    int m_type; // map type
+   int p_type; // Pyramid type (0 -> Bergot, 1 -> Fuentes)
    char d_name[32];
    ScalarFiniteElement *L2_Elements[Geometry::NumGeom];
    ScalarFiniteElement *Tr_Elements[Geometry::NumGeom];
@@ -379,7 +384,7 @@ public:
    L2_FECollection(const int p, const int dim,
                    const int btype = BasisType::GaussLegendre,
                    const int map_type = FiniteElement::VALUE,
-                   const int pyrtype = 1);
+                   const int pyr_type = ScalarPyramid::DefaultType);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -409,7 +414,7 @@ public:
    int GetBasisType() const { return b_type; }
 
    FiniteElementCollection *Clone(int p) const override
-   { return new L2_FECollection(p, dim, b_type, m_type); }
+   { return new L2_FECollection(p, dim, b_type, m_type, p_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }

fem/gslib.cpp

@@ -556,7 +556,7 @@ void obboxsurf_calc_3(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, n);
    lag(I0, work, n, 1, 0);
 
-   for (int ie = 0; ie < nel; ie++,x+=n2,y+=n2,z+=n2)
+   for (int ie = 0; (unsigned)ie < nel; ie++,x+=n2,y+=n2,z+=n2)
    {
       struct gslib::dbl_range ab[3];
       struct gslib::dbl_range tb[3];
@@ -780,7 +780,7 @@ void obboxedge_calc_2(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, nr);
    lag(I0r, work, nr,1, 0);
 
-   for (int ie = 0; ie < nel; ie++,x+=nr,y+=nr)
+   for (int ie = 0; (unsigned)ie < nel; ie++,x+=nr,y+=nr)
    {
       double x0[2], A[4];
       struct gslib::dbl_range ab[2], tb[2];
@@ -892,7 +892,7 @@ void obboxedge_calc_3(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, nr);
    lag(I0r, work, nr, 1, 0);
 
-   for (int ie = 0; ie < nel; ie++,x+=nr,y+=nr,z+=nr)
+   for (int ie = 0; (unsigned)ie < nel; ie++,x+=nr,y+=nr,z+=nr)
    {
       double x0[3], A[9], Ai[9];
       struct gslib::dbl_range ab[3], tb[3];
@@ -4518,7 +4518,7 @@ Mesh* FindPointsGSLIB::GetBoundingBoxMesh(int type)
    int eidx = 0;
    if (myid == save_rank)
    {
-      for (int p = 0; p < gsl_comm->np; p++)
+      for (int p = 0; (unsigned)p < gsl_comm->np; p++)
       {
          if (static_cast<unsigned int>(p) != save_rank)
          {

fem/integ/bilininteg_convection_pa.cpp

@@ -178,6 +178,8 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    // Assumes tensor-product elements
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
+   MFEM_VERIFY(el.GetMapType() == FiniteElement::VALUE,
+               "Only value map type currently supported");
    ElementTransformation &Trans = *mesh->GetTypicalElementTransformation();
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Trans);
    if (DeviceCanUseCeed())

fem/integ/bilininteg_elasticity_kernels.cpp

@@ -91,15 +91,15 @@ void ElasticityAddMultPA(const int dim, const int nDofs,
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
+                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag)
 {
    switch (dim)
    {
       case 2:
-         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, QVec, diag);
+         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, ir, diag);
          break;
       case 3:
-         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, QVec, diag);
+         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, ir, diag);
          break;
       default:
          MFEM_ABORT("Only dimensions 2 and 3 supported.");

fem/integ/bilininteg_elasticity_kernels.hpp

@@ -38,7 +38,6 @@
 #include "../../linalg/vector.hpp"
 #include "../../linalg/tensor.hpp"
 #include "../quadinterpolator.hpp"
-#include "../bilininteg.hpp"
 #include "../coefficient.hpp"
 #include "../qfunction.hpp"
 
@@ -133,12 +132,12 @@ void ElasticityAssembleEA(const int dim, const int i_block, const int j_block,
 /// @param[in] mu Quadrature function for second Lame param.
 /// @param[in] geom Geometric factors corresponding to fespace.
 /// @param[in] maps DofToQuad maps for one element (assume elements all same).
-/// @param QVec Scratch Q-Vector. nQuad x dim x dim x dim x dim x numEls.
+/// @param[in] ir Integration rule.
 /// @param[out] diag diagonal of A. nDofs x dim x numEls.
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag);
+                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag);
 
 /// Templated implementation of ElasticityAddMultPA.
 template<int dim, int i_block = -1, int j_block = -1>
@@ -280,77 +279,67 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
 template<int dim>
 void ElasticityAssembleDiagonalPA_(const int nDofs,
                                    const CoefficientVector &lambda,
-                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                   const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
+                                   const CoefficientVector &mu,
+                                   const GeometricFactors &geom,
+                                   const DofToQuad &maps,
+                                   const IntegrationRule &ir,
+                                   Vector &diag)
 {
-   using future::tensor;
-   using future::make_tensor;
    using future::det;
    using future::inv;
+   using future::make_tensor;
+   using future::tensor;
 
    // Assuming all elements are the same
-   const auto &ir = QVec.GetIntRule(0);
    static constexpr int d = dim;
    const int numPoints = ir.GetNPoints();
-   const int numEls = lambda.Size()/numPoints;
+   const int numEls = lambda.Size() / numPoints;
+
    const auto lamDev = Reshape(lambda.Read(), numPoints, numEls);
    const auto muDev = Reshape(mu.Read(), numPoints, numEls);
    const auto J = Reshape(geom.J.Read(), numPoints, d, d, numEls);
-   auto Q = Reshape(QVec.ReadWrite(), numPoints, d,d, d, numEls);
    const real_t *ipWeights = ir.GetWeights().Read();
-   mfem::forall_2D(numEls, numPoints,1, [=] MFEM_HOST_DEVICE (int e)
-   {
-      MFEM_FOREACH_THREAD(p, x,numPoints)
-      {
-         auto invJ = inv(make_tensor<d, d>(
-         [&](int i, int j) { return J(p, i, j, e); }));
-         const real_t w = ipWeights[p] /det(invJ);
-         for (int n = 0; n < d; n++)
-         {
-            for (int m = 0; m < d; m++)
-            {
-               for (int q = 0; q < d; q++)
-               {
-                  // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-                  // this contraction could be made slightly cheaper using Voigt
-                  // notation, but repeated entries are summed for simplicity.
-                  real_t contraction = 0.;
-                  for (int a = 0; a < d; a++)
-                  {
-                     for (int b = 0; b < d; b++)
-                     {
-                        contraction += ((a == q)*invJ(m,b) + (b==q)*invJ(m,a))*((a == q)
-                                                                                *invJ(n, b) + (b==q)*invJ(n,a));
-                     }
-                  }
-                  // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
-                  // contraction = 4*sym(grad(u))sym(grad(v))
-                  Q(p,m,n,q,e) = w*(lamDev(p, e)*invJ(m,q)*invJ(n,q)
-                                    + 0.5*muDev(p, e)*contraction);
-               }
-            }
-         }
-      }
-   });
-
-   // Reduce quadrature function to an E-Vector
-   const auto QRead = Reshape(QVec.Read(), numPoints, d, d, d, numEls);
-   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
    const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
+   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
+
    mfem::forall_2D(numEls, d, nDofs, [=] MFEM_HOST_DEVICE (int e)
    {
-      MFEM_FOREACH_THREAD(i, y, nDofs)
+      MFEM_FOREACH_THREAD_DIRECT(i, y, nDofs)
       {
-         MFEM_FOREACH_THREAD(q, x, d)
+         MFEM_FOREACH_THREAD_DIRECT(q, x, d)
          {
-            real_t sum = 0.;
-            for (int n = 0; n < d; n++)
+            real_t sum = 0.0;
+            for (int p = 0; p < numPoints; p++)
             {
-               for (int m = 0; m < d; m++)
+               const auto invJ = inv(make_tensor<d, d>([&](int r, int c)
                {
-                  for (int p = 0; p < numPoints; p++ )
+                  return J(p, r, c, e);
+               }));
+               const real_t w = ipWeights[p] / det(invJ);
+
+               for (int n = 0; n < d; n++)
+               {
+                  for (int m = 0; m < d; m++)
                   {
-                     sum += QRead(p,m,n,q,e)*G(p,m,i)*G(p,n,i);
+                     // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                     // this contraction could be made slightly cheaper using Voigt
+                     // notation, but repeated entries are summed for simplicity.
+                     real_t contraction = 0.0;
+                     for (int a = 0; a < d; a++)
+                     {
+                        for (int b = 0; b < d; b++)
+                        {
+                           contraction +=
+                              ((a == q) * invJ(m, b) + (b == q) * invJ(m, a)) *
+                              ((a == q) * invJ(n, b) + (b == q) * invJ(n, a));
+                        }
+                     }
+                     // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
+                     // contraction = 4*sym(grad(u))sym(grad(v))
+                     const real_t Q =
+                        w * (lamDev(p, e) * invJ(m, q) * invJ(n, q)
+                             + 0.5 * muDev(p, e) * contraction);
+                     sum += Q * G(p, m, i) * G(p, n, i);
                   }
                }
             }

fem/integ/bilininteg_elasticity_pa.cpp

@@ -10,7 +10,6 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
-#include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 #include "bilininteg_elasticity_kernels.hpp"
 
@@ -59,9 +58,8 @@ void ElasticityIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
 void ElasticityIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   q_vec->SetVDim(vdim*vdim*vdim*vdim);
    internal::ElasticityAssembleDiagonalPA(vdim, ndofs, *lambda_quad, *mu_quad,
-                                          *geom, *maps, *q_vec, diag);
+                                          *geom, *maps, *IntRule, diag);
 }
 
 void ElasticityIntegrator::AddMultPA(const Vector &x, Vector &y) const

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -147,18 +147,16 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHcurlMassApply2D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y)
+void PAHcurlMassApply2D(const int NE, const bool symmetric,
+                        [[maybe_unused]] const bool scalar_coeff,
+                        const Array<real_t> &bo, const Array<real_t> &bc,
+                        const Array<real_t> &bot, const Array<real_t> &bct,
+                        const Vector &pa_data, const Vector &x, Vector &y,
+                        const int D1D, [[maybe_unused]] const int TestD1D,
+                        const int Q1D)
 {
+   MFEM_ASSERT(D1D == TestD1D,
+               "Trial and Test space must have the same number of dofs");
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(bc.Read(), Q1D, D1D);
    auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
@@ -277,18 +275,16 @@ void PAHcurlMassApply2D(const int D1D,
    }); // end of element loop
 }
 
-void PAHcurlMassApply3D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y)
+void PAHcurlMassApply3D(const int NE, const bool symmetric,
+                        [[maybe_unused]] const bool scalar_coeff,
+                        const Array<real_t> &bo, const Array<real_t> &bc,
+                        const Array<real_t> &bot, const Array<real_t> &bct,
+                        const Vector &pa_data, const Vector &x, Vector &y,
+                        const int D1D, [[maybe_unused]] const int TestD1D,
+                        const int Q1D)
 {
+   MFEM_VERIFY(D1D == TestD1D,
+               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
@@ -789,6 +785,23 @@ void PAHcurlL2Setup2D(const int Q1D,
    });
 }
 
+void PAHcurlL2IntSetup2D(const int Q1D, const int NE, const Array<real_t> &w,
+                         Vector &coeff, const Vector &detJ, Vector &op)
+{
+   const int NQ = Q1D*Q1D;
+   auto W = w.Read();
+   auto C = Reshape(coeff.Read(), NQ, NE);
+   auto J = Reshape(detJ.Read(), NQ, NE);
+   auto y = Reshape(op.Write(), NQ, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int q = 0; q < NQ; ++q)
+      {
+         y(q,e) = W[q] * C(q,e) / J(q,e);
+      }
+   });
+}
+
 void PAHcurlL2Setup3D(const int NQ,
                       const int coeffDim,
                       const int NE,

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -181,228 +181,312 @@ inline void SmemPAHcurlMassAssembleDiagonal3D(const int d1d,
 }
 
 // PA H(curl) Mass Apply 2D kernel
-void PAHcurlMassApply2D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y);
+void PAHcurlMassApply2D(const int NE, const bool symmetric,
+                        const bool scalar_coeff, const Array<real_t> &bo,
+                        const Array<real_t> &bc, const Array<real_t> &bot,
+                        const Array<real_t> &bct, const Vector &pa_data,
+                        const Vector &x, Vector &y, const int TrialD1D,
+                        const int TestD1D, const int Q1D);
 
 // PA H(curl) Mass Apply 3D kernel
-void PAHcurlMassApply3D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y);
+void PAHcurlMassApply3D(const int NE, const bool symmetric,
+                        [[maybe_unused]] const bool scalar_coeff,
+                        const Array<real_t> &bo, const Array<real_t> &bc,
+                        const Array<real_t> &bot, const Array<real_t> &bct,
+                        const Vector &pa_data, const Vector &x, Vector &y,
+                        const int TrialD1D, [[maybe_unused]] const int TestD1D,
+                        const int Q1D);
 
 // Shared memory PA H(curl) Mass Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHcurlMassApply3D(const int d1d,
-                                   const int q1d,
-                                   const int NE,
-                                   const bool symmetric,
-                                   const Array<real_t> &bo,
-                                   const Array<real_t> &bc,
-                                   const Array<real_t> &bot,
-                                   const Array<real_t> &bct,
-                                   const Vector &pa_data,
-                                   const Vector &x,
-                                   Vector &y)
+template <int T_D1D = 0, int T_Q1D = 0, int TBATCH = 0, bool ACCUMULATE = true>
+inline void SmemPAHcurlMassApply3D(
+   const int NE, const bool symmetric, [[maybe_unused]] const bool scalar_coeff,
+   const Array<real_t> &bo, const Array<real_t> &bc,
+   [[maybe_unused]] const Array<real_t> &bot,
+   [[maybe_unused]] const Array<real_t> &bct, const Vector &pa_data,
+   const Vector &x, Vector &y, const int d1d = 0,
+   [[maybe_unused]] const int test_d1d = 0, const int q1d = 0)
 {
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
    MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: d1d > HCURL_MAX_D1D");
    MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
                "Error: q1d > HCURL_MAX_Q1D");
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
+   MFEM_ASSERT(Q1D >= D1D, "Expected Q1D >= D1D");
    const int dataSize = symmetric ? 6 : 9;
 
-   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
-   auto Bc = Reshape(bc.Read(), Q1D, D1D);
-   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
-   auto X = Reshape(x.Read(), 3*(D1D-1)*D1D*D1D, NE);
-   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
+   // assume trial space == test space
+   auto Bo = bo.Read();
+   auto Bc = bc.Read();
+   auto op =
+      Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
+   auto X_ = Reshape(x.Read(), 3 * (D1D - 1) * D1D * D1D, NE);
+   auto y_ = y.ReadWrite();
 
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   constexpr int MD_ = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
+   constexpr int MQ_ = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+   constexpr int MDQ_ = std::max(MD_, MQ_);
+   constexpr int MB_ = TBATCH ? TBATCH : 1;
+
+   mfem::forall_2D_batch<MDQ_ * MDQ_ * MDQ_ * MB_>(
+      NE, MDQ_ * MDQ_ * MDQ_, 1, MB_, [=] MFEM_HOST_DEVICE(int e)
    {
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = TBATCH ? TBATCH : 1;
+      int tidz = MFEM_THREAD_ID(z);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+
       constexpr int VDIM = 3;
       constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
       constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MDQ = std::max(MD1D, MQ1D);
 
-      MFEM_SHARED real_t sBo[MQ1D][MD1D];
-      MFEM_SHARED real_t sBc[MQ1D][MD1D];
+      // nvcc limit work-around: can't have Y_ be captured first in
+      // if constexpr, so capture y_ and construct Y_ locally
+      // only works on GPU
+      auto Y = Reshape(y_, VDIM * (D1D - 1) * D1D * D1D, NE);
 
-      real_t op9[9];
-      MFEM_SHARED real_t sop[9*MQ1D*MQ1D];
-      MFEM_SHARED real_t mass[MQ1D][MQ1D][3];
+      MFEM_SHARED real_t sBo[MDQ * (MD1D - 1)];
+      MFEM_SHARED real_t sBc[MDQ * MD1D];
+      auto BO = Reshape(sBo, Q1D, D1D - 1);
+      auto BC = Reshape(sBc, Q1D, D1D);
 
-      MFEM_SHARED real_t sX[MD1D][MD1D][MD1D];
+      MFEM_SHARED real_t sX[nbz * VDIM * (MD1D - 1) * MD1D * MD1D];
+      MFEM_SHARED real_t sm0[nbz * VDIM * MDQ * MDQ * MDQ];
+      MFEM_SHARED real_t sm1[nbz * VDIM * MDQ * MDQ * MDQ];
 
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      real_t(*X)[nbz][(MD1D - 1) * MD1D * MD1D] =
+         (real_t(*)[nbz][(MD1D - 1) * MD1D * MD1D])(sX);
+      // shapes of buffers always use MQ1D to mitigate shared memory bank
+      // conflicts
+      real_t(*DDQ)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
+      real_t(*DQQ)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
+      real_t(*QQQ)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
+      real_t(*QQD)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
+      real_t(*QDD)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
+
+      // load dofs into smem
+      const int offset = (D1D - 1) * D1D * D1D;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         for (int dim = 0; dim < VDIM; ++dim)
          {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               for (int i=0; i<dataSize; ++i)
-               {
-                  op9[i] = op(qx,qy,qz,i,e);
-               }
-            }
+            X[dim][tidz][ix] = X_(ix + dim * offset, e);
          }
       }
-
-      const int tidx = MFEM_THREAD_ID(x);
-      const int tidy = MFEM_THREAD_ID(y);
-      const int tidz = MFEM_THREAD_ID(z);
-
+      // load basis functions data
       if (tidz == 0)
       {
-         MFEM_FOREACH_THREAD(d,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(ix, x, D1D * Q1D) { sBc[ix] = Bc[ix]; }
+         MFEM_FOREACH_THREAD_DIRECT(ix, x, (D1D - 1) * Q1D)
          {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
+            sBo[ix] = Bo[ix];
+         }
+      }
+
+      for (int dim0 = 0; dim0 < VDIM; ++dim0)
+      {
+         MFEM_SYNC_THREAD;
+         // sum factor to QQQ = Q_{dim0,dim1} B X_{dim1}
+         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         {
+            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, Q1D, D1Dy, D1Dz,
+                                                 Q1D, Q1D, Q1D)
             {
-               sBc[q][d] = Bc(q,d);
-               if (d < D1D-1)
+               real_t u = 0;
+               for (int dx = 0; dx < D1Dx; ++dx)
                {
-                  sBo[q][d] = Bo(q,d);
+                  real_t b;
+                  if (dim1 == 0)
+                  {
+                     b = BO(qx, dx);
+                  }
+                  else
+                  {
+                     b = BC(qx, dx);
+                  }
+                  u += X[dim1][tidz][dx + (dy + dz * D1Dy) * D1Dx] * b;
+               }
+               DDQ[dim1][tidz][dz][dy][qx] = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         {
+            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
+            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, Q1D, Q1D, D1Dz,
+                                                 Q1D, Q1D, Q1D)
+            {
+               real_t u = 0;
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  real_t b;
+                  if (dim1 == 1)
+                  {
+                     b = BO(qy, dy);
+                  }
+                  else
+                  {
+                     b = BC(qy, dy);
+                  }
+                  u += DDQ[dim1][tidz][dz][dy][qx] * b;
+               }
+               DQQ[dim1][tidz][dz][qy][qx] = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         {
+            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
+            // const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
+            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+            MFEM_FOREACH_THREAD_DIRECT_3D(qx, qy, qz, x, Q1D, Q1D, Q1D)
+            {
+               real_t u = 0;
+               for (int dz = 0; dz < D1Dz; ++dz)
+               {
+                  real_t b;
+                  if (dim1 == 2)
+                  {
+                     b = BO(qz, dz);
+                  }
+                  else
+                  {
+                     b = BC(qz, dz);
+                  }
+                  u += DQQ[dim1][tidz][dz][qy][qx] * b;
+               }
+               // pa_data is row major
+               int idx;
+               if (symmetric)
+               {
+                  int row;
+                  int col;
+                  if (dim0 > dim1)
+                  {
+                     row = dim1;
+                     col = dim0;
+                  }
+                  else
+                  {
+                     row = dim0;
+                     col = dim1;
+                  }
+                  idx = col + VDIM * row - row * (row + 1) / 2;
+               }
+               else
+               {
+                  idx = dim0 * VDIM + dim1;
+               }
+               QQQ[dim1][tidz][qz][qy][qx] = op(qx, qy, qz, idx, e) * u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         // sum factor back to Y
+         // Assume bot and bct == bo^t and bc^t respectively (i.e. test ==
+         // trial functions), skip loading them again.
+         {
+            const int D1Dz = (dim0 == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (dim0 == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (dim0 == 0) ? D1D - 1 : D1D;
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, D1Dz, Q1D, Q1D,
+                                                 Q1D, Q1D, Q1D)
+            {
+               for (int dim1 = 0; dim1 < VDIM; ++dim1)
+               {
+                  real_t u = 0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     real_t b = 0;
+                     if (dim0 == 2)
+                     {
+                        b = BO(qz, dz);
+                     }
+                     else
+                     {
+                        b = BC(qz, dz);
+                     }
+                     u += QQQ[dim1][tidz][qz][qy][qx] * b;
+                  }
+                  QQD[dim1][tidz][qy][qx][dz] = u;
+               }
+            }
+            MFEM_SYNC_THREAD;
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, D1Dy, D1Dz, Q1D,
+                                                 Q1D, Q1D, Q1D)
+            {
+               for (int dim1 = 0; dim1 < VDIM; ++dim1)
+               {
+                  real_t u = 0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     real_t b;
+                     if (dim0 == 1)
+                     {
+                        b = BO(qy, dy);
+                     }
+                     else
+                     {
+                        b = BC(qy, dy);
+                     }
+                     u += QQD[dim1][tidz][qy][qx][dz] * b;
+                  }
+                  QDD[dim1][tidz][qx][dz][dy] = u;
+               }
+            }
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD_DIRECT_3D(dx, dy, dz, x, D1Dx, D1Dy, D1Dz)
+            {
+               int ix = dx + D1Dx * (dy + D1Dy * dz);
+               real_t u = 0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  real_t b;
+                  if (dim0 == 0)
+                  {
+                     b = BO(qx, dx);
+                  }
+                  else
+                  {
+                     b = BC(qx, dx);
+                  }
+                  for (int dim1 = 0; dim1 < VDIM; ++dim1)
+                  {
+                     u += QDD[dim1][tidz][qx][dz][dy] * b;
+                  }
+               }
+               if constexpr (ACCUMULATE)
+               {
+                  Y(ix + dim0 * offset, e) += u;
+               }
+               else
+               {
+                  Y(ix + dim0 * offset, e) = u;
                }
             }
          }
       }
-      MFEM_SYNC_THREAD;
-
-      for (int qz=0; qz < Q1D; ++qz)
-      {
-         int osc = 0;
-         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
-         {
-            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
-
-            MFEM_FOREACH_THREAD(dz,z,D1Dz)
-            {
-               MFEM_FOREACH_THREAD(dy,y,D1Dy)
-               {
-                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
-                  {
-                     sX[dz][dy][dx] = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            if (tidz == qz)
-            {
-               for (int i=0; i<dataSize; ++i)
-               {
-                  sop[i + (dataSize*tidx) + (dataSize*Q1D*tidy)] = op9[i];
-               }
-
-               MFEM_FOREACH_THREAD(qy,y,Q1D)
-               {
-                  MFEM_FOREACH_THREAD(qx,x,Q1D)
-                  {
-                     real_t u = 0.0;
-
-                     for (int dz = 0; dz < D1Dz; ++dz)
-                     {
-                        const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
-                        for (int dy = 0; dy < D1Dy; ++dy)
-                        {
-                           const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
-                           for (int dx = 0; dx < D1Dx; ++dx)
-                           {
-                              const real_t t = sX[dz][dy][dx];
-                              const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
-                              u += t * wx * wy * wz;
-                           }
-                        }
-                     }
-
-                     mass[qy][qx][c] = u;
-                  } // qx
-               } // qy
-            } // tidz == qz
-
-            osc += D1Dx * D1Dy * D1Dz;
-            MFEM_SYNC_THREAD;
-         } // c
-
-         MFEM_SYNC_THREAD;  // Sync mass[qy][qx][d] and sop
-
-         osc = 0;
-         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
-         {
-            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
-
-            real_t dxyz = 0.0;
-
-            MFEM_FOREACH_THREAD(dz,z,D1Dz)
-            {
-               const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
-
-               MFEM_FOREACH_THREAD(dy,y,D1Dy)
-               {
-                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
-                  {
-                     for (int qy = 0; qy < Q1D; ++qy)
-                     {
-                        const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
-                        for (int qx = 0; qx < Q1D; ++qx)
-                        {
-                           const int os = (dataSize*qx) + (dataSize*Q1D*qy);
-                           const int id1 = os + ((c == 0) ? 0 : ((c == 1) ? (symmetric ? 1 : 3) :
-                                                                 (symmetric ? 2 : 6))); // O11, O21, O31
-                           const int id2 = os + ((c == 0) ? 1 : ((c == 1) ? (symmetric ? 3 : 4) :
-                                                                 (symmetric ? 4 : 7))); // O12, O22, O32
-                           const int id3 = os + ((c == 0) ? 2 : ((c == 1) ? (symmetric ? 4 : 5) :
-                                                                 (symmetric ? 5 : 8))); // O13, O23, O33
-
-                           const real_t m_c = (sop[id1] * mass[qy][qx][0]) + (sop[id2] * mass[qy][qx][1]) +
-                                              (sop[id3] * mass[qy][qx][2]);
-
-                           const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
-                           dxyz += m_c * wx * wy * wz;
-                        }
-                     }
-                  }
-               }
-            }
-
-            MFEM_SYNC_THREAD;
-
-            MFEM_FOREACH_THREAD(dz,z,D1Dz)
-            {
-               MFEM_FOREACH_THREAD(dy,y,D1Dy)
-               {
-                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
-                  {
-                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) += dxyz;
-                  }
-               }
-            }
-
-            osc += D1Dx * D1Dy * D1Dz;
-         } // c loop
-      } // qz
    }); // end of element loop
 }
 
@@ -1805,13 +1889,17 @@ inline void SmemPACurlCurlApply3D(const int d1d,
    ForallWrap<3>(true, NE, device_kernel, host_kernel, Q1D, Q1D, Q1D);
 }
 
-// PA H(curl)-L2 Assemble 2D kernel
+// PA H(curl)-L2 value Assemble 2D kernel
 void PAHcurlL2Setup2D(const int Q1D,
                       const int NE,
                       const Array<real_t> &w,
                       Vector &coeff,
                       Vector &op);
 
+// PA H(curl)-L2 integral Assemble 2D kernel
+void PAHcurlL2IntSetup2D(const int Q1D, const int NE, const Array<real_t> &w,
+                         Vector &coeff, const Vector &detJ, Vector &op);
+
 // PA H(curl)-L2 Assemble 3D kernel
 void PAHcurlL2Setup3D(const int NQ,
                       const int coeffDim,

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -62,6 +62,30 @@ void PAHcurlHdivMassApply2D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
+/// H(curl) test, H(div) trial
+inline void
+PAHcurlHdivMassApply2D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false,
+                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
+/// H(div) test, H(curl) trial
+inline void
+PAHdivHcurlMassApply2D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true,
+                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
 // PA H(curl)-H(div) Mass Apply 3D kernel
 void PAHcurlHdivMassApply3D(const int D1D,
                             const int D1Dtest,
@@ -78,6 +102,30 @@ void PAHcurlHdivMassApply3D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
+/// H(curl) test, H(div) trial
+inline void
+PAHcurlHdivMassApply3D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false, false, Bo_,
+                          Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
+/// H(div) test, H(curl) trial
+inline void
+PAHdivHcurlMassApply3D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true, false, Bo_,
+                          Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
 // PA H(curl)-H(div) Curl Apply 3D kernel
 template<int T_D1D = 0, int T_D1D_TEST = 0, int T_Q1D = 0>
 inline void PAHcurlHdivApply3D(const int d1d,
@@ -816,8 +864,656 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
    }); // end of element loop
 }
 
+namespace curlinterp
+{
+constexpr int NBZ3D(int ndof_o, int nquad_o, int mdq)
+{
+   if (ndof_o <= 0 || nquad_o <= 0)
+   {
+      return 1;
+   }
+   int ndof_c = ndof_o + 1;
+   int nquad_c = nquad_o + 1;
+   // z dimension is capped at 64 on nvidia and amd gpus
+   int tmp =
+      std::min((128 + mdq * mdq * (mdq - 1) - 1) / (mdq * mdq * (mdq - 1)), 64);
+   int smem_req =
+      sizeof(mfem::real_t) *
+      ((3 * ndof_c * ndof_c * ndof_o + 2 * 2 * mdq * mdq * mdq) * tmp +
+       ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
+   // assume GPU has at least 48k shared memory
+   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
+}
+}
+
+template <int T_NDOF_O, int T_NQUAD_O>
+void CurlInterpolatorApply3DSmem(const int ne, const int ndof_o,
+                                 const int nquad_o, const Vector &pa,
+                                 const Vector &x_, Vector &y_)
+{
+   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+   constexpr int mnq_o =
+      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
+   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
+   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
+   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
+   int mnq = std::max(ndof_o + 1, nquad_o + 1);
+   auto pa_data = pa.Read();
+   auto x_d = x_.Read();
+   auto y_d = y_.ReadWrite();
+   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
+      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MND_O =
+         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+      constexpr int MNQ_O =
+         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
+      int tidz = MFEM_THREAD_ID(z);
+      // Make mnq a local variable since capturing would result in different
+      // captures between host/device versions, and spuriously fails
+      int mnq = std::max(ndof_o + 1, nquad_o + 1);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
+      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
+      const int NDOF_C = NDOF_O + 1;
+      const int NQUAD_C = NQUAD_O + 1;
+      MFEM_SHARED real_t
+      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
+      auto X_ = Reshape(x_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
+      auto Y = Reshape(y_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
+      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
+      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
+      auto Boo =
+         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
+      MFEM_SHARED real_t X[3][nbz][MND_O * (MND_O + 1) * (MND_O + 1)];
+      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
+      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
+
+      // shapes of buffers always use MNDQ to mitigate shared memory bank
+      // conflicts
+      real_t(*DDQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      real_t(*DQQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
+      real_t(*QQQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      const int offset = NDOF_O * NDOF_C * NDOF_C;
+      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
+      {
+         for (int dim = 0; dim < 3; ++dim)
+         {
+            X[dim][tidz][ix] = X_(ix + dim * offset, e);
+         }
+      }
+      // load basis functions data
+      if (tidz == 0)
+      {
+         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
+         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
+      }
+      MFEM_SYNC_THREAD;
+
+      // x: Vz Bcc Gco Boo - Vy Bcc Boo Gco
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Bcc(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Bcc(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Gco(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_O; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Boo(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_O; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Boo(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Gco(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_O) * NQUAD_C, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // y: Vx Boo Bcc Gco - Vz Gco Bcc Boo
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_O; ++dx)
+         {
+            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Gco(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Bcc(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
+                                           NDOF_O, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Bcc(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Gco(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_O; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Boo(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_C) * NQUAD_O + offsetq, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // z: Vy Gco Boo Bcc - Vx Boo Gco Bcc
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Gco(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_O; ++dx)
+         {
+            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_O; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Boo(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Gco(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Bcc(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Bcc(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_O) * NQUAD_O + 2 * offsetq, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+template <int T_NDOF_O, int T_NQUAD_O>
+void CurlInterpolatorTApply3DSmem(const int ne, const int ndof_o,
+                                  const int nquad_o, const Vector &pa,
+                                  const Vector &x_, Vector &y_)
+{
+   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+   constexpr int mnq_o =
+      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
+   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
+   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
+   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
+   int mnq = std::max(ndof_o + 1, nquad_o + 1);
+   auto pa_data = pa.Read();
+   auto x_d = x_.Read();
+   auto y_d = y_.ReadWrite();
+   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
+      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MND_O =
+         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+      constexpr int MNQ_O =
+         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
+      int tidz = MFEM_THREAD_ID(z);
+      // Make mnq a local variable since capturing would result in different
+      // captures between host/device versions, and spuriously fails
+      int mnq = std::max(ndof_o + 1, nquad_o + 1);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
+      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
+      const int NDOF_C = NDOF_O + 1;
+      const int NQUAD_C = NQUAD_O + 1;
+      MFEM_SHARED real_t
+      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
+      auto X_ = Reshape(x_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
+      auto Y = Reshape(y_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
+      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
+      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
+      auto Boo =
+         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
+      MFEM_SHARED real_t X[3][nbz][MNQ_O * MNQ_O * (MNQ_O + 1)];
+      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
+      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
+
+      // shapes of buffers always use MNDQ to mitigate shared memory bank
+      // conflicts
+      real_t(*QQD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      real_t(*QDD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
+      real_t(*DDD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      const int offset = NDOF_O * NDOF_C * NDOF_C;
+      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offsetq)
+      {
+         for (int dim = 0; dim < 3; ++dim)
+         {
+            X[dim][tidz][ix] = X_(ix + dim * offsetq, e);
+         }
+      }
+      // load basis functions data
+      if (tidz == 0)
+      {
+         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
+         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
+      }
+      MFEM_SYNC_THREAD;
+
+      // x: Vy Boo Bcc Gco - Vz Boo Gco Bcc
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Gco(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_C; ++qz)
+         {
+            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_C; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Bcc(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Gco(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Boo(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Boo(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_C) * NDOF_O, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // y: Vz Gco Boo Bcc - Vx Bcc Boo Gco
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_C; ++qz)
+         {
+            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Gco(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Boo(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
+                                           NQUAD_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Boo(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Gco(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_C; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Bcc(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_O) * NDOF_C + offset, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // z: Vx Bcc Gco Boo - Vy Gco Bcc Boo
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Boo(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Boo(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Gco(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_C; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Bcc(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_C; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Bcc(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Gco(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_C) * NDOF_C + 2 * offset, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
 } // namespace internal
 
+template <int DIM, int NDOF_O, int NQUAD_O>
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyPAKernels::Kernel()
+{
+   if constexpr (DIM == 3)
+   {
+      return internal::CurlInterpolatorApply3DSmem<NDOF_O, NQUAD_O>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+template <int DIM, int NDOF_O, int NQUAD_O>
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyTPAKernels::Kernel()
+{
+   if constexpr (DIM == 3)
+   {
+      return internal::CurlInterpolatorTApply3DSmem<NDOF_O, NQUAD_O>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
 } // namespace mfem
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_hdiv_kernels.cpp

@@ -294,61 +294,14 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const bool symmetric,
-                     const Array<real_t> &Bo,
-                     const Array<real_t> &Bc,
-                     const Array<real_t> &Bot,
-                     const Array<real_t> &Bct,
-                     const Vector &op,
-                     const Vector &x,
-                     Vector &y)
-{
-   const int id = (D1D << 4) | Q1D;
-
-   if (dim == 2)
-   {
-      switch (id)
-      {
-         case 0x22: return SmemPAHdivMassApply2D<2,2>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x33: return SmemPAHdivMassApply2D<3,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x44: return SmemPAHdivMassApply2D<4,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x55: return SmemPAHdivMassApply2D<5,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         default: // fallback
-            return PAHdivMassApply2D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch (id)
-      {
-         case 0x23: return SmemPAHdivMassApply3D<2,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x34: return SmemPAHdivMassApply3D<3,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x45: return SmemPAHdivMassApply3D<4,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x56: return SmemPAHdivMassApply3D<5,6>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x67: return SmemPAHdivMassApply3D<6,7>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x78: return SmemPAHdivMassApply3D<7,8>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         default: // fallback
-            return PAHdivMassApply3D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-      }
-   }
-}
-
-void PAHdivMassApply2D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_)
+void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int TestD1D, const int Q1D)
 {
+   MFEM_VERIFY(D1D == TestD1D,
+               "Trial and test spaces must have same number of dofs");
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
@@ -468,18 +421,14 @@ void PAHdivMassApply2D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivMassApply3D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_)
+void PAHdivMassApply3D(const int NE, const bool symmetric, const bool,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int TestD1D, const int Q1D)
 {
+   MFEM_VERIFY(D1D == TestD1D,
+               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
                "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,

fem/integ/bilininteg_hdiv_kernels.hpp

@@ -66,58 +66,29 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
                                   const Vector &op_,
                                   Vector &diag_);
 
-void PAHdivMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const bool symmetric,
-                     const Array<real_t> &Bo,
-                     const Array<real_t> &Bc,
-                     const Array<real_t> &Bot,
-                     const Array<real_t> &Bct,
-                     const Vector &op,
-                     const Vector &x,
-                     Vector &y);
-
 // PA H(div) Mass Apply 2D kernel
-void PAHdivMassApply2D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_);
+void PAHdivMassApply2D(const int NE, const bool symmetric,
+                       const bool scalar_coeff, const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_, const Vector &op_,
+                       const Vector &x_, Vector &y_, const int D1D,
+                       const int TestD1D, const int Q1D);
 
 // PA H(div) Mass Apply 3D kernel
-void PAHdivMassApply3D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_);
+void PAHdivMassApply3D(const int NE, const bool symmetric,
+                       const bool scalar_coeff, const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_, const Vector &op_,
+                       const Vector &x_, Vector &y_, const int D1D,
+                       const int TestD1D, const int Q1D);
 
 // Shared memory PA H(div) Mass Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHdivMassApply2D(const int NE,
-                                  const bool symmetric,
-                                  const Array<real_t> &Bo_,
-                                  const Array<real_t> &Bc_,
-                                  const Array<real_t> &Bot_,
-                                  const Array<real_t> &Bct_,
-                                  const Vector &op_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply2D(
+   const int NE, const bool symmetric, const bool, const Array<real_t> &Bo_,
+   const Array<real_t> &Bc_, const Array<real_t> &Bot_,
+   const Array<real_t> &Bct_, const Vector &op_, const Vector &x_, Vector &y_,
+   const int d1d = 0, const int = 0, const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);
@@ -280,18 +251,13 @@ inline void SmemPAHdivMassApply2D(const int NE,
 }
 
 // Shared memory PA H(div) Mass Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHdivMassApply3D(const int NE,
-                                  const bool symmetric,
-                                  const Array<real_t> &Bo_,
-                                  const Array<real_t> &Bc_,
-                                  const Array<real_t> &Bot_,
-                                  const Array<real_t> &Bct_,
-                                  const Vector &op_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void
+SmemPAHdivMassApply3D(const int NE, const bool symmetric, const bool,
+                      const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                      const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                      const Vector &op_, const Vector &x_, Vector &y_,
+                      const int d1d = 0, const int = 0, const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);

fem/integ/bilininteg_interp_pa.cpp

@@ -14,9 +14,218 @@
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 
+#include "bilininteg_hcurlhdiv_kernels.hpp"
+
 namespace mfem
 {
 
+namespace
+{
+
+void PAHcurlApplyCurl2D(const int c_dofs1D,
+                        const int o_dofs1D,
+                        const int NE,
+                        const Array<real_t> &Bo_,
+                        const Array<real_t> &Gc_,
+                        const Vector &x_,
+                        Vector &y_)
+{
+   auto Bo = Reshape(Bo_.Read(), o_dofs1D, o_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), o_dofs1D, o_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int iy = 0; iy < c_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < o_dofs1D; ++ix)
+         {
+            const real_t xv = X(ix + iy * o_dofs1D, e);
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t gy = Gc(oy, iy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  Y(ox, oy, e) -= Bo(ox, ix) * gy * xv;
+               }
+            }
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int iy = 0; iy < o_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < c_dofs1D; ++ix)
+         {
+            const real_t xv = X(y_nd + ix + iy * c_dofs1D, e);
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t by = Bo(oy, iy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  Y(ox, oy, e) += Gc(ox, ix) * by * xv;
+               }
+            }
+         }
+      }
+   });
+}
+
+void PAHcurlApplyCurlTranspose2D(const int c_dofs1D,
+                                 const int o_dofs1D,
+                                 const int NE,
+                                 const Array<real_t> &Bo_,
+                                 const Array<real_t> &Gc_,
+                                 const Vector &x_,
+                                 Vector &y_)
+{
+   auto Bo = Reshape(Bo_.Read(), o_dofs1D, o_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), o_dofs1D, o_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int dy = 0; dy < c_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < o_dofs1D; ++dx)
+         {
+            real_t sum = 0.0;
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t gy = Gc(oy, dy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  sum -= Bo(ox, dx) * gy * X(ox, oy, e);
+               }
+            }
+            Y(dx + dy * o_dofs1D, e) += sum;
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int dy = 0; dy < o_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < c_dofs1D; ++dx)
+         {
+            real_t sum = 0.0;
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t by = Bo(oy, dy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  sum += Gc(ox, dx) * by * X(ox, oy, e);
+               }
+            }
+            Y(y_nd + dx + dy * c_dofs1D, e) += sum;
+         }
+      }
+   });
+}
+
+void PAHdivApplyCurl2D(const int c_dofs1D,
+                       const int o_dofs1D,
+                       const int NE,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Gc_,
+                       const Vector &x_,
+                       Vector &y_)
+{
+   auto Bc = Reshape(Bc_.Read(), c_dofs1D, c_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), c_dofs1D, c_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int iy = 0; iy < c_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < c_dofs1D; ++ix)
+         {
+            const real_t xv = X(ix, iy, e);
+            for (int oy = 0; oy < o_dofs1D; ++oy)
+            {
+               const real_t gy = Gc(oy, iy);
+               for (int ox = 0; ox < c_dofs1D; ++ox)
+               {
+                  Y(ox + oy * c_dofs1D, e) += Bc(ox, ix) * gy * xv;
+               }
+            }
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int iy = 0; iy < c_dofs1D; ++iy)
+      {
+         for (int ix = 0; ix < c_dofs1D; ++ix)
+         {
+            const real_t xv = X(ix, iy, e);
+            for (int oy = 0; oy < c_dofs1D; ++oy)
+            {
+               const real_t by = Bc(oy, iy);
+               for (int ox = 0; ox < o_dofs1D; ++ox)
+               {
+                  Y(y_nd + ox + oy * o_dofs1D, e) -= Gc(ox, ix) * by * xv;
+               }
+            }
+         }
+      }
+   });
+}
+
+void PAHdivApplyCurlTranspose2D(const int c_dofs1D,
+                                const int o_dofs1D,
+                                const int NE,
+                                const Array<real_t> &Bc_,
+                                const Array<real_t> &Gc_,
+                                const Vector &x_,
+                                Vector &y_)
+{
+   auto Bc = Reshape(Bc_.Read(), c_dofs1D, c_dofs1D);
+   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
+   auto X = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), c_dofs1D, c_dofs1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int dy = 0; dy < o_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < c_dofs1D; ++dx)
+         {
+            const real_t xv = X(dx + dy * c_dofs1D, e);
+            for (int iy = 0; iy < c_dofs1D; ++iy)
+            {
+               const real_t gy = Gc(dy, iy);
+               for (int ix = 0; ix < c_dofs1D; ++ix)
+               {
+                  Y(ix, iy, e) += Bc(dx, ix) * gy * xv;
+               }
+            }
+         }
+      }
+
+      const int y_nd = c_dofs1D * o_dofs1D;
+      for (int dy = 0; dy < c_dofs1D; ++dy)
+      {
+         for (int dx = 0; dx < o_dofs1D; ++dx)
+         {
+            const real_t xv = X(y_nd + dx + dy * o_dofs1D, e);
+            for (int iy = 0; iy < c_dofs1D; ++iy)
+            {
+               const real_t by = Bc(dy, iy);
+               for (int ix = 0; ix < c_dofs1D; ++ix)
+               {
+                  Y(ix, iy, e) -= Gc(dx, ix) * by * xv;
+               }
+            }
+         }
+      }
+   });
+}
+
+}
+
 // Apply to x corresponding to DOFs in H^1 (domain) the (topological) gradient
 // to get a dof in H(curl) (range). You can think of the range as the "test" space
 // and the domain as the "trial" space, but there's no integration.
@@ -1950,4 +2159,266 @@ void IdentityInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
    }
 }
 
+void CurlInterpolator::AssemblePA(const FiniteElementSpace &dom_fes,
+                                  const FiniteElementSpace &ran_fes)
+{
+   Mesh *mesh = dom_fes.GetMesh();
+   dim = mesh->Dimension();
+   ne = dom_fes.GetNE();
+   pa_mode_2d = 0;
+   MFEM_VERIFY(ne == ran_fes.GetNE(),
+               "Different meshes for domain and range spaces");
+
+   if (dim == 2)
+   {
+      pa_data.SetSize(0);
+      const FiniteElement *dom_fel = dom_fes.GetTypicalFE();
+      const FiniteElement *ran_fel = ran_fes.GetTypicalFE();
+      const bool hcurl_to_scalar =
+         dynamic_cast<const VectorTensorFiniteElement*>(dom_fel) != NULL &&
+         dom_fel->GetDerivType() == FiniteElement::CURL &&
+         dynamic_cast<const TensorBasisElement*>(ran_fel) != NULL &&
+         ran_fel->GetRangeType() == FiniteElement::SCALAR;
+      const bool scalar_to_hdiv =
+         dynamic_cast<const TensorBasisElement*>(dom_fel) != NULL &&
+         dom_fel->GetRangeType() == FiniteElement::SCALAR &&
+         dynamic_cast<const VectorTensorFiniteElement*>(ran_fel) != NULL &&
+         ran_fel->GetDerivType() == FiniteElement::DIV;
+
+      MFEM_VERIFY(hcurl_to_scalar || scalar_to_hdiv,
+                  "2D CurlInterpolator PA supports H(curl)->scalar and scalar->H(div) only.");
+
+      int closed_basis_type = -1;
+      int open_basis_type = -1;
+      if (hcurl_to_scalar)
+      {
+         const auto *trial_fec = dynamic_cast<const ND_FECollection*>(dom_fes.FEColl());
+         const auto *range_fec = dynamic_cast<const L2_FECollection*>(ran_fes.FEColl());
+         MFEM_VERIFY(trial_fec != NULL, "H(curl) domain must use ND_FECollection.");
+         MFEM_VERIFY(range_fec != NULL, "Scalar range must use L2_FECollection.");
+         MFEM_VERIFY(ran_fel->GetMapType() == FiniteElement::INTEGRAL,
+                     "2D H(curl)->scalar CurlInterpolator PA supports integral-map scalar range spaces only.");
+         closed_basis_type = trial_fec->GetClosedBasisType();
+         open_basis_type = trial_fec->GetOpenBasisType();
+         MFEM_VERIFY(range_fec->GetBasisType() == open_basis_type,
+                     "Domain/range open basis types do not match.");
+         pa_mode_2d = 1;
+      }
+      else
+      {
+         const auto *trial_fec = dynamic_cast<const H1_FECollection*>(dom_fes.FEColl());
+         const auto *range_fec = dynamic_cast<const RT_FECollection*>(ran_fes.FEColl());
+         MFEM_VERIFY(trial_fec != NULL, "Scalar domain must use H1_FECollection.");
+         MFEM_VERIFY(range_fec != NULL, "H(div) range must use RT_FECollection.");
+         closed_basis_type = trial_fec->GetBasisType();
+         open_basis_type = range_fec->GetOpenBasisType();
+         MFEM_VERIFY(range_fec->GetClosedBasisType() == closed_basis_type,
+                     "Domain/range closed basis types do not match.");
+         pa_mode_2d = 2;
+      }
+
+      const int order = hcurl_to_scalar
+                        ? dynamic_cast<const VectorTensorFiniteElement*>(dom_fel)->GetOrder()
+                        : dynamic_cast<const NodalTensorFiniteElement*>(dom_fel)->GetOrder();
+      c_dofs1D = order + 1;
+      o_dofs1D = order;
+
+      closed_dofquad_fe.reset(new H1_SegmentElement(order, closed_basis_type));
+      open_dofquad_fe.reset(new L2_SegmentElement(order - 1, open_basis_type));
+
+      mfem::QuadratureFunctions1D qf1d;
+      mfem::IntegrationRule closed_ir;
+      closed_ir.SetSize(c_dofs1D);
+      qf1d.GaussLobatto(c_dofs1D, &closed_ir);
+
+      mfem::IntegrationRule open_ir;
+      open_ir.SetSize(o_dofs1D);
+      qf1d.GaussLegendre(o_dofs1D, &open_ir);
+
+      maps_C_C = &closed_dofquad_fe->GetDofToQuad(closed_ir, DofToQuad::TENSOR);
+      maps_O_C = &closed_dofquad_fe->GetDofToQuad(open_ir, DofToQuad::TENSOR);
+      maps_O_O = &open_dofquad_fe->GetDofToQuad(open_ir, DofToQuad::TENSOR);
+
+      MFEM_VERIFY(maps_C_C->ndof == c_dofs1D && maps_C_C->nqpt == c_dofs1D, "");
+      MFEM_VERIFY(maps_O_C->ndof == c_dofs1D && maps_O_C->nqpt == o_dofs1D, "");
+      MFEM_VERIFY(maps_O_O->ndof == o_dofs1D && maps_O_O->nqpt == o_dofs1D, "");
+      return;
+   }
+
+   closed_dofquad_fe.reset();
+   open_dofquad_fe.reset();
+   maps_C_C = nullptr;
+   maps_O_C = nullptr;
+   maps_O_O = nullptr;
+
+   const VectorTensorFiniteElement *dom_el =
+      dynamic_cast<const VectorTensorFiniteElement *>(dom_fes.GetTypicalFE());
+   const VectorTensorFiniteElement *ran_el =
+      dynamic_cast<const VectorTensorFiniteElement *>(ran_fes.GetTypicalFE());
+   MFEM_VERIFY(dom_el != NULL, "Only VectorTensorFiniteElement is supported!");
+   MFEM_VERIFY(ran_el != NULL, "Only VectorTensorFiniteElement is supported!");
+   MFEM_VERIFY(dom_el->GetDerivType() == FiniteElement::CURL,
+               "Domain space must be H(curl)");
+   MFEM_VERIFY(ran_el->GetDerivType() == FiniteElement::DIV,
+               "Range space must be H(div)");
+
+   const int dims = dom_el->GetDim();
+   MFEM_VERIFY(dims == 3, "");
+
+   ndof_o = dom_el->GetOrder();
+   int ndof_c = ndof_o + 1;
+   nquad_o = ran_el->GetOrder();
+   int nquad_c = nquad_o + 1;
+
+   // extract the tensor product range dof locations
+   std::vector<real_t> qc(nquad_c);
+   std::vector<real_t> qo(nquad_o);
+   {
+      const IntegrationRule &ran_nodes = ran_el->GetNodes();
+      const Array<int> &quad_map = ran_el->GetDofMap();
+      for (int i = 0; i < nquad_c; ++i)
+      {
+         int idx = UnsignIndex(quad_map[i]);
+         qc[i] = ran_nodes.IntPoint(idx).x;
+      }
+      int offset = ndof_c * ndof_o * ndof_o;
+      for (int i = 0; i < nquad_o; ++i)
+      {
+         int idx = UnsignIndex(quad_map[i + offset]);
+         qo[i] = ran_nodes.IntPoint(idx).x;
+      }
+   }
+
+   // evaluate closed/open 1D basis (and their derivatives) at closed and
+   // open quads
+   // storage order: GCO, BCC, BOO
+   pa_data.SetSize(ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
+   auto ptr = pa_data.HostWrite();
+   auto &cbasis1d = dom_el->GetBasis1D();
+   auto &obasis1d = dom_el->GetOpenBasis1D();
+   Vector b, g;
+   b.SetSize(ndof_c);
+   g.SetSize(ndof_c);
+   for (int j = 0; j < nquad_o; ++j)
+   {
+      cbasis1d.Eval(qo[j], b, g);
+      for (int i = 0; i < ndof_c; ++i)
+      {
+         ptr[j + i * nquad_o] = g[i];
+      }
+   }
+   ptr += nquad_o * ndof_c;
+
+   for (int j = 0; j < nquad_c; ++j)
+   {
+      cbasis1d.Eval(qc[j], b);
+      for (int i = 0; i < ndof_c; ++i)
+      {
+         ptr[j + i * nquad_c] = b[i];
+      }
+   }
+   ptr += ndof_c * nquad_c;
+
+   b.SetSize(ndof_o);
+   for (int j = 0; j < nquad_o; ++j)
+   {
+      obasis1d.Eval(qo[j], b);
+      for (int i = 0; i < ndof_o; ++i)
+      {
+         ptr[j + i * nquad_o] = b[i];
+      }
+   }
+}
+
+CurlInterpolator::Kernels::Kernels()
+{
+   CurlInterpolator::AddSpecialization<3, 1, 1>();
+   CurlInterpolator::AddSpecialization<3, 2, 2>();
+   CurlInterpolator::AddSpecialization<3, 3, 3>();
+   CurlInterpolator::AddSpecialization<3, 4, 4>();
+   CurlInterpolator::AddSpecialization<3, 5, 5>();
+}
+
+CurlInterpolator::CurlInterpolator() { static Kernels kernels{}; }
+
+void CurlInterpolator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (dim == 2)
+   {
+      MFEM_VERIFY(maps_C_C != nullptr && maps_O_C != nullptr,
+                  "2D CurlInterpolator PA data is not assembled.");
+      if (pa_mode_2d == 1)
+      {
+         MFEM_VERIFY(maps_O_O != nullptr,
+                     "2D CurlInterpolator scalar curl map is not assembled.");
+         PAHcurlApplyCurl2D(c_dofs1D, o_dofs1D, ne, maps_O_O->B, maps_O_C->G,
+                            x, y);
+      }
+      else if (pa_mode_2d == 2)
+      {
+         PAHdivApplyCurl2D(c_dofs1D, o_dofs1D, ne, maps_C_C->B, maps_O_C->G,
+                           x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported 2D CurlInterpolator mode.");
+      }
+      return;
+   }
+
+   ApplyPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x, y);
+}
+
+void CurlInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
+{
+   if (dim == 2)
+   {
+      MFEM_VERIFY(maps_C_C != nullptr && maps_O_C != nullptr,
+                  "2D CurlInterpolator PA data is not assembled.");
+      if (pa_mode_2d == 1)
+      {
+         MFEM_VERIFY(maps_O_O != nullptr,
+                     "2D CurlInterpolator scalar curl map is not assembled.");
+         PAHcurlApplyCurlTranspose2D(c_dofs1D, o_dofs1D, ne, maps_O_O->B,
+                                     maps_O_C->G, x, y);
+      }
+      else if (pa_mode_2d == 2)
+      {
+         PAHdivApplyCurlTranspose2D(c_dofs1D, o_dofs1D, ne, maps_C_C->B,
+                                    maps_O_C->G, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported 2D CurlInterpolator mode.");
+      }
+      return;
+   }
+
+   ApplyTPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x, y);
+}
+
+/// \cond DO_NOT_DOCUMENT
+
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 3)
+   {
+      return internal::CurlInterpolatorApply3DSmem<0, 0>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyTPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 3)
+   {
+      return internal::CurlInterpolatorTApply3DSmem<0, 0>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
 } // namespace mfem

fem/integ/bilininteg_mass_ea.cpp

@@ -16,6 +16,71 @@
 
 namespace mfem
 {
+namespace
+{
+
+void AssembleEATriangularImpl(const int dim, const int ne,
+                              const int dofs1D, const int quad1D,
+                              const Array<real_t> &B,
+                              const Vector &pa_data,
+                              Vector &data,
+                              const bool add)
+{
+   using internal::EAMassAssembleTriangular1DLower;
+   using internal::EAMassAssembleTriangular2DLower;
+   using internal::EAMassAssembleTriangular3DLower;
+
+   if (dim == 1)
+   {
+      auto kernel = EAMassAssembleTriangular1DLower<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
+      {
+         case 0x22: kernel = EAMassAssembleTriangular1DLower<2,2>; break;
+         case 0x33: kernel = EAMassAssembleTriangular1DLower<3,3>; break;
+         case 0x44: kernel = EAMassAssembleTriangular1DLower<4,4>; break;
+         case 0x55: kernel = EAMassAssembleTriangular1DLower<5,5>; break;
+         case 0x66: kernel = EAMassAssembleTriangular1DLower<6,6>; break;
+         case 0x77: kernel = EAMassAssembleTriangular1DLower<7,7>; break;
+         case 0x88: kernel = EAMassAssembleTriangular1DLower<8,8>; break;
+         case 0x99: kernel = EAMassAssembleTriangular1DLower<9,9>; break;
+      }
+      return kernel(ne, B, pa_data, data, add, dofs1D, quad1D);
+   }
+   else if (dim == 2)
+   {
+      auto kernel = EAMassAssembleTriangular2DLower<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
+      {
+         case 0x22: kernel = EAMassAssembleTriangular2DLower<2,2>; break;
+         case 0x33: kernel = EAMassAssembleTriangular2DLower<3,3>; break;
+         case 0x44: kernel = EAMassAssembleTriangular2DLower<4,4>; break;
+         case 0x55: kernel = EAMassAssembleTriangular2DLower<5,5>; break;
+         case 0x66: kernel = EAMassAssembleTriangular2DLower<6,6>; break;
+         case 0x77: kernel = EAMassAssembleTriangular2DLower<7,7>; break;
+         case 0x88: kernel = EAMassAssembleTriangular2DLower<8,8>; break;
+         case 0x99: kernel = EAMassAssembleTriangular2DLower<9,9>; break;
+      }
+      return kernel(ne, B, pa_data, data, add, dofs1D, quad1D);
+   }
+   else if (dim == 3)
+   {
+      auto kernel = EAMassAssembleTriangular3DLower<0,0>;
+      switch ((dofs1D << 4 ) | quad1D)
+      {
+         case 0x23: kernel = EAMassAssembleTriangular3DLower<2,3>; break;
+         case 0x34: kernel = EAMassAssembleTriangular3DLower<3,4>; break;
+         case 0x45: kernel = EAMassAssembleTriangular3DLower<4,5>; break;
+         case 0x56: kernel = EAMassAssembleTriangular3DLower<5,6>; break;
+         case 0x67: kernel = EAMassAssembleTriangular3DLower<6,7>; break;
+         case 0x78: kernel = EAMassAssembleTriangular3DLower<7,8>; break;
+         case 0x89: kernel = EAMassAssembleTriangular3DLower<8,9>; break;
+      }
+      return kernel(ne, B, pa_data, data, add, dofs1D, quad1D);
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+}
 
 void MassIntegrator::AssembleEA_(Vector &ea_data,
                                  const bool add)
@@ -75,6 +140,16 @@ void MassIntegrator::AssembleEA_(Vector &ea_data,
    MFEM_ABORT("Unknown kernel.");
 }
 
+void MassIntegrator::AssembleEATriangular_(
+   TriPackLowerMatrix &ea_data,
+   const bool add)
+{
+   Vector &data = ea_data.Data();
+   const Array<real_t> &B = maps->B;
+   return AssembleEATriangularImpl(dim, ne, dofs1D, quad1D, B, pa_data, data,
+                                   add);
+}
+
 void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
                                 Vector &ea_data,
                                 const bool add)
@@ -83,6 +158,35 @@ void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
    if (ne > 0) { AssembleEA_(ea_data, add); }
 }
 
+void MassIntegrator::AssembleEATriangular(const FiniteElementSpace &fes,
+                                          TriPackLowerMatrix &ea_data,
+                                          const bool add)
+{
+   AssemblePA(fes);
+   if (ne == 0) { return; }
+
+   int elem_dofs = 1;
+   for (int d = 0; d < dim; ++d)
+   {
+      elem_dofs *= dofs1D;
+   }
+
+   if (add)
+   {
+      MFEM_VERIFY(ea_data.GetNumRows() == elem_dofs,
+                  "Invalid triangular EA element size.");
+      MFEM_VERIFY(ea_data.GetNumMatrices() == ne,
+                  "Invalid triangular EA element count.");
+   }
+   else
+   {
+      ea_data.SetSize(elem_dofs, ne);
+      ea_data.UseDevice(true);
+   }
+
+   AssembleEATriangular_(ea_data, add);
+}
+
 void MassIntegrator::AssembleEABoundary(const FiniteElementSpace &fes,
                                         Vector &ea_data,
                                         const bool add)

fem/integ/bilininteg_mass_kernels.hpp

@@ -1402,6 +1402,480 @@ inline void EAMassAssemble3D(const int NE,
    });
 }
 
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void EAMassAssembleTriangular1DLower(const int NE,
+                                            const Array<real_t> &basis,
+                                            const Vector &padata,
+                                            Vector &eadata,
+                                            const bool add,
+                                            const int d1d = 0,
+                                            const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(basis.Read(), Q1D, D1D);
+   auto D = Reshape(padata.Read(), Q1D, NE);
+   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(),
+                    TriPackLowerMatrix::PackedSize(D1D), NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      for (int i1 = 0; i1 < D1D; ++i1)
+      {
+         for (int j1 = 0; j1 <= i1; ++j1)
+         {
+            real_t val = 0.0;
+            for (int k1 = 0; k1 < Q1D; ++k1)
+            {
+               val += B(k1, i1) * B(k1, j1) * D(k1, e);
+            }
+            const int idx = TriPackLowerMatrix::LowerIndex(i1, j1, D1D);
+            if (add)
+            {
+               M(idx, e) += val;
+            }
+            else
+            {
+               M(idx, e) = val;
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void EAMassAssembleTriangular2DLower(const int NE,
+                                            const Array<real_t> &basis,
+                                            const Vector &padata,
+                                            Vector &eadata,
+                                            const bool add,
+                                            const int d1d = 0,
+                                            const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const int ndofs = D1D*D1D;
+   auto B = Reshape(basis.Read(), Q1D, D1D);
+   auto D = Reshape(padata.Read(), Q1D, Q1D, NE);
+   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(),
+                    TriPackLowerMatrix::PackedSize(ndofs), NE);
+
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      const int ndofs = D1D*D1D;
+      real_t r_B[MQ1][MD1];
+      for (int d = 0; d < D1D; d++)
+      {
+         for (int q = 0; q < Q1D; q++)
+         {
+            r_B[q][d] = B(q,d);
+         }
+      }
+      MFEM_SHARED real_t s_D[MQ1][MQ1];
+      MFEM_FOREACH_THREAD(k1,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(k2,y,Q1D)
+         {
+            s_D[k1][k2] = D(k1,k2,e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(i1,x,D1D)
+      {
+         MFEM_FOREACH_THREAD(i2,y,D1D)
+         {
+            const int row = i1 + D1D*i2;
+            for (int j2 = 0; j2 < D1D; ++j2)
+            {
+               for (int j1 = 0; j1 < D1D; ++j1)
+               {
+                  const int col = j1 + D1D*j2;
+                  if (row < col)
+                  {
+                     continue;
+                  }
+                  real_t val = 0.0;
+                  for (int k1 = 0; k1 < Q1D; ++k1)
+                  {
+                     for (int k2 = 0; k2 < Q1D; ++k2)
+                     {
+                        val += r_B[k1][i1] * r_B[k1][j1]
+                               * r_B[k2][i2] * r_B[k2][j2]
+                               * s_D[k1][k2];
+                     }
+                  }
+                  const int idx = TriPackLowerMatrix::LowerIndex(row, col, ndofs);
+                  if (add)
+                  {
+                     M(idx, e) += val;
+                  }
+                  else
+                  {
+                     M(idx, e) = val;
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D, int T_Q1D, int T_COLB, int T_NT = 32>
+inline void EAMassAssembleTriangular3D_LowerBlockCols_Impl(
+   const int NE,
+   const Array<real_t> &basis,
+   const Vector &padata,
+   Vector &eadata,
+   const bool add,
+   const int,
+   const int)
+{
+   static_assert(T_D1D > 0 && T_Q1D > 0, "");
+   // Specialized packed lower-triangular hex mass assembly using block-column
+   // sum-factorization.
+   constexpr int D1D = T_D1D;
+   constexpr int Q1D = T_Q1D;
+   constexpr int COLB = T_COLB;
+   constexpr int NT = T_NT;
+   constexpr int ND = D1D*D1D*D1D;
+   constexpr int NQ = Q1D*Q1D*Q1D;
+
+   auto B = Reshape(basis.Read(), Q1D, D1D);
+   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
+   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(),
+                    TriPackLowerMatrix::PackedSize(ND), NE);
+
+   mfem::forall_3D_grid(NE, NT, 1, 1, 0, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int tid = MFEM_THREAD_ID(x);
+
+      MFEM_SHARED real_t s_B[Q1D][D1D];
+      MFEM_SHARED real_t uW[NQ*COLB];
+      MFEM_SHARED real_t t1[D1D*Q1D*Q1D*COLB];
+      MFEM_SHARED real_t t2[D1D*D1D*Q1D*COLB];
+
+      for (int qb = tid; qb < Q1D*D1D; qb += NT)
+      {
+         const int q = qb % Q1D;
+         const int d = qb / Q1D;
+         s_B[q][d] = B(q, d);
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int j0 = 0; j0 < ND; j0 += COLB)
+      {
+         const int b = (j0 + COLB <= ND) ? COLB : (ND - j0);
+         int j1[COLB], j2[COLB], j3[COLB];
+         for (int c = 0; c < COLB; ++c)
+         {
+            if (c < b)
+            {
+               const int jj = j0 + c;
+               j1[c] = jj % D1D;
+               const int tmp = jj / D1D;
+               j2[c] = tmp % D1D;
+               j3[c] = tmp / D1D;
+            }
+         }
+
+         for (int q = tid; q < NQ; q += NT)
+         {
+            const int q1 = q % Q1D;
+            const int tmp = q / Q1D;
+            const int q2 = tmp % Q1D;
+            const int q3 = tmp / Q1D;
+            const real_t Dq = D(q1, q2, q3, e);
+
+            for (int c = 0; c < b; ++c)
+            {
+               uW[q + NQ*c] = s_B[q1][j1[c]] * s_B[q2][j2[c]]
+                              * s_B[q3][j3[c]] * Dq;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         constexpr int T1S = D1D*Q1D*Q1D;
+         for (int a = tid; a < T1S; a += NT)
+         {
+            const int i1 = a % D1D;
+            const int tmp = a / D1D;
+            const int q2 = tmp % Q1D;
+            const int q3 = tmp / Q1D;
+
+            for (int c = 0; c < b; ++c)
+            {
+               real_t sum = 0.0;
+               for (int q1 = 0; q1 < Q1D; ++q1)
+               {
+                  const int q = q1 + Q1D*(q2 + Q1D*q3);
+                  sum += s_B[q1][i1] * uW[q + NQ*c];
+               }
+               t1[a + T1S*c] = sum;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         constexpr int T2S = D1D*D1D*Q1D;
+         for (int a = tid; a < T2S; a += NT)
+         {
+            const int i1 = a % D1D;
+            const int tmp = a / D1D;
+            const int i2 = tmp % D1D;
+            const int q3 = tmp / D1D;
+
+            for (int c = 0; c < b; ++c)
+            {
+               real_t sum = 0.0;
+               for (int q2 = 0; q2 < Q1D; ++q2)
+               {
+                  const int a1 = i1 + D1D*(q2 + Q1D*q3);
+                  sum += s_B[q2][i2] * t1[a1 + T1S*c];
+               }
+               t2[a + T2S*c] = sum;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         for (int c = 0; c < b; ++c)
+         {
+            const int col = j0 + c;
+            const int jj1 = j1[c];
+            const int jj2 = j2[c];
+            const int jj3 = j3[c];
+
+            for (int i3 = jj3 + 1; i3 < D1D; ++i3)
+            {
+               for (int a = tid; a < D1D*D1D; a += NT)
+               {
+                  const int i1 = a % D1D;
+                  const int i2 = a / D1D;
+                  real_t sum = 0.0;
+                  for (int q3 = 0; q3 < Q1D; ++q3)
+                  {
+                     const int a2 = i1 + D1D*(i2 + D1D*q3);
+                     sum += s_B[q3][i3] * t2[a2 + T2S*c];
+                  }
+                  const int row = i1 + D1D*(i2 + D1D*i3);
+                  const int idx =
+                     TriPackLowerMatrix::LowerIndex(row, col, ND);
+                  if (add) { M(idx, e) += sum; }
+                  else { M(idx, e) = sum; }
+               }
+               MFEM_SYNC_THREAD;
+            }
+
+            for (int i2 = jj2 + 1; i2 < D1D; ++i2)
+            {
+               const int i3 = jj3;
+               for (int i1 = tid; i1 < D1D; i1 += NT)
+               {
+                  real_t sum = 0.0;
+                  for (int q3 = 0; q3 < Q1D; ++q3)
+                  {
+                     const int a2 = i1 + D1D*(i2 + D1D*q3);
+                     sum += s_B[q3][i3] * t2[a2 + T2S*c];
+                  }
+                  const int row = i1 + D1D*(i2 + D1D*i3);
+                  const int idx =
+                     TriPackLowerMatrix::LowerIndex(row, col, ND);
+                  if (add) { M(idx, e) += sum; }
+                  else { M(idx, e) = sum; }
+               }
+               MFEM_SYNC_THREAD;
+            }
+
+            {
+               const int i3 = jj3;
+               const int i2 = jj2;
+               for (int i1 = tid + jj1; i1 < D1D; i1 += NT)
+               {
+                  real_t sum = 0.0;
+                  for (int q3 = 0; q3 < Q1D; ++q3)
+                  {
+                     const int a2 = i1 + D1D*(i2 + D1D*q3);
+                     sum += s_B[q3][i3] * t2[a2 + T2S*c];
+                  }
+                  const int row = i1 + D1D*(i2 + D1D*i3);
+                  const int idx =
+                     TriPackLowerMatrix::LowerIndex(row, col, ND);
+                  if (add) { M(idx, e) += sum; }
+                  else { M(idx, e) = sum; }
+               }
+               MFEM_SYNC_THREAD;
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D, int T_Q1D, int T_NT = 32>
+inline void EAMassAssembleTriangular3D_LowerBlockCols(
+   const int NE,
+   const Array<real_t> &basis,
+   const Vector &padata,
+   Vector &eadata,
+   const bool add,
+   const int d1d = 0,
+   const int q1d = 0)
+{
+   static_assert(T_D1D > 0 && T_Q1D > 0, "");
+
+   constexpr int D1D = T_D1D;
+   constexpr int Q1D = T_Q1D;
+   constexpr int NQ = Q1D*Q1D*Q1D;
+   constexpr int SharedBytesPerCol =
+      sizeof(real_t)*(NQ + D1D*Q1D*Q1D + D1D*D1D*Q1D);
+   constexpr int SharedBytesBase = sizeof(real_t)*(Q1D*D1D);
+   constexpr int MaxSharedBytes = 48*1024;
+   constexpr int COLB =
+      (SharedBytesBase + 4*SharedBytesPerCol <= MaxSharedBytes) ? 4 :
+      (SharedBytesBase + 2*SharedBytesPerCol <= MaxSharedBytes) ? 2 : 1;
+
+   return EAMassAssembleTriangular3D_LowerBlockCols_Impl<T_D1D, T_Q1D, COLB, T_NT>(
+             NE, basis, padata, eadata, add, d1d, q1d);
+}
+
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void EAMassAssembleTriangular3DLower(const int NE,
+                                            const Array<real_t> &basis,
+                                            const Vector &padata,
+                                            Vector &eadata,
+                                            const bool add,
+                                            const int d1d = 0,
+                                            const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const int ndofs = D1D*D1D*D1D;
+   auto B = Reshape(basis.Read(), Q1D, D1D);
+   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
+   auto M = Reshape(add ? eadata.ReadWrite() : eadata.Write(),
+                    TriPackLowerMatrix::PackedSize(ndofs), NE);
+
+   if constexpr (T_D1D > 0 && T_Q1D > 0)
+   {
+      // Use the sum-factorized packed paths when the tensor dimensions are
+      // known at compile time. The generic path below handles dynamic sizes.
+      return EAMassAssembleTriangular3D_LowerBlockCols<T_D1D, T_Q1D>(
+                NE, basis, padata, eadata, add, d1d, q1d);
+   }
+   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int DQ = T_D1D * T_Q1D;
+      const int ndofs = D1D*D1D*D1D;
+
+      constexpr bool USE_REG = DQ != 0 && DQ <= 12;
+      constexpr int MD1r = USE_REG ? MD1 : 1;
+      constexpr int MQ1r = USE_REG ? MQ1 : 1;
+      constexpr int MD1s = USE_REG ? 1 : MD1;
+      constexpr int MQ1s = USE_REG ? 1 : MQ1;
+
+      MFEM_SHARED real_t s_B[MQ1s][MD1s];
+      real_t r_B[MQ1r][MD1r];
+      real_t (*l_B)[MD1] = nullptr;
+      if (USE_REG)
+      {
+         for (int d = 0; d < D1D; d++)
+         {
+            for (int q = 0; q < Q1D; q++)
+            {
+               r_B[q][d] = B(q,d);
+            }
+         }
+         l_B = (real_t (*)[MD1])r_B;
+      }
+      else
+      {
+         if (MFEM_THREAD_ID(z) == 0)
+         {
+            MFEM_FOREACH_THREAD(d,x,D1D)
+            {
+               MFEM_FOREACH_THREAD(q,y,Q1D)
+               {
+                  s_B[q][d] = B(q,d);
+               }
+            }
+         }
+         l_B = (real_t (*)[MD1])s_B;
+      }
+
+      MFEM_SHARED real_t s_D[MQ1][MQ1][MQ1];
+      MFEM_FOREACH_THREAD(k1,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(k2,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(k3,z,Q1D)
+            {
+               s_D[k1][k2][k3] = D(k1,k2,k3,e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(i1,x,D1D)
+      {
+         MFEM_FOREACH_THREAD(i2,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(i3,z,D1D)
+            {
+               const int row = i1 + D1D*(i2 + D1D*i3);
+               for (int j3 = 0; j3 < D1D; ++j3)
+               {
+                  for (int j2 = 0; j2 < D1D; ++j2)
+                  {
+                     for (int j1 = 0; j1 < D1D; ++j1)
+                     {
+                        const int col = j1 + D1D*(j2 + D1D*j3);
+                        if (row < col)
+                        {
+                           continue;
+                        }
+                        real_t val = 0.0;
+                        for (int k1 = 0; k1 < Q1D; ++k1)
+                        {
+                           for (int k2 = 0; k2 < Q1D; ++k2)
+                           {
+                              for (int k3 = 0; k3 < Q1D; ++k3)
+                              {
+                                 val += l_B[k1][i1] * l_B[k1][j1]
+                                        * l_B[k2][i2] * l_B[k2][j2]
+                                        * l_B[k3][i3] * l_B[k3][j3]
+                                        * s_D[k1][k2][k3];
+                              }
+                           }
+                        }
+                        const int idx = TriPackLowerMatrix::LowerIndex(row, col, ndofs);
+                        if (add)
+                        {
+                           M(idx, e) += val;
+                        }
+                        else
+                        {
+                           M(idx, e) = val;
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
 } // namespace internal
 
 namespace

fem/integ/bilininteg_vecdiv_pa.hpp

@@ -0,0 +1,365 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../config/config.hpp"
+#include "../../general/array.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/vector.hpp"
+#include "../bilininteg.hpp"
+#include "../kernels.hpp"
+
+namespace mfem
+{
+
+/// \cond DO_NOT_DOCUMENT
+
+namespace internal
+{
+
+// Shared memory PA Divergence Apply 2D kernel
+template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
+inline void SmemPADivergenceApply2D(const int NE,
+                                    const Array<real_t> &b_,
+                                    const Array<real_t> &g_,
+                                    const Array<real_t> &bt_,
+                                    const Vector &q_,
+                                    const Vector &x_,
+                                    Vector &y_,
+                                    const int tr_d1d = 0,
+                                    const int te_d1d = 0,
+                                    const int q1d = 0)
+{
+   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
+   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
+   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, 2, NE);
+   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, 1, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
+
+      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
+      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
+
+      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
+      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
+
+      kernels::internal::LoadDofs2d(e, TR_D1D, X, g0);
+      kernels::internal::Grad2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
+
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            r0[0][qy][qx] =
+               g1[0][0][qy][qx] * Q(qx, qy, 0, 0, e) +
+               g1[0][1][qy][qx] * Q(qx, qy, 1, 0, e) +
+               g1[1][0][qy][qx] * Q(qx, qy, 0, 1, e) +
+               g1[1][1][qy][qx] * Q(qx, qy, 1, 1, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      kernels::internal::LoadMatrix<MQ1,true>(TE_D1D, Q1D, Bt, sB);
+      kernels::internal::EvalTranspose2d(TE_D1D, Q1D, smem, sB, r0, r1);
+      kernels::internal::WriteDofs2d(e, TE_D1D, r1, Y);
+   });
+}
+
+// Shared memory PA Divergence Apply 2D kernel transpose
+template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
+inline void SmemPADivergenceApplyTranspose2D(const int NE,
+                                             const Array<real_t> &bt,
+                                             const Array<real_t> &gt,
+                                             const Array<real_t> &b,
+                                             const Vector &q_,
+                                             const Vector &x_,
+                                             Vector &y_,
+                                             const int tr_d1d = 0,
+                                             const int te_d1d = 0,
+                                             const int q1d = 0)
+{
+   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
+   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
+   const auto X  = Reshape(x_.Read(), TE_D1D, TE_D1D, 1, NE);
+   auto Y  = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, 2, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
+
+      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
+      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
+
+      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
+      kernels::internal::LoadDofs2d(e, TE_D1D, X, r0);
+      kernels::internal::Eval2d(TE_D1D, Q1D, smem, sB, r0, r1);
+
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            g0[0][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 0, e);
+            g0[0][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 0, e);
+            g0[1][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 1, e);
+            g0[1][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 1, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
+      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
+      kernels::internal::GradTranspose2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
+      kernels::internal::WriteDofs2d(e, TR_D1D, g1, Y);
+   });
+}
+
+// Shared memory PA Divergence Apply 3D kernel transpose
+template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
+inline void SmemPADivergenceApplyTranspose3D(const int NE,
+                                             const Array<real_t> &bt,
+                                             const Array<real_t> &gt,
+                                             const Array<real_t> &b,
+                                             const Vector &q_,
+                                             const Vector &x_,
+                                             Vector &y_,
+                                             int tr_d1d = 0,
+                                             int te_d1d = 0,
+                                             int q1d = 0)
+{
+   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
+   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
+   const auto X = Reshape(x_.Read(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
+   auto Y = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
+
+      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
+      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
+
+      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
+      kernels::internal::LoadDofs3d(e, TE_D1D, X, r0);
+      kernels::internal::Eval3d(TE_D1D, Q1D, smem, sB, r0, r1);
+
+      for (int qz = 0; qz < Q1D; qz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               const auto r = r1[0][qz][qy][qx];
+               g0[0][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 0, e);
+               g0[0][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 0, e);
+               g0[0][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 0, e);
+
+               g0[1][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 1, e);
+               g0[1][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 1, e);
+               g0[1][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 1, e);
+
+               g0[2][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 2, e);
+               g0[2][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 2, e);
+               g0[2][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 2, e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
+      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
+      kernels::internal::GradTranspose3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
+      kernels::internal::WriteDofs3d(e, TR_D1D, g1, Y);
+   });
+}
+
+// Shared memory PA Divergence Apply 3D kernel
+template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
+inline void SmemPADivergenceApply3D(const int NE,
+                                    const Array<real_t> &b_,
+                                    const Array<real_t> &g_,
+                                    const Array<real_t> &bt_,
+                                    const Vector &q_,
+                                    const Vector &x_,
+                                    Vector &y_,
+                                    const int tr_d1d = 0,
+                                    const int te_d1d = 0,
+                                    const int q1d = 0)
+{
+   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
+   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
+   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3,3, NE);
+   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
+   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
+
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
+
+      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
+      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
+
+      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
+      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
+
+      kernels::internal::LoadDofs3d(e, TR_D1D, X, g0);
+      kernels::internal::Grad3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
+
+      for (int qz = 0; qz < Q1D; qz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               r0[0][qz][qy][qx] =
+                  // c = 0
+                  g1[0][0][qz][qy][qx] * Q(qx, qy, qz, 0, 0, e) +
+                  g1[0][1][qz][qy][qx] * Q(qx, qy, qz, 1, 0, e) +
+                  g1[0][2][qz][qy][qx] * Q(qx, qy, qz, 2, 0, e) +
+                  // c = 1
+                  g1[1][0][qz][qy][qx] * Q(qx, qy, qz, 0, 1, e) +
+                  g1[1][1][qz][qy][qx] * Q(qx, qy, qz, 1, 1, e) +
+                  g1[1][2][qz][qy][qx] * Q(qx, qy, qz, 2, 1, e) +
+                  // c = 2
+                  g1[2][0][qz][qy][qx] * Q(qx, qy, qz, 0, 2, e) +
+                  g1[2][1][qz][qy][qx] * Q(qx, qy, qz, 1, 2, e) +
+                  g1[2][2][qz][qy][qx] * Q(qx, qy, qz, 2, 2, e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      kernels::internal::LoadMatrix<MQ1, true>(TE_D1D, Q1D, Bt, sB);
+      kernels::internal::EvalTranspose3d(TE_D1D, Q1D, smem, sB, r0, r1);
+      kernels::internal::WriteDofs3d(e, TE_D1D, r1, Y);
+   });
+}
+
+} // namespace internal
+
+template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
+VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
+VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Kernel()
+{
+   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
+   if constexpr (DIM == 2)
+   {
+      return internal::SmemPADivergenceApply2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      return internal::SmemPADivergenceApply3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+inline VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
+VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Fallback
+(int dim, int tr_d1d, int te_d1d, int q1d)
+{
+   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
+   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   if (dim == 2)
+   {
+      return internal::SmemPADivergenceApply2D;
+   }
+   else if (dim == 3)
+   {
+      return internal::SmemPADivergenceApply3D;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
+VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
+VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Kernel()
+{
+   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
+   if constexpr (DIM == 2)
+   {
+      return internal::SmemPADivergenceApplyTranspose2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      return internal::SmemPADivergenceApplyTranspose3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+inline VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
+VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Fallback
+(int dim, int tr_d1d, int te_d1d, int q1d)
+{
+   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
+   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   if (dim == 2)
+   {
+      return internal::SmemPADivergenceApplyTranspose2D;
+   }
+   else if (dim == 3)
+   {
+      return internal::SmemPADivergenceApplyTranspose3D;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
+} // namespace mfem

fem/integ/bilininteg_vecmass_pa.cpp

@@ -22,6 +22,8 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
+   MFEM_VERIFY(el.GetMapType() == FiniteElement::VALUE,
+               "Only value map type supported");
    ElementTransformation &Trans = *mesh->GetTypicalElementTransformation();
    const auto *ir = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, Trans);
 
@@ -205,157 +207,40 @@ void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 
 }
 
-template <const int T_D1D = 0, const int T_Q1D = 0>
-static void PAVectorMassAssembleDiagonal2D(const int NE,
-                                           const Array<real_t> &b,
-                                           const Vector &pa_data, Vector &diag,
-                                           const int d1d = 0, const int q1d = 0)
-{
-   constexpr int VDIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   const auto B = Reshape(b.Read(), Q1D, D1D);
-   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, VDIM, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      real_t temp[max_Q1D][max_D1D];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            temp[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               temp[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            real_t temp1 = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               temp1 += B(qx, dx) * B(qx, dx) * temp[qx][dy];
-            }
-            Y(dx, dy, 0, e) = temp1;
-            Y(dx, dy, 1, e) = temp1;
-         }
-      }
-   });
-}
-
-template <const int T_D1D = 0, const int T_Q1D = 0>
-static void PAVectorMassAssembleDiagonal3D(const int NE,
-                                           const Array<real_t> &B_,
-                                           const Vector &pa_data, Vector &diag,
-                                           const int d1d = 0, const int q1d = 0)
-{
-   constexpr int VDIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   const auto B = Reshape(B_.Read(), Q1D, D1D);
-   MFEM_VERIFY(pa_data.Size() == Q1D * Q1D * Q1D * NE, "pa_data size error");
-   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      real_t temp[max_Q1D][max_Q1D][max_D1D];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               temp[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  temp[qx][qy][dz] +=
-                     B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      real_t temp2[max_Q1D][max_D1D][max_D1D];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               temp2[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  temp2[qx][dy][dz] +=
-                     B(qy, dy) * B(qy, dy) * temp[qx][qy][dz];
-               }
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               real_t temp3 = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  temp3 += B(qx, dx) * B(qx, dx) * temp2[qx][dy][dz];
-               }
-               Y(dx, dy, dz, 0, e) = temp3;
-               Y(dx, dy, dz, 1, e) = temp3;
-               Y(dx, dy, dz, 2, e) = temp3;
-            }
-         }
-      }
-   });
-}
-
-static void PAVectorMassAssembleDiagonal(const int dim, const int D1D,
-                                         const int Q1D, const int NE,
-                                         const Array<real_t> &B,
-                                         const Vector &pa_data,
-                                         Vector &diag)
-{
-   if (dim == 2)
-   {
-      return PAVectorMassAssembleDiagonal2D(NE, B, pa_data, diag, D1D, Q1D);
-   }
-   else if (dim == 3)
-   {
-      return PAVectorMassAssembleDiagonal3D(NE, B, pa_data, diag, D1D, Q1D);
-   }
-   MFEM_ABORT("Dimension not implemented.");
-}
-
 void VectorMassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   if (DeviceCanUseCeed()) { ceedOp->GetDiagonal(diag); }
-   else
-   {
-      MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
-      MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
-      PAVectorMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
-   }
+   if (DeviceCanUseCeed()) { return ceedOp->GetDiagonal(diag); }
+
+   MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
+   MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
+
+   // Add the VectorMassAssembleDiagonalPA specializations
+   static const auto vector_mass_assemble_diagonal_kernel_specializations =
+      ( // 2D
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 2>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 3>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 4>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 5>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 6>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 7>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 8>::Add(),
+         // 3D
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 2>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 3>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 4>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 5>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 6>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 7>::Add(),
+         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 8>::Add(),
+         true);
+   MFEM_CONTRACT_VAR(vector_mass_assemble_diagonal_kernel_specializations);
+
+   VectorMassAssembleDiagonalPA::Run(dim, quad1D, // templated arguments
+                                     ne, dofs1D, quad1D,
+                                     maps->B.Read(),
+                                     pa_data.Read(),
+                                     diag.ReadWrite());
+
 }
 
 } // namespace mfem

fem/integ/bilininteg_vecmass_pa.hpp

@@ -176,8 +176,146 @@ void SmemPAVectorMassApply3D(const int NE,
    });
 }
 
+template <int T_Q1D = 0, int T_MDQ = 16>
+static void SmemPAVectorMassAssembleDiagonal2D(const int ne,
+                                               const int d1d,
+                                               const int q1d,
+                                               const real_t *b_r,
+                                               const real_t *d_r,
+                                               real_t *y_rw)
+{
+   constexpr int VDIM = 2;
+
+   const int D1D = d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
+
+   const auto B = Reshape(b_r, Q1D, D1D);
+   const auto D = Reshape(d_r, Q1D, Q1D, ne);
+   auto Y = Reshape(y_rw, D1D, D1D, VDIM, ne);
+
+   mfem::forall_2D<T_Q1D*T_Q1D>(
+      ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
+
+      MFEM_SHARED real_t sm[MQ1][MQ1];
+
+      MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+         {
+            real_t u = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               u += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
+            }
+            sm[qx][dy] = u;
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+         {
+            real_t u = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               u += B(qx, dx) * B(qx, dx) * sm[qx][dy];
+            }
+            Y(dx, dy, 0, e) += u;
+            Y(dx, dy, 1, e) += u;
+         }
+      }
+   });
+}
+
+// T_MDQ <= 10 so the Q1D^3 thread block stays within the 1024/block GPU limit
+template <int T_Q1D = 0, int T_MDQ = 10>
+static void SmemPAVectorMassAssembleDiagonal3D(const int ne,
+                                               const int d1d,
+                                               const int q1d,
+                                               const real_t *b_r,
+                                               const real_t *d_r,
+                                               real_t *y_rw)
+{
+   constexpr int VDIM = 3;
+
+   const int D1D = d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
+
+   const auto B = Reshape(b_r, Q1D, D1D);
+   const auto D = Reshape(d_r, Q1D, Q1D, Q1D, ne);
+   auto Y = Reshape(y_rw, D1D, D1D, D1D, VDIM, ne);
+
+   mfem::forall_3D<T_Q1D*T_Q1D*T_Q1D>(
+      ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
+
+      MFEM_SHARED real_t sm[2][MQ1][MQ1][MQ1];
+
+      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               real_t u = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
+               }
+               sm[0][dz][qy][qx] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               real_t u = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  u += B(qy, dy) * B(qy, dy) * sm[0][dz][qy][qx];
+               }
+               sm[1][dz][dy][qx] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+            {
+               real_t u = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  u += B(qx, dx) * B(qx, dx) * sm[1][dz][dy][qx];
+               }
+               Y(dx, dy, dz, 0, e) += u;
+               Y(dx, dy, dz, 1, e) += u;
+               Y(dx, dy, dz, 2, e) += u;
+            }
+         }
+      }
+   });
+}
+
 } // namespace internal
 
+// AddMultPA kernels
 template<int DIM, int T_D1D, int T_Q1D>
 VectorMassIntegrator::VectorMassAddMultPAType
 VectorMassIntegrator::VectorMassAddMultPA::Kernel()
@@ -194,7 +332,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Kernel()
 }
 
 inline VectorMassIntegrator::VectorMassAddMultPAType
-VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int d1d, int q1d)
+VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
 {
    if (dim == 2)
    {
@@ -204,7 +342,37 @@ VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int d1d, int q1d)
    {
       return internal::SmemPAVectorMassApply3D;
    }
-   else { MFEM_ABORT("Unsupported kernel"); }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+// DiagonalPA kernels
+template<int DIM, int T_Q1D>
+VectorMassIntegrator::VectorMassAssembleDiagonalPAType
+VectorMassIntegrator::VectorMassAssembleDiagonalPA::Kernel()
+{
+   if constexpr (DIM == 2)
+   {
+      return internal::SmemPAVectorMassAssembleDiagonal2D<T_Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      return internal::SmemPAVectorMassAssembleDiagonal3D<T_Q1D>;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+inline VectorMassIntegrator::VectorMassAssembleDiagonalPAType
+VectorMassIntegrator::VectorMassAssembleDiagonalPA::Fallback(int dim, int)
+{
+   if (dim == 2)
+   {
+      return internal::SmemPAVectorMassAssembleDiagonal2D;
+   }
+   else if (dim == 3)
+   {
+      return internal::SmemPAVectorMassAssembleDiagonal3D;
+   }
+   MFEM_ABORT("Unsupported kernel");
 }
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_vectorfemass_kernels.hpp

@@ -0,0 +1,113 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
+#define MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
+
+#include "../../config/config.hpp"
+#include "../bilininteg.hpp"
+
+#include "bilininteg_diffusion_kernels.hpp"
+#include "bilininteg_hcurl_kernels.hpp"
+#include "bilininteg_hdiv_kernels.hpp"
+#include "bilininteg_hcurlhdiv_kernels.hpp"
+
+namespace mfem
+{
+/// \cond DO_NOT_DOCUMENT
+namespace internal
+{
+namespace hcurlmass
+{
+constexpr int NBZ3D(int d1d, int q1d)
+{
+   if (d1d <= 1 || q1d <= 0)
+   {
+      return 1;
+   }
+   // assume q1d >= d1d
+   // z dimension is capped at 64 on nvidia and amd gpus
+   int tmp = std::min((128 + q1d * q1d * q1d - 1) / (q1d * q1d * q1d), 64);
+   int smem_req =
+      sizeof(mfem::real_t) *
+      (3 * ((d1d - 1) * d1d * d1d + 2 * q1d * q1d * q1d) * tmp +
+       q1d * (d1d - 1) + q1d * d1d);
+   // assume GPU has at least 48k shared memory
+   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
+}
+} // namespace hcurlmass
+} // namespace internal
+
+template <FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
+          int DIM, int TrialD1D, int TestD1D, int Q1D>
+VectorFEMassIntegrator::ApplyKernelType
+VectorFEMassIntegrator::ApplyPAKernels::Kernel()
+{
+   constexpr bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
+   constexpr bool trial_div = (TrialType == mfem::FiniteElement::DIV);
+   constexpr bool test_curl = (TestType == mfem::FiniteElement::CURL);
+   constexpr bool test_div = (TestType == mfem::FiniteElement::DIV);
+
+   if constexpr (DIM == 3)
+   {
+      if constexpr (trial_curl && test_curl)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            // assume TrialD1D == TestD1D
+            return internal::SmemPAHcurlMassApply3D<
+                   TrialD1D, Q1D, internal::hcurlmass::NBZ3D(TrialD1D, Q1D)>;
+         }
+         else
+         {
+            return internal::PAHcurlMassApply3D;
+         }
+      }
+      else if constexpr (trial_div && test_div)
+      {
+         // assumes TrialD1D == TestD1D
+         return internal::SmemPAHdivMassApply3D<TrialD1D, Q1D>;
+      }
+      else if constexpr (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply3D;
+      }
+      else if constexpr (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply3D;
+      }
+   }
+   else if constexpr (DIM == 2) // 2D
+   {
+      if constexpr (trial_curl && test_curl)
+      {
+         return internal::PAHcurlMassApply2D;
+      }
+      else if constexpr (trial_div && test_div)
+      {
+         // assumes TrialD1D == TestD1D
+         return internal::SmemPAHdivMassApply2D<TrialD1D, Q1D>;
+      }
+      else if constexpr (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply2D;
+      }
+      else if constexpr (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply2D;
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+/// \endcond DO_NOT_DOCUMENT
+}
+
+#endif

fem/integ/bilininteg_vectorfemass_pa.cpp

@@ -10,15 +10,123 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
-#include "../gridfunc.hpp"
-#include "../qfunction.hpp"
-#include "bilininteg_diffusion_kernels.hpp"
-#include "bilininteg_hcurl_kernels.hpp"
-#include "bilininteg_hdiv_kernels.hpp"
-#include "bilininteg_hcurlhdiv_kernels.hpp"
+#include "bilininteg_vectorfemass_kernels.hpp"
 
 namespace mfem
 {
+/// \cond DO_NOT_DOCUMENT
+VectorFEMassIntegrator::ApplyKernelType
+VectorFEMassIntegrator::ApplyPAKernels::Fallback(
+   FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
+   int dim, int, int, int)
+{
+   const bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
+   const bool trial_div = (TrialType == mfem::FiniteElement::DIV);
+   const bool test_curl = (TestType == mfem::FiniteElement::CURL);
+   const bool test_div = (TestType == mfem::FiniteElement::DIV);
+
+   if (dim == 3)
+   {
+      if (trial_curl && test_curl)
+      {
+         return internal::PAHcurlMassApply3D;
+      }
+      else if (trial_div && test_div)
+      {
+         return internal::PAHdivMassApply3D;
+      }
+      else if (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply3D;
+      }
+      else if (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply3D;
+      }
+   }
+   else if (dim == 2) // 2D
+   {
+      if (trial_curl && test_curl)
+      {
+         return internal::PAHcurlMassApply2D;
+      }
+      else if (trial_div && test_div)
+      {
+         return internal::PAHdivMassApply2D;
+      }
+      else if (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply2D;
+      }
+      else if (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply2D;
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+/// \endcond DO_NOT_DOCUMENT
+
+VectorFEMassIntegrator::Kernels::Kernels()
+{
+   // h(curl), h(curl)
+   // Q = P + 1 (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 2, 2, 3>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 3, 3, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 4, 4, 5>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 5, 5, 6>();
+   // Q = P + 2 (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 2, 2, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 3, 3, 5>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 4, 4, 6>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 5, 5, 7>();
+   // Q = P + 4 (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 2, 2, 6>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 3, 3, 7>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 4, 4, 8>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 5, 5, 9>();
+   // h(div), h(div)
+   // Q = P (2D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 2, 2, 2>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 3, 3, 3>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 4, 4, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 5, 5, 5>();
+
+   // Q = P + 1 (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 2, 2, 3>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 3, 3, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 4, 4, 5>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 5, 5, 6>();
+}
+
+void VectorFEMassIntegrator::Init(Coefficient *q, DiagonalMatrixCoefficient *dq,
+                                  MatrixCoefficient *mq)
+{
+   static Kernels kernels{};
+   Q = q;
+   DQ = dq;
+   MQ = mq;
+}
 
 void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
@@ -67,8 +175,8 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
 
-   trial_fetype = trial_el->GetDerivType();
-   test_fetype = test_el->GetDerivType();
+   trial_fetype = static_cast<FiniteElement::DerivType>(trial_el->GetDerivType());
+   test_fetype = static_cast<FiniteElement::DerivType>(test_el->GetDerivType());
 
    const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
    const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
@@ -215,225 +323,34 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
 
 void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
-   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
-   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
-   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
-
-   if (dim == 3)
-   {
-      if (trial_curl && test_curl)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            const int ID = (dofs1D << 4) | quad1D;
-            switch (ID)
-            {
-               case 0x23:
-                  return internal::SmemPAHcurlMassApply3D<2,3>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               case 0x34:
-                  return internal::SmemPAHcurlMassApply3D<3,4>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               case 0x45:
-                  return internal::SmemPAHcurlMassApply3D<4,5>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               case 0x56:
-                  return internal::SmemPAHcurlMassApply3D<5,6>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               default:
-                  return internal::SmemPAHcurlMassApply3D(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-            }
-         }
-         else
-         {
-            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                         mapsO->Bt, mapsC->Bt, pa_data, x, y);
-         }
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                   mapsO->Bt, mapsC->Bt, pa_data, x, y);
-      }
-      else if (trial_curl && test_div)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                          true, false, mapsO->B, mapsC->B, mapsOtest->Bt,
-                                          mapsCtest->Bt, pa_data, x, y);
-      }
-      else if (trial_div && test_curl)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                          false, false, mapsO->B, mapsC->B, mapsOtest->Bt,
-                                          mapsCtest->Bt, pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
-   else // 2D
-   {
-      if (trial_curl && test_curl)
-      {
-         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                      mapsO->Bt, mapsC->Bt, pa_data, x, y);
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                   mapsO->Bt,
-                                   mapsC->Bt, pa_data, x, y);
-      }
-      else if ((trial_curl && test_div) || (trial_div && test_curl))
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                          trial_curl, false, mapsO->B, mapsC->B,
-                                          mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
+   const bool scalar_coeff = !(DQ || MQ);
+   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
+                       quad1D, ne, symmetric, scalar_coeff, mapsO->B, mapsC->B,
+                       mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y, dofs1D,
+                       dofs1Dtest, quad1D);
 }
 
 void VectorFEMassIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
 {
-   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
-   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
-   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
-   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
+   const bool scalar_coeff = !(DQ || MQ);
 
    Vector abs_pa_data(pa_data);
    abs_pa_data.Abs();
 
    Array<real_t> absBo(mapsO->B);
    Array<real_t> absBc(mapsC->B);
-   Array<real_t> absBto(mapsO->Bt);
-   Array<real_t> absBtc(mapsC->Bt);
    Array<real_t> absBto_t(mapsOtest->Bt);
    Array<real_t> absBtc_t(mapsCtest->Bt);
 
    absBo.Abs();
    absBc.Abs();
-   absBto.Abs();
-   absBtc.Abs();
    absBto_t.Abs();
    absBtc_t.Abs();
 
-   if (dim == 3)
-   {
-      if (trial_curl && test_curl)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            const int ID = (dofs1D << 4) | quad1D;
-            switch (ID)
-            {
-               case 0x23:
-                  return internal::SmemPAHcurlMassApply3D<2,3>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               case 0x34:
-                  return internal::SmemPAHcurlMassApply3D<3,4>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               case 0x45:
-                  return internal::SmemPAHcurlMassApply3D<4,5>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               case 0x56:
-                  return internal::SmemPAHcurlMassApply3D<5,6>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               default:
-                  return internal::SmemPAHcurlMassApply3D(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-            }
-         }
-         else
-         {
-            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric,
-                                         absBo, absBc, absBto, absBtc,
-                                         abs_pa_data, x, y);
-         }
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric,
-                                   absBo, absBc, absBto, absBtc,
-                                   abs_pa_data, x, y);
-      }
-      else if (trial_curl && test_div)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
-                                          scalarCoeff, true, false,
-                                          absBo, absBc, absBto_t, absBtc_t,
-                                          abs_pa_data, x, y);
-      }
-      else if (trial_div && test_curl)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
-                                          scalarCoeff, false, false,
-                                          absBo, absBc, absBto_t, absBtc_t,
-                                          abs_pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
-   else // 2D
-   {
-      if (trial_curl && test_curl)
-      {
-         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric,
-                                      absBo, absBc, absBto, absBtc,
-                                      abs_pa_data, x, y);
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric,
-                                   absBo, absBc, absBto, absBtc,
-                                   abs_pa_data, x, y);
-      }
-      else if ((trial_curl && test_div) || (trial_div && test_curl))
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne,
-                                          scalarCoeff, trial_curl, false,
-                                          absBo, absBc, absBto_t, absBtc_t,
-                                          abs_pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
+   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
+                       quad1D, ne, symmetric, scalar_coeff, absBo, absBc,
+                       absBto_t, absBtc_t, abs_pa_data, x, y, dofs1D,
+                       dofs1Dtest, quad1D);
 }
 
 void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,

fem/integ/nonlininteg_vecconvection_pa.cpp

@@ -9,21 +9,51 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../../general/forall.hpp"
-#include "../nonlininteg.hpp"
 #include "../ceed/integrators/nlconvection/nlconvection.hpp"
+#include "./nonlininteg_vecconvection_pa.hpp" // IWYU pragma: keep
+#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
+#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
 
 namespace mfem
 {
 
+VectorConvectionNLFIntegrator::Kernels::Kernels()
+{
+   // 2D
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 2>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 3>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 4>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 5>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 5>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 6>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 7>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 8>();
+   VectorConvectionNLFIntegrator::AddSpecialization<2, 6, 8>();
+   // 3D
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 3>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 4>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 5>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 4>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 5>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 6>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 5>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 6>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 7>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 8>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 6>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 7>();
+   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 8>();
+}
+
 void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES,
                "PA Only supports Ordering::byNODES!");
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
-   ElementTransformation &T = *mesh->GetTypicalElementTransformation();
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, T);
+   ElementTransformation &Tr = *mesh->GetTypicalElementTransformation();
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Tr);
+
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
@@ -39,769 +69,124 @@ void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
       }
       return;
    }
-   dim = mesh->Dimension();
-   ne = fes.GetMesh()->GetNE();
+
+   ne = mesh->GetNE();
    nq = ir->GetNPoints();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   dim = mesh->Dimension();
+   MFEM_VERIFY(dim == 2 || dim == 3, "Dimension not supported");
+
+   const MemoryType mt = pa_mt == MemoryType::DEFAULT
+                         ? Device::GetDeviceMemoryType()
+                         : pa_mt;
+   pa_adj.SetSize(ne * nq * dim * dim, mt);
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   pa_data.SetSize(ne * nq * dim * dim, Device::GetMemoryType());
-   real_t COEFF = 1.0;
-   if (Q)
-   {
-      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient *>(Q);
-      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
-      COEFF = cQ->constant;
-   }
-   const int NE = ne;
-   const int NQ = nq;
-   auto W = ir->GetWeights().Read();
-   if (dim == 1)
-   {
-      MFEM_ABORT("dim==1 not supported!");
-   }
+   d1d = maps->ndof;
+   q1d = maps->nqpt;
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+
+   const int nq1d = q1d * q1d * (dim==3 ? q1d : 1);
+   MFEM_VERIFY(coeff.Size() == 1 || coeff.Size() == nq1d*ne, "Invalid coeff");
+   MFEM_VERIFY(ir->GetWeights().Size() == nq1d, "Invalid weights size");
+
+   const auto w_r = ir->GetWeights().Read();
+   const bool const_coeff = coeff.Size() == 1;
+
    if (dim == 2)
    {
-      auto J = Reshape(geom->J.Read(), NQ, 2, 2, NE);
-      auto G = Reshape(pa_data.Write(), NQ, 2, 2, NE);
-      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+      const int Q1D = q1d;
+      constexpr int VDIM = 2, DIM = 2;
+      const auto W = Reshape(w_r, Q1D, Q1D);
+      const auto C = const_coeff ?
+                     Reshape(coeff.Read(), 1, 1, 1) :
+                     Reshape(coeff.Read(), Q1D, Q1D, ne);
+      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, VDIM, DIM, ne);
+      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, ne);
+
+      mfem::forall_2D(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
       {
-         for (int q = 0; q < NQ; ++q)
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
          {
-            const real_t J11 = J(q, 0, 0, e);
-            const real_t J12 = J(q, 0, 1, e);
-            const real_t J21 = J(q, 1, 0, e);
-            const real_t J22 = J(q, 1, 1, e);
-            // Store wq * Q * adj(J)
-            G(q, 0, 0, e) = W[q] * COEFF * J22;  // 1,1
-            G(q, 0, 1, e) = W[q] * COEFF * -J12; // 1,2
-            G(q, 1, 0, e) = W[q] * COEFF * -J21; // 2,1
-            G(q, 1, 1, e) = W[q] * COEFF * J11;  // 2,2
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               const real_t J11 = J(qx, qy, 0, 0, e), J12 = J(qx, qy, 0, 1, e);
+               const real_t J21 = J(qx, qy, 1, 0, e), J22 = J(qx, qy, 1, 1, e);
+               // adj(J)
+               const real_t A11 = +J22, A12 = -J12;
+               const real_t A21 = -J21, A22 = +J11;
+               // Store w * coeff * adj(J)
+               const real_t w = W(qx, qy);
+               const real_t c = const_coeff ? C(0, 0, 0) : C(qx, qy, e);
+               A(0, 0, qx, qy, e) = w * c * A11;
+               A(1, 0, qx, qy, e) = w * c * A12;
+               A(0, 1, qx, qy, e) = w * c * A21;
+               A(1, 1, qx, qy, e) = w * c * A22;
+            }
          }
       });
    }
-   if (dim == 3)
+   else if (dim == 3)
    {
-      auto J = Reshape(geom->J.Read(), NQ, 3, 3, NE);
-      auto G = Reshape(pa_data.Write(), NQ, 3, 3, NE);
-      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+      const int Q1D = q1d;
+      constexpr int VDIM = 3, DIM = 3;
+      const auto W = Reshape(w_r, Q1D, Q1D, Q1D);
+      const auto C = const_coeff ?
+                     Reshape(coeff.Read(), 1, 1, 1, 1) :
+                     Reshape(coeff.Read(), Q1D, Q1D, Q1D, ne);
+      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, Q1D, VDIM, DIM, ne);
+      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, Q1D, ne);
+
+      mfem::forall_3D(ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
       {
-         for (int q = 0; q < NQ; ++q)
+         MFEM_FOREACH_THREAD_DIRECT(qz, z, Q1D)
          {
-            const real_t J11 = J(q, 0, 0, e);
-            const real_t J21 = J(q, 1, 0, e);
-            const real_t J31 = J(q, 2, 0, e);
-            const real_t J12 = J(q, 0, 1, e);
-            const real_t J22 = J(q, 1, 1, e);
-            const real_t J32 = J(q, 2, 1, e);
-            const real_t J13 = J(q, 0, 2, e);
-            const real_t J23 = J(q, 1, 2, e);
-            const real_t J33 = J(q, 2, 2, e);
-            const real_t cw = W[q] * COEFF;
-            // adj(J)
-            const real_t A11 = (J22 * J33) - (J23 * J32);
-            const real_t A12 = (J32 * J13) - (J12 * J33);
-            const real_t A13 = (J12 * J23) - (J22 * J13);
-            const real_t A21 = (J31 * J23) - (J21 * J33);
-            const real_t A22 = (J11 * J33) - (J13 * J31);
-            const real_t A23 = (J21 * J13) - (J11 * J23);
-            const real_t A31 = (J21 * J32) - (J31 * J22);
-            const real_t A32 = (J31 * J12) - (J11 * J32);
-            const real_t A33 = (J11 * J22) - (J12 * J21);
-            // Store wq * Q * adj(J)
-            G(q, 0, 0, e) = cw * A11; // 1,1
-            G(q, 0, 1, e) = cw * A12; // 1,2
-            G(q, 0, 2, e) = cw * A13; // 1,3
-            G(q, 1, 0, e) = cw * A21; // 2,1
-            G(q, 1, 1, e) = cw * A22; // 2,2
-            G(q, 1, 2, e) = cw * A23; // 2,3
-            G(q, 2, 0, e) = cw * A31; // 3,1
-            G(q, 2, 1, e) = cw * A32; // 3,2
-            G(q, 2, 2, e) = cw * A33; // 3,3
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  const real_t J11 = J(qx, qy, qz, 0, 0, e),
+                               J12 = J(qx, qy, qz, 0, 1, e),
+                               J13 = J(qx, qy, qz, 0, 2, e);
+                  const real_t J21 = J(qx, qy, qz, 1, 0, e),
+                               J22 = J(qx, qy, qz, 1, 1, e),
+                               J23 = J(qx, qy, qz, 1, 2, e);
+                  const real_t J31 = J(qx, qy, qz, 2, 0, e),
+                               J32 = J(qx, qy, qz, 2, 1, e),
+                               J33 = J(qx, qy, qz, 2, 2, e);
+                  const real_t c =
+                     const_coeff ? C(0, 0, 0, 0) : C(qx, qy, qz, e);
+                  const real_t cw = W(qx, qy, qz) * c;
+                  // adj(J)
+                  const real_t A11 = (J22 * J33) - (J23 * J32);
+                  const real_t A12 = (J32 * J13) - (J12 * J33);
+                  const real_t A13 = (J12 * J23) - (J22 * J13);
+                  const real_t A21 = (J31 * J23) - (J21 * J33);
+                  const real_t A22 = (J11 * J33) - (J13 * J31);
+                  const real_t A23 = (J21 * J13) - (J11 * J23);
+                  const real_t A31 = (J21 * J32) - (J31 * J22);
+                  const real_t A32 = (J31 * J12) - (J11 * J32);
+                  const real_t A33 = (J11 * J22) - (J12 * J21);
+                  // Store wq * coeff * adj(J)
+                  A(0, 0, qx, qy, qz, e) = cw * A11;
+                  A(1, 0, qx, qy, qz, e) = cw * A12;
+                  A(2, 0, qx, qy, qz, e) = cw * A13;
+                  A(0, 1, qx, qy, qz, e) = cw * A21;
+                  A(1, 1, qx, qy, qz, e) = cw * A22;
+                  A(2, 1, qx, qy, qz, e) = cw * A23;
+                  A(0, 2, qx, qy, qz, e) = cw * A31;
+                  A(1, 2, qx, qy, qz, e) = cw * A32;
+                  A(2, 2, qx, qy, qz, e) = cw * A33;
+               }
+            }
          }
       });
    }
-}
-
-// PA Convection NL 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAConvectionNLApply2D(const int NE,
-                                  const Array<real_t> &b,
-                                  const Array<real_t> &g,
-                                  const Array<real_t> &bt,
-                                  const Vector &q_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto Q = Reshape(q_.Read(), Q1D * Q1D, 2, 2, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, 2, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   else
    {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      real_t data[max_Q1D][max_Q1D][2];
-      real_t grad0[max_Q1D][max_Q1D][2];
-      real_t grad1[max_Q1D][max_Q1D][2];
-      real_t Z[max_Q1D][max_Q1D][2];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            data[qy][qx][0] = 0.0;
-            data[qy][qx][1] = 0.0;
-            grad0[qy][qx][0] = 0.0;
-            grad0[qy][qx][1] = 0.0;
-            grad1[qy][qx][0] = 0.0;
-            grad1[qy][qx][1] = 0.0;
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         real_t dataX[max_Q1D][2];
-         real_t gradX0[max_Q1D][2];
-         real_t gradX1[max_Q1D][2];
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            dataX[qx][0] = 0.0;
-            dataX[qx][1] = 0.0;
-            gradX0[qx][0] = 0.0;
-            gradX0[qx][1] = 0.0;
-            gradX1[qx][0] = 0.0;
-            gradX1[qx][1] = 0.0;
-         }
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            const real_t s0 = x(dx, dy, 0, e);
-            const real_t s1 = x(dx, dy, 1, e);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const real_t Bx = B(qx, dx);
-               const real_t Gx = G(qx, dx);
-               dataX[qx][0] += s0 * Bx;
-               dataX[qx][1] += s1 * Bx;
-               gradX0[qx][0] += s0 * Gx;
-               gradX0[qx][1] += s0 * Bx;
-               gradX1[qx][0] += s1 * Gx;
-               gradX1[qx][1] += s1 * Bx;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            const real_t By = B(qy, dy);
-            const real_t Gy = G(qy, dy);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               data[qy][qx][0] += dataX[qx][0] * By;
-               data[qy][qx][1] += dataX[qx][1] * By;
-               grad0[qy][qx][0] += gradX0[qx][0] * By;
-               grad0[qy][qx][1] += gradX0[qx][1] * Gy;
-               grad1[qy][qx][0] += gradX1[qx][0] * By;
-               grad1[qy][qx][1] += gradX1[qx][1] * Gy;
-            }
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const int q = qx + qy * Q1D;
-            const real_t u1 = data[qy][qx][0];
-            const real_t u2 = data[qy][qx][1];
-            const real_t grad00 = grad0[qy][qx][0];
-            const real_t grad01 = grad0[qy][qx][1];
-            const real_t grad10 = grad1[qy][qx][0];
-            const real_t grad11 = grad1[qy][qx][1];
-            const real_t Dxu1 = grad00 * Q(q, 0, 0, e) + grad01 * Q(q, 1, 0, e);
-            const real_t Dyu1 = grad00 * Q(q, 0, 1, e) + grad01 * Q(q, 1, 1, e);
-            const real_t Dxu2 = grad10 * Q(q, 0, 0, e) + grad11 * Q(q, 1, 0, e);
-            const real_t Dyu2 = grad10 * Q(q, 0, 1, e) + grad11 * Q(q, 1, 1, e);
-            Z[qy][qx][0] = u1 * Dxu1 + u2 * Dyu1;
-            Z[qy][qx][1] = u1 * Dxu2 + u2 * Dyu2;
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         real_t Y[max_D1D][2];
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            Y[dx][0] = 0.0;
-            Y[dx][1] = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const real_t Btx = Bt(dx, qx);
-               Y[dx][0] += Btx * Z[qy][qx][0];
-               Y[dx][1] += Btx * Z[qy][qx][1];
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const real_t Bty = Bt(dy, qy);
-               y(dx, dy, 0, e) += Bty * Y[dx][0];
-               y(dx, dy, 1, e) += Bty * Y[dx][1];
-            }
-         }
-      }
-   });
-}
-
-// PA Convection NL 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAConvectionNLApply3D(const int NE,
-                                  const Array<real_t> &b,
-                                  const Array<real_t> &g,
-                                  const Array<real_t> &bt,
-                                  const Vector &q_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
-{
-   constexpr int VDIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto Q = Reshape(q_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int VDIM = 3;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      real_t data[max_Q1D][max_Q1D][max_Q1D][VDIM];
-      real_t grad0[max_Q1D][max_Q1D][max_Q1D][VDIM];
-      real_t grad1[max_Q1D][max_Q1D][max_Q1D][VDIM];
-      real_t grad2[max_Q1D][max_Q1D][max_Q1D][VDIM];
-      real_t Z[max_Q1D][max_Q1D][max_Q1D][VDIM];
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               data[qz][qy][qx][0] = 0.0;
-               data[qz][qy][qx][1] = 0.0;
-               data[qz][qy][qx][2] = 0.0;
-
-               grad0[qz][qy][qx][0] = 0.0;
-               grad0[qz][qy][qx][1] = 0.0;
-               grad0[qz][qy][qx][2] = 0.0;
-
-               grad1[qz][qy][qx][0] = 0.0;
-               grad1[qz][qy][qx][1] = 0.0;
-               grad1[qz][qy][qx][2] = 0.0;
-
-               grad2[qz][qy][qx][0] = 0.0;
-               grad2[qz][qy][qx][1] = 0.0;
-               grad2[qz][qy][qx][2] = 0.0;
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         real_t dataXY[max_Q1D][max_Q1D][VDIM];
-         real_t gradXY0[max_Q1D][max_Q1D][VDIM];
-         real_t gradXY1[max_Q1D][max_Q1D][VDIM];
-         real_t gradXY2[max_Q1D][max_Q1D][VDIM];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               dataXY[qy][qx][0] = 0.0;
-               dataXY[qy][qx][1] = 0.0;
-               dataXY[qy][qx][2] = 0.0;
-
-               gradXY0[qy][qx][0] = 0.0;
-               gradXY0[qy][qx][1] = 0.0;
-               gradXY0[qy][qx][2] = 0.0;
-
-               gradXY1[qy][qx][0] = 0.0;
-               gradXY1[qy][qx][1] = 0.0;
-               gradXY1[qy][qx][2] = 0.0;
-
-               gradXY2[qy][qx][0] = 0.0;
-               gradXY2[qy][qx][1] = 0.0;
-               gradXY2[qy][qx][2] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            real_t dataX[max_Q1D][VDIM];
-            real_t gradX0[max_Q1D][VDIM];
-            real_t gradX1[max_Q1D][VDIM];
-            real_t gradX2[max_Q1D][VDIM];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               dataX[qx][0] = 0.0;
-               dataX[qx][1] = 0.0;
-               dataX[qx][2] = 0.0;
-
-               gradX0[qx][0] = 0.0;
-               gradX0[qx][1] = 0.0;
-               gradX0[qx][2] = 0.0;
-
-               gradX1[qx][0] = 0.0;
-               gradX1[qx][1] = 0.0;
-               gradX1[qx][2] = 0.0;
-
-               gradX2[qx][0] = 0.0;
-               gradX2[qx][1] = 0.0;
-               gradX2[qx][2] = 0.0;
-            }
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const real_t s0 = x(dx, dy, dz, 0, e);
-               const real_t s1 = x(dx, dy, dz, 1, e);
-               const real_t s2 = x(dx, dy, dz, 2, e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const real_t Bx = B(qx, dx);
-                  const real_t Gx = G(qx, dx);
-
-                  dataX[qx][0] += s0 * Bx;
-                  dataX[qx][1] += s1 * Bx;
-                  dataX[qx][2] += s2 * Bx;
-
-                  gradX0[qx][0] += s0 * Gx;
-                  gradX0[qx][1] += s0 * Bx;
-                  gradX0[qx][2] += s0 * Bx;
-
-                  gradX1[qx][0] += s1 * Gx;
-                  gradX1[qx][1] += s1 * Bx;
-                  gradX1[qx][2] += s1 * Bx;
-
-                  gradX2[qx][0] += s2 * Gx;
-                  gradX2[qx][1] += s2 * Bx;
-                  gradX2[qx][2] += s2 * Bx;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const real_t By = B(qy, dy);
-               const real_t Gy = G(qy, dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  dataXY[qy][qx][0] += dataX[qx][0] * By;
-                  dataXY[qy][qx][1] += dataX[qx][1] * By;
-                  dataXY[qy][qx][2] += dataX[qx][2] * By;
-
-                  gradXY0[qy][qx][0] += gradX0[qx][0] * By;
-                  gradXY0[qy][qx][1] += gradX0[qx][1] * Gy;
-                  gradXY0[qy][qx][2] += gradX0[qx][2] * By;
-
-                  gradXY1[qy][qx][0] += gradX1[qx][0] * By;
-                  gradXY1[qy][qx][1] += gradX1[qx][1] * Gy;
-                  gradXY1[qy][qx][2] += gradX1[qx][2] * By;
-
-                  gradXY2[qy][qx][0] += gradX2[qx][0] * By;
-                  gradXY2[qy][qx][1] += gradX2[qx][1] * Gy;
-                  gradXY2[qy][qx][2] += gradX2[qx][2] * By;
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            const real_t Bz = B(qz, dz);
-            const real_t Gz = G(qz, dz);
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  data[qz][qy][qx][0] += dataXY[qy][qx][0] * Bz;
-                  data[qz][qy][qx][1] += dataXY[qy][qx][1] * Bz;
-                  data[qz][qy][qx][2] += dataXY[qy][qx][2] * Bz;
-
-                  grad0[qz][qy][qx][0] += gradXY0[qy][qx][0] * Bz;
-                  grad0[qz][qy][qx][1] += gradXY0[qy][qx][1] * Bz;
-                  grad0[qz][qy][qx][2] += gradXY0[qy][qx][2] * Gz;
-
-                  grad1[qz][qy][qx][0] += gradXY1[qy][qx][0] * Bz;
-                  grad1[qz][qy][qx][1] += gradXY1[qy][qx][1] * Bz;
-                  grad1[qz][qy][qx][2] += gradXY1[qy][qx][2] * Gz;
-
-                  grad2[qz][qy][qx][0] += gradXY2[qy][qx][0] * Bz;
-                  grad2[qz][qy][qx][1] += gradXY2[qy][qx][1] * Bz;
-                  grad2[qz][qy][qx][2] += gradXY2[qy][qx][2] * Gz;
-               }
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const int q = qx + Q1D * (qy + qz * Q1D);
-
-               const real_t u1 = data[qz][qy][qx][0];
-               const real_t u2 = data[qz][qy][qx][1];
-               const real_t u3 = data[qz][qy][qx][2];
-
-               const real_t grad00 = grad0[qz][qy][qx][0];
-               const real_t grad01 = grad0[qz][qy][qx][1];
-               const real_t grad02 = grad0[qz][qy][qx][2];
-
-               const real_t grad10 = grad1[qz][qy][qx][0];
-               const real_t grad11 = grad1[qz][qy][qx][1];
-               const real_t grad12 = grad1[qz][qy][qx][2];
-
-               const real_t grad20 = grad2[qz][qy][qx][0];
-               const real_t grad21 = grad2[qz][qy][qx][1];
-               const real_t grad22 = grad2[qz][qy][qx][2];
-
-               const real_t Dxu1 = grad00 * Q(q, 0, 0, e)
-                                   + grad01 * Q(q, 1, 0, e)
-                                   + grad02 * Q(q, 2, 0, e);
-               const real_t Dyu1 = grad00 * Q(q, 0, 1, e)
-                                   + grad01 * Q(q, 1, 1, e)
-                                   + grad02 * Q(q, 2, 1, e);
-               const real_t Dzu1 = grad00 * Q(q, 0, 2, e)
-                                   + grad01 * Q(q, 1, 2, e)
-                                   + grad02 * Q(q, 2, 2, e);
-
-               const real_t Dxu2 = grad10 * Q(q, 0, 0, e)
-                                   + grad11 * Q(q, 1, 0, e)
-                                   + grad12 * Q(q, 2, 0, e);
-               const real_t Dyu2 = grad10 * Q(q, 0, 1, e)
-                                   + grad11 * Q(q, 1, 1, e)
-                                   + grad12 * Q(q, 2, 1, e);
-               const real_t Dzu2 = grad10 * Q(q, 0, 2, e)
-                                   + grad11 * Q(q, 1, 2, e)
-                                   + grad12 * Q(q, 2, 2, e);
-
-               const real_t Dxu3 = grad20 * Q(q, 0, 0, e)
-                                   + grad21 * Q(q, 1, 0, e)
-                                   + grad22 * Q(q, 2, 0, e);
-               const real_t Dyu3 = grad20 * Q(q, 0, 1, e)
-                                   + grad21 * Q(q, 1, 1, e)
-                                   + grad22 * Q(q, 2, 1, e);
-               const real_t Dzu3 = grad20 * Q(q, 0, 2, e)
-                                   + grad21 * Q(q, 1, 2, e)
-                                   + grad22 * Q(q, 2, 2, e);
-
-               Z[qz][qy][qx][0] = u1 * Dxu1 + u2 * Dyu1 + u3 * Dzu1;
-               Z[qz][qy][qx][1] = u1 * Dxu2 + u2 * Dyu2 + u3 * Dzu2;
-               Z[qz][qy][qx][2] = u1 * Dxu3 + u2 * Dyu3 + u3 * Dzu3;
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         real_t opXY[max_D1D][max_D1D][VDIM];
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               opXY[dy][dx][0] = 0.0;
-               opXY[dy][dx][1] = 0.0;
-               opXY[dy][dx][2] = 0.0;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            real_t opX[max_D1D][VDIM];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               opX[dx][0] = 0.0;
-               opX[dx][1] = 0.0;
-               opX[dx][2] = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const real_t Btx = Bt(dx, qx);
-                  opX[dx][0] += Btx * Z[qz][qy][qx][0];
-                  opX[dx][1] += Btx * Z[qz][qy][qx][1];
-                  opX[dx][2] += Btx * Z[qz][qy][qx][2];
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const real_t Bty = Bt(dy, qy);
-                  opXY[dy][dx][0] += Bty * opX[dx][0];
-                  opXY[dy][dx][1] += Bty * opX[dx][1];
-                  opXY[dy][dx][2] += Bty * opX[dx][2];
-               }
-            }
-         }
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const real_t Btz = Bt(dz, qz);
-                  y(dx, dy, dz, 0, e) += Btz * opXY[dy][dx][0];
-                  y(dx, dy, dz, 1, e) += Btz * opXY[dy][dx][1];
-                  y(dx, dy, dz, 2, e) += Btz * opXY[dy][dx][2];
-               }
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, int T_MAX_D1D = 0, int T_MAX_Q1D = 0>
-static void SmemPAConvectionNLApply3D(const int NE,
-                                      const Array<real_t> &b_,
-                                      const Array<real_t> &g_,
-                                      const Vector &d_,
-                                      const Vector &x_,
-                                      Vector &y_,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
-{
-   constexpr int VDIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
-      MFEM_SHARED real_t BG[2][MQ1 * MD1];
-      real_t(*B)[MD1] = (real_t(*)[MD1])(BG + 0);
-      real_t(*G)[MD1] = (real_t(*)[MD1])(BG + 1);
-      real_t(*Bt)[MQ1] = (real_t(*)[MQ1])(BG + 0);
-      MFEM_SHARED real_t U[2][MQ1][MQ1][MQ1];
-      MFEM_SHARED real_t sm0[3][MQ1 * MQ1 * MQ1];
-      MFEM_SHARED real_t sm1[3][MQ1 * MQ1 * MQ1];
-      real_t(*DDQ0)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 0);
-      real_t(*DDQ1)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 1);
-      real_t(*X)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 2);
-      real_t(*DQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 0);
-      real_t(*DQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 1);
-      real_t(*DQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 2);
-      real_t(*QQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 0);
-      real_t(*QQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 1);
-      real_t(*QQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 2);
-      real_t(*QQD0)[MQ1][MD1] = (real_t(*)[MQ1][MD1])(sm1 + 0);
-      real_t(*QDD0)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 0);
-      MFEM_SHARED real_t Z[MQ1][MQ1][MQ1];
-
-      for (int cy = 0; cy < VDIM; ++cy)
-      {
-         if (tidz == 0)
-         {
-            MFEM_FOREACH_THREAD(q, x, Q1D)
-            {
-               MFEM_FOREACH_THREAD(d, y, D1D)
-               {
-                  B[q][d] = b(q, d);
-                  G[q][d] = g(q, d);
-               }
-            }
-         }
-         MFEM_FOREACH_THREAD(qz, z, Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD(qx, x, Q1D) { Z[qz][qy][qx] = 0.0; }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int c = 0; c < VDIM; ++c)
-         {
-            MFEM_FOREACH_THREAD(dz, z, D1D)
-            {
-               MFEM_FOREACH_THREAD(dy, y, D1D)
-               {
-                  MFEM_FOREACH_THREAD(dx, x, D1D)
-                  {
-                     X[dz][dy][dx] = x(dx, dy, dz, cy, e);
-                     U[0][dz][dy][dx] = x(dx, dy, dz, c, e);
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            MFEM_FOREACH_THREAD(dz, z, D1D)
-            {
-               MFEM_FOREACH_THREAD(dy, y, D1D)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, Q1D)
-                  {
-                     real_t u = 0.0;
-                     real_t v = 0.0;
-                     real_t z = 0.0;
-                     for (int dx = 0; dx < D1D; ++dx)
-                     {
-                        const real_t coord = X[dz][dy][dx];
-                        const real_t value = U[0][dz][dy][dx];
-                        u += coord * B[qx][dx];
-                        v += coord * G[qx][dx];
-                        z += value * B[qx][dx];
-                     }
-                     DDQ0[dz][dy][qx] = u;
-                     DDQ1[dz][dy][qx] = v;
-                     U[1][dz][dy][qx] = z;
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            MFEM_FOREACH_THREAD(dz, z, D1D)
-            {
-               MFEM_FOREACH_THREAD(qy, y, Q1D)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, Q1D)
-                  {
-                     real_t u = 0.0;
-                     real_t v = 0.0;
-                     real_t w = 0.0;
-                     real_t z = 0.0;
-                     for (int dy = 0; dy < D1D; ++dy)
-                     {
-                        u += DDQ1[dz][dy][qx] * B[qy][dy];
-                        v += DDQ0[dz][dy][qx] * G[qy][dy];
-                        w += DDQ0[dz][dy][qx] * B[qy][dy];
-                        z += U[1][dz][dy][qx] * B[qy][dy];
-                     }
-                     DQQ0[dz][qy][qx] = u;
-                     DQQ1[dz][qy][qx] = v;
-                     DQQ2[dz][qy][qx] = w;
-                     U[0][dz][qy][qx] = z;
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            MFEM_FOREACH_THREAD(qz, z, Q1D)
-            {
-               MFEM_FOREACH_THREAD(qy, y, Q1D)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, Q1D)
-                  {
-                     real_t u = 0.0;
-                     real_t v = 0.0;
-                     real_t w = 0.0;
-                     real_t z = 0.0;
-                     for (int dz = 0; dz < D1D; ++dz)
-                     {
-                        u += DQQ0[dz][qy][qx] * B[qz][dz];
-                        v += DQQ1[dz][qy][qx] * B[qz][dz];
-                        w += DQQ2[dz][qy][qx] * G[qz][dz];
-                        z += U[0][dz][qy][qx] * B[qz][dz];
-                     }
-                     QQQ0[qz][qy][qx] = u;
-                     QQQ1[qz][qy][qx] = v;
-                     QQQ2[qz][qy][qx] = w;
-                     U[1][qz][qy][qx] = z;
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            MFEM_FOREACH_THREAD(qz, z, Q1D)
-            {
-               MFEM_FOREACH_THREAD(qy, y, Q1D)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, Q1D)
-                  {
-                     const int q = qx + (qy + qz * Q1D) * Q1D;
-                     const real_t z = U[1][qz][qy][qx];
-                     const real_t gX = QQQ0[qz][qy][qx];
-                     const real_t gY = QQQ1[qz][qy][qx];
-                     const real_t gZ = QQQ2[qz][qy][qx];
-                     const real_t d = gX * D(q, 0, c, e) + gY * D(q, 1, c, e)
-                                      + gZ * D(q, 2, c, e);
-                     Z[qz][qy][qx] += z * d;
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-         } // for each conv component
-         if (tidz == 0)
-         {
-            MFEM_FOREACH_THREAD(d, y, D1D)
-            {
-               MFEM_FOREACH_THREAD(q, x, Q1D) { Bt[d][q] = b(q, d); }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qz, z, Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD(dx, x, D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     u += Z[qz][qy][qx] * Bt[dx][qx];
-                  }
-                  QQD0[qz][qy][dx] = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qz, z, Q1D)
-         {
-            MFEM_FOREACH_THREAD(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD(dx, x, D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     u += QQD0[qz][qy][dx] * Bt[dy][qy];
-                  }
-                  QDD0[qz][dy][dx] = u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dz, z, D1D)
-         {
-            MFEM_FOREACH_THREAD(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD(dx, x, D1D)
-               {
-                  real_t u = 0.0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u += QDD0[qz][dy][dx] * Bt[dz][qz];
-                  }
-                  Y(dx, dy, dz, cy, e) += u;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
+      MFEM_ABORT("dim " << dim << " not supported!");
+   }
 }
 
 void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
@@ -812,26 +197,13 @@ void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
    else
    {
-      const int NE = ne;
-      const int D1D = maps->ndof;
-      const int Q1D = maps->nqpt;
-      const Vector &QV = pa_data;
-      const Array<real_t> &B = maps->B;
-      const Array<real_t> &G = maps->G;
-      const Array<real_t> &Bt = maps->Bt;
-      if (dim == 2)
-      {
-         return PAConvectionNLApply2D(NE, B, G, Bt, QV, x, y, D1D, Q1D);
-      }
-      if (dim == 3)
-      {
-         constexpr int T_MAX_D1D = 8;
-         constexpr int T_MAX_Q1D = 8;
-         MFEM_VERIFY(D1D <= T_MAX_D1D && Q1D <= T_MAX_Q1D, "Not yet implemented!");
-         return SmemPAConvectionNLApply3D<0, 0, T_MAX_D1D, T_MAX_Q1D>
-                (NE, B, G, QV, x, y, D1D, Q1D);
-      }
-      MFEM_ABORT("Not yet implemented!");
+      AddMultPAKernels::Run(dim, d1d, q1d, ne,
+                            maps->B.Read(),
+                            maps->G.Read(),
+                            pa_adj.Read(),
+                            x.Read(),
+                            y.ReadWrite(),
+                            d1d, q1d);
    }
 }
 

fem/integ/nonlininteg_vecconvection_pa.hpp

@@ -0,0 +1,209 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../config/config.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../kernels.hpp"
+#include "../nonlininteg.hpp"
+
+namespace mfem
+{
+
+/// \cond DO_NOT_DOCUMENT
+
+namespace internal
+{
+
+// PA Convection NL 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLApply2D(const int NE,
+                                      const real_t *b,
+                                      const real_t *g,
+                                      const real_t *a,
+                                      const real_t *x,
+                                      real_t *y,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
+{
+   static constexpr int VDIM = 2, DIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
+   const auto X = Reshape(x, D1D, D1D, VDIM, NE);
+   auto Y = Reshape(y, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
+      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
+      kernels::internal::v_regs2d_t<VDIM, MQ1> s0, s1;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
+
+      kernels::internal::LoadDofs2d(e, D1D, X, r0);
+      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
+      kernels::internal::LoadDofs2d(e, D1D, X, g0);
+      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
+
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            const future::tensor<real_t, 2> U =
+            {
+               r1[0][qy][qx], r1[1][qy][qx]
+            };
+            const future::tensor<real_t, 2,2> gradU = {{
+                  {g1[0][0][qy][qx], g1[1][0][qy][qx]},
+                  {g1[0][1][qy][qx], g1[1][1][qy][qx]},
+               }
+            };
+            const future::tensor<real_t, 2,2> Q = {{
+                  {A(0,0,qx,qy,e), A(1,0,qx,qy,e)},
+                  {A(0,1,qx,qy,e), A(1,1,qx,qy,e)},
+               }
+            };
+            const future::tensor<real_t, 2> conv = transpose(gradU) * (Q * U);
+            s0[0][qy][qx] = conv[0];
+            s0[1][qy][qx] = conv[1];
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, s0, s1);
+      kernels::internal::WriteDofs2d(e, D1D, s1, Y);
+   });
+}
+
+// PA Convection NL 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLApply3D(const int NE,
+                                      const real_t *b,
+                                      const real_t *g,
+                                      const real_t *a,
+                                      const real_t *x,
+                                      real_t *y,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
+{
+   static constexpr int VDIM = 3, DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto X = Reshape(x, D1D, D1D, D1D, VDIM, NE);
+   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
+      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
+      kernels::internal::v_regs3d_t<VDIM, MQ1> s0, s1;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
+
+      kernels::internal::LoadDofs3d(e, D1D, X, r0);
+      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
+      kernels::internal::LoadDofs3d(e, D1D, X, g0);
+      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
+
+      for (int qz = 0; qz < Q1D; qz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               const future::tensor<real_t, 3> U =
+               {
+                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
+               };
+               const future::tensor<real_t, 3,3> gradU = {{
+                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
+                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
+                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
+                  }
+               };
+               const future::tensor<real_t, 3,3> Q = {{
+                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
+                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
+                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
+                  }
+               };
+               const future::tensor<real_t, 3> conv = transpose(gradU) * (Q * U);
+               s0[0][qz][qy][qx] = conv[0];
+               s0[1][qz][qy][qx] = conv[1];
+               s0[2][qz][qy][qx] = conv[2];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, s0, s1);
+      kernels::internal::WriteDofs3d(e, D1D, s1, Y);
+   });
+}
+
+} // namespace internal
+
+template<int DIM, int T_D1D, int T_Q1D>
+VectorConvectionNLFIntegrator::AddMultPAType
+VectorConvectionNLFIntegrator::AddMultPAKernels::Kernel()
+{
+   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
+   if constexpr (DIM == 2)
+   {
+      return internal::SmemPAConvectionNLApply2D<T_D1D, T_Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      return internal::SmemPAConvectionNLApply3D<T_D1D, T_Q1D>;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+inline VectorConvectionNLFIntegrator::AddMultPAType
+VectorConvectionNLFIntegrator::AddMultPAKernels::Fallback
+(int dim, int d1d, int q1d)
+{
+   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   if (dim == 2)
+   {
+      return internal::SmemPAConvectionNLApply2D<>;
+   }
+   else if (dim == 3)
+   {
+      return internal::SmemPAConvectionNLApply3D<>;
+   }
+   MFEM_ABORT("Unsupported kernel");
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
+} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.cpp

@@ -0,0 +1,50 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../ceed/interface/util.hpp"
+#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
+
+namespace mfem
+{
+
+void VectorConvectionNLFIntegrator::AssembleGradDiagonalPA(Vector &de) const
+{
+   MFEM_VERIFY(!DeviceCanUseCeed(),
+               "VectorConvectionNLFIntegrator PA gradients are not supported "
+               "with the libCEED backend");
+
+   if (dim == 2)
+   {
+      GradDiagPA2D::Run(d1d, q1d, ne,
+                        maps->B.Read(),
+                        maps->G.Read(),
+                        pa_adj.Read(),
+                        pa_u.Read(),
+                        de.ReadWrite(),
+                        d1d, q1d);
+   }
+   else if (dim == 3)
+   {
+      GradDiagPA3D::Run(d1d, q1d, ne,
+                        maps->B.Read(),
+                        maps->G.Read(),
+                        pa_adj.Read(),
+                        pa_u.Read(),
+                        de.ReadWrite(),
+                        d1d, q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+   }
+}
+
+} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.hpp

@@ -0,0 +1,302 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../config/config.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../kernels.hpp"
+#include "../nonlininteg.hpp"
+
+namespace mfem
+{
+
+/// \cond DO_NOT_DOCUMENT
+
+namespace internal
+{
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLGradDiagonal2D(const int NE,
+                                             const real_t *b,
+                                             const real_t *g,
+                                             const real_t *a,
+                                             const real_t *u,
+                                             real_t *de,
+                                             const int d1d,
+                                             const int q1d)
+{
+   static constexpr int VDIM = 2, DIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
+   const auto U = Reshape(u, D1D, D1D, VDIM, NE);
+   auto D = Reshape(de, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t sM[3][MQ1][MQ1], sQ[3][MQ1][MQ1];
+      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
+      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
+
+      kernels::internal::LoadDofs2d(e, D1D, U, r0);
+      kernels::internal::Eval2d(D1D, Q1D, sM[0], sB, r0, r1);
+
+      kernels::internal::LoadDofs2d(e, D1D, U, g0);
+      kernels::internal::Grad2d(D1D, Q1D, sM[0], sB, sG, g0, g1);
+
+      for (int v = 0; v < VDIM; ++v)
+      {
+         future::tensor<real_t, VDIM> e_v = {};
+         e_v[v] = real_t(1);
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               const future::tensor<real_t, VDIM> u_val =
+               {
+                  r1[0][qy][qx], r1[1][qy][qx]
+               };
+               const future::tensor<real_t, VDIM, DIM> Q_adj =
+               {
+                  {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
+                     { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
+                  }
+               };
+               const future::tensor<real_t, VDIM, DIM> grad_U =
+               {
+                  {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
+                     { g1[0][1][qy][qx], g1[1][1][qy][qx] }
+                  }
+               };
+               const auto one = Q_adj * u_val;
+               const auto two = transpose(grad_U) * (Q_adj * e_v);
+               sQ[0][qx][qy] = one[0];
+               sQ[1][qx][qy] = one[1];
+               sQ[2][qx][qy] = two[v];
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               real_t s[3] = {};
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const real_t By = sB[dy][qy], Gy = sG[dy][qy];
+                  s[0] += By * By * sQ[0][qx][qy];
+                  s[1] += Gy * By * sQ[1][qx][qy];
+                  s[2] += By * By * sQ[2][qx][qy];
+               }
+               sM[0][qx][dy] = s[0];
+               sM[1][qx][dy] = s[1];
+               sM[2][qx][dy] = s[2];
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+            {
+               real_t d = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
+                  d += Gx * Bx * sM[0][qx][dy] +
+                       Bx * Bx * sM[1][qx][dy] +
+                       Bx * Bx * sM[2][qx][dy];
+               }
+               D(dx, dy, v, e) += d;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLGradDiagonal3D(const int NE,
+                                             const real_t *b,
+                                             const real_t *g,
+                                             const real_t *a,
+                                             const real_t *u,
+                                             real_t *de,
+                                             const int d1d,
+                                             const int q1d)
+{
+   static constexpr int VDIM = 3, DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
+   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, NE);
+   auto D = Reshape(de, D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t sM[4][MQ1][MQ1], sQ[4][MQ1][MQ1];
+      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
+      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
+
+      kernels::internal::LoadDofs3d(e, D1D, U, r0);
+      kernels::internal::Eval3d(D1D, Q1D, sM[0], sB, r0, r1);
+
+      kernels::internal::LoadDofs3d(e, D1D, U, g0);
+      kernels::internal::Grad3d(D1D, Q1D, sM[0], sB, sG, g0, g1);
+
+      for (int v = 0; v < VDIM; ++v)
+      {
+         future::tensor<real_t, VDIM> e_v = {};
+         e_v[v] = real_t(1);
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  real_t s[4] = {};
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     const future::tensor<real_t, VDIM> u_val =
+                     {
+                        r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
+                     };
+                     const future::tensor<real_t, VDIM, DIM> Q_adj = {{
+                           {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
+                           {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
+                           {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
+                        }
+                     };
+                     const future::tensor<real_t, VDIM, DIM> grad_U = {{
+                           {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
+                           {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
+                           {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
+                        }
+                     };
+                     const auto one = Q_adj * u_val;
+                     const auto two = transpose(grad_U) * (Q_adj * e_v);
+
+                     const real_t Bz = sB[dz][qz], Gz = sG[dz][qz];
+                     s[0] += one[0] * Bz * Bz;
+                     s[1] += one[1] * Bz * Bz;
+                     s[2] += one[2] * Bz * Gz;
+                     s[3] += two[v] * Bz * Bz;
+                  }
+                  sQ[0][qx][qy] = s[0];
+                  sQ[1][qx][qy] = s[1];
+                  sQ[2][qx][qy] = s[2];
+                  sQ[3][qx][qy] = s[3];
+               }
+            }
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  real_t s[4] = {};
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     const real_t By = sB[dy][qy], Gy = sG[dy][qy];
+                     s[0] += By * By * sQ[0][qx][qy];
+                     s[1] += Gy * By * sQ[1][qx][qy];
+                     s[2] += By * By * sQ[2][qx][qy];
+                     s[3] += By * By * sQ[3][qx][qy];
+                  }
+                  sM[0][dy][qx] = s[0];
+                  sM[1][dy][qx] = s[1];
+                  sM[2][dy][qx] = s[2];
+                  sM[3][dy][qx] = s[3];
+               }
+            }
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+               {
+                  real_t d = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
+                     d += Gx * Bx * sM[0][dy][qx];
+                     d += Bx * Bx * sM[1][dy][qx];
+                     d += Bx * Bx * sM[2][dy][qx];
+                     d += Bx * Bx * sM[3][dy][qx];
+                  }
+                  D(dx, dy, dz, v, e) += d;
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   });
+}
+
+} // namespace internal
+
+template<int T_D1D, int T_Q1D>
+VectorConvectionNLFIntegrator::GradDiagPAType
+VectorConvectionNLFIntegrator::GradDiagPA2D::Kernel()
+{
+   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
+   return internal::SmemPAConvectionNLGradDiagonal2D<T_D1D, T_Q1D>;
+}
+
+inline VectorConvectionNLFIntegrator::GradDiagPAType
+VectorConvectionNLFIntegrator::GradDiagPA2D::Fallback(int d1d, int q1d)
+{
+   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   return internal::SmemPAConvectionNLGradDiagonal2D<>;
+}
+
+template<int T_D1D, int T_Q1D>
+VectorConvectionNLFIntegrator::GradDiagPAType
+VectorConvectionNLFIntegrator::GradDiagPA3D::Kernel()
+{
+   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
+   return internal::SmemPAConvectionNLGradDiagonal3D<T_D1D, T_Q1D>;
+}
+
+inline VectorConvectionNLFIntegrator::GradDiagPAType
+VectorConvectionNLFIntegrator::GradDiagPA3D::Fallback(int d1d, int q1d)
+{
+   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   return internal::SmemPAConvectionNLGradDiagonal3D<>;
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
+} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.cpp

@@ -0,0 +1,64 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../ceed/interface/util.hpp"
+#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
+
+namespace mfem
+{
+
+void VectorConvectionNLFIntegrator::AssembleGradPA(
+   const Vector &u, const FiniteElementSpace &fes)
+{
+   MFEM_VERIFY(!DeviceCanUseCeed(),
+               "VectorConvectionNLFIntegrator PA gradients are not supported "
+               "with the libCEED backend");
+
+   this->pa_u = u;
+   AssemblePA(fes);
+}
+
+void VectorConvectionNLFIntegrator::AddMultGradPA(const Vector &x,
+                                                  Vector &y) const
+{
+   MFEM_VERIFY(!DeviceCanUseCeed(),
+               "VectorConvectionNLFIntegrator PA gradients are not supported "
+               "with the libCEED backend");
+
+   if (dim == 2)
+   {
+      AddMultGradPA2D::Run(d1d, q1d, ne,
+                           maps->B.Read(),
+                           maps->G.Read(),
+                           pa_adj.Read(),
+                           pa_u.Read(),
+                           x.Read(),
+                           y.ReadWrite(),
+                           d1d, q1d);
+   }
+   else if (dim == 3)
+   {
+      AddMultGradPA3D::Run(d1d, q1d, ne,
+                           maps->B.Read(),
+                           maps->G.Read(),
+                           pa_adj.Read(),
+                           pa_u.Read(),
+                           x.Read(),
+                           y.ReadWrite(),
+                           d1d, q1d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension");
+   }
+}
+
+} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.hpp

@@ -0,0 +1,257 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "../../config/config.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../kernels.hpp"
+#include "../nonlininteg.hpp"
+
+namespace mfem
+{
+
+/// \cond DO_NOT_DOCUMENT
+
+namespace internal
+{
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLGradApply2D(const int ne,
+                                          const real_t *b,
+                                          const real_t *g,
+                                          const real_t *a,
+                                          const real_t *u,
+                                          const real_t *du,
+                                          real_t *y,
+                                          const int d1d,
+                                          const int q1d)
+{
+   static constexpr int VDIM = 2, DIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, ne);
+   const auto U = Reshape(u, D1D, D1D, VDIM, ne);
+   const auto dU = Reshape(du, D1D, D1D, VDIM, ne);
+   auto Y = Reshape(y, D1D, D1D, VDIM, ne);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1, g2;
+      kernels::internal::v_regs2d_t<DIM, MQ1> r0, r1, r2;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
+
+      kernels::internal::LoadDofs2d(e, D1D, dU, g0);
+      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
+
+      kernels::internal::LoadDofs2d(e, D1D, U, r0);
+      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r2); // u value
+
+      kernels::internal::LoadDofs2d(e, D1D, dU, r0);
+      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // δu value
+
+      kernels::internal::LoadDofs2d(e, D1D, U, g0);
+      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
+
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            // First part of the Jacobian: u·∇δu
+            const future::tensor<real_t, DIM> u_val =
+            {
+               r2[0][qy][qx], r2[1][qy][qx]
+            };
+            const future::tensor<real_t, VDIM, DIM> Q_adj =
+            {
+               {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
+                  { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
+               }
+            };
+            const future::tensor<real_t, VDIM, DIM> grad_dU =
+            {
+               {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
+                  { g1[0][1][qy][qx], g1[1][1][qy][qx] }
+               }
+            };
+            const auto one = transpose(grad_dU) * (Q_adj * u_val);
+
+            // Second part of the Jacobian: δu·∇u
+            const future::tensor<real_t, DIM> du_val =
+            {
+               r1[0][qy][qx], r1[1][qy][qx]
+            };
+            const future::tensor<real_t, VDIM, DIM> grad_U =
+            {
+               {  { g2[0][0][qy][qx], g2[1][0][qy][qx] },
+                  { g2[0][1][qy][qx], g2[1][1][qy][qx] }
+               }
+            };
+            const auto two = transpose(grad_U) * (Q_adj * du_val);
+
+            // u⋅∇δu + δu⋅∇u
+            r0[0][qy][qx] = one[0] + two[0];
+            r0[1][qy][qx] = one[1] + two[1];
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, r0, r1);
+      kernels::internal::WriteDofs2d(e, D1D, r1, Y);
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAConvectionNLGradApply3D(const int ne,
+                                          const real_t *b,
+                                          const real_t *g,
+                                          const real_t *a,
+                                          const real_t *u,
+                                          const real_t *du,
+                                          real_t *y,
+                                          const int d1d,
+                                          const int q1d)
+{
+   static constexpr int VDIM = 3, DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, ne);
+   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, ne);
+   const auto dU = Reshape(du, D1D, D1D, D1D, VDIM, ne);
+   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, ne);
+
+   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
+
+      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1, r2;
+      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1, g2;
+
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
+
+      kernels::internal::LoadDofs3d(e, D1D, dU, g0);
+      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
+
+      kernels::internal::LoadDofs3d(e, D1D, U, r0);
+      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r2); // u value
+
+      kernels::internal::LoadDofs3d(e, D1D, dU, r0);
+      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // δu value
+
+      kernels::internal::LoadDofs3d(e, D1D, U, g0);
+      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
+
+      for (int qz = 0; qz < Q1D; qz++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               // First part of the Jacobian: u·∇δu
+               const future::tensor<real_t, DIM> u_val =
+               {
+                  r2[0][qz][qy][qx],
+                  r2[1][qz][qy][qx],
+                  r2[2][qz][qy][qx]
+               };
+               const future::tensor<real_t, VDIM, DIM> Q_adj = {{
+                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
+                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
+                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
+                  }
+               };
+               const future::tensor<real_t, DIM, DIM> grad_dU = {{
+                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
+                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
+                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
+                  }
+               };
+               const auto one = transpose(grad_dU) * (Q_adj * u_val);
+
+               // Second part of the Jacobian: δu·∇u
+               const future::tensor<real_t, DIM> du_val =
+               {
+                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
+               };
+               const future::tensor<real_t, VDIM, DIM> grad_U = {{
+                     {g2[0][0][qz][qy][qx], g2[1][0][qz][qy][qx], g2[2][0][qz][qy][qx]},
+                     {g2[0][1][qz][qy][qx], g2[1][1][qz][qy][qx], g2[2][1][qz][qy][qx]},
+                     {g2[0][2][qz][qy][qx], g2[1][2][qz][qy][qx], g2[2][2][qz][qy][qx]}
+                  }
+               };
+               const auto two = transpose(grad_U) * (Q_adj * du_val);
+
+               // u⋅∇δu + δu⋅∇u
+               r0[0][qz][qy][qx] = one[0] + two[0];
+               r0[1][qz][qy][qx] = one[1] + two[1];
+               r0[2][qz][qy][qx] = one[2] + two[2];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, r0, r1);
+      kernels::internal::WriteDofs3d(e, D1D, r1, Y);
+   });
+}
+
+} // namespace internal
+
+template<int T_D1D, int T_Q1D>
+VectorConvectionNLFIntegrator::AddMultGradPAType
+VectorConvectionNLFIntegrator::AddMultGradPA2D::Kernel()
+{
+   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
+   return internal::SmemPAConvectionNLGradApply2D<T_D1D, T_Q1D>;
+}
+
+inline VectorConvectionNLFIntegrator::AddMultGradPAType
+VectorConvectionNLFIntegrator::AddMultGradPA2D::Fallback(int d1d, int q1d)
+{
+   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   return internal::SmemPAConvectionNLGradApply2D<>;
+}
+
+template<int T_D1D, int T_Q1D>
+VectorConvectionNLFIntegrator::AddMultGradPAType
+VectorConvectionNLFIntegrator::AddMultGradPA3D::Kernel()
+{
+   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
+   return internal::SmemPAConvectionNLGradApply3D<T_D1D, T_Q1D>;
+}
+
+inline VectorConvectionNLFIntegrator::AddMultGradPAType
+VectorConvectionNLFIntegrator::AddMultGradPA3D::Fallback(int d1d, int q1d)
+{
+   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   return internal::SmemPAConvectionNLGradApply3D<>;
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
+} // namespace mfem

fem/intrules.cpp

@@ -542,7 +542,10 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
          return;
    }
 
-#ifndef MFEM_USE_MPFR
+#ifdef MFEM_USE_MPFR
+   MFEM_WARNING("MPFR implementation of Gauss-Jacobi quadrature not implemented yet. Falling "
+                "back to double precision implementation...");
+#endif
 
    const int n = np;
    // common constants for Jacobi polynomials
@@ -611,13 +614,6 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
                                                 ab + 1) / ((1.0 - xi*xi)*pp*pp) / pow(2, ab);
       // map nodes and weights to the interval [0,1]
    }
-
-#else // MFEM_USE_MPFR is defined
-
-   MFEM_ABORT("MPFR implementation of Gauss-Jacobi quadrature not defined yet");
-
-#endif // MFEM_USE_MPFR
-
 }
 
 

fem/kernel_dispatch.hpp

@@ -61,7 +61,7 @@ namespace mfem
 #define MFEM_REGISTER_KERNELS_1(KernelName, KernelType, Params)                \
    MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, (), Params)
 
-// Version of MFEM_REGISTER_KERNELS without any optional (non-dispatch)
+// Version of MFEM_REGISTER_KERNELS with optional (non-dispatch)
 // parameters (e.g. NBZ).
 #define MFEM_REGISTER_KERNELS_2(KernelName, KernelType, Params, OptParams)     \
    MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, OptParams,           \

fem/kernels.hpp

@@ -83,7 +83,7 @@ constexpr int SetMaxOf(int n) { return NextMultipleOf<4>(n); }
 #endif // CUDA/HIP && DEVICE_COMPILE
 
 /// Load 2D matrix into shared memory
-template <int MQ1>
+template <int MQ1, bool TRANSPOSE = false>
 inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d,
                                         const real_t *M, real_t (*N)[MQ1])
 {
@@ -91,7 +91,14 @@ inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d,
    {
       MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
       {
-         N[dy][qx] = M[dy * q1d + qx];
+         if constexpr (TRANSPOSE)
+         {
+            N[dy][qx] = M[qx * d1d + dy];
+         }
+         else
+         {
+            N[dy][qx] = M[dy * q1d + qx];
+         }
       }
    }
    MFEM_SYNC_THREAD;

fem/lor/lor_ams.cpp

@@ -94,10 +94,10 @@ void BatchedLOR_AMS::Form2DEdgeToVertex_RT(Array<int> &edge2vert)
          const int iv0 = ix + iy*op1;
          const int iv1 = ix1 + iy1*op1;
 
-         // Rotated gradient in 2D (-dy, dx), so flip the sign for the first
-         // component (c == 0).
-         e2v(0, iedge) = (c == 1) ? iv0 : iv1;
-         e2v(1, iedge) = (c == 1) ? iv1 : iv0;
+         // 2D curl (dy, -dx), so flip the sign for the second
+         // component (c == 1).
+         e2v(0, iedge) = (c == 0) ? iv0 : iv1;
+         e2v(1, iedge) = (c == 0) ? iv1 : iv0;
       }
    }
 }

fem/lor/lor_batched.cpp

@@ -142,8 +142,6 @@ static MFEM_HOST_DEVICE int GetAndIncrementNnzIndex(const int i_L, int* I)
 
 int BatchedLORAssembly::FillI(SparseMatrix &A) const
 {
-   static constexpr int Max = 16;
-
    const int nvdof = fes_ho.GetVSize();
 
    const int ndof_per_el = fes_ho.GetTypicalFE()->GetDof();
@@ -165,6 +163,8 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
    const auto K = dof_glob2loc_offsets_.Read();
    const auto map = Reshape(sparse_mapping.Read(), nnz_per_row, ndof_per_el);
 
+   Array<int> ij_elts(dof_glob2loc_.Size() * 2);
+   auto d_ij_elts = Reshape(ij_elts.Write(), dof_glob2loc_.Size(), 2);
 
    auto I = A.WriteI();
 
@@ -176,10 +176,10 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
       const int sii = el_dof_lex(ii_el, iel_ho);
       const int ii = (sii >= 0) ? sii : -1 -sii;
       // Get number and list of elements containing this DOF
-      int i_elts[Max];
       const int i_offset = K[ii];
       const int i_next_offset = K[ii+1];
       const int i_ne = i_next_offset - i_offset;
+      int *i_elts = &d_ij_elts(i_offset, 0);
       for (int e_i = 0; e_i < i_ne; ++e_i)
       {
          const int si_E = dof_glob2loc[i_offset+e_i]; // signed
@@ -202,7 +202,7 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
          }
          else // assembly required
          {
-            int j_elts[Max];
+            int *j_elts = &d_ij_elts(j_offset, 1);
             for (int e_j = 0; e_j < j_ne; ++e_j)
             {
                const int sj_E = dof_glob2loc[j_offset+e_j]; // signed
@@ -269,7 +269,8 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
       mfem::forall(nvdof + 1, [=] MFEM_HOST_DEVICE (int i) { I[i] = I2[i]; });
    }
 
-   static constexpr int Max = 16;
+   Array<int> ij_B_el(dof_glob2loc_.Size() * 4);
+   auto d_ij_B_el = Reshape(ij_B_el.Write(), dof_glob2loc_.Size(), 4);
 
    mfem::forall(ndof_per_el*nel_ho, [=] MFEM_HOST_DEVICE (int i)
    {
@@ -279,11 +280,13 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
       const int sii = el_dof_lex(ii_el, iel_ho); // signed
       const int ii = (sii >= 0) ? sii : -1 - sii;
       // Get number and list of elements containing this DOF
-      int i_elts[Max];
-      int i_B[Max];
       const int i_offset = K[ii];
       const int i_next_offset = K[ii+1];
       const int i_ne = i_next_offset - i_offset;
+
+      int *i_elts = &d_ij_B_el(i_offset, 0);
+      int *i_B = &d_ij_B_el(i_offset, 1);
+
       for (int e_i = 0; e_i < i_ne; ++e_i)
       {
          const int si_E = dof_glob2loc[i_offset+e_i]; // signed
@@ -312,8 +315,8 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
          }
          else // assembly required
          {
-            int j_elts[Max];
-            int j_B[Max];
+            int *j_elts = &d_ij_B_el(j_offset, 2);
+            int *j_B = &d_ij_B_el(j_offset, 3);
             for (int e_j = 0; e_j < j_ne; ++e_j)
             {
                const int sj_E = dof_glob2loc[j_offset+e_j]; // signed

fem/nonlinearform_ext.cpp

@@ -100,6 +100,17 @@ PANonlinearFormExtension::Gradient::Gradient(const PANonlinearFormExtension &e):
 
 void PANonlinearFormExtension::Gradient::AssembleGrad(const Vector &g)
 {
+   if (DeviceCanUseCeed())
+   {
+      for (int i = 0; i < ext.dnfi.Size(); ++i)
+      {
+         MFEM_VERIFY(dynamic_cast<VectorConvectionNLFIntegrator *>
+                     (ext.dnfi[i]) == nullptr,
+                     "VectorConvectionNLFIntegrator PA gradients are not supported "
+                     "with the libCEED backend");
+      }
+   }
+
    ext.elemR->Mult(g, ext.xe);
    for (int i = 0; i < ext.dnfi.Size(); ++i)
    {

fem/nonlininteg.cpp

@@ -954,4 +954,74 @@ void SkewSymmetricVectorConvectionNLFIntegrator::AssembleElementGrad(
    }
 }
 
+void ConvectiveVectorConvectionNLFIntegrator::AssemblePA(
+   const FiniteElementSpace &)
+{
+   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void ConvectiveVectorConvectionNLFIntegrator::AssembleGradPA(
+   const Vector &, const FiniteElementSpace &)
+{
+   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void ConvectiveVectorConvectionNLFIntegrator::AddMultPA(
+   const Vector &, Vector &) const
+{
+   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void ConvectiveVectorConvectionNLFIntegrator::AddMultGradPA(
+   const Vector &, Vector &) const
+{
+   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void ConvectiveVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
+   Vector &) const
+{
+   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void SkewSymmetricVectorConvectionNLFIntegrator::AssemblePA(
+   const FiniteElementSpace &)
+{
+   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradPA(
+   const Vector &, const FiniteElementSpace &)
+{
+   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void SkewSymmetricVectorConvectionNLFIntegrator::AddMultPA(
+   const Vector &, Vector &) const
+{
+   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void SkewSymmetricVectorConvectionNLFIntegrator::AddMultGradPA(
+   const Vector &, Vector &) const
+{
+   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
+void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
+   Vector &) const
+{
+   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
+              "partial assembly; use VectorConvectionNLFIntegrator");
+}
+
 }

fem/nonlininteg.hpp

@@ -18,6 +18,7 @@
 #include "fespace.hpp"
 #include "ceed/interface/operator.hpp"
 #include "integrator.hpp"
+#include "kernel_dispatch.hpp"
 
 namespace mfem
 {
@@ -384,15 +385,17 @@ private:
    DenseMatrix dshape, dshapex, EF, gradEF, ELV, elmat_comp;
    Vector shape;
    // PA extension
-   Vector pa_data;
+   int dim, ne, nq, d1d, q1d;
+   Vector pa_adj, pa_u;
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq;
 
 public:
-   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { }
+   struct Kernels { Kernels(); };
 
-   VectorConvectionNLFIntegrator() = default;
+   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { static Kernels kernels; }
+
+   VectorConvectionNLFIntegrator() { static Kernels kernels; }
 
    static const IntegrationRule &GetRule(const FiniteElement &fe,
                                          const ElementTransformation &T);
@@ -411,12 +414,55 @@ public:
 
    void AssemblePA(const FiniteElementSpace &fes) override;
 
-   void AssembleMF(const FiniteElementSpace &fes) override;
+   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
 
    void AddMultPA(const Vector &x, Vector &y) const override;
 
-   void AddMultMF(const Vector &x, Vector &y) const override;
+   using AddMultPAType =
+      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
+              const real_t *x, real_t *y,
+              const int d1d, const int q1d);
+   MFEM_REGISTER_KERNELS(AddMultPAKernels, AddMultPAType, (int, int, int));
 
+   void AddMultGradPA(const Vector &x, Vector &y) const override;
+
+   using AddMultGradPAType =
+      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
+              const real_t *u, const real_t *x, real_t *y,
+              const int d1d, const int q1d);
+
+   MFEM_REGISTER_KERNELS(AddMultGradPA2D, AddMultGradPAType, (int, int));
+   MFEM_REGISTER_KERNELS(AddMultGradPA3D, AddMultGradPAType, (int, int));
+
+   void AssembleGradDiagonalPA(Vector &) const override;
+
+   using GradDiagPAType =
+      void (*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
+               const real_t *u, real_t *y,
+               const int d1d, const int q1d);
+
+   MFEM_REGISTER_KERNELS(GradDiagPA2D, GradDiagPAType, (int, int));
+   MFEM_REGISTER_KERNELS(GradDiagPA3D, GradDiagPAType, (int, int));
+
+   template <int DIM, int D1D, int Q1D>
+   static void AddSpecialization()
+   {
+      AddMultPAKernels::Specialization<DIM, D1D, Q1D>::Add();
+      if constexpr (DIM == 2)
+      {
+         AddMultGradPA2D::Specialization<D1D, Q1D>::Add();
+         GradDiagPA2D::Specialization<D1D, Q1D>::Add();
+      }
+      else if constexpr (DIM == 3)
+      {
+         AddMultGradPA3D::Specialization<D1D, Q1D>::Add();
+         GradDiagPA3D::Specialization<D1D, Q1D>::Add();
+      }
+   }
+
+   void AssembleMF(const FiniteElementSpace &fes) override;
+
+   void AddMultMF(const Vector &x, Vector &y) const override;
 
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
@@ -430,7 +476,8 @@ protected:
 
 
 /** This class is used to assemble the convective form of the nonlinear term
-    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$ */
+    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$.
+    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
 class ConvectiveVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -448,12 +495,20 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
+
+   using NonlinearFormIntegrator::AssemblePA;
+   void AssemblePA(const FiniteElementSpace &fes) override;
+   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
+   void AddMultPA(const Vector &x, Vector &y) const override;
+   void AddMultGradPA(const Vector &x, Vector &y) const override;
+   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 
 /** This class is used to assemble the skew-symmetric form of the nonlinear term
     arising in the Navier-Stokes equations
-    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$ */
+    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$.
+    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
 class SkewSymmetricVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -471,6 +526,13 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
+
+   using NonlinearFormIntegrator::AssemblePA;
+   void AssemblePA(const FiniteElementSpace &fes) override;
+   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
+   void AddMultPA(const Vector &x, Vector &y) const override;
+   void AddMultGradPA(const Vector &x, Vector &y) const override;
+   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 }

fem/pgridfunc.cpp

@@ -22,10 +22,20 @@ using namespace std;
 namespace mfem
 {
 
-ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf)
+ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
+                                 bool preserve)
 {
    fes = pfes = pf;
    SetDataAndSize(gf->GetData(), gf->Size());
+
+   if (pfes->HaveDofSigns())
+   {
+      MFEM_VERIFY(!preserve, "Differing sign conventions for the serial and "
+                  "parallel grid functions will prevent preserving the serial "
+                  "GridFunctions in this context.");
+
+      pfes->ApplyDofSigns(HostReadWrite());
+   }
 }
 
 ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, HypreParVector *tv)

fem/pgridfunc.hpp

@@ -100,8 +100,12 @@ public:
    /// Construct a ParGridFunction using a GridFunction as external data.
    /** The parallel space @a *pf and the space used by @a *gf should match. The
        data from @a *gf is used as the local data of the ParGridFunction on each
-       processor. The ParGridFunction does not assume ownership of the data. */
-   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf);
+       processor. The ParGridFunction does not assume ownership of the data.
+       The boolean, @a preserve, indicates that the data stored in @a *gf should
+       remain unchanged. An error will occur if @a preserve is true and
+       construction of a valid ParGridFunction requires the data to change. */
+   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
+                   bool preserve = true);
 
    /** @brief Creates grid function on (all) dofs from a given vector on the
        true dofs, i.e. P tv. */

fem/qinterp/det.cpp

@@ -21,24 +21,23 @@ namespace quadrature_interpolator
 
 void InitDetKernels()
 {
-   using k = QuadratureInterpolator::DetKernels;
    // 2D
-   k::Specialization<2,2,2,2>::Add();
-   k::Specialization<2,2,2,3>::Add();
-   k::Specialization<2,2,2,4>::Add();
-   k::Specialization<2,2,2,6>::Add();
-   k::Specialization<2,2,3,4>::Add();
-   k::Specialization<2,2,3,6>::Add();
-   k::Specialization<2,2,4,4>::Add();
-   k::Specialization<2,2,4,6>::Add();
-   k::Specialization<2,2,5,6>::Add();
+   QuadratureInterpolator::AddDetSpecializations<2,2,2,2>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,2,3>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,2,4>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,2,6>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,3,4>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,3,6>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,4,4>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,4,6>();
+   QuadratureInterpolator::AddDetSpecializations<2,2,5,6>();
    // 3D
-   k::Specialization<3,3,2,4>::Add();
-   k::Specialization<3,3,3,3>::Add();
-   k::Specialization<3,3,3,5>::Add();
-   k::Specialization<3,3,3,6>::Add();
-   k::Specialization<3,3,4,6>::Add();
-   k::Specialization<3,3,3,4>::Add();
+   QuadratureInterpolator::AddDetSpecializations<3,3,2,4>();
+   QuadratureInterpolator::AddDetSpecializations<3,3,3,3>();
+   QuadratureInterpolator::AddDetSpecializations<3,3,3,5>();
+   QuadratureInterpolator::AddDetSpecializations<3,3,3,6>();
+   QuadratureInterpolator::AddDetSpecializations<3,3,4,6>();
+   QuadratureInterpolator::AddDetSpecializations<3,3,3,4>();
 }
 
 } // namespace quadrature_interpolator
@@ -47,8 +46,8 @@ void InitDetKernels()
 /// @cond Suppress_Doxygen_warnings
 
 QuadratureInterpolator::DetKernelType
-QuadratureInterpolator::DetKernels::Fallback(
-   int DIM, int SDIM, int D1D, int Q1D)
+QuadratureInterpolator::DetKernels::Fallback(int DIM, int SDIM, int D1D,
+                                             int Q1D)
 {
    if (DIM == 1)
    {

fem/qinterp/eval.hpp

@@ -30,23 +30,18 @@ namespace internal
 namespace quadrature_interpolator
 {
 
-template<QVectorLayout Q_LAYOUT>
-static void Values1D(const int NE,
-                     const real_t *b_,
-                     const real_t *x_,
-                     real_t *y_,
-                     const int vdim,
-                     const int d1d,
-                     const int q1d)
+template <QVectorLayout Q_LAYOUT, bool Integral>
+static void ImplValues1D(const int NE, const real_t *b_, const real_t *detJ_,
+                         const real_t *x_, real_t *y_, const int vdim,
+                         const int d1d, const int q1d)
 {
-   const auto b = Reshape(b_, q1d, d1d);
-   const auto x = Reshape(x_, d1d, vdim, NE);
-   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
-            Reshape(y_, q1d, vdim, NE):
-            Reshape(y_, vdim, q1d, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
    {
+      const auto b = Reshape(b_, q1d, d1d);
+      const auto x = Reshape(x_, d1d, vdim, NE);
+      const auto detJ = Reshape(detJ_, q1d, NE);
+      auto y = Q_LAYOUT == QVectorLayout::byNODES ? Reshape(y_, q1d, vdim, NE)
+               : Reshape(y_, vdim, q1d, NE);
       for (int c = 0; c < vdim; c++)
       {
          for (int q = 0; q < q1d; q++)
@@ -56,24 +51,36 @@ static void Values1D(const int NE,
             {
                u += b(q, d) * x(d, c, e);
             }
-            if (Q_LAYOUT == QVectorLayout::byVDIM)  { y(c, q, e) = u; }
-            if (Q_LAYOUT == QVectorLayout::byNODES) { y(q, c, e) = u; }
+            if constexpr (Integral)
+            {
+               u /= detJ(q, e);
+            }
+            if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
+            {
+               y(c, q, e) = u;
+            }
+            if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
+            {
+               y(q, c, e) = u;
+            }
          }
       }
    });
 }
 
+template <QVectorLayout Q_LAYOUT>
+static void Values1D(const int NE, const real_t *b_, const real_t *x_,
+                     real_t *y_, const int vdim, const int d1d, const int q1d)
+{
+   ImplValues1D<Q_LAYOUT, false>(NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
+}
+
 // Template compute kernel for Values in 2D: tensor product version.
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-         int T_NBZ = 1>
-static void Values2D(const int NE,
-                     const real_t *b_,
-                     const real_t *x_,
-                     real_t *y_,
-                     const int vdim = 0,
-                     const int d1d = 0,
-                     const int q1d = 0)
+template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
+          int T_Q1D = 0, int T_NBZ = 1>
+static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
+                         const real_t *x_, real_t *y_, const int vdim = 0,
+                         const int d1d = 0, const int q1d = 0)
 {
    static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
 
@@ -82,13 +89,14 @@ static void Values2D(const int NE,
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
-   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
-   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
-            Reshape(y_, Q1D, Q1D, VDIM, NE):
-            Reshape(y_, VDIM, Q1D, Q1D, NE);
 
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int e)
    {
+      const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
+      const auto detJ = Reshape(detJ_, Q1D, Q1D, NE);
+      auto y = Q_LAYOUT == QVectorLayout::byNODES
+               ? Reshape(y_, Q1D, Q1D, VDIM, NE)
+               : Reshape(y_, VDIM, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -110,16 +118,33 @@ static void Values2D(const int NE,
 
       for (int c = 0; c < VDIM; c++)
       {
-         kernels::internal::LoadX(e,D1D,c,x,DD);
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx, x, D1D)
+            {
+               DD(dx, dy) = x(dx, dy, c, e);
+            }
+         }
+         MFEM_SYNC_THREAD;
          kernels::internal::EvalX(D1D,Q1D,B,DD,DQ);
          kernels::internal::EvalY(D1D,Q1D,B,DQ,QQ);
          MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
             MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               real_t u = QQ(qx,qy);
-               if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,e) = u; }
-               if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,c,e) = u; }
+               real_t u = QQ(qx, qy);
+               if constexpr (Integral)
+               {
+                  u /= detJ(qx, qy, e);
+               }
+               if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
+               {
+                  y(c, qx, qy, e) = u;
+               }
+               if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
+               {
+                  y(qx, qy, c, e) = u;
+               }
             }
          }
          MFEM_SYNC_THREAD;
@@ -127,29 +152,37 @@ static void Values2D(const int NE,
    });
 }
 
-// Template compute kernel for Values in 3D: tensor product version.
-template<QVectorLayout Q_LAYOUT,
-         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
-static void Values3D(const int NE,
-                     const real_t *b_,
-                     const real_t *x_,
-                     real_t *y_,
-                     const int vdim = 0,
-                     const int d1d = 0,
+// Template compute kernel for Values in 2D: tensor product version.
+template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+          int T_NBZ = 1>
+static void Values2D(const int NE, const real_t *b_, const real_t *x_,
+                     real_t *y_, const int vdim = 0, const int d1d = 0,
                      const int q1d = 0)
+{
+   return ImplValues2D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D, T_NBZ>(
+             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
+}
+
+// Template compute kernel for Values in 3D: tensor product version.
+template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
+          int T_Q1D = 0>
+static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
+                         const real_t *x_, real_t *y_, const int vdim = 0,
+                         const int d1d = 0, const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
-   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
-   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
-            Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE):
-            Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
 
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
    {
+      const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
+      const auto detJ = Reshape(detJ_, Q1D, Q1D, Q1D, NE);
+      auto y = Q_LAYOUT == QVectorLayout::byNODES
+               ? Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE)
+               : Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -171,7 +204,17 @@ static void Values3D(const int NE,
 
       for (int c = 0; c < VDIM; c++)
       {
-         kernels::internal::LoadX(e,D1D,c,x,DDD);
+         MFEM_FOREACH_THREAD(dz, z, D1D)
+         {
+            MFEM_FOREACH_THREAD(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  DDD(dx, dy, dz) = x(dx, dy, dz, c, e);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
          kernels::internal::EvalX(D1D,Q1D,B,DDD,DDQ);
          kernels::internal::EvalY(D1D,Q1D,B,DDQ,DQQ);
          kernels::internal::EvalZ(D1D,Q1D,B,DQQ,QQQ);
@@ -181,9 +224,19 @@ static void Values3D(const int NE,
             {
                MFEM_FOREACH_THREAD(qx,x,Q1D)
                {
-                  const real_t u = QQQ(qz,qy,qx);
-                  if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,qz,e) = u; }
-                  if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,qz,c,e) = u; }
+                  real_t u = QQQ(qz,qy,qx);
+                  if constexpr (Integral)
+                  {
+                     u /= detJ(qx, qy, qz, e);
+                  }
+                  if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
+                  {
+                     y(c, qx, qy, qz, e) = u;
+                  }
+                  if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
+                  {
+                     y(qx, qy, qz, c, e) = u;
+                  }
                }
             }
          }
@@ -192,14 +245,431 @@ static void Values3D(const int NE,
    });
 }
 
+// Template compute kernel for Values in 3D: tensor product version.
+template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
+static void Values3D(const int NE, const real_t *b_, const real_t *x_,
+                     real_t *y_, const int vdim = 0, const int d1d = 0,
+                     const int q1d = 0)
+{
+   return ImplValues3D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D>(
+             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
+}
+
+template <bool Integral>
+void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
+                const real_t *detJ, const GeometricFactors *geom,
+                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+                Vector &q_der, Vector &q_det, const int eval_flags);
+
+inline void Eval1D(const int NE, const int vdim, const QVectorLayout q_layout,
+                   const GeometricFactors *geom, const DofToQuad &maps,
+                   const Vector &e_vec, Vector &q_val, Vector &q_der, Vector &q_det,
+                   const int eval_flags)
+{
+   ImplEval1D<false>(NE, vdim, q_layout, nullptr, geom, maps, e_vec, q_val,
+                     q_der, q_det, eval_flags);
+}
+
+// Template compute kernel for 2D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
+static void ImplEval2D(const int NE, const int vdim,
+                       const QVectorLayout q_layout, const real_t *detJ_,
+                       const GeometricFactors *geom, const DofToQuad &maps,
+                       const Vector &e_vec, Vector &q_val, Vector &q_der,
+                       Vector &q_det, const int eval_flags)
+{
+   using QI = QuadratureInterpolator;
+
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+   const int ND = T_ND ? T_ND : nd;
+   const int NQ = T_NQ ? T_NQ : nq;
+   const int NMAX = NQ > ND ? NQ : ND;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
+   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
+   MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
+   MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
+   if constexpr(Integral)
+   {
+      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
+                                  QI::DETERMINANTS)),
+                  "Integral FE does not support computing derivatives");
+   }
+   const auto B = Reshape(maps.B.Read(), NQ, ND);
+   const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
+   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
+   const auto E_ = e_vec.Read();
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ, VDIM, NE):
+              Reshape(q_val.Write(), VDIM, NQ, NE);
+   auto der = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_der.Write(), NQ, VDIM, 2, NE):
+              Reshape(q_der.Write(), VDIM, 2, NQ, NE);
+   auto det = Reshape(q_det.Write(), NQ, NE);
+   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const auto E = Reshape(E_, ND, VDIM, NE);
+      const auto detJ = Reshape(detJ_, NQ, NE);
+      const int ND = T_ND ? T_ND : nd;
+      const int NQ = T_NQ ? T_NQ : nq;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND2D;
+      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM2D;
+      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
+      MFEM_FOREACH_THREAD(d, x, ND)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            s_E[c + d * VDIM] = E(d, c, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(q, x, NQ)
+      {
+         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
+         {
+            real_t ed[max_VDIM];
+            for (int c = 0; c < VDIM; c++)
+            {
+               ed[c] = 0.0;
+            }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t b = B(q,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  ed[c] += b * s_E[c + d * VDIM];
+               }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               if constexpr (Integral)
+               {
+                  ed[c] /= detJ(q, e);
+               }
+               if (q_layout == QVectorLayout::byVDIM)
+               {
+                  val(c, q, e) = ed[c];
+               }
+               if (q_layout == QVectorLayout::byNODES)
+               {
+                  val(q, c, e) = ed[c];
+               }
+            }
+         }
+         if ((eval_flags & QI::DERIVATIVES) ||
+             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
+             (eval_flags & QI::DETERMINANTS))
+         {
+            // use MAX_VDIM2D to avoid "subscript out of range" warnings
+            real_t D[QI::MAX_VDIM2D*2];
+            for (int i = 0; i < 2*VDIM; i++)
+            {
+               D[i] = 0.0;
+            }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t wx = G(q,0,d);
+               const real_t wy = G(q,1,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  real_t s_e = s_E[c+d*VDIM];
+                  D[c+VDIM*0] += s_e * wx;
+                  D[c+VDIM*1] += s_e * wy;
+               }
+            }
+            if (eval_flags & QI::DERIVATIVES)
+            {
+               for (int c = 0; c < VDIM; c++)
+               {
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = D[c+VDIM*0];
+                     der(c,1,q,e) = D[c+VDIM*1];
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = D[c+VDIM*0];
+                     der(q,c,1,e) = D[c+VDIM*1];
+                  }
+               }
+            }
+            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
+            {
+               real_t Jloc[4], Jinv[4];
+               Jloc[0] = J(q,0,0,e);
+               Jloc[1] = J(q,1,0,e);
+               Jloc[2] = J(q,0,1,e);
+               Jloc[3] = J(q,1,1,e);
+               kernels::CalcInverse<2>(Jloc, Jinv);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  const real_t u = D[c+VDIM*0];
+                  const real_t v = D[c+VDIM*1];
+                  const real_t JiU = Jinv[0]*u + Jinv[1]*v;
+                  const real_t JiV = Jinv[2]*u + Jinv[3]*v;
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = JiU;
+                     der(c,1,q,e) = JiV;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = JiU;
+                     der(q,c,1,e) = JiV;
+                  }
+               }
+            }
+            if (eval_flags & QI::DETERMINANTS)
+            {
+               if (VDIM == 2)
+               {
+                  det(q, e) = kernels::Det<2>(D);
+               }
+               else
+               {
+                  DeviceTensor<2> j(D, 3, 2);
+                  const real_t dE = j(0,0)*j(0,0) + j(1,0)*j(1,0) + j(2,0)*j(2,0);
+                  const real_t dF = j(0,0)*j(0,1) + j(1,0)*j(1,1) + j(2,0)*j(2,1);
+                  const real_t dG = j(0,1)*j(0,1) + j(1,1)*j(1,1) + j(2,1)*j(2,1);
+                  det(q,e) = std::sqrt(dE*dG - dF*dF);
+               }
+            }
+         }
+      }
+   });
+}
+
+// Template compute kernel for 2D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template <const int T_VDIM, const int T_ND, const int T_NQ>
+static void Eval2D(const int NE, const int vdim, const QVectorLayout q_layout,
+                   const GeometricFactors *geom, const DofToQuad &maps,
+                   const Vector &e_vec, Vector &q_val, Vector &q_der,
+                   Vector &q_det, const int eval_flags)
+{
+   ImplEval2D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
+                                         maps, e_vec, q_val, q_der, q_det,
+                                         eval_flags);
+}
+
+// Template compute kernel for 3D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
+static void ImplEval3D(const int NE, const int vdim,
+                       const QVectorLayout q_layout, const real_t *detJ_,
+                       const GeometricFactors *geom, const DofToQuad &maps,
+                       const Vector &e_vec, Vector &q_val, Vector &q_der,
+                       Vector &q_det, const int eval_flags)
+{
+   using QI = QuadratureInterpolator;
+
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+   const int ND = T_ND ? T_ND : nd;
+   const int NQ = T_NQ ? T_NQ : nq;
+   const int NMAX = NQ > ND ? NQ : ND;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
+   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 3, "");
+   MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
+   MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
+   MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
+   if constexpr(Integral)
+   {
+      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
+                                  QI::DETERMINANTS)),
+                  "Integral FE does not support computing derivatives");
+   }
+   const auto B = Reshape(maps.B.Read(), NQ, ND);
+   const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
+   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
+   auto E_ = e_vec.Read();
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ, VDIM, NE):
+              Reshape(q_val.Write(), VDIM, NQ, NE);
+   auto der = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_der.Write(), NQ, VDIM, 3, NE):
+              Reshape(q_der.Write(), VDIM, 3, NQ, NE);
+   auto det = Reshape(q_det.Write(), NQ, NE);
+   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const auto E = Reshape(E_, ND, VDIM, NE);
+      const auto detJ = Reshape(detJ_, NQ, NE);
+      const int ND = T_ND ? T_ND : nd;
+      const int NQ = T_NQ ? T_NQ : nq;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND3D;
+      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM3D;
+      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
+      MFEM_FOREACH_THREAD(d, x, ND)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            s_E[c + d * VDIM] = E(d, c, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(q, x, NQ)
+      {
+         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
+         {
+            real_t ed[max_VDIM];
+            for (int c = 0; c < VDIM; c++)
+            {
+               ed[c] = 0.0;
+            }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t b = B(q,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  ed[c] += b * s_E[c + d * VDIM];
+               }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               if constexpr (Integral)
+               {
+                  ed[c] /= detJ(q, e);
+               }
+               if (q_layout == QVectorLayout::byVDIM)
+               {
+                  val(c, q, e) = ed[c];
+               }
+               if (q_layout == QVectorLayout::byNODES)
+               {
+                  val(q, c, e) = ed[c];
+               }
+            }
+         }
+         if ((eval_flags & QI::DERIVATIVES) ||
+             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
+             (eval_flags & QI::DETERMINANTS))
+         {
+            // use MAX_VDIM3D to avoid "subscript out of range" warnings
+            real_t D[QI::MAX_VDIM3D*3];
+            for (int i = 0; i < 3*VDIM; i++)
+            {
+               D[i] = 0.0;
+            }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t wx = G(q,0,d);
+               const real_t wy = G(q,1,d);
+               const real_t wz = G(q,2,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  real_t s_e = s_E[c+d*VDIM];
+                  D[c+VDIM*0] += s_e * wx;
+                  D[c+VDIM*1] += s_e * wy;
+                  D[c+VDIM*2] += s_e * wz;
+               }
+            }
+            if (eval_flags & QI::DERIVATIVES)
+            {
+               for (int c = 0; c < VDIM; c++)
+               {
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = D[c+VDIM*0];
+                     der(c,1,q,e) = D[c+VDIM*1];
+                     der(c,2,q,e) = D[c+VDIM*2];
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = D[c+VDIM*0];
+                     der(q,c,1,e) = D[c+VDIM*1];
+                     der(q,c,2,e) = D[c+VDIM*2];
+                  }
+               }
+            }
+            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
+            {
+               real_t Jloc[9], Jinv[9];
+               for (int col = 0; col < 3; col++)
+               {
+                  for (int row = 0; row < 3; row++)
+                  {
+                     Jloc[row+3*col] = J(q,row,col,e);
+                  }
+               }
+               kernels::CalcInverse<3>(Jloc, Jinv);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  const real_t u = D[c+VDIM*0];
+                  const real_t v = D[c+VDIM*1];
+                  const real_t w = D[c+VDIM*2];
+                  const real_t JiU = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
+                  const real_t JiV = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
+                  const real_t JiW = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = JiU;
+                     der(c,1,q,e) = JiV;
+                     der(c,2,q,e) = JiW;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = JiU;
+                     der(q,c,1,e) = JiV;
+                     der(q,c,2,e) = JiW;
+                  }
+               }
+            }
+            if (VDIM == 3 && (eval_flags & QI::DETERMINANTS))
+            {
+               // The check (VDIM == 3) should eliminate this block when VDIM is
+               // known at compile time and (VDIM != 3).
+               det(q,e) = kernels::Det<3>(D);
+            }
+         }
+      }
+   });
+}
+
+// Template compute kernel for 3D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template <const int T_VDIM, const int T_ND, const int T_NQ>
+static void Eval3D(const int NE, const int vdim, const QVectorLayout q_layout,
+                   const GeometricFactors *geom, const DofToQuad &maps,
+                   const Vector &e_vec, Vector &q_val, Vector &q_der,
+                   Vector &q_det, const int eval_flags)
+{
+   ImplEval3D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
+                                         maps, e_vec, q_val, q_der, q_det,
+                                         eval_flags);
+}
+
 } // namespace quadrature_interpolator
 
 } // namespace internal
 
 /// @cond Suppress_Doxygen_warnings
 
-template<int DIM, QVectorLayout Q_LAYOUT,
-         int VDIM, int D1D, int Q1D, int NBZ>
+template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
+QuadratureInterpolator::IntTensorEvalKernelType
+QuadratureInterpolator::IntTensorEvalKernels::Kernel()
+{
+   if constexpr (DIM == 1) { return internal::quadrature_interpolator::ImplValues1D<Q_LAYOUT, true>; }
+   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::ImplValues2D<Q_LAYOUT, true, VDIM, D1D, Q1D, NBZ>; }
+   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::ImplValues3D<Q_LAYOUT, true, VDIM, D1D, Q1D>; }
+   MFEM_ABORT("");
+}
+
+template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
 QuadratureInterpolator::TensorEvalKernelType
 QuadratureInterpolator::TensorEvalKernels::Kernel()
 {
@@ -209,6 +679,28 @@ QuadratureInterpolator::TensorEvalKernels::Kernel()
    MFEM_ABORT("");
 }
 
+template <int DIM, int VDIM, int ND, int NQ>
+QuadratureInterpolator::IntEvalKernelType
+QuadratureInterpolator::IntEvalKernels::Kernel()
+{
+   using namespace internal::quadrature_interpolator;
+   if constexpr (DIM == 1) { return ImplEval1D<true>; }
+   else if constexpr (DIM == 2) { return ImplEval2D<true,VDIM,ND,NQ>; }
+   else if constexpr (DIM == 3) { return ImplEval3D<true,VDIM,ND,NQ>; }
+   MFEM_ABORT("");
+}
+
+template <int DIM, int VDIM, int ND, int NQ>
+QuadratureInterpolator::EvalKernelType
+QuadratureInterpolator::EvalKernels::Kernel()
+{
+   using namespace internal::quadrature_interpolator;
+   if constexpr (DIM == 1) { return Eval1D; }
+   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
+   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
+   MFEM_ABORT("");
+}
+
 /// @endcond
 
 } // namespace mfem

fem/qinterp/eval_by_nodes.cpp

@@ -21,63 +21,105 @@ namespace quadrature_interpolator
 
 void InitEvalByNodesKernels()
 {
-   using k = QuadratureInterpolator::TensorEvalKernels;
-
    // 2D
-   k::Specialization<2,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,3,2>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 3, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 3, 2, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 3, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 3, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 4, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 1, 4, 4, 1>();
 
-   k::Specialization<2,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,2,4>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,2,5>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,2,6>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 2, 2, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 2, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 2, 5, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 2, 6, 1>();
 
-   k::Specialization<2,QVectorLayout::byNODES,2,3,3>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,3,6>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 3, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 3, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 3, 6, 1>();
 
-   k::Specialization<2,QVectorLayout::byNODES,2,4,3>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,4,4>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,4,5>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,4,6>::Opt<1>::Add();
-   k::Specialization<2,QVectorLayout::byNODES,2,4,7>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 4, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 4, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 4, 5, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 4, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 4, 7, 1>();
 
-   k::Specialization<2,QVectorLayout::byNODES,2,5,6>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byNODES, 2, 5, 6, 1>();
 
    // 3D
-   k::Specialization<3,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,1,4,8>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 3, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 3, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 3, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 4, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 4, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 1, 4, 8, 1>();
 
-   k::Specialization<3,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 2, 2, 2, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 2, 2, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 2, 3, 4, 1>();
 
-   k::Specialization<3,QVectorLayout::byNODES,3,2,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,2,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,2,5>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,2,6>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 2, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 2, 5, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 2, 6, 1>();
 
-   k::Specialization<3,QVectorLayout::byNODES,3,3,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,3,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,3,5>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,3,6>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 3, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 3, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 3, 5, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 3, 6, 1>();
 
-   k::Specialization<3,QVectorLayout::byNODES,3,4,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,4,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,4,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,4,7>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byNODES,3,4,8>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 4, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 4, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 4, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 4, 7, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byNODES, 3, 4, 8, 1>();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/eval_by_vdim.cpp

@@ -21,36 +21,59 @@ namespace quadrature_interpolator
 
 void InitEvalByVDimKernels()
 {
-   using k = QuadratureInterpolator::TensorEvalKernels;
    // 2D
-   k::Specialization<2,QVectorLayout::byVDIM,1,2,4>::Opt<8>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,1,3,6>::Opt<4>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,1,4,8>::Opt<2>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 1, 2, 4, 8>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 1, 3, 6, 4>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 1, 4, 8, 2>();
 
-   k::Specialization<2,QVectorLayout::byVDIM,2,2,4>::Opt<8>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,2,3,4>::Opt<8>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,2,3,6>::Opt<4>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,2,4,6>::Opt<2>::Add();
-   k::Specialization<2,QVectorLayout::byVDIM,2,4,8>::Opt<2>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 2, 2, 4, 8>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 2, 3, 4, 8>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 2, 3, 6, 4>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 2, 4, 6, 2>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   2, QVectorLayout::byVDIM, 2, 4, 8, 2>();
    // 3D
-   k::Specialization<3,QVectorLayout::byVDIM,1,2,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,1,3,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,1,4,8>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,2,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,3,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,4,8>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 1, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 1, 3, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 1, 4, 8, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 2, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 3, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 4, 8, 1>();
 
-   k::Specialization<3,QVectorLayout::byVDIM,3,2,2>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,3,3>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,4,4>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,5,5>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,6,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,7,7>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,8,8>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,9,9>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 2, 2, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 3, 3, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 4, 4, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 5, 5, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 6, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 7, 7, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 8, 8, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 9, 9, 1>();
 
-   k::Specialization<3,QVectorLayout::byVDIM,3,4,6>::Opt<1>::Add();
-   k::Specialization<3,QVectorLayout::byVDIM,3,3,4>::Opt<1>::Add();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 4, 6, 1>();
+   QuadratureInterpolator::AddTensorEvalSpecializations<
+   3, QVectorLayout::byVDIM, 3, 3, 4, 1>();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/grad.hpp

@@ -268,8 +268,9 @@ static void Derivatives3D(const int NE,
       DeviceMatrix B(BG[0], D1D, Q1D);
       DeviceMatrix G(BG[1], D1D, Q1D);
 
-      MFEM_SHARED real_t sm0[3][MQ1*MQ1*MQ1];
-      MFEM_SHARED real_t sm1[3][MQ1*MQ1*MQ1];
+      constexpr int MDQ = MD1 > MQ1 ? MD1 : MQ1;
+      MFEM_SHARED real_t sm0[3][MD1*MD1*MDQ];
+      MFEM_SHARED real_t sm1[3][MD1*MQ1*MQ1];
       DeviceTensor<3> X(sm0[2], D1D, D1D, D1D);
       DeviceTensor<3> DDQ0(sm0[0], D1D, D1D, Q1D);
       DeviceTensor<3> DDQ1(sm0[1], D1D, D1D, Q1D);

fem/qinterp/grad_by_nodes.cpp

@@ -22,74 +22,71 @@ namespace quadrature_interpolator
 template <bool P>
 void InitGradByNodesKernels()
 {
-   using k = QuadratureInterpolator::GradKernels;
    constexpr auto L = QVectorLayout::byNODES;
    // 2D
-   k::Specialization<2,L,P,1,3,3>::template Opt<16>::Add();
-   k::Specialization<2,L,P,1,3,4>::template Opt<16>::Add();
-   k::Specialization<2,L,P,1,4,3>::template Opt<16>::Add();
-   k::Specialization<2,L,P,1,4,4>::template Opt<16>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,3,16>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,4,16>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,3,16>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,4,16>();
 
-   k::Specialization<2,L,P,2,2,2>::template Opt<16>::Add();
-   k::Specialization<2,L,P,2,2,3>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,2,4>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,2,5>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,2,6>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,2,16>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,3,8>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,4,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,5,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,6,2>();
 
-   k::Specialization<2,L,P,2,3,3>::template Opt<2>::Add();
-   k::Specialization<2,L,P,2,3,4>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,4,3>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,3,6>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,3,2>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,4,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,3,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,6,2>();
 
-   k::Specialization<2,L,P,2,4,4>::template Opt<2>::Add();
-   k::Specialization<2,L,P,2,4,5>::template Opt<2>::Add();
-   k::Specialization<2,L,P,2,4,6>::template Opt<2>::Add();
-   k::Specialization<2,L,P,2,4,7>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,4,2>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,5,2>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,6,2>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,7,2>();
 
-   k::Specialization<2,L,P,2,5,6>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,5,6,2>();
    // 3D
-   k::Specialization<3,L,P,1,2,4>::Add();
-   k::Specialization<3,L,P,1,3,3>::Add();
-   k::Specialization<3,L,P,1,3,4>::Add();
-   k::Specialization<3,L,P,1,3,6>::Add();
-   k::Specialization<3,L,P,1,4,4>::Add();
-   k::Specialization<3,L,P,1,4,8>::Add();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,2,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,3>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,6>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,8>();
 
-   k::Specialization<3,L,P,3,2,3>::Add();
-   k::Specialization<3,L,P,3,2,4>::Add();
-   k::Specialization<3,L,P,3,2,5>::Add();
-   k::Specialization<3,L,P,3,2,6>::Add();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,3>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,5>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,6>();
 
-   k::Specialization<3,L,P,3,3,3>::Add();
-   k::Specialization<3,L,P,3,3,4>::Add();
-   k::Specialization<3,L,P,3,3,5>::Add();
-   k::Specialization<3,L,P,3,3,6>::Add();
-   k::Specialization<3,L,P,3,4,4>::Add();
-   k::Specialization<3,L,P,3,4,6>::Add();
-   k::Specialization<3,L,P,3,4,7>::Add();
-   k::Specialization<3,L,P,3,4,8>::Add();
-
-   using k2 = QuadratureInterpolator::CollocatedGradKernels;
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,3>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,5>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,6>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,6>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,7>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,8>();
 
    // 2D
-   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
-   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,2,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,3,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,4,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,2,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,3,4>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,4,2>();
 
-   k2::Specialization<3,L,P,1,2>::Add();
-   k2::Specialization<3,L,P,1,3>::Add();
-   k2::Specialization<3,L,P,1,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,4>();
 
-   k2::Specialization<3,L,P,2,2>::Add();
-   k2::Specialization<3,L,P,2,3>::Add();
-   k2::Specialization<3,L,P,2,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,4>();
 
-   k2::Specialization<3,L,P,3,2>::Add();
-   k2::Specialization<3,L,P,3,3>::Add();
-   k2::Specialization<3,L,P,3,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,4>();
 }
 
 template void InitGradByNodesKernels<true>();

fem/qinterp/grad_by_vdim.cpp

@@ -22,47 +22,45 @@ namespace quadrature_interpolator
 template <bool P>
 void InitGradByVDimKernels()
 {
-   using k = QuadratureInterpolator::GradKernels;
    constexpr auto L = QVectorLayout::byVDIM;
    // 2D
-   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,4,8>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,6,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,5,8,2>();
 
-   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
-   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
-   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,3,8>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,4,8>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,6,4>();
+   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,5,8,2>();
    // 3D
-   k::Specialization<3,L,P,1,3,4>::Add();
-   k::Specialization<3,L,P,1,4,6>::Add();
-   k::Specialization<3,L,P,1,5,8>::Add();
-   k::Specialization<3,L,P,3,3,4>::Add();
-   k::Specialization<3,L,P,3,4,6>::Add();
-   k::Specialization<3,L,P,3,5,8>::Add();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,6>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,5,8>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,4>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,6>();
+   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,5,8>();
 
-   using k2 = QuadratureInterpolator::CollocatedGradKernels;
    // 2D
-   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,2,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,3,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,4,16>();
 
-   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
-   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
-   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,2,16>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,3,4>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,4,2>();
 
    // 3D
-   k2::Specialization<3,L,P,1,2>::Add();
-   k2::Specialization<3,L,P,1,3>::Add();
-   k2::Specialization<3,L,P,1,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,4>();
 
-   k2::Specialization<3,L,P,2,2>::Add();
-   k2::Specialization<3,L,P,2,3>::Add();
-   k2::Specialization<3,L,P,2,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,4>();
 
-   k2::Specialization<3,L,P,3,2>::Add();
-   k2::Specialization<3,L,P,3,3>::Add();
-   k2::Specialization<3,L,P,3,4>::Add();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,2>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,3>();
+   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,4>();
 }
 
 template void InitGradByVDimKernels<true>();

fem/quadinterpolator.cpp

@@ -69,8 +69,9 @@ QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
 
    d_buffer.UseDevice(true);
    if (fespace->GetNE() == 0) { return; }
-   MFEM_VERIFY(SupportsFESpace(fes),
-               "Only elements with MapType VALUE and H_DIV are supported!");
+   MFEM_VERIFY(
+      SupportsFESpace(fes),
+      "Only elements with MapType VALUE, INTEGRAL, or H_DIV are supported!");
 }
 
 QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
@@ -84,8 +85,9 @@ QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
 {
    d_buffer.UseDevice(true);
    if (fespace->GetNE() == 0) { return; }
-   MFEM_VERIFY(SupportsFESpace(fes),
-               "Only elements with MapType VALUE and H_DIV are supported!");
+   MFEM_VERIFY(
+      SupportsFESpace(fes),
+      "Only elements with MapType VALUE, INTEGRAL, or H_DIV are supported!");
 }
 
 bool QuadratureInterpolator::SupportsFESpace(const FiniteElementSpace &fespace)
@@ -93,9 +95,9 @@ bool QuadratureInterpolator::SupportsFESpace(const FiniteElementSpace &fespace)
    const FiniteElement *fe = fespace.GetTypicalFE();
    const Mesh &mesh = *fespace.GetMesh();
    return (fe->GetMapType() == FiniteElement::MapType::VALUE ||
-           fe->GetMapType() == FiniteElement::MapType::H_DIV)
-          && (!fespace.IsVariableOrder())
-          && (!mesh.IsMixedMesh());
+           fe->GetMapType() == FiniteElement::MapType::INTEGRAL ||
+           fe->GetMapType() == FiniteElement::MapType::H_DIV) &&
+          (!fespace.IsVariableOrder()) && (!mesh.IsMixedMesh());
 }
 
 namespace internal
@@ -108,16 +110,11 @@ namespace quadrature_interpolator
 // * non-tensor product version,
 // * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
 // * assumes 'maps.mode == FULL'.
-static void Eval1D(const int NE,
-                   const int vdim,
-                   const QVectorLayout q_layout,
-                   const GeometricFactors *geom,
-                   const DofToQuad &maps,
-                   const Vector &e_vec,
-                   Vector &q_val,
-                   Vector &q_der,
-                   Vector &q_det,
-                   const int eval_flags)
+template <bool Integral>
+void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
+                const real_t *detJ_, const GeometricFactors *geom,
+                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+                Vector &q_der, Vector &q_det, const int eval_flags)
 {
    using QI = QuadratureInterpolator;
 
@@ -126,13 +123,16 @@ static void Eval1D(const int NE,
    MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
    MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 1, "");
    MFEM_VERIFY(vdim == 1 || !(eval_flags & QI::DETERMINANTS), "");
-   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
-               "'geom' must be given (non-null) only when evaluating physical"
-               " derivatives");
-   const auto B = Reshape(maps.B.Read(), nq, nd);
-   const auto G = Reshape(maps.G.Read(), nq, nd);
+   if constexpr(Integral)
+   {
+      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
+                                  QI::DETERMINANTS)),
+                  "Integral FE does not support computing derivatives");
+   }
+   const auto B_ = maps.B.Read();
+   const auto G_ = maps.G.Read();
    const auto J = Reshape(geom ? geom->J.Read() : nullptr, nq, NE);
-   const auto E = Reshape(e_vec.Read(), nd, vdim, NE);
+   const auto E_ = e_vec.Read();
    auto val = q_layout == QVectorLayout::byNODES ?
               Reshape(q_val.Write(), nq, vdim, NE):
               Reshape(q_val.Write(), vdim, nq, NE);
@@ -140,8 +140,12 @@ static void Eval1D(const int NE,
               Reshape(q_der.Write(), nq, vdim, NE):
               Reshape(q_der.Write(), vdim, nq, NE);
    auto det = Reshape(q_det.Write(), nq, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
    {
+      const auto B = Reshape(B_, nq, nd);
+      const auto G = Reshape(G_, nq, nd);
+      const auto E = Reshape(E_, nd, vdim, NE);
+      const auto detJ = Reshape(detJ_, nq, NE);
       for (int q = 0; q < nq; ++q)
       {
          if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
@@ -151,10 +155,20 @@ static void Eval1D(const int NE,
                real_t q_val = 0.0;
                for (int d = 0; d < nd; ++d)
                {
-                  q_val += B(q,d)*E(d,c,e);
+                  q_val += B(q, d) * E(d, c, e);
+               }
+               if constexpr (Integral)
+               {
+                  q_val /= detJ(q, e);
+               }
+               if (q_layout == QVectorLayout::byVDIM)
+               {
+                  val(c, q, e) = q_val;
+               }
+               if (q_layout == QVectorLayout::byNODES)
+               {
+                  val(q, c, e) = q_val;
                }
-               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = q_val; }
-               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = q_val; }
             }
          }
          if ((eval_flags & QI::DERIVATIVES) ||
@@ -166,7 +180,7 @@ static void Eval1D(const int NE,
                real_t q_d = 0.0;
                for (int d = 0; d < nd; ++d)
                {
-                  q_d += G(q,d)*E(d,c,e);
+                  q_d += G(q, d) * E(d, c, e);
                }
                if (eval_flags & QI::PHYSICAL_DERIVATIVES)
                {
@@ -174,8 +188,14 @@ static void Eval1D(const int NE,
                }
                if (eval_flags & QI::DERIVATIVES || eval_flags & QI::PHYSICAL_DERIVATIVES)
                {
-                  if (q_layout == QVectorLayout::byVDIM) { der(c,q,e) = q_d; }
-                  if (q_layout == QVectorLayout::byNODES) { der(q,c,e) = q_d; }
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c, q, e) = q_d;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q, c, e) = q_d;
+                  }
                }
                if (vdim == 1 && (eval_flags & QI::DETERMINANTS))
                {
@@ -187,317 +207,17 @@ static void Eval1D(const int NE,
    });
 }
 
-// Template compute kernel for 2D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template<const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval2D(const int NE,
-                   const int vdim,
-                   const QVectorLayout q_layout,
-                   const GeometricFactors *geom,
-                   const DofToQuad &maps,
-                   const Vector &e_vec,
-                   Vector &q_val,
-                   Vector &q_der,
-                   Vector &q_det,
-                   const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
+template void
+ImplEval1D<true>(const int NE, const int vdim, const QVectorLayout q_layout,
+                 const real_t *detJ, const GeometricFactors *geom,
+                 const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+                 Vector &q_der, Vector &q_det, const int eval_flags);
 
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
-   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
-               "'geom' must be given (non-null) only when evaluating physical"
-               " derivatives");
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
-   const auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 2, NE):
-              Reshape(q_der.Write(), VDIM, 2, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE (int e)
-   {
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND2D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM2D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c+d*VDIM] = E(d,c,e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = ed[c]; }
-               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = ed[c]; }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM2D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM2D*2];
-            for (int i = 0; i < 2*VDIM; i++) { D[i] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[4], Jinv[4];
-               Jloc[0] = J(q,0,0,e);
-               Jloc[1] = J(q,1,0,e);
-               Jloc[2] = J(q,0,1,e);
-               Jloc[3] = J(q,1,1,e);
-               kernels::CalcInverse<2>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v;
-                  const real_t JiV = Jinv[2]*u + Jinv[3]*v;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                  }
-               }
-            }
-            if (eval_flags & QI::DETERMINANTS)
-            {
-               if (VDIM == 2) { det(q,e) = kernels::Det<2>(D); }
-               else
-               {
-                  DeviceTensor<2> j(D, 3, 2);
-                  const double E = j(0,0)*j(0,0) + j(1,0)*j(1,0) + j(2,0)*j(2,0);
-                  const double F = j(0,0)*j(0,1) + j(1,0)*j(1,1) + j(2,0)*j(2,1);
-                  const double G = j(0,1)*j(0,1) + j(1,1)*j(1,1) + j(2,1)*j(2,1);
-                  det(q,e) = std::sqrt(E*G - F*F);
-               }
-            }
-         }
-      }
-   });
-}
-
-// Template compute kernel for 3D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template<const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval3D(const int NE,
-                   const int vdim,
-                   const QVectorLayout q_layout,
-                   const GeometricFactors *geom,
-                   const DofToQuad &maps,
-                   const Vector &e_vec,
-                   Vector &q_val,
-                   Vector &q_der,
-                   Vector &q_det,
-                   const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
-
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 3, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
-   MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
-   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
-               "'geom' must be given (non-null) only when evaluating physical"
-               " derivatives");
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
-   const auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 3, NE):
-              Reshape(q_der.Write(), VDIM, 3, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE (int e)
-   {
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND3D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM3D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c+d*VDIM] = E(d,c,e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = ed[c]; }
-               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = ed[c]; }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM3D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM3D*3];
-            for (int i = 0; i < 3*VDIM; i++) { D[i] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               const real_t wz = G(q,2,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-                  D[c+VDIM*2] += s_e * wz;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                     der(c,2,q,e) = D[c+VDIM*2];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                     der(q,c,2,e) = D[c+VDIM*2];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[9], Jinv[9];
-               for (int col = 0; col < 3; col++)
-               {
-                  for (int row = 0; row < 3; row++)
-                  {
-                     Jloc[row+3*col] = J(q,row,col,e);
-                  }
-               }
-               kernels::CalcInverse<3>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t w = D[c+VDIM*2];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
-                  const real_t JiV = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
-                  const real_t JiW = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                     der(c,2,q,e) = JiW;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                     der(q,c,2,e) = JiW;
-                  }
-               }
-            }
-            if (VDIM == 3 && (eval_flags & QI::DETERMINANTS))
-            {
-               // The check (VDIM == 3) should eliminate this block when VDIM is
-               // known at compile time and (VDIM != 3).
-               det(q,e) = kernels::Det<3>(D);
-            }
-         }
-      }
-   });
-}
+template void
+ImplEval1D<false>(const int NE, const int vdim, const QVectorLayout q_layout,
+                  const real_t *detJ, const GeometricFactors *geom,
+                  const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+                  Vector &q_der, Vector &q_det, const int eval_flags);
 
 } // namespace quadrature_interpolator
 
@@ -535,10 +255,20 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    const int nd = maps.ndof;
    const int nq = maps.nqpt;
    const GeometricFactors *geom = nullptr;
-   if (eval_flags & PHYSICAL_DERIVATIVES)
    {
-      const int jacobians = GeometricFactors::JACOBIANS;
-      geom = fespace->GetMesh()->GetGeometricFactors(*ir, jacobians);
+      int jac_factors = 0;
+      if (eval_flags & PHYSICAL_DERIVATIVES)
+      {
+         jac_factors = GeometricFactors::JACOBIANS;
+      }
+      if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
+      {
+         jac_factors |= GeometricFactors::DETERMINANTS;
+      }
+      if (jac_factors)
+      {
+         geom = fespace->GetMesh()->GetGeometricFactors(*ir, jac_factors);
+      }
    }
 
    MFEM_ASSERT(!(eval_flags & DETERMINANTS) || dim == vdim ||
@@ -552,29 +282,61 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    {
       if (eval_flags & (VALUES | PHYSICAL_VALUES))
       {
-         TensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne, maps.B.Read(),
-                                e_vec.Read(), q_val.Write(), vdim, nd, nq);
+         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
+         {
+            IntTensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne,
+                                      maps.B.Read(), geom->detJ.Read(),
+                                      e_vec.Read(), q_val.Write(), vdim, nd, nq);
+         }
+         else
+         {
+            TensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne,
+                                   maps.B.Read(), e_vec.Read(), q_val.Write(),
+                                   vdim, nd, nq);
+         }
       }
       if (eval_flags & (DERIVATIVES | PHYSICAL_DERIVATIVES))
       {
          const bool phys = (eval_flags & PHYSICAL_DERIVATIVES);
          const real_t *J = phys ? geom->J.Read() : nullptr;
          const int s_dim = phys ? sdim : dim;
-         GradKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
-                          maps.B.Read(), maps.G.Read(), J, e_vec.Read(),
-                          q_der.Write(), s_dim, vdim, nd, nq);
+         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
+         {
+            MFEM_ABORT("");
+         }
+         else
+         {
+            GradKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
+                             maps.B.Read(), maps.G.Read(), J, e_vec.Read(),
+                             q_der.Write(), s_dim, vdim, nd, nq);
+         }
       }
       if (eval_flags & DETERMINANTS)
       {
-         DetKernels::Run(dim, vdim, nd, nq, ne, maps.B.Read(),
-                         maps.G.Read(), e_vec.Read(), q_det.Write(), nd,
-                         nq, &d_buffer);
+         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
+         {
+            MFEM_ABORT("");
+         }
+         else
+         {
+            DetKernels::Run(dim, vdim, nd, nq, ne, maps.B.Read(), maps.G.Read(),
+                            e_vec.Read(), q_det.Write(), nd, nq, &d_buffer);
+         }
       }
    }
    else // use_tensor_eval == false
    {
-      EvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne,vdim, q_layout,
-                       geom, maps, e_vec, q_val, q_der, q_det, eval_flags);
+      if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
+      {
+         IntEvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne, vdim,
+                             q_layout, geom->detJ.Read(), geom, maps, e_vec,
+                             q_val, q_der, q_det, eval_flags);
+      }
+      else
+      {
+         EvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne, vdim, q_layout,
+                          geom, maps, e_vec, q_val, q_der, q_det, eval_flags);
+      }
    }
 }
 
@@ -700,22 +462,41 @@ namespace
 
 using namespace internal::quadrature_interpolator;
 
-using EvalKernel = QuadratureInterpolator::EvalKernelType;
-using TensorEvalKernel = QuadratureInterpolator::TensorEvalKernelType;
-using GradKernel = QuadratureInterpolator::GradKernelType;
-using CollocatedGradKernel = QuadratureInterpolator::CollocatedGradKernelType;
-
-template <QVectorLayout Q_LAYOUT>
-TensorEvalKernel FallbackTensorEvalKernel(int DIM)
+template <QVectorLayout Q_LAYOUT> auto IntFallbackTensorEvalKernel(int DIM)
 {
-   if (DIM == 1) { return Values1D<Q_LAYOUT>; }
-   else if (DIM == 2) { return Values2D<Q_LAYOUT>; }
-   else if (DIM == 3) { return Values3D<Q_LAYOUT>; }
-   else { MFEM_ABORT(""); }
+   if (DIM == 1)
+   {
+      return ImplValues1D<Q_LAYOUT, true>;
+   }
+   else if (DIM == 2)
+   {
+      return ImplValues2D<Q_LAYOUT, true>;
+   }
+   else if (DIM == 3)
+   {
+      return ImplValues3D<Q_LAYOUT, true>;
+   }
+   MFEM_ABORT("");
 }
 
-template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
-GradKernel GetGradKernel(int DIM)
+template <QVectorLayout Q_LAYOUT> auto FallbackTensorEvalKernel(int DIM)
+{
+   if (DIM == 1)
+   {
+      return Values1D<Q_LAYOUT>;
+   }
+   else if (DIM == 2)
+   {
+      return Values2D<Q_LAYOUT>;
+   }
+   else if (DIM == 3)
+   {
+      return Values3D<Q_LAYOUT>;
+   }
+   MFEM_ABORT("");
+}
+
+template <QVectorLayout Q_LAYOUT, bool GRAD_PHYS> auto GetGradKernel(int DIM)
 {
    if (DIM == 1) { return Derivatives1D<Q_LAYOUT, GRAD_PHYS>; }
    else if (DIM == 2) { return Derivatives2D<Q_LAYOUT, GRAD_PHYS>; }
@@ -723,79 +504,185 @@ GradKernel GetGradKernel(int DIM)
    else { MFEM_ABORT(""); }
 }
 
-
-template<QVectorLayout Q_LAYOUT>
-GradKernel GetGradKernel(int DIM, bool GRAD_PHYS)
+template <QVectorLayout Q_LAYOUT> auto GetGradKernel(int DIM, bool GRAD_PHYS)
 {
    if (GRAD_PHYS) { return GetGradKernel<Q_LAYOUT, true>(DIM); }
    else { return GetGradKernel<Q_LAYOUT, false>(DIM); }
 }
 
 template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
-CollocatedGradKernel GetCollocatedGradKernel(int DIM)
+auto GetCollocatedGradKernel(int DIM)
 {
-   if (DIM == 1) { return CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 2) { return CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS>; }
-   else if (DIM == 3) { return CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS>; }
-   else { MFEM_ABORT(""); }
+   if (DIM == 1)
+   {
+      return CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>;
+   }
+   else if (DIM == 2)
+   {
+      return CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS>;
+   }
+   else if (DIM == 3)
+   {
+      return CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS>;
+   }
+   MFEM_ABORT("");
 }
 
-template<QVectorLayout Q_LAYOUT>
-CollocatedGradKernel GetCollocatedGradKernel(int DIM, bool GRAD_PHYS)
+template <QVectorLayout Q_LAYOUT>
+auto GetCollocatedGradKernel(int DIM, bool GRAD_PHYS)
 {
    if (GRAD_PHYS) { return GetCollocatedGradKernel<Q_LAYOUT, true>(DIM); }
    else { return GetCollocatedGradKernel<Q_LAYOUT, false>(DIM); }
 }
+
+auto GetCollocatedGradKernel(int DIM, bool GRAD_PHYS, QVectorLayout Q_LAYOUT)
+{
+   if (Q_LAYOUT == QVectorLayout::byNODES)
+   {
+      return GetCollocatedGradKernel<QVectorLayout::byNODES>(
+                DIM, GRAD_PHYS);
+   }
+   else
+   {
+      return GetCollocatedGradKernel<QVectorLayout::byVDIM>(
+                DIM, GRAD_PHYS);
+   }
+}
 } // namespace
 
-template <int DIM, int VDIM, int ND, int NQ>
-EvalKernel QuadratureInterpolator::EvalKernels::Kernel()
+template <int DIM, bool Integral>
+auto GetEvalKernelVDimFallback(int VDIM)
 {
-   using namespace internal::quadrature_interpolator;
-   if constexpr (DIM == 1) { return Eval1D; }
-   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
-   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
+   if constexpr (Integral)
+   {
+      using EvalKernels = QuadratureInterpolator::IntEvalKernels;
+      if (VDIM == 1)
+      {
+         return EvalKernels::Kernel<DIM, 1, 0, 0>();
+      }
+      else if (VDIM == 2)
+      {
+         return EvalKernels::Kernel<DIM, 2, 0, 0>();
+      }
+      else if (VDIM == 3)
+      {
+         return EvalKernels::Kernel<DIM, 3, 0, 0>();
+      }
+   }
+   if constexpr (!Integral)
+   {
+      using EvalKernels = QuadratureInterpolator::EvalKernels;
+      if (VDIM == 1)
+      {
+         return EvalKernels::Kernel<DIM, 1, 0, 0>();
+      }
+      else if (VDIM == 2)
+      {
+         return EvalKernels::Kernel<DIM, 2, 0, 0>();
+      }
+      else if (VDIM == 3)
+      {
+         return EvalKernels::Kernel<DIM, 3, 0, 0>();
+      }
+   }
    MFEM_ABORT("");
 }
 
-template <int DIM>
-EvalKernel GetEvalKernelVDimFallback(int VDIM)
+template auto GetEvalKernelVDimFallback<1, true>(int VDIM);
+template auto GetEvalKernelVDimFallback<1, false>(int VDIM);
+template auto GetEvalKernelVDimFallback<2, true>(int VDIM);
+template auto GetEvalKernelVDimFallback<2, false>(int VDIM);
+template auto GetEvalKernelVDimFallback<3, true>(int VDIM);
+template auto GetEvalKernelVDimFallback<3, false>(int VDIM);
+
+QuadratureInterpolator::IntEvalKernelType
+QuadratureInterpolator::IntEvalKernels::Fallback(int DIM, int VDIM, int ND,
+                                                 int NQ)
 {
-   using EvalKernels = QuadratureInterpolator::EvalKernels;
-   if (VDIM == 1) { return EvalKernels::Kernel<DIM,1,0,0>(); }
-   else if (VDIM == 2) { return EvalKernels::Kernel<DIM,2,0,0>(); }
-   else if (VDIM == 3) { return EvalKernels::Kernel<DIM,3,0,0>(); }
-   else { MFEM_ABORT(""); }
+   if (DIM == 1)
+   {
+      return GetEvalKernelVDimFallback<1, true>(VDIM);
+   }
+   else if (DIM == 2)
+   {
+      return GetEvalKernelVDimFallback<2, true>(VDIM);
+   }
+   else if (DIM == 3)
+   {
+      return GetEvalKernelVDimFallback<3, true>(VDIM);
+   }
+   else
+   {
+      MFEM_ABORT("");
+   }
 }
 
-EvalKernel QuadratureInterpolator::EvalKernels::Fallback(
-   int DIM, int VDIM, int ND, int NQ)
+QuadratureInterpolator::EvalKernelType
+QuadratureInterpolator::EvalKernels::Fallback(int DIM, int VDIM, int ND, int NQ)
 {
-   if (DIM == 1) { return GetEvalKernelVDimFallback<1>(VDIM); }
-   else if (DIM == 2) { return GetEvalKernelVDimFallback<2>(VDIM); }
-   else if (DIM == 3) { return GetEvalKernelVDimFallback<3>(VDIM); }
-   else { MFEM_ABORT(""); }
+   if (DIM == 1)
+   {
+      return GetEvalKernelVDimFallback<1, false>(VDIM);
+   }
+   else if (DIM == 2)
+   {
+      return GetEvalKernelVDimFallback<2, false>(VDIM);
+   }
+   else if (DIM == 3)
+   {
+      return GetEvalKernelVDimFallback<3, false>(VDIM);
+   }
+   else
+   {
+      MFEM_ABORT("");
+   }
 }
 
-TensorEvalKernel QuadratureInterpolator::TensorEvalKernels::Fallback(
-   int DIM, QVectorLayout Q_LAYOUT, int, int, int)
+QuadratureInterpolator::IntTensorEvalKernelType
+QuadratureInterpolator::IntTensorEvalKernels::Fallback(int DIM,
+                                                       QVectorLayout Q_LAYOUT,
+                                                       int, int, int)
 {
-   if (Q_LAYOUT == QVectorLayout::byNODES) { return FallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM); }
-   else { return FallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM); }
+   if (Q_LAYOUT == QVectorLayout::byNODES)
+   {
+      return IntFallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM);
+   }
+   else
+   {
+      return IntFallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM);
+   }
 }
 
-GradKernel QuadratureInterpolator::GradKernels::Fallback(
-   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int, int)
+QuadratureInterpolator::TensorEvalKernelType
+QuadratureInterpolator::TensorEvalKernels::Fallback(int DIM,
+                                                    QVectorLayout Q_LAYOUT, int,
+                                                    int, int)
+{
+   if (Q_LAYOUT == QVectorLayout::byNODES)
+   {
+      return FallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM);
+   }
+   else
+   {
+      return FallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM);
+   }
+}
+
+QuadratureInterpolator::GradKernelType
+QuadratureInterpolator::GradKernels::Fallback(int DIM, QVectorLayout Q_LAYOUT,
+                                              bool GRAD_PHYS, int, int, int)
 {
    if (Q_LAYOUT == QVectorLayout::byNODES) { return GetGradKernel<QVectorLayout::byNODES>(DIM, GRAD_PHYS); }
    else { return GetGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
 }
 
-CollocatedGradKernel QuadratureInterpolator::CollocatedGradKernels::Fallback(
-   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int)
+QuadratureInterpolator::CollocatedGradKernelType
+QuadratureInterpolator::CollocatedGradKernels::Fallback(int DIM,
+                                                        QVectorLayout Q_LAYOUT,
+                                                        bool GRAD_PHYS, int,
+                                                        int)
 {
-   if (Q_LAYOUT == QVectorLayout::byNODES) { return GetCollocatedGradKernel<QVectorLayout::byNODES>(DIM, GRAD_PHYS); }
-   else { return GetCollocatedGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
+   return GetCollocatedGradKernel(DIM, GRAD_PHYS, Q_LAYOUT);
 }
 
 /// @endcond
@@ -806,98 +693,97 @@ namespace quadrature_interpolator
 {
 void InitEvalKernels()
 {
-   using k = QuadratureInterpolator::EvalKernels;
    // 2D, VDIM = 1
-   k::Specialization<2,1,1,1>::Add();
-   k::Specialization<2,1,1,4>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,1,1>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,1,4>();
    // Q1
-   k::Specialization<2,1,4,4>::Add();
-   k::Specialization<2,1,4,9>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,4,4>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,4,9>();
    // Q2
-   k::Specialization<2,1,9,9>::Add();
-   k::Specialization<2,1,9,16>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,9,9>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,9,16>();
    // Q3
-   k::Specialization<2,1,16,16>::Add();
-   k::Specialization<2,1,16,25>::Add();
-   k::Specialization<2,1,16,36>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,16,16>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,16,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,16,36>();
    // Q4
-   k::Specialization<2,1,25,25>::Add();
-   k::Specialization<2,1,25,36>::Add();
-   k::Specialization<2,1,25,49>::Add();
-   k::Specialization<2,1,25,64>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,25,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,25,36>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,25,49>();
+   QuadratureInterpolator::AddEvalSpecializations<2,1,25,64>();
 
    // 3D, VDIM = 1
    // Q0
-   k::Specialization<3,1,1,1>::Add();
-   k::Specialization<3,1,1,8>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,1,1>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,1,8>();
    // Q1
-   k::Specialization<3,1,8,8>::Add();
-   k::Specialization<3,1,8,27>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,8,8>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,8,27>();
    // Q2
-   k::Specialization<3,1,27,27>::Add();
-   k::Specialization<3,1,27,64>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,27,27>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,27,64>();
    // Q3
-   k::Specialization<3,1,64,64>::Add();
-   k::Specialization<3,1,64,125>::Add();
-   k::Specialization<3,1,64,216>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,64,64>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,64,125>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,64,216>();
    // Q4
-   k::Specialization<3,1,125,125>::Add();
-   k::Specialization<3,1,125,216>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,125,125>();
+   QuadratureInterpolator::AddEvalSpecializations<3,1,125,216>();
 
    // 2D, VDIM = 3
    // Q0
-   k::Specialization<2,3,1,1>::Add();
-   k::Specialization<2,3,1,4>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,1,1>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,1,4>();
    // Q1
-   k::Specialization<2,3,4,4>::Add();
-   k::Specialization<2,3,4,9>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,4,4>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,4,9>();
    // Q2
-   k::Specialization<2,3,9,4>::Add();
-   k::Specialization<2,3,9,9>::Add();
-   k::Specialization<2,3,9,16>::Add();
-   k::Specialization<2,3,9,25>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,9,4>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,9,9>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,9,16>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,9,25>();
    // Q3
-   k::Specialization<2,3,16,16>::Add();
-   k::Specialization<2,3,16,25>::Add();
-   k::Specialization<2,3,16,36>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,16,16>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,16,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,16,36>();
    // Q4
-   k::Specialization<2,3,25,25>::Add();
-   k::Specialization<2,3,25,36>::Add();
-   k::Specialization<2,3,25,49>::Add();
-   k::Specialization<2,3,25,64>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,25,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,25,36>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,25,49>();
+   QuadratureInterpolator::AddEvalSpecializations<2,3,25,64>();
 
    // 2D, VDIM = 2
    // Q1
-   k::Specialization<2,2,4,4>::Add();
-   k::Specialization<2,2,4,9>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,4,4>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,4,9>();
    // Q2
-   k::Specialization<2,2,9,9>::Add();
-   k::Specialization<2,2,9,16>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,9,9>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,9,16>();
    // Q3
-   k::Specialization<2,2,16,16>::Add();
-   k::Specialization<2,2,16,25>::Add();
-   k::Specialization<2,2,16,36>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,16,16>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,16,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,16,36>();
    // Q4
-   k::Specialization<2,2,25,25>::Add();
-   k::Specialization<2,2,25,36>::Add();
-   k::Specialization<2,2,25,49>::Add();
-   k::Specialization<2,2,25,64>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,25,25>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,25,36>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,25,49>();
+   QuadratureInterpolator::AddEvalSpecializations<2,2,25,64>();
 
    // 3D, VDIM = 3
    // Q1
-   k::Specialization<3,3,8,8>::Add();
-   k::Specialization<3,3,8,27>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,8,8>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,8,27>();
    // Q2
-   k::Specialization<3,3,27,27>::Add();
-   k::Specialization<3,3,27,64>::Add();
-   k::Specialization<3,3,27,125>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,27,27>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,27,64>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,27,125>();
    // Q3
-   k::Specialization<3,3,64,64>::Add();
-   k::Specialization<3,3,64,125>::Add();
-   k::Specialization<3,3,64,216>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,64,64>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,64,125>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,64,216>();
    // Q4
-   k::Specialization<3,3,125,125>::Add();
-   k::Specialization<3,3,125,216>::Add();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,125,125>();
+   QuadratureInterpolator::AddEvalSpecializations<3,3,125,216>();
 }
 
 } // namespace quadrature_Interpolator

fem/quadinterpolator.hpp

@@ -117,6 +117,10 @@ public:
        FiniteElementSpace is a vector space) and their determinants are computed
        and stored in @a q_det.
 
+       For Integral spaces, the flags VALUES requests the computation of the
+       scalar field values. The result is stored in @a q_val. Derivative types
+       are not supported.
+
        For H(div)-conforming spaces, the flags VALUES / PHYSICAL_VALUES request
        the computation of the vector field values in reference or physical
        space, respectively. The flag PHYSICAL_MAGNITUDES requests the
@@ -159,26 +163,49 @@ public:
    /// QuadratureInterpolator.
    static bool SupportsFESpace(const FiniteElementSpace &fespace);
 
-   using TensorEvalKernelType = void(*)(const int, const real_t *, const real_t *,
-                                        real_t *, const int, const int, const int);
-   using GradKernelType = void(*)(const int, const real_t *, const real_t *,
-                                  const real_t *, const real_t *, real_t *,
-                                  const int, const int, const int, const int);
-   using CollocatedGradKernelType = void(*)(const int, const real_t *,
-                                            const real_t *, const real_t *,
-                                            real_t *, const int, const int,
-                                            const int);
-   using DetKernelType = void(*)(const int NE, const real_t *, const real_t *,
-                                 const real_t *, real_t *, const int, const int,
-                                 Vector *);
-   using EvalKernelType = void(*)(const int, const int, const QVectorLayout,
-                                  const GeometricFactors *, const DofToQuad &,
-                                  const Vector &, Vector &, Vector &, Vector &,
-                                  const int);
+   // value map types
+   using TensorEvalKernelType = void (*)(const int ne, const real_t *B,
+                                         const real_t *e_vec, real_t *q_val,
+                                         const int vdim, const int nd,
+                                         const int nq);
+   using GradKernelType = void (*)(const int ne, const real_t *B,
+                                   const real_t *G, const real_t *J,
+                                   const real_t *e_vec, real_t *q_der,
+                                   const int s_dim, const int v_dim,
+                                   const int nd, const int nq);
+   using CollocatedGradKernelType = void (*)(const int ne, const real_t *G,
+                                             const real_t *J,
+                                             const real_t *e_vec, real_t *q_der,
+                                             const int sdim, const int vdim,
+                                             const int d1d);
+   using DetKernelType = void (*)(const int NE, const real_t *B,
+                                  const real_t *G, const real_t *e_vec,
+                                  real_t *q_det, const int nd, const int nq,
+                                  Vector *d_buffer);
+   using EvalKernelType = void (*)(const int NE, const int vdim,
+                                   const QVectorLayout q_layout,
+                                   const GeometricFactors *geom,
+                                   const DofToQuad &maps, const Vector &e_vec,
+                                   Vector &q_val, Vector &q_der, Vector &q_det,
+                                   const int eval_flags);
+
+   // integral map types
+   using IntTensorEvalKernelType = void (*)(const int ne, const real_t *B,
+                                            const real_t *detJ,
+                                            const real_t *e_vec, real_t *q_val,
+                                            const int vdim, const int nd,
+                                            const int nq);
+   using IntEvalKernelType =
+      void (*)(const int NE, const int vdim, const QVectorLayout q_layout,
+               const real_t *detJ, const GeometricFactors *geom,
+               const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+               Vector &q_der, Vector &q_det, const int eval_flags);
+
    using TensorEvalHDivKernelType =
       void(*)(const int, const real_t *, const real_t *, const real_t *,
               const real_t *, real_t *, const int, const int);
 
+   // value-type mapping
    MFEM_REGISTER_KERNELS(TensorEvalKernels, TensorEvalKernelType,
                          (int, QVectorLayout, int, int, int), (int));
    MFEM_REGISTER_KERNELS(GradKernels, GradKernelType,
@@ -187,8 +214,84 @@ public:
    MFEM_REGISTER_KERNELS(EvalKernels, EvalKernelType, (int, int, int, int));
    MFEM_REGISTER_KERNELS(CollocatedGradKernels, CollocatedGradKernelType,
                          (int, QVectorLayout, bool, int, int), (int));
+
+   // integral-type mapping
+   MFEM_REGISTER_KERNELS(IntTensorEvalKernels, IntTensorEvalKernelType,
+                         (int, QVectorLayout, int, int, int), (int));
+   MFEM_REGISTER_KERNELS(IntEvalKernels, IntEvalKernelType, (int, int, int, int));
+
    MFEM_REGISTER_KERNELS(TensorEvalHDivKernels, TensorEvalHDivKernelType,
                          (int, QVectorLayout, unsigned, int, int));
+
+   /// Adds specializations for TensorEvalKernels
+   template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D,
+             int NBZ = 0>
+   static void AddTensorEvalSpecializations()
+   {
+      if constexpr (NBZ)
+      {
+         IntTensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
+                              Q1D>::template Opt<NBZ>::Add();
+         TensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
+                           Q1D>::template Opt<NBZ>::Add();
+      }
+      else if constexpr (NBZ == 0)
+      {
+         IntTensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
+                              Q1D>::Add();
+         TensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
+                           Q1D>::Add();
+      }
+   }
+
+   /// Adds specializations for EvalKernels
+   template <int DIM, int VDIM, int ND, int NQ>
+   static void AddEvalSpecializations()
+   {
+      IntEvalKernels::Specialization<DIM, VDIM, ND, NQ>::Add();
+      EvalKernels::Specialization<DIM, VDIM, ND, NQ>::Add();
+   }
+
+   /// Adds specializations for GradKernels
+   template <int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
+             int Q1D, int NBZ = 0>
+   static void AddGradSpecializations()
+   {
+      if constexpr (NBZ)
+      {
+         GradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM, D1D,
+                     Q1D>::template Opt<NBZ>::Add();
+      }
+      else if constexpr (NBZ == 0)
+      {
+         GradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM, D1D,
+                     Q1D>::Add();
+      }
+   }
+
+   /// Adds specializations for CollocatedGradKernels
+   template <int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
+             int NBZ = 0>
+   static void AddCollocatedGradSpecializations()
+   {
+      if constexpr (NBZ)
+      {
+         CollocatedGradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM,
+                               D1D>::template Opt<NBZ>::Add();
+      }
+      else if constexpr (NBZ == 0)
+      {
+         CollocatedGradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM,
+                               D1D>::Add();
+      }
+   }
+
+   /// Adds specializations for DetKernels
+   template <int DIM, int SDIM, int D1D, int Q1D>
+   static void AddDetSpecializations()
+   {
+      DetKernels::Specialization<DIM, SDIM, D1D, Q1D>::Add();
+   }
 };
 
 }

general/backends.hpp

@@ -14,7 +14,7 @@
 
 #include "../config/config.hpp"
 
-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#if defined(MFEM_USE_CUDA)
 #include <cusparse.h>
 #include <library_types.h>
 #include <cuda_runtime.h>
@@ -22,7 +22,7 @@
 #endif
 #include "cuda.hpp"
 
-#if defined(MFEM_USE_HIP) && defined(__HIP__)
+#if defined(MFEM_USE_HIP)
 #include <hip/hip_runtime.h>
 #endif
 #include "hip.hpp"
@@ -45,15 +45,17 @@
 #endif
 
 #if !defined(MFEM_USE_CUDA_OR_HIP)
-constexpr bool mfem_use_gpu = false;
-#define MFEM_DEVICE
-#define MFEM_HOST
-#define MFEM_LAMBDA
-// #define MFEM_HOST_DEVICE // defined in config/config.hpp
 // MFEM_DEVICE_SYNC is made available for debugging purposes
 #define MFEM_DEVICE_SYNC
 // MFEM_STREAM_SYNC is used for UVM and MPI GPU-Aware kernels
 #define MFEM_STREAM_SYNC
+#endif
+
+#if !defined(MFEM_USE_CUDA_OR_HIP_LANG)
+#define MFEM_DEVICE
+#define MFEM_HOST
+#define MFEM_LAMBDA
+// #define MFEM_HOST_DEVICE // defined in config/config.hpp
 #define MFEM_LAUNCH_BOUNDS(...)
 #endif
 
@@ -66,6 +68,23 @@ constexpr bool mfem_use_gpu = false;
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) MFEM_FOREACH_THREAD(i,k,N)
+// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
+// Example (3,2,1) block:
+// 0 (0,0), 1 (1,0), 2 (2,0)
+// 3 (1,0), 4 (1,1), 5 (2,1)
+#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
+   for (int iz = 0; iz < SZ; ++iz)                                             \
+      for (int iy = 0; iy < SY; ++iy)                                          \
+         for (int ix = 0; ix < SX; ++ix)
+// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
+// contiguous in x. This intentionally offsets threads within the block to avoid
+// shared memory bank conflicts.
+// Example (3,2,1) block assigned to work on (2,2,1) items:
+// 0 (0,0), 1 (1,0), 2 (N/A)
+// 3 (1,0), 4 (1,1), 5 (N/A)
+#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
+                                             OY, OZ)                           \
+   MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)
 #endif
 
 // 'double' and 'float' atomicAdd implementation for previous versions of CUDA
@@ -109,4 +128,23 @@ MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
 #endif
 }
 
+namespace mfem::internal
+{
+
+#if defined(MFEM_USE_CUDA_OR_HIP) && !defined(MFEM_USE_CUDA_OR_HIP_LANG)
+static constexpr bool can_compile_kernels = false;
+#else
+static constexpr bool can_compile_kernels = true;
+#endif
+
+template <bool can_compile_kernels = can_compile_kernels>
+void RequireKernelCompilation()
+{
+   static_assert(
+      can_compile_kernels,
+      "The calling function needs to be compiled with CUDA/HIP language!");
+}
+
+}
+
 #endif // MFEM_BACKENDS_HPP

general/cuda.hpp

@@ -18,14 +18,8 @@
 // CUDA block size used by MFEM.
 #define MFEM_CUDA_BLOCKS 256
 
-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#if defined(MFEM_USE_CUDA)
 #define MFEM_USE_CUDA_OR_HIP
-constexpr bool mfem_use_gpu = true;
-#define MFEM_DEVICE __device__
-#define MFEM_HOST __host__
-#define MFEM_LAMBDA __host__
-#define MFEM_LAUNCH_BOUNDS __launch_bounds__
-// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
 #define MFEM_DEVICE_SYNC MFEM_GPU_CHECK(cudaDeviceSynchronize())
 #define MFEM_STREAM_SYNC MFEM_GPU_CHECK(cudaStreamSynchronize(0))
 // Define a CUDA error check macro, MFEM_GPU_CHECK(x), where x returns/is of
@@ -40,6 +34,15 @@ constexpr bool mfem_use_gpu = true;
     }                                                                          \
   } while (0)
 
+// Macros defined only when compiling with CUDA language
+#if defined(__CUDACC__)
+#define MFEM_USE_CUDA_OR_HIP_LANG
+#define MFEM_DEVICE __device__
+#define MFEM_HOST __host__
+#define MFEM_LAMBDA __host__
+#define MFEM_LAUNCH_BOUNDS __launch_bounds__
+// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
+
 // Define the MFEM inner threading macros
 #if defined(__CUDA_ARCH__)
 #define MFEM_SHARED __shared__
@@ -49,13 +52,31 @@ constexpr bool mfem_use_gpu = true;
 #define MFEM_THREAD_SIZE(k) blockDim.k
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=threadIdx.k; i<N; i+=blockDim.k)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) if(const int i=threadIdx.k; i<N)
+// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
+// Example (3,2,1) block:
+// 0 (0,0), 1 (1,0), 2 (2,0)
+// 3 (1,0), 4 (1,1), 5 (2,1)
+#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
+   if (int ix = threadIdx.k % (SX), iy = threadIdx.k / (SX), iz = iy / (SY);   \
+       (iy %= (SY)), (threadIdx.k < (SX) * (SY) * (SZ)))
+// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
+// contiguous in x. This intentionally offsets threads within the block to avoid
+// shared memory bank conflicts.
+// Example (3,2,1) block assigned to work on (2,2,1) items:
+// 0 (0,0), 1 (1,0), 2 (N/A)
+// 3 (1,0), 4 (1,1), 5 (N/A)
+#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
+                                             OY, OZ)                           \
+   if (int ix = threadIdx.k % (OX), iy = threadIdx.k / (OX), iz = iy / (OY);   \
+       (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__CUDA_ARCH__)
-#endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#endif // defined(__CUDACC__)
+#endif // defined(MFEM_USE_CUDA)
 
 namespace mfem
 {
 
-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#if defined(MFEM_USE_CUDA)
 // Function used by the macro MFEM_GPU_CHECK.
 void mfem_cuda_error(cudaError_t err, const char *expr, const char *func,
                      const char *file, int line);

general/error.cpp

@@ -171,7 +171,7 @@ void mfem_error(const char *msg)
 #ifdef MFEM_USE_EXCEPTIONS
    if (mfem_error_action == MFEM_ERROR_THROW)
    {
-      throw ErrorException(msg);
+      throw ErrorException(msg ? msg : "");
    }
 #endif
 

general/error.hpp

@@ -15,7 +15,7 @@
 #include "../config/config.hpp"
 #include <iomanip>
 #include <sstream>
-#ifdef MFEM_USE_HIP
+#if defined(MFEM_USE_HIP)
 #include <hip/hip_runtime.h>
 #endif
 
@@ -153,21 +153,13 @@ void mfem_warning(const char *msg = NULL);
 
 
 // Additional abort functions for HIP
-#if defined(MFEM_USE_HIP)
-#ifndef __HIP_DEVICE_COMPILE__
-template<typename T>
-__host__ void abort_msg(T & msg)
-{
-   MFEM_ABORT(msg);
-}
-#else
+#if defined(__HIP_DEVICE_COMPILE__)
 template<typename T>
 __device__ void abort_msg(T & msg)
 {
    abort();
 }
 #endif
-#endif
 
 // Abort inside a device kernel
 #if defined(__CUDA_ARCH__)

general/forall.hpp

@@ -480,8 +480,8 @@ template <typename DBODY>
 void RajaHipWrap1D(const int N, DBODY &&d_body)
 {
    //true denotes asynchronous kernel
-   RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS,true> >(RAJA::RangeSegment(0,N),
-                                                       d_body);
+   RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS, true> >(
+      Device::GetRajaResource(), RAJA::RangeSegment(0, N), d_body);
 }
 
 template <typename DBODY>
@@ -1044,6 +1044,8 @@ inline void ForallWrap(const bool use_dev, const int N,
                        const int X=0, const int Y=0, const int Z=0,
                        const int G=0)
 {
+   internal::RequireKernelCompilation();
+
    MFEM_CONTRACT_VAR(X);
    MFEM_CONTRACT_VAR(Y);
    MFEM_CONTRACT_VAR(Z);
@@ -1276,6 +1278,9 @@ inline void hypre_forall_cpu(int N, lambda &&body)
 template<typename lambda>
 inline void hypre_forall_gpu(int N, lambda &&body)
 {
+   internal::RequireKernelCompilation();
+
+#if defined(MFEM_USE_CUDA_OR_HIP_LANG)
 #if defined(HYPRE_USING_CUDA)
    CuWrap1D(N, body);
 #elif defined(HYPRE_USING_HIP)
@@ -1283,6 +1288,7 @@ inline void hypre_forall_gpu(int N, lambda &&body)
 #else
 #error Unknown HYPRE GPU backend!
 #endif
+#endif
 }
 #endif
 

general/hip.hpp

@@ -18,14 +18,8 @@
 // HIP block size used by MFEM.
 #define MFEM_HIP_BLOCKS 256
 
-#if defined(MFEM_USE_HIP) && defined(__HIP__)
+#if defined(MFEM_USE_HIP)
 #define MFEM_USE_CUDA_OR_HIP
-constexpr bool mfem_use_gpu = true;
-#define MFEM_DEVICE __device__
-#define MFEM_HOST __host__
-#define MFEM_LAMBDA __host__ __device__
-#define MFEM_LAUNCH_BOUNDS __launch_bounds__
-// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
 #define MFEM_DEVICE_SYNC MFEM_GPU_CHECK(hipDeviceSynchronize())
 #define MFEM_STREAM_SYNC MFEM_GPU_CHECK(hipStreamSynchronize(0))
 // Define a HIP error check macro, MFEM_GPU_CHECK(x), where x returns/is of
@@ -40,6 +34,15 @@ constexpr bool mfem_use_gpu = true;
     }                                                                          \
   } while (0)
 
+// Macros defined only when compiling with HIP language
+#if defined(__HIP__)
+#define MFEM_USE_CUDA_OR_HIP_LANG
+#define MFEM_DEVICE __device__
+#define MFEM_HOST __host__
+#define MFEM_LAMBDA __host__ __device__
+#define MFEM_LAUNCH_BOUNDS __launch_bounds__
+// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
+
 // Define the MFEM inner threading macros
 #if defined(__HIP_DEVICE_COMPILE__)
 #define MFEM_SHARED __shared__
@@ -51,8 +54,28 @@ constexpr bool mfem_use_gpu = true;
    for(int i=hipThreadIdx_ ##k; i<N; i+=hipBlockDim_ ##k)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) \
    if(const int i=hipThreadIdx_ ##k; i<N)
+// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
+// Example (3,2,1) block:
+// 0 (0,0), 1 (1,0), 2 (2,0)
+// 3 (1,0), 4 (1,1), 5 (2,1)
+#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
+   if (int ix = hipThreadIdx_##k % (SX), iy = hipThreadIdx_##k / (SX),         \
+       iz = iy / (SY);                                                         \
+       (iy %= (SY)), (hipThreadIdx_##k < (SX) * (SY) * (SZ)))
+// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
+// contiguous in x. This intentionally offsets threads within the block to avoid
+// shared memory bank conflicts.
+// Example (3,2,1) block assigned to work on (2,2,1) items:
+// 0 (0,0), 1 (1,0), 2 (N/A)
+// 3 (1,0), 4 (1,1), 5 (N/A)
+#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
+                                             OY, OZ)                           \
+   if (int ix = hipThreadIdx_##k % (OX), iy = hipThreadIdx_##k / (OX),         \
+       iz = iy / (OY);                                                         \
+       (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__HIP_DEVICE_COMPILE__)
-#endif // defined(MFEM_USE_HIP) && defined(__HIP__)
+#endif // defined(__HIP__)
+#endif // defined(MFEM_USE_HIP)
 
 namespace mfem
 {

general/reducers.hpp

@@ -550,10 +550,10 @@ void reduce(int N, T &res, B &&body, const R &reducer, bool use_dev,
 
       int num_mp = Device::NumMultiprocessors(Device::GetId());
 #if defined(MFEM_USE_CUDA)
-      // good value of mp_sat found experimentally on Lassen
+      // good value of mp_sat found experimentally on Lassen (V100)
       constexpr int mp_sat = 8;
 #elif defined(MFEM_USE_HIP)
-      // good value of mp_sat found experimentally on Tuolumne
+      // good value of mp_sat found experimentally on Tuolumne (MI300A)
       constexpr int mp_sat = 4;
 #else
       num_mp = 1;

general/scan.hpp

@@ -15,6 +15,10 @@
 #include "backends.hpp"
 #include "forall.hpp"
 
+#if defined(MFEM_USE_CUDA_OR_HIP) && !defined(MFEM_USE_CUDA_OR_HIP_LANG)
+#error "This header requires compilation with CUDA/HIP language!"
+#else
+
 #ifdef MFEM_USE_CUDA
 #include <cub/device/device_scan.cuh>
 #include <cub/device/device_select.cuh>
@@ -406,4 +410,6 @@ void CopyUnique(bool use_dev, InputIt d_in, OutputIt d_out,
 
 #undef MFEM_CUB_NAMESPACE
 
-#endif
+#endif // defined(MFEM_USE_CUDA_OR_HIP) && !defined(MFEM_USE_CUDA_OR_HIP_LANG)
+
+#endif // MFEM_SCAN_HPP

linalg/CMakeLists.txt

@@ -27,6 +27,7 @@ list(APPEND SRCS
   handle.cpp
   matrix.cpp
   mma.cpp
+  multivector.cpp
   ode.cpp
   operator.cpp
   ordering.cpp
@@ -35,6 +36,8 @@ list(APPEND SRCS
   sparsemat.cpp
   sparsesmoothers.cpp
   symmat.cpp
+  tripack.cpp
+  tripack_magma.cpp
   vector.cpp
   )
 
@@ -63,6 +66,7 @@ list(APPEND HDRS
   linalg.hpp
   matrix.hpp
   mma.hpp
+  multivector.hpp
   ode.hpp
   operator.hpp
   ordering.hpp
@@ -71,6 +75,8 @@ list(APPEND HDRS
   sparsemat.hpp
   sparsesmoothers.hpp
   symmat.hpp
+  tripack.hpp
+  tripack_magma.hpp
   tlayout.hpp
   tmatrix.hpp
   ttensor.hpp