Added support for MultiVector and cleaned up.

.github/workflows/builds-and-tests.yml

@@ -142,10 +142,6 @@ jobs:
 
     continue-on-error: ${{ matrix.enzyme && true || false }}
 
-    # Enable ccache for all jobs except Windows (would need sccache).
-    env:
-      USE_CCACHE: ${{ matrix.os != 'windows-latest' }}
-
     steps:
       # Fix 'No space left on device' errors for Ubuntu builds.
       - name: Run Actions Cleaner
@@ -294,52 +290,6 @@ jobs:
           echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
           echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
 
-      # Restore the compiler cache (ccache). The key embeds the run id, so new
-      # runs save a fresh snapshot; the restore-keys prefix warm-starts from the
-      # most recent prior run (incl. the base branch for PRs).
-      - name: cache ccache
-        if: ${{ env.USE_CCACHE == 'true' }}
-        uses: actions/cache@v5
-        with:
-          path: .ccache
-          key: ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-${{ github.run_id }}
-          restore-keys: |
-            ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-
-
-      # Configure ccache and select how it is injected into the MFEM build:
-      # - make:  set CXX="ccache g++"; for MPI, OMPI_CXX="ccache g++" so mpicxx
-      #          runs ccache around g++ (not ccache around the mpicxx wrapper).
-      # - cmake: set CMAKE_<LANG>_COMPILER_LAUNCHER=ccache.
-      # - enzyme: wrap the brew clang++ via OMPI_CXX.
-      # The chosen options are passed through build-mfem's 'config-options'
-      # input (see the build step below).
-      - name: configure ccache
-        if: ${{ env.USE_CCACHE == 'true' }}
-        run: |
-          command -v ccache >/dev/null 2>&1 || {
-            if [[ "${{ runner.os }}" == "Linux" ]]; then
-              sudo apt-get update && sudo apt-get install -y ccache
-            else
-              brew install ccache
-            fi
-          }
-          echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
-          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
-          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
-          # Ignore header timestamps (restamped by each checkout) so direct mode hits.
-          echo "CCACHE_SLOPPINESS=include_file_mtime,include_file_ctime,time_macros" >> $GITHUB_ENV
-          # Hash absolute paths relative to the workspace.
-          echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
-          if [[ "${{ matrix.enzyme }}" == "true" ]]; then
-            echo "OMPI_CXX=ccache $LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
-          elif [[ "${{ matrix.build-system }}" == "cmake" ]]; then
-            echo 'CCACHE_CONFIG_OPTS=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache' >> $GITHUB_ENV
-          else
-            echo "OMPI_CXX=ccache g++" >> $GITHUB_ENV
-            echo 'CCACHE_CONFIG_OPTS=CXX="ccache g++" MPICXX="mpicxx"' >> $GITHUB_ENV
-          fi
-        shell: bash
-
       # MFEM build and test
       - name: build
         uses: mfem/github-actions/build-mfem@v2.7
@@ -355,14 +305,9 @@ jobs:
           metis-dir: ${{ env.METIS_TOP_DIR }}
           mfem-dir: ${{ env.MFEM_TOP_DIR }}
           precision: ${{ matrix.precision }}
-          config-options: ${{ matrix.config-opts }} ${{ env.CCACHE_CONFIG_OPTS }}
+          config-options: ${{ matrix.config-opts }}
           library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
 
-      - name: ccache stats
-        if: ${{ env.USE_CCACHE == 'true' }}
-        run: ccache -s
-        shell: bash
-
       # Run checks (and only checks) on debug targets
       - name: checks
         if: matrix.build-system == 'make' && matrix.target == 'dbg'

.github/workflows/cleanup-pr-caches.yml

@@ -1,42 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-# A closed PR's caches can never be restored again, so delete them to free
-# space against the 10 GB per-repo cache limit.
-name: Cleanup PR caches
-
-on:
-  pull_request:
-    types: [closed]
-
-permissions:
-  actions: write
-
-jobs:
-  cleanup:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Delete caches for the closed PR
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GH_REPO: ${{ github.repository }}
-          PR_REF: refs/pull/${{ github.event.pull_request.number }}/merge
-        run: |
-          echo "Deleting caches for $PR_REF"
-          while :; do
-            ids=$(gh cache list --ref "$PR_REF" --limit 100 --json id --jq '.[].id')
-            [ -n "$ids" ] || break
-            echo "$ids" | while read -r id; do
-              [ -n "$id" ] || continue
-              echo "Deleting cache $id"
-              gh cache delete "$id" || echo "  (already gone)"
-            done
-          done

.github/workflows/repo-check.yml

@@ -13,7 +13,6 @@ name: "Checks"
 
 permissions:
   actions: write
-  pull-requests: read
 
 on:
   push:
@@ -30,11 +29,6 @@ concurrency:
 # by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
 # and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
 # github.repository").
-#
-# The logic for the branch-history check is slightly different, since that check
-# also inspects the PR's labels to allow for overriding failures. In this case,
-# we run on all 'pull_request' triggers, but only run for 'push' triggers that
-# do not correspond to any open PRs.
 
 jobs:
   file-headers-check:
@@ -134,7 +128,10 @@ jobs:
 
   branch-history:
     if: |
-      github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
+      github.ref != 'refs/heads/next' &&
+      github.ref != 'refs/heads/master' &&
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
     runs-on: ubuntu-latest
     steps:
       - name: checkout mfem
@@ -142,27 +139,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: check for pull request
-        id: check_pr
-        if: github.event_name == 'push'
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          pr_exists=$(gh pr list --repo "$GITHUB_REPOSITORY" \
-            --head "$GITHUB_REF_NAME" \
-            --state open \
-            --json number \
-            --jq 'length > 0')
-          echo "pr_exists=$pr_exists" >> "$GITHUB_OUTPUT"
-
       - name: branch-history
-        id: branch_history
-        if: |
-          (github.event_name == 'pull_request' ||
-           github.event_name == 'workflow_dispatch' ||
-           steps.check_pr.outputs.pr_exists == 'false')
-        continue-on-error: ${{ contains(github.event.pull_request.labels.*.name,
-                                        'branch-history-override') }}
         run: |
           # We override origin to make sure we point to the main repo.
           # This is to have consistent test results on PRs from forks.
@@ -170,9 +147,3 @@ jobs:
           git remote add origin https://github.com/mfem/mfem.git
           git checkout -b gh-actions-branch-history
           ./config/githooks/pre-push --history
-
-      - name: report branch-history override
-        if: steps.branch_history.outcome == 'failure'
-        run: |
-          echo "::warning::branch-history check failed, but the" \
-            "'branch-history-override' label is set."

.gitignore

@@ -260,7 +260,6 @@ miniapps/meshing/polar-nc
 miniapps/meshing/mesh-quality
 miniapps/meshing/hpref
 miniapps/meshing/phpref
-miniapps/meshing/pref321
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh

.gitlab/dane-baseline.yml

@@ -102,14 +102,12 @@ report_baseline:
         mkdir -p ${MACHINE_NAME}
         rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
         rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        status=0
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir} || { status=1; }
+        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
         printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
           >> ${rundir}/pipeline.txt
         # We create an autotest-email.html file, because that's how we signal
         # that there was an error / diff (temporary).
-        if [[ $status -ne 0 ]] || \
-           [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
+        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
            [[ -f ${rundir}/${BASELINE_TEST}-${MACHINE_NAME}.diff ]]; then
           cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
         fi

CHANGELOG

@@ -46,29 +46,8 @@ Discretization improvements
 
 - Extend FindPointsGSLIB to support surface meshes.
 
-- Added support for complex-valued mixed bilinear forms via the new classes
-  MixedSesquilinearForm and ParMixedSesquilinearForm, mirroring the existing
-  SesquilinearForm classes. Rectangular complex operators are now also
-  handled correctly by ComplexSparseMatrix::GetSystemMatrix and
-  ComplexHypreParMatrix::GetSystemMatrix, which previously assumed equal
-  trial and test spaces.
-
-- Added FiniteElementSpace::GetBoundaryLoopEdgeDofs to extract the edge DOFs on
-  the perimeter loop of a set of boundary elements, with a ParFiniteElementSpace
-  overload that reconciles the selection across processor boundaries so the
-  result is partition invariant. This is useful for imposing boundary conditions
-  on boundary edge DOFs.
-
-- Added a MaxAbs reduction to GroupCommunicator that selects the signed value of
-  largest magnitude across a group, keeping its sign. Equal-magnitude ties
-  resolve deterministically to the positive value.
-
 Meshing improvements
 --------------------
-- Added support for nonuniform anisotropic mesh refinement on parallel quad/hex
-  meshes with arbitrary spacing in each direction. This enables in particular
-  3:1 refinement in parallel, as demonstrated in the new meshing miniapp pref321.
-
 - Added option to guarantee mesh validity during TMOP-based r-adaptivity, using
   bounds on the determinant of the mesh transformation Jacobian.
 
@@ -89,55 +68,16 @@ Linear and nonlinear solvers
   PRefinement multigrid methods for problems posed on trace spaces (see e.g. the
   DPG miniapps).
 
-- Added new class MultiVector: an array of Vectors of different sizes where each
-  Vector can be allocated independently. Also, added associated methods in class
-  Operator: MultMV, MultTransposeMV, and GetGradientMV, that use MultiVector
-  objects for input and/or output parameters. [PR #5249]
-
 GPU computing
 -------------
-- Improved partial assembly for VectorDivergenceIntegrator with shared-memory
-  kernels, kernel registration, and transpose support.
-
-- Improved partial-assembly diagonal kernels for VectorMassIntegrator (shared-
-  memory specializations) and ElasticityIntegrator (no scratch Q-vector).
-
-- Added PA gradient and diagonal support for VectorConvectionNLFIntegrator
- (AssembleGradPA, AddMultGradPA, AssembleGradDiagonalPA).
-
 - Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
 
-- Added partial assembly support for MixedScalarWeakGradientIntegrator.
-
-- Added partial assembly support for MixedDotProductIntegrator.
-
-- Added partial assembly support for MixedScalarCrossProductIntegrator.
-
-- Added partial assembly support for MixedScalarWeakCrossProductIntegrator.
-
-- Added partial assembly support for MixedVectorGradientIntegrator for H1->RT.
-
-- Added support for device partial assembly CurlInterpolator.
-  This supports 2D and 3D variants:
-    2D H1 (out-of-plane) to RT (in-plane)
-    2D ND (in-plane) to Integral L2 (out-of-plane)
-    3D ND to RT
-
 - Added NVIDIA cuDSS library interface. Implementation examples have been
   added to ex1 and ex1p. See https://developer.nvidia.com/cudss for more
   details. Supported versions >= 0.6.0.
 
 - Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
 
-- Changed VectorFEMassIntegrator to use kernel specialization dispatch for
-  partial assembly.
-
-- Added support for FiniteElement::MapType::INTEGRAL spaces to
-  QuadratureInterpolator.
-
-- Added support for FiniteElement::MapType::INTEGRAL spaces to
-  MixedScalarCurlIntegrator.
-
 New and updated examples and miniapps
 -------------------------------------
 - The Lorentz miniapp (in miniapps/electromagnetics) has been updated to
@@ -152,20 +92,6 @@ Miscellaneous
   using the new method ApplyDofSigns() in class ParFiniteElementSpace: the
   method will return immediately if no sign flips are needed.
 
-- Added support for coefficient-weighted LOR transfer in
-  L2ProjectionGridTransfer. The transfer conserves the weighted mass, for
-  example when transferring velocity while conserving density-weighted momentum.
-  This is illustrated in the lor-transfer and plor-transfer miniapps.
-
-- Added support for saving DataCollection output on the node-local storage,
-  instead of requiring that the filesystem is shared among all the ranks.
-
-API changes
------------
-- Removed ProjectGrad from 2D RT elements. Users should use ProjectCurl instead.
-  This also fixes a bug where ProjectCurl was returning the negative curl,
-  identical to ProjectGrad.
-
 
 Version 4.9, released on Dec 11, 2025
 =====================================

CMakeLists.txt

@@ -88,9 +88,18 @@ if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
-# RAJA requires C++20:
-if ((MFEM_USE_UMPIRE OR MFEM_USE_RAJA) AND ("${CMAKE_CXX_STANDARD}" LESS "20"))
-  set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard to use." FORCE)
+# Ginkgo requires C++17:
+if ((MFEM_USE_GINKGO) AND ("${CMAKE_CXX_STANDARD}" LESS "17"))
+  set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to use." FORCE)
+# Google Benchmark, SUNDIALS, STRUMPACK, Tribol, RAJA and Umpire require C++14:
+elseif ((MFEM_USE_BENCHMARK OR
+         MFEM_USE_SUNDIALS OR
+         MFEM_USE_STRUMPACK OR
+         MFEM_USE_TRIBOL OR
+         MFEM_USE_RAJA OR
+         MFEM_USE_UMPIRE) AND
+    ("${CMAKE_CXX_STANDARD}" LESS "14"))
+  set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ standard to use." FORCE)
 endif()
 
 # Include xSDK default CMake file.
@@ -230,13 +239,6 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
-# Shadow warnings for clang only; GCC's -Wshadow flags more.
-if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall -Wshadow")
-elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall")
-endif()
-
 # Shared build on Windows
 if (WIN32 AND BUILD_SHARED_LIBS)
   # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?

config/defaults.mk

@@ -27,10 +27,7 @@ MPICXX = mpicxx
 
 BASE_FLAGS  = -std=c++17
 OPTIM_FLAGS = -O3 $(BASE_FLAGS)
-
-# The variable WARNING_FLAGS depends on which compiler is used, and is defined
-# later in this file.
-DEBUG_FLAGS = $(strip -g $(addprefix $(XCOMPILER),$(WARNING_FLAGS)) $(BASE_FLAGS))
+DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS)
 
 # Prefixes for passing flags to the compiler and linker when using CXX or MPICXX
 CXX_XCOMPILER =
@@ -49,10 +46,6 @@ SHARED = NO
 #
 # If you set MFEM_USE_ENZYME=YES, must use CUDA_CXX=clang++
 CUDA_CXX = nvcc
-# CUDA compute capability used during compilation, e.g. sm_60. Multiple
-# architectures can be requested as a comma-separated list, e.g. sm_70,sm_80.
-# A single value may also be one of the nvcc special values "all",
-# "all-major", or "native".
 CUDA_ARCH = sm_60
 # Base CUDA install directory, only needed if building with clang+cuda:
 # The default setting is:
@@ -61,23 +54,11 @@ CUDA_ARCH = sm_60
 # 3. Use /usr/local/cuda
 CUDA_DIR = $(or $(CUDA_HOME),$(patsubst %/,%,$(dir \
  $(patsubst %/,%,$(dir $(shell command -v nvcc))))),/usr/local/cuda)
-# Derive nvcc/clang architecture flags from CUDA_ARCH. A comma-separated list
-# expands into one -gencode / --cuda-gpu-arch flag per architecture; otherwise
-# use the -arch / --cuda-gpu-arch shorthand.
-MFEM_COMMA := ,
-CUDA_ARCH_NUMS = $(patsubst sm_%,%,$(subst $(MFEM_COMMA), ,$(CUDA_ARCH)))
-NVCC_ARCH_FLAGS = $(strip $(if $(findstring $(MFEM_COMMA),$(CUDA_ARCH)),\
- $(foreach arch,$(CUDA_ARCH_NUMS),\
- -gencode arch=compute_$(arch)$(MFEM_COMMA)code=sm_$(arch)),\
- -arch=$(CUDA_ARCH)))
-CLANG_ARCH_FLAGS = $(strip $(if $(findstring $(MFEM_COMMA),$(CUDA_ARCH)),\
- $(foreach arch,$(CUDA_ARCH_NUMS),--cuda-gpu-arch=sm_$(arch)),\
- --cuda-gpu-arch=$(CUDA_ARCH)))
 # flags for clang+cuda
-CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) $(CLANG_ARCH_FLAGS)
+CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) --cuda-gpu-arch=$(CUDA_ARCH)
 # flags for nvcc
 NVCC_FLAGS = -x=cu --expt-extended-lambda --expt-relaxed-constexpr \
- $(NVCC_ARCH_FLAGS) -isystem "$(CUDA_DIR)/include"
+ -arch=$(CUDA_ARCH) -isystem "$(CUDA_DIR)/include"
 # Prefixes for passing flags to the host compiler and linker when using
 # CUDA_CXX=nvcc
 CUDA_XCOMPILER = -Xcompiler=
@@ -395,7 +376,7 @@ CUDSS_LIBRARY_DIR = $(CUDSS_DIR)/lib
 CUDSS_OPT         = -I$(CUDSS_INCLUDE_DIR)
 CUDSS_LIB         = \
  $(XLINKER)-rpath,$(CUDSS_LIBRARY_DIR) -L$(CUDSS_LIBRARY_DIR) -lcudss
-# The cuDSS communication and threading libraries.
+# The cuDSS communication and threading libraries. 
 MFEM_CUDSS_COMM_LIB = $(abspath $(wildcard $(or $(CUDSS_COMM_LIB),\
    $(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so))))
 MFEM_CUDSS_THREADING_LIB = $(abspath $(wildcard $(or $(CUDSS_THREADING_LIB),\
@@ -678,15 +659,3 @@ VERBOSE = NO
 
 # Optional build tag
 MFEM_BUILD_TAG = $(shell uname -snm)
-
-# Enable -pedantic flag only for gcc or clang. nvcc complains with -pedantic
-# because of line directives.
-PEDANTIC_FLAG = $(if \
-   $(findstring NVIDIA,$(shell $(MFEM_CXX) --version 2>&1)),, \
-   $(if $(or \
-      $(findstring gcc version,$(shell $(MFEM_CXX) -v 2>&1)), \
-      $(findstring clang version,$(shell $(MFEM_CXX) -v 2>&1))),-pedantic,))
-# Enable shadow warnings for clang only; GCC's -Wshadow flags more.
-SHADOW_WARNING_FLAG = $(if $(findstring clang,\
- $(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
-WARNING_FLAGS = $(PEDANTIC_FLAG) -Wall $(SHADOW_WARNING_FLAG)

config/githooks/README.md

@@ -39,8 +39,3 @@ when a picture was added for documentation.
 If that is the case, make sure the failure is indeed justified, and rerun the
 push command with the `--no-verify` option. This will skip the hooks, allowing
 you to push those changes.
-
-The `branch-history` check is run automatically through GitHub Actions. If a
-branch is known to have a large number of changes that are legitimate, the
-check can be overridden by setting  the label 'branch-history-override' on the
-pull request.

data/nested_cubes.geo

@@ -1,131 +0,0 @@
-// Define the cube sizes
-L_outer = 1.0;
-L_inner = 0.5;
-
-// Set mesh size and algorithm
-mesh_size = 0.4;
-Mesh.Algorithm3D = 1; // Delaunay algorithm for 3D mesh
-Mesh.CharacteristicLengthFactor = 1.0;
-Mesh.MshFileVersion = 2.2;
-
-// Define center point for concentric cubes
-cx = 0.5;
-cy = 0.5;
-cz = 0.5;
-
-// Define the points (vertices of the outer cube)
-Point(1) = {cx-L_outer/2, cy-L_outer/2, cz-L_outer/2, mesh_size};
-Point(2) = {cx+L_outer/2, cy-L_outer/2, cz-L_outer/2, mesh_size};
-Point(3) = {cx+L_outer/2, cy+L_outer/2, cz-L_outer/2, mesh_size};
-Point(4) = {cx-L_outer/2, cy+L_outer/2, cz-L_outer/2, mesh_size};
-Point(5) = {cx-L_outer/2, cy-L_outer/2, cz+L_outer/2, mesh_size};
-Point(6) = {cx+L_outer/2, cy-L_outer/2, cz+L_outer/2, mesh_size};
-Point(7) = {cx+L_outer/2, cy+L_outer/2, cz+L_outer/2, mesh_size};
-Point(8) = {cx-L_outer/2, cy+L_outer/2, cz+L_outer/2, mesh_size};
-
-// Define the points (vertices of the inner cube)
-Point(9) = {cx-L_inner/2, cy-L_inner/2, cz-L_inner/2, mesh_size};
-Point(10) = {cx+L_inner/2, cy-L_inner/2, cz-L_inner/2, mesh_size};
-Point(11) = {cx+L_inner/2, cy+L_inner/2, cz-L_inner/2, mesh_size};
-Point(12) = {cx-L_inner/2, cy+L_inner/2, cz-L_inner/2, mesh_size};
-Point(13) = {cx-L_inner/2, cy-L_inner/2, cz+L_inner/2, mesh_size};
-Point(14) = {cx+L_inner/2, cy-L_inner/2, cz+L_inner/2, mesh_size};
-Point(15) = {cx+L_inner/2, cy+L_inner/2, cz+L_inner/2, mesh_size};
-Point(16) = {cx-L_inner/2, cy+L_inner/2, cz+L_inner/2, mesh_size};
-
-// Define the lines (edges of the outer cube)
-Line(1) = {1, 2};
-Line(2) = {2, 3};
-Line(3) = {3, 4};
-Line(4) = {4, 1};
-Line(5) = {5, 6};
-Line(6) = {6, 7};
-Line(7) = {7, 8};
-Line(8) = {8, 5};
-Line(9) = {1, 5};
-Line(10) = {2, 6};
-Line(11) = {3, 7};
-Line(12) = {4, 8};
-
-// Define the lines (edges of the inner cube)
-Line(13) = {9, 10};
-Line(14) = {10, 11};
-Line(15) = {11, 12};
-Line(16) = {12, 9};
-Line(17) = {13, 14};
-Line(18) = {14, 15};
-Line(19) = {15, 16};
-Line(20) = {16, 13};
-Line(21) = {9, 13};
-Line(22) = {10, 14};
-Line(23) = {11, 15};
-Line(24) = {12, 16};
-
-// Define the surfaces (faces of the outer cube)
-Line Loop(1) = {1, 2, 3, 4};
-Plane Surface(1) = {1};
-
-Line Loop(2) = {5, 6, 7, 8};
-Plane Surface(2) = {2};
-
-Line Loop(3) = {9, 5, -10, -1};
-Plane Surface(3) = {3};
-
-Line Loop(4) = {10, 6, -11, -2};
-Plane Surface(4) = {4};
-
-Line Loop(5) = {11, 7, -12, -3};
-Plane Surface(5) = {5};
-
-Line Loop(6) = {12, 8, -9, -4};
-Plane Surface(6) = {6};
-
-// Define the surfaces (faces of the inner cube)
-Line Loop(7) = {13, 14, 15, 16};
-Plane Surface(7) = {7};
-
-Line Loop(8) = {17, 18, 19, 20};
-Plane Surface(8) = {8};
-
-Line Loop(9) = {21, 17, -22, -13};
-Plane Surface(9) = {9};
-
-Line Loop(10) = {22, 18, -23, -14};
-Plane Surface(10) = {10};
-
-Line Loop(11) = {23, 19, -24, -15};
-Plane Surface(11) = {11};
-
-Line Loop(12) = {24, 20, -21, -16};
-Plane Surface(12) = {12};
-
-// Define the volumes
-Surface Loop(1) = {1, 2, 3, 4, 5, 6};
-Surface Loop(2) = {7, 8, 9, 10, 11, 12};
-Volume(1) = {1, 2}; // Outer volume with inner hole
-Volume(2) = {2};    // Inner volume
-
-// Assign physical groups
-Physical Volume(1) = {1}; // Outer volume
-Physical Volume(2) = {2}; // Inner volume
-
-// Outer cube surfaces
-Physical Surface(1) = {1}; // Outer bottom
-Physical Surface(2) = {2}; // Outer top
-Physical Surface(3) = {3}; // Outer front
-Physical Surface(4) = {4}; // Outer right
-Physical Surface(5) = {5}; // Outer back
-Physical Surface(6) = {6}; // Outer left
-
-// Inner cube surfaces
-Physical Surface(7) = {7};  // Inner bottom (-xy)
-Physical Surface(8) = {8};  // Inner top (+xy)
-Physical Surface(9) = {9};  // Inner front (-xz)
-Physical Surface(10) = {10}; // Inner right (+yz)
-Physical Surface(11) = {11}; // Inner back (+xz)
-Physical Surface(12) = {12}; // Inner left (-yz)
-
-// Mesh control
-Mesh.OptimizeNetgen = 1;
-Mesh.Optimize = 1;
-Mesh.ElementOrder = 1;

data/nested_cubes.msh

@@ -1,907 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-138
-1 0 0 0
-2 1 0 0
-3 1 1 0
-4 0 1 0
-5 0 0 1
-6 1 0 1
-7 1 1 1
-8 0 1 1
-9 0.25 0.25 0.25
-10 0.75 0.25 0.25
-11 0.75 0.75 0.25
-12 0.25 0.75 0.25
-13 0.25 0.25 0.75
-14 0.75 0.25 0.75
-15 0.75 0.75 0.75
-16 0.25 0.75 0.75
-17 0.3333333333325025 0 0
-18 0.6666666666657889 0 0
-19 1 0.3333333333325025 0
-20 1 0.6666666666657889 0
-21 0.6666666666675911 1 0
-22 0.3333333333347203 1 0
-23 0 0.6666666666675911 0
-24 0 0.3333333333347203 0
-25 0.3333333333325025 0 1
-26 0.6666666666657889 0 1
-27 1 0.3333333333325025 1
-28 1 0.6666666666657889 1
-29 0.6666666666675911 1 1
-30 0.3333333333347203 1 1
-31 0 0.6666666666675911 1
-32 0 0.3333333333347203 1
-33 0 0 0.3333333333325025
-34 0 0 0.6666666666657889
-35 1 0 0.3333333333325025
-36 1 0 0.6666666666657889
-37 1 1 0.3333333333325025
-38 1 1 0.6666666666657889
-39 0 1 0.3333333333325025
-40 0 1 0.6666666666657889
-41 0.5000000000003468 0.25 0.25
-42 0.75 0.5000000000003468 0.25
-43 0.5000000000013763 0.75 0.25
-44 0.25 0.5000000000013763 0.25
-45 0.5000000000003468 0.25 0.75
-46 0.75 0.5000000000003468 0.75
-47 0.5000000000013763 0.75 0.75
-48 0.25 0.5000000000013763 0.75
-49 0.25 0.25 0.5000000000003468
-50 0.75 0.25 0.5000000000003468
-51 0.75 0.75 0.5000000000003468
-52 0.25 0.75 0.5000000000003468
-53 0.7113248654055673 0.4999999999991457 0
-54 0.2886751345942123 0.5000000000011557 0
-55 0.5000000000006117 0.7525600817161773 0
-56 0.4999999999993867 0.2474399182839603 0
-57 0.2423197548524782 0.7576802451481532 0
-58 0.757680245147464 0.2423197548520695 0
-59 0.2423197548507857 0.2423197548513912 0
-60 0.7576802451491019 0.7576802451486099 0
-61 0.7113248654055673 0.4999999999991457 1
-62 0.2886751345942123 0.5000000000011557 1
-63 0.5000000000006117 0.7525600817161773 1
-64 0.4999999999993867 0.2474399182839603 1
-65 0.2423197548524782 0.7576802451481532 1
-66 0.757680245147464 0.2423197548520695 1
-67 0.2423197548507857 0.2423197548513912 1
-68 0.7576802451491019 0.7576802451486099 1
-69 0.4999999999993203 0 0.301447615129799
-70 0.4999999999992795 0 0.7028666213189801
-71 0.7525600817158393 0 0.5007190394076877
-72 0.2474399182836191 0 0.5007190394076877
-73 0.7576802451479793 0 0.7576802451479793
-74 0.2423197548517962 0 0.7576802451477375
-75 0.7576802451484569 0 0.2423197548510767
-76 0.2423197548513188 0 0.2423197548513187
-77 1 0.4999999999993203 0.301447615129799
-78 1 0.4999999999992795 0.7028666213189801
-79 1 0.7525600817158394 0.5007190394076877
-80 1 0.2474399182836191 0.5007190394076877
-81 1 0.7576802451479794 0.7576802451479794
-82 1 0.2423197548517962 0.7576802451477376
-83 1 0.7576802451484569 0.2423197548510768
-84 1 0.2423197548513188 0.2423197548513188
-85 0.5000000000008327 1 0.3014476151298047
-86 0.500000000000961 1 0.7028666213191928
-87 0.2474399182842484 1 0.5007190394077241
-88 0.7525600817164384 1 0.5007190394078933
-89 0.2423197548520873 1 0.7576802451480517
-90 0.7576802451481496 1 0.2423197548518761
-91 0.2423197548516099 1 0.2423197548510044
-92 0.7576802451486874 1 0.7576802451481952
-93 0 0.5000000000008327 0.3014476151298047
-94 0 0.500000000000961 0.7028666213191928
-95 0 0.2474399182842484 0.5007190394077241
-96 0 0.7525600817164384 0.5007190394078933
-97 0 0.2423197548520873 0.7576802451480517
-98 0 0.7576802451481496 0.2423197548518761
-99 0 0.2423197548516099 0.2423197548510044
-100 0 0.7576802451486874 0.7576802451481952
-101 0.3968750000003409 0.603125000000244 0.25
-102 0.4374999999998713 0.4375000000001287 0.25
-103 0.5739583333335919 0.5718750000001767 0.25
-104 0.6093749999999631 0.3906250000003402 0.25
-105 0.3968750000003409 0.603125000000244 0.75
-106 0.4374999999998713 0.4375000000001287 0.75
-107 0.5739583333335919 0.5718750000001767 0.75
-108 0.6093749999999631 0.3906250000003402 0.75
-109 0.3806942419826734 0.25 0.3806942419826734
-110 0.5625000000001735 0.25 0.4375000000000001
-111 0.4254282069971791 0.25 0.5712615403304835
-112 0.6093749999998808 0.25 0.6093749999998808
-113 0.75 0.3806942419826734 0.3806942419826734
-114 0.75 0.5625000000001735 0.4375000000000001
-115 0.75 0.4254282069971791 0.5712615403304835
-116 0.75 0.6093749999998808 0.6093749999998808
-117 0.3968750000004991 0.75 0.3968750000002804
-118 0.4375000000000001 0.75 0.5625000000004308
-119 0.5739583333336153 0.75 0.4281250000000707
-120 0.6093749999998166 0.75 0.6093749999993661
-121 0.25 0.3968750000004991 0.3968750000002804
-122 0.25 0.4375000000000001 0.5625000000004308
-123 0.25 0.5739583333336153 0.4281250000000707
-124 0.25 0.6093749999998166 0.6093749999993661
-125 0.4962939304035875 0.5214350017087855 0.4925553109323813
-126 0.3432581985549767 0.6471275530923523 0.3554206606168006
-127 0.6442168181713744 0.5929232373214773 0.3593785632300704
-128 0.625174517421737 0.3491579444372839 0.3604636129462503
-129 0.6130544111091688 0.6576364353993367 0.5046720648819553
-130 0.4281518698369243 0.3632662294430484 0.3548726263205102
-131 0.3639383531355198 0.3520949250221278 0.4978389662613994
-132 0.629585530087249 0.3489878162230438 0.5124654846339122
-133 0.3710853378652663 0.6517586292698121 0.6382643241302075
-134 0.5917018263727056 0.6522525456211955 0.6390764961119443
-135 0.3530810314228338 0.4582477062424107 0.6430239639637545
-136 0.6571010904289998 0.5300774811423468 0.6603970500567977
-137 0.6484596018596915 0.361399676127967 0.6360267588157177
-138 0.4782020887035478 0.3534611476388013 0.6141275027013793
-$EndNodes
-$Elements
-760
-1 2 2 1 1 1 17 59
-2 2 2 1 1 24 1 59
-3 2 2 1 1 18 2 58
-4 2 2 1 1 2 19 58
-5 2 2 1 1 20 3 60
-6 2 2 1 1 3 21 60
-7 2 2 1 1 22 4 57
-8 2 2 1 1 4 23 57
-9 2 2 1 1 17 18 56
-10 2 2 1 1 17 56 59
-11 2 2 1 1 56 18 58
-12 2 2 1 1 19 20 53
-13 2 2 1 1 19 53 58
-14 2 2 1 1 53 20 60
-15 2 2 1 1 21 22 55
-16 2 2 1 1 21 55 60
-17 2 2 1 1 55 22 57
-18 2 2 1 1 23 24 54
-19 2 2 1 1 23 54 57
-20 2 2 1 1 54 24 59
-21 2 2 1 1 54 53 55
-22 2 2 1 1 53 54 56
-23 2 2 1 1 55 53 60
-24 2 2 1 1 53 56 58
-25 2 2 1 1 54 55 57
-26 2 2 1 1 56 54 59
-27 2 2 2 2 5 25 67
-28 2 2 2 2 32 5 67
-29 2 2 2 2 26 6 66
-30 2 2 2 2 6 27 66
-31 2 2 2 2 28 7 68
-32 2 2 2 2 7 29 68
-33 2 2 2 2 30 8 65
-34 2 2 2 2 8 31 65
-35 2 2 2 2 25 26 64
-36 2 2 2 2 25 64 67
-37 2 2 2 2 64 26 66
-38 2 2 2 2 27 28 61
-39 2 2 2 2 27 61 66
-40 2 2 2 2 61 28 68
-41 2 2 2 2 29 30 63
-42 2 2 2 2 29 63 68
-43 2 2 2 2 63 30 65
-44 2 2 2 2 31 32 62
-45 2 2 2 2 31 62 65
-46 2 2 2 2 62 32 67
-47 2 2 2 2 62 61 63
-48 2 2 2 2 61 62 64
-49 2 2 2 2 63 61 68
-50 2 2 2 2 61 64 66
-51 2 2 2 2 62 63 65
-52 2 2 2 2 64 62 67
-53 2 2 3 3 17 1 76
-54 2 2 3 3 1 33 76
-55 2 2 3 3 2 18 75
-56 2 2 3 3 35 2 75
-57 2 2 3 3 5 25 74
-58 2 2 3 3 34 5 74
-59 2 2 3 3 26 6 73
-60 2 2 3 3 6 36 73
-61 2 2 3 3 18 17 69
-62 2 2 3 3 69 17 76
-63 2 2 3 3 18 69 75
-64 2 2 3 3 25 26 70
-65 2 2 3 3 25 70 74
-66 2 2 3 3 70 26 73
-67 2 2 3 3 33 34 72
-68 2 2 3 3 33 72 76
-69 2 2 3 3 72 34 74
-70 2 2 3 3 36 35 71
-71 2 2 3 3 71 35 75
-72 2 2 3 3 36 71 73
-73 2 2 3 3 69 70 71
-74 2 2 3 3 70 69 72
-75 2 2 3 3 69 71 75
-76 2 2 3 3 72 69 76
-77 2 2 3 3 71 70 73
-78 2 2 3 3 70 72 74
-79 2 2 4 4 19 2 84
-80 2 2 4 4 2 35 84
-81 2 2 4 4 3 20 83
-82 2 2 4 4 37 3 83
-83 2 2 4 4 6 27 82
-84 2 2 4 4 36 6 82
-85 2 2 4 4 28 7 81
-86 2 2 4 4 7 38 81
-87 2 2 4 4 20 19 77
-88 2 2 4 4 77 19 84
-89 2 2 4 4 20 77 83
-90 2 2 4 4 27 28 78
-91 2 2 4 4 27 78 82
-92 2 2 4 4 78 28 81
-93 2 2 4 4 35 36 80
-94 2 2 4 4 35 80 84
-95 2 2 4 4 80 36 82
-96 2 2 4 4 38 37 79
-97 2 2 4 4 79 37 83
-98 2 2 4 4 38 79 81
-99 2 2 4 4 77 78 79
-100 2 2 4 4 78 77 80
-101 2 2 4 4 77 79 83
-102 2 2 4 4 80 77 84
-103 2 2 4 4 79 78 81
-104 2 2 4 4 78 80 82
-105 2 2 5 5 21 3 90
-106 2 2 5 5 3 37 90
-107 2 2 5 5 4 22 91
-108 2 2 5 5 39 4 91
-109 2 2 5 5 7 29 92
-110 2 2 5 5 38 7 92
-111 2 2 5 5 30 8 89
-112 2 2 5 5 8 40 89
-113 2 2 5 5 22 21 85
-114 2 2 5 5 85 21 90
-115 2 2 5 5 22 85 91
-116 2 2 5 5 29 30 86
-117 2 2 5 5 29 86 92
-118 2 2 5 5 86 30 89
-119 2 2 5 5 37 38 88
-120 2 2 5 5 37 88 90
-121 2 2 5 5 88 38 92
-122 2 2 5 5 40 39 87
-123 2 2 5 5 87 39 91
-124 2 2 5 5 40 87 89
-125 2 2 5 5 85 86 87
-126 2 2 5 5 86 85 88
-127 2 2 5 5 85 87 91
-128 2 2 5 5 88 85 90
-129 2 2 5 5 87 86 89
-130 2 2 5 5 86 88 92
-131 2 2 6 6 1 24 99
-132 2 2 6 6 33 1 99
-133 2 2 6 6 23 4 98
-134 2 2 6 6 4 39 98
-135 2 2 6 6 32 5 97
-136 2 2 6 6 5 34 97
-137 2 2 6 6 8 31 100
-138 2 2 6 6 40 8 100
-139 2 2 6 6 24 23 93
-140 2 2 6 6 93 23 98
-141 2 2 6 6 24 93 99
-142 2 2 6 6 31 32 94
-143 2 2 6 6 31 94 100
-144 2 2 6 6 94 32 97
-145 2 2 6 6 34 33 95
-146 2 2 6 6 95 33 99
-147 2 2 6 6 34 95 97
-148 2 2 6 6 39 40 96
-149 2 2 6 6 39 96 98
-150 2 2 6 6 96 40 100
-151 2 2 6 6 93 94 95
-152 2 2 6 6 94 93 96
-153 2 2 6 6 93 95 99
-154 2 2 6 6 96 93 98
-155 2 2 6 6 95 94 97
-156 2 2 6 6 94 96 100
-157 2 2 7 7 9 41 102
-158 2 2 7 7 44 9 102
-159 2 2 7 7 41 10 104
-160 2 2 7 7 10 42 104
-161 2 2 7 7 42 11 103
-162 2 2 7 7 11 43 103
-163 2 2 7 7 43 12 101
-164 2 2 7 7 12 44 101
-165 2 2 7 7 102 41 104
-166 2 2 7 7 42 103 104
-167 2 2 7 7 43 101 103
-168 2 2 7 7 101 44 102
-169 2 2 7 7 101 102 103
-170 2 2 7 7 103 102 104
-171 2 2 8 8 13 45 106
-172 2 2 8 8 48 13 106
-173 2 2 8 8 45 14 108
-174 2 2 8 8 14 46 108
-175 2 2 8 8 46 15 107
-176 2 2 8 8 15 47 107
-177 2 2 8 8 47 16 105
-178 2 2 8 8 16 48 105
-179 2 2 8 8 106 45 108
-180 2 2 8 8 46 107 108
-181 2 2 8 8 47 105 107
-182 2 2 8 8 105 48 106
-183 2 2 8 8 105 106 107
-184 2 2 8 8 107 106 108
-185 2 2 9 9 41 9 109
-186 2 2 9 9 9 49 109
-187 2 2 9 9 10 41 110
-188 2 2 9 9 50 10 110
-189 2 2 9 9 13 45 111
-190 2 2 9 9 49 13 111
-191 2 2 9 9 45 14 112
-192 2 2 9 9 14 50 112
-193 2 2 9 9 41 109 110
-194 2 2 9 9 111 45 112
-195 2 2 9 9 109 49 111
-196 2 2 9 9 50 110 112
-197 2 2 9 9 110 109 111
-198 2 2 9 9 110 111 112
-199 2 2 10 10 42 10 113
-200 2 2 10 10 10 50 113
-201 2 2 10 10 11 42 114
-202 2 2 10 10 51 11 114
-203 2 2 10 10 14 46 115
-204 2 2 10 10 50 14 115
-205 2 2 10 10 46 15 116
-206 2 2 10 10 15 51 116
-207 2 2 10 10 42 113 114
-208 2 2 10 10 115 46 116
-209 2 2 10 10 113 50 115
-210 2 2 10 10 51 114 116
-211 2 2 10 10 114 113 115
-212 2 2 10 10 114 115 116
-213 2 2 11 11 43 11 119
-214 2 2 11 11 11 51 119
-215 2 2 11 11 12 43 117
-216 2 2 11 11 52 12 117
-217 2 2 11 11 15 47 120
-218 2 2 11 11 51 15 120
-219 2 2 11 11 47 16 118
-220 2 2 11 11 16 52 118
-221 2 2 11 11 117 43 119
-222 2 2 11 11 47 118 120
-223 2 2 11 11 119 51 120
-224 2 2 11 11 52 117 118
-225 2 2 11 11 118 117 119
-226 2 2 11 11 118 119 120
-227 2 2 12 12 9 44 121
-228 2 2 12 12 49 9 121
-229 2 2 12 12 44 12 123
-230 2 2 12 12 12 52 123
-231 2 2 12 12 48 13 122
-232 2 2 12 12 13 49 122
-233 2 2 12 12 16 48 124
-234 2 2 12 12 52 16 124
-235 2 2 12 12 121 44 123
-236 2 2 12 12 48 122 124
-237 2 2 12 12 49 121 122
-238 2 2 12 12 123 52 124
-239 2 2 12 12 122 121 123
-240 2 2 12 12 122 123 124
-241 4 2 1 1 105 62 106 107
-242 4 2 1 1 102 54 101 103
-243 4 2 1 1 118 86 120 119
-244 4 2 1 1 124 94 123 122
-245 4 2 1 1 52 39 12 96
-246 4 2 1 1 52 12 39 87
-247 4 2 1 1 88 38 15 51
-248 4 2 1 1 79 15 38 51
-249 4 2 1 1 80 14 50 36
-250 4 2 1 1 71 50 14 36
-251 4 2 1 1 120 88 15 51
-252 4 2 1 1 116 15 79 51
-253 4 2 1 1 50 14 112 71
-254 4 2 1 1 52 117 12 87
-255 4 2 1 1 111 69 109 110
-256 4 2 1 1 114 77 115 113
-257 4 2 1 1 124 123 94 96
-258 4 2 1 1 120 86 88 119
-259 4 2 1 1 14 45 64 108
-260 4 2 1 1 103 54 101 55
-261 4 2 1 1 62 105 63 107
-262 4 2 1 1 63 29 15 47
-263 4 2 1 1 14 45 26 64
-264 4 2 1 1 51 11 88 37
-265 4 2 1 1 80 50 10 35
-266 4 2 1 1 49 33 9 72
-267 4 2 1 1 71 10 50 35
-268 4 2 1 1 79 11 51 37
-269 4 2 1 1 49 9 33 95
-270 4 2 1 1 12 123 52 96
-271 4 2 1 1 96 16 52 40
-272 4 2 1 1 49 13 34 72
-273 4 2 1 1 87 52 16 40
-274 4 2 1 1 13 49 34 95
-275 4 2 1 1 43 101 12 55
-276 4 2 1 1 43 12 22 55
-277 4 2 1 1 106 61 108 107
-278 4 2 1 1 102 103 104 53
-279 4 2 1 1 50 115 14 80
-280 4 2 1 1 80 10 50 113
-281 4 2 1 1 9 109 49 72
-282 4 2 1 1 47 15 63 107
-283 4 2 1 1 124 16 52 96
-284 4 2 1 1 49 121 9 95
-285 4 2 1 1 15 28 68 46
-286 4 2 1 1 81 28 15 46
-287 4 2 1 1 92 15 29 47
-288 4 2 1 1 27 82 14 46
-289 4 2 1 1 14 66 27 46
-290 4 2 1 1 26 45 14 73
-291 4 2 1 1 78 115 116 114
-292 4 2 1 1 112 70 111 110
-293 4 2 1 1 79 11 114 51
-294 4 2 1 1 71 50 10 110
-295 4 2 1 1 10 41 104 56
-296 4 2 1 1 77 80 115 113
-297 4 2 1 1 69 109 72 111
-298 4 2 1 1 63 16 30 47
-299 4 2 1 1 25 45 13 64
-300 4 2 1 1 52 16 118 87
-301 4 2 1 1 95 13 49 122
-302 4 2 1 1 57 12 23 44
-303 4 2 1 1 119 11 88 51
-304 4 2 1 1 41 17 9 56
-305 4 2 1 1 18 41 10 56
-306 4 2 1 1 21 11 43 55
-307 4 2 1 1 63 105 16 47
-308 4 2 1 1 13 45 106 64
-309 4 2 1 1 9 102 41 56
-310 4 2 1 1 72 49 13 111
-311 4 2 1 1 11 103 43 55
-312 4 2 1 1 70 112 71 110
-313 4 2 1 1 116 79 78 114
-314 4 2 1 1 32 13 48 67
-315 4 2 1 1 16 89 30 47
-316 4 2 1 1 32 48 13 97
-317 4 2 1 1 48 16 100 31
-318 4 2 1 1 121 123 93 122
-319 4 2 1 1 119 117 118 85
-320 4 2 1 1 10 19 42 84
-321 4 2 1 1 17 9 76 41
-322 4 2 1 1 24 9 59 44
-323 4 2 1 1 42 19 10 58
-324 4 2 1 1 10 41 18 75
-325 4 2 1 1 20 83 11 42
-326 4 2 1 1 90 11 43 21
-327 4 2 1 1 53 104 102 56
-328 4 2 1 1 106 61 64 108
-329 4 2 1 1 87 118 117 85
-330 4 2 1 1 95 121 93 122
-331 4 2 1 1 80 50 115 113
-332 4 2 1 1 109 49 72 111
-333 4 2 1 1 78 27 28 46
-334 4 2 1 1 31 94 48 32
-335 4 2 1 1 19 77 20 42
-336 4 2 1 1 23 24 44 93
-337 4 2 1 1 123 124 52 96
-338 4 2 1 1 88 120 119 51
-339 4 2 1 1 91 43 85 117
-340 4 2 1 1 39 98 12 96
-341 4 2 1 1 12 91 39 87
-342 4 2 1 1 15 92 38 88
-343 4 2 1 1 38 81 15 79
-344 4 2 1 1 36 80 14 82
-345 4 2 1 1 14 71 36 73
-346 4 2 1 1 103 102 54 53
-347 4 2 1 1 61 106 62 107
-348 4 2 1 1 86 29 30 47
-349 4 2 1 1 26 45 70 25
-350 4 2 1 1 21 85 22 43
-351 4 2 1 1 94 93 123 122
-352 4 2 1 1 119 118 86 85
-353 4 2 1 1 12 96 93 123
-354 4 2 1 1 115 78 77 114
-355 4 2 1 1 69 111 70 110
-356 4 2 1 1 28 27 61 46
-357 4 2 1 1 48 62 31 32
-358 4 2 1 1 88 15 92 120
-359 4 2 1 1 79 81 15 116
-360 4 2 1 1 71 14 112 73
-361 4 2 1 1 114 116 79 51
-362 4 2 1 1 112 50 71 110
-363 4 2 1 1 91 12 117 87
-364 4 2 1 1 80 115 14 82
-365 4 2 1 1 63 15 29 68
-366 4 2 1 1 64 26 14 66
-367 4 2 1 1 47 63 105 107
-368 4 2 1 1 43 103 101 55
-369 4 2 1 1 35 10 80 84
-370 4 2 1 1 33 76 9 72
-371 4 2 1 1 37 11 88 90
-372 4 2 1 1 35 71 10 75
-373 4 2 1 1 37 79 11 83
-374 4 2 1 1 9 99 33 95
-375 4 2 1 1 68 63 15 107
-376 4 2 1 1 99 44 93 121
-377 4 2 1 1 18 69 41 17
-378 4 2 1 1 10 35 2 84
-379 4 2 1 1 15 7 28 81
-380 4 2 1 1 3 37 90 11
-381 4 2 1 1 108 64 14 66
-382 4 2 1 1 51 38 79 37
-383 4 2 1 1 12 4 23 98
-384 4 2 1 1 12 39 91 4
-385 4 2 1 1 49 33 72 34
-386 4 2 1 1 10 71 110 75
-387 4 2 1 1 114 11 79 83
-388 4 2 1 1 40 96 16 100
-389 4 2 1 1 34 13 74 72
-390 4 2 1 1 16 87 40 89
-391 4 2 1 1 34 97 13 95
-392 4 2 1 1 20 19 42 53
-393 4 2 1 1 54 23 24 44
-394 4 2 1 1 87 16 118 89
-395 4 2 1 1 95 97 13 122
-396 4 2 1 1 57 22 12 55
-397 4 2 1 1 41 102 104 56
-398 4 2 1 1 64 45 106 108
-399 4 2 1 1 101 57 12 55
-400 4 2 1 1 74 13 5 25
-401 4 2 1 1 13 34 74 5
-402 4 2 1 1 8 40 16 100
-403 4 2 1 1 16 31 65 8
-404 4 2 1 1 82 36 6 14
-405 4 2 1 1 102 9 59 56
-406 4 2 1 1 106 67 13 64
-407 4 2 1 1 119 88 11 90
-408 4 2 1 1 95 49 121 122
-409 4 2 1 1 52 118 117 87
-410 4 2 1 1 124 16 96 100
-411 4 2 1 1 80 10 113 84
-412 4 2 1 1 109 9 76 72
-413 4 2 1 1 72 13 70 111
-414 4 2 1 1 99 9 121 95
-415 4 2 1 1 30 65 16 63
-416 4 2 1 1 13 67 25 64
-417 4 2 1 1 11 55 53 103
-418 4 2 1 1 123 12 44 93
-419 4 2 1 1 56 104 10 58
-420 4 2 1 1 10 18 56 58
-421 4 2 1 1 21 60 11 55
-422 4 2 1 1 17 59 9 56
-423 4 2 1 1 16 63 62 105
-424 4 2 1 1 48 105 16 62
-425 4 2 1 1 111 45 13 70
-426 4 2 1 1 119 86 88 85
-427 4 2 1 1 123 93 94 96
-428 4 2 1 1 115 78 80 77
-429 4 2 1 1 111 72 69 70
-430 4 2 1 1 55 103 54 53
-431 4 2 1 1 63 61 62 107
-432 4 2 1 1 14 27 66 6
-433 4 2 1 1 99 9 1 24
-434 4 2 1 1 49 33 34 95
-435 4 2 1 1 96 52 39 40
-436 4 2 1 1 22 12 91 4
-437 4 2 1 1 20 11 83 3
-438 4 2 1 1 29 15 92 7
-439 4 2 1 1 18 10 75 2
-440 4 2 1 1 36 50 80 35
-441 4 2 1 1 18 17 41 56
-442 4 2 1 1 51 88 38 37
-443 4 2 1 1 62 31 16 48
-444 4 2 1 1 62 16 31 65
-445 4 2 1 1 26 45 25 64
-446 4 2 1 1 104 58 42 10
-447 4 2 1 1 29 63 30 47
-448 4 2 1 1 77 78 79 114
-449 4 2 1 1 71 69 70 110
-450 4 2 1 1 42 103 11 53
-451 4 2 1 1 93 94 95 122
-452 4 2 1 1 118 87 86 85
-453 4 2 1 1 61 62 106 64
-454 4 2 1 1 53 102 54 56
-455 4 2 1 1 78 82 27 46
-456 4 2 1 1 94 48 32 97
-457 4 2 1 1 86 30 89 47
-458 4 2 1 1 78 28 81 46
-459 4 2 1 1 26 70 45 73
-460 4 2 1 1 100 94 48 31
-461 4 2 1 1 86 92 29 47
-462 4 2 1 1 76 69 17 41
-463 4 2 1 1 42 19 77 84
-464 4 2 1 1 77 83 20 42
-465 4 2 1 1 18 41 69 75
-466 4 2 1 1 43 85 90 21
-467 4 2 1 1 91 43 117 12
-468 4 2 1 1 48 62 32 67
-469 4 2 1 1 27 66 61 46
-470 4 2 1 1 68 28 61 46
-471 4 2 1 1 54 24 59 44
-472 4 2 1 1 19 42 53 58
-473 4 2 1 1 23 54 57 44
-474 4 2 1 1 12 93 96 98
-475 4 2 1 1 99 44 121 9
-476 4 2 1 1 42 58 104 53
-477 4 2 1 1 21 43 22 55
-478 4 2 1 1 53 20 11 42
-479 4 2 1 1 53 11 20 60
-480 4 2 1 1 106 67 48 13
-481 4 2 1 1 36 71 50 35
-482 4 2 1 1 52 87 39 40
-483 4 2 1 1 48 67 106 62
-484 4 2 1 1 72 70 13 74
-485 4 2 1 1 9 33 99 1
-486 4 2 1 1 3 37 11 83
-487 4 2 1 1 13 34 5 97
-488 4 2 1 1 12 39 4 98
-489 4 2 1 1 14 36 6 73
-490 4 2 1 1 15 38 92 7
-491 4 2 1 1 8 40 89 16
-492 4 2 1 1 75 35 2 10
-493 4 2 1 1 30 16 65 8
-494 4 2 1 1 14 27 6 82
-495 4 2 1 1 26 14 66 6
-496 4 2 1 1 16 31 8 100
-497 4 2 1 1 32 13 67 5
-498 4 2 1 1 13 67 5 25
-499 4 2 1 1 68 7 28 15
-500 4 2 1 1 29 15 7 68
-501 4 2 1 1 57 4 23 12
-502 4 2 1 1 22 12 4 57
-503 4 2 1 1 10 19 84 2
-504 4 2 1 1 9 59 1 24
-505 4 2 1 1 17 9 59 1
-506 4 2 1 1 18 10 2 58
-507 4 2 1 1 11 21 90 3
-508 4 2 1 1 20 11 3 60
-509 4 2 1 1 16 62 63 65
-510 4 2 1 1 11 53 55 60
-511 4 2 1 1 78 81 79 116
-512 4 2 1 1 71 112 70 73
-513 4 2 1 1 86 88 92 120
-514 4 2 1 1 96 94 124 100
-515 4 2 1 1 94 97 95 122
-516 4 2 1 1 89 118 87 86
-517 4 2 1 1 15 38 7 81
-518 4 2 1 1 102 44 59 9
-519 4 2 1 1 59 44 102 54
-520 4 2 1 1 110 71 69 75
-521 4 2 1 1 79 77 114 83
-522 4 2 1 1 76 69 109 72
-523 4 2 1 1 113 77 80 84
-524 4 2 1 1 93 99 121 95
-525 4 2 1 1 87 117 91 85
-526 4 2 1 1 90 119 43 11
-527 4 2 1 1 80 78 115 82
-528 4 2 1 1 88 119 85 90
-529 4 2 1 1 90 43 119 85
-530 4 2 1 1 46 108 66 61
-531 4 2 1 1 46 66 108 14
-532 4 2 1 1 26 14 6 73
-533 4 2 1 1 44 101 57 12
-534 4 2 1 1 44 57 101 54
-535 4 2 1 1 46 107 68 15
-536 4 2 1 1 46 68 107 61
-537 4 2 1 1 9 33 1 76
-538 4 2 1 1 100 124 48 94
-539 4 2 1 1 100 48 124 16
-540 4 2 1 1 25 70 13 45
-541 4 2 1 1 56 53 104 58
-542 4 2 1 1 61 64 108 66
-543 4 2 1 1 106 62 67 64
-544 4 2 1 1 102 59 54 56
-545 4 2 1 1 32 13 5 97
-546 4 2 1 1 30 16 8 89
-547 4 2 1 1 13 70 25 74
-548 4 2 1 1 97 122 48 13
-549 4 2 1 1 75 110 41 69
-550 4 2 1 1 54 57 101 55
-551 4 2 1 1 97 48 122 94
-552 4 2 1 1 47 118 89 86
-553 4 2 1 1 47 89 118 16
-554 4 2 1 1 17 9 1 76
-555 4 2 1 1 11 21 3 60
-556 4 2 1 1 10 19 2 58
-557 4 2 1 1 75 41 110 10
-558 4 2 1 1 61 63 68 107
-559 4 2 1 1 70 111 45 112
-560 4 2 1 1 78 115 46 116
-561 4 2 1 1 77 114 42 113
-562 4 2 1 1 69 41 109 110
-563 4 2 1 1 107 61 108 46
-564 4 2 1 1 103 42 104 53
-565 4 2 1 1 118 86 47 120
-566 4 2 1 1 94 124 48 122
-567 4 2 1 1 117 119 43 85
-568 4 2 1 1 123 44 121 93
-569 4 2 1 1 101 54 102 44
-570 4 2 1 1 48 105 62 106
-571 4 2 1 1 91 43 12 22
-572 4 2 1 1 91 43 22 85
-573 4 2 1 1 93 12 23 98
-574 4 2 1 1 93 23 12 44
-575 4 2 1 1 46 81 116 15
-576 4 2 1 1 46 116 81 78
-577 4 2 1 1 92 47 120 15
-578 4 2 1 1 120 47 92 86
-579 4 2 1 1 46 82 115 78
-580 4 2 1 1 46 115 82 14
-581 4 2 1 1 73 45 112 70
-582 4 2 1 1 73 112 45 14
-583 4 2 1 1 99 44 9 24
-584 4 2 1 1 99 44 24 93
-585 4 2 1 1 84 42 113 77
-586 4 2 1 1 84 113 42 10
-587 4 2 1 1 76 41 109 69
-588 4 2 1 1 109 41 76 9
-589 4 2 1 1 42 83 114 77
-590 4 2 1 1 42 114 83 11
-591 4 2 2 2 135 13 122 131
-592 4 2 2 2 132 138 125 110
-593 4 2 2 2 138 108 125 106
-594 4 2 2 2 136 108 125 137
-595 4 2 2 2 110 138 125 131
-596 4 2 2 2 136 107 134 125
-597 4 2 2 2 122 135 131 125
-598 4 2 2 2 13 49 122 131
-599 4 2 2 2 136 108 137 46
-600 4 2 2 2 110 138 131 111
-601 4 2 2 2 132 138 110 112
-602 4 2 2 2 106 107 108 125
-603 4 2 2 2 138 112 137 45
-604 4 2 2 2 104 127 125 128
-605 4 2 2 2 113 115 114 125
-606 4 2 2 2 131 122 125 121
-607 4 2 2 2 112 45 14 137
-608 4 2 2 2 110 131 125 130
-609 4 2 2 2 115 116 114 125
-610 4 2 2 2 132 138 112 137
-611 4 2 2 2 50 137 115 132
-612 4 2 2 2 104 127 128 42
-613 4 2 2 2 105 134 125 133
-614 4 2 2 2 129 120 51 119
-615 4 2 2 2 104 130 128 125
-616 4 2 2 2 105 134 133 47
-617 4 2 2 2 135 106 138 125
-618 4 2 2 2 106 105 107 125
-619 4 2 2 2 138 108 106 45
-620 4 2 2 2 104 130 125 102
-621 4 2 2 2 104 127 42 103
-622 4 2 2 2 109 131 49 111
-623 4 2 2 2 106 135 48 105
-624 4 2 2 2 126 101 43 125
-625 4 2 2 2 138 137 125 108
-626 4 2 2 2 104 127 103 125
-627 4 2 2 2 116 114 129 51
-628 4 2 2 2 136 108 46 107
-629 4 2 2 2 136 108 107 125
-630 4 2 2 2 52 124 123 125
-631 4 2 2 2 132 112 110 50
-632 4 2 2 2 44 121 125 123
-633 4 2 2 2 131 122 121 49
-634 4 2 2 2 105 134 47 107
-635 4 2 2 2 106 13 135 138
-636 4 2 2 2 52 133 125 118
-637 4 2 2 2 103 43 101 125
-638 4 2 2 2 117 43 119 125
-639 4 2 2 2 105 134 107 125
-640 4 2 2 2 137 115 132 125
-641 4 2 2 2 50 137 132 112
-642 4 2 2 2 106 135 105 125
-643 4 2 2 2 102 101 44 125
-644 4 2 2 2 104 130 102 41
-645 4 2 2 2 138 45 13 111
-646 4 2 2 2 131 138 135 13
-647 4 2 2 2 138 112 45 111
-648 4 2 2 2 131 138 13 111
-649 4 2 2 2 107 134 15 136
-650 4 2 2 2 118 52 117 125
-651 4 2 2 2 101 126 12 44
-652 4 2 2 2 104 130 41 128
-653 4 2 2 2 138 137 108 45
-654 4 2 2 2 133 118 16 47
-655 4 2 2 2 44 101 126 125
-656 4 2 2 2 133 134 118 47
-657 4 2 2 2 133 134 125 118
-658 4 2 2 2 124 122 123 125
-659 4 2 2 2 129 118 119 125
-660 4 2 2 2 102 104 103 125
-661 4 2 2 2 135 48 13 106
-662 4 2 2 2 123 122 121 125
-663 4 2 2 2 103 127 43 125
-664 4 2 2 2 106 45 13 138
-665 4 2 2 2 103 11 127 42
-666 4 2 2 2 127 51 129 114
-667 4 2 2 2 13 49 131 111
-668 4 2 2 2 137 108 14 46
-669 4 2 2 2 15 47 134 107
-670 4 2 2 2 10 41 128 104
-671 4 2 2 2 16 118 133 52
-672 4 2 2 2 46 107 15 136
-673 4 2 2 2 41 102 9 130
-674 4 2 2 2 128 50 132 110
-675 4 2 2 2 10 41 110 128
-676 4 2 2 2 50 137 112 14
-677 4 2 2 2 130 102 9 44
-678 4 2 2 2 105 133 16 47
-679 4 2 2 2 127 11 103 43
-680 4 2 2 2 128 130 41 110
-681 4 2 2 2 116 134 15 51
-682 4 2 2 2 137 45 14 108
-683 4 2 2 2 12 126 101 43
-684 4 2 2 2 133 48 135 105
-685 4 2 2 2 128 42 10 104
-686 4 2 2 2 131 9 109 49
-687 4 2 2 2 118 117 119 125
-688 4 2 2 2 102 103 101 125
-689 4 2 2 2 129 118 125 134
-690 4 2 2 2 125 129 116 114
-691 4 2 2 2 117 126 43 125
-692 4 2 2 2 126 117 12 52
-693 4 2 2 2 52 126 117 125
-694 4 2 2 2 12 117 126 43
-695 4 2 2 2 127 51 114 11
-696 4 2 2 2 114 129 127 125
-697 4 2 2 2 127 119 43 125
-698 4 2 2 2 119 11 127 43
-699 4 2 2 2 127 113 114 125
-700 4 2 2 2 127 113 42 114
-701 4 2 2 2 128 130 110 125
-702 4 2 2 2 15 47 120 134
-703 4 2 2 2 127 11 114 42
-704 4 2 2 2 120 47 118 134
-705 4 2 2 2 44 130 102 125
-706 4 2 2 2 44 126 12 123
-707 4 2 2 2 123 44 126 125
-708 4 2 2 2 129 118 134 120
-709 4 2 2 2 110 132 128 125
-710 4 2 2 2 128 127 125 113
-711 4 2 2 2 128 127 113 42
-712 4 2 2 2 128 50 110 10
-713 4 2 2 2 113 42 10 128
-714 4 2 2 2 129 134 125 116
-715 4 2 2 2 127 51 11 119
-716 4 2 2 2 129 134 116 51
-717 4 2 2 2 127 51 119 129
-718 4 2 2 2 129 119 127 125
-719 4 2 2 2 110 131 130 109
-720 4 2 2 2 109 130 9 131
-721 4 2 2 2 121 130 9 44
-722 4 2 2 2 122 48 13 135
-723 4 2 2 2 44 121 130 125
-724 4 2 2 2 110 138 111 112
-725 4 2 2 2 131 138 125 135
-726 4 2 2 2 115 137 14 46
-727 4 2 2 2 126 52 12 123
-728 4 2 2 2 123 126 52 125
-729 4 2 2 2 50 137 14 115
-730 4 2 2 2 136 137 115 46
-731 4 2 2 2 121 9 131 49
-732 4 2 2 2 121 131 130 125
-733 4 2 2 2 131 130 9 121
-734 4 2 2 2 113 132 115 125
-735 4 2 2 2 113 50 115 132
-736 4 2 2 2 128 50 10 113
-737 4 2 2 2 132 113 128 125
-738 4 2 2 2 128 50 113 132
-739 4 2 2 2 133 105 135 125
-740 4 2 2 2 52 124 125 133
-741 4 2 2 2 133 48 105 16
-742 4 2 2 2 16 133 124 52
-743 4 2 2 2 136 137 125 115
-744 4 2 2 2 132 138 137 125
-745 4 2 2 2 41 130 9 109
-746 4 2 2 2 134 120 15 51
-747 4 2 2 2 129 118 120 119
-748 4 2 2 2 129 120 134 51
-749 4 2 2 2 135 122 124 125
-750 4 2 2 2 135 48 124 122
-751 4 2 2 2 133 48 16 124
-752 4 2 2 2 133 135 124 125
-753 4 2 2 2 133 48 124 135
-754 4 2 2 2 116 136 134 125
-755 4 2 2 2 115 136 116 125
-756 4 2 2 2 46 115 136 116
-757 4 2 2 2 136 134 15 116
-758 4 2 2 2 46 136 15 116
-759 4 2 2 2 109 41 130 110
-760 4 2 2 2 110 131 109 111
-$EndElements

data/square_in_square.geo

@@ -1,77 +0,0 @@
-// Square-in-square 2D geometry for MFEM
-// Creates concentric squares with different material attributes
-
-// Define the square sizes
-L_outer = 2.0;
-L_inner = 0.5;
-
-// Set mesh size and algorithm
-mesh_size = 1.0;
-Mesh.Algorithm = 6; // Frontal-Delaunay for 2D triangular mesh
-Mesh.CharacteristicLengthFactor = 1.0;
-Mesh.MshFileVersion = 2.2;
-
-// Define center point for concentric squares
-cx = 0.0;
-cy = 0.0;
-
-// Define the points (vertices of the outer square)
-Point(1) = {cx-L_outer/2, cy-L_outer/2, 0, mesh_size}; // bottom-left outer
-Point(2) = {cx+L_outer/2, cy-L_outer/2, 0, mesh_size}; // bottom-right outer
-Point(3) = {cx+L_outer/2, cy+L_outer/2, 0, mesh_size}; // top-right outer
-Point(4) = {cx-L_outer/2, cy+L_outer/2, 0, mesh_size}; // top-left outer
-
-// Define the points (vertices of the inner square)
-Point(5) = {cx-L_inner/2, cy-L_inner/2, 0, mesh_size}; // bottom-left inner
-Point(6) = {cx+L_inner/2, cy-L_inner/2, 0, mesh_size}; // bottom-right inner
-Point(7) = {cx+L_inner/2, cy+L_inner/2, 0, mesh_size}; // top-right inner
-Point(8) = {cx-L_inner/2, cy+L_inner/2, 0, mesh_size}; // top-left inner
-
-// Define the lines (edges of the outer square)
-Line(1) = {1, 2}; // bottom edge
-Line(2) = {2, 3}; // right edge
-Line(3) = {3, 4}; // top edge
-Line(4) = {4, 1}; // left edge
-
-// Define the lines (edges of the inner square)
-Line(5) = {5, 6}; // bottom edge
-Line(6) = {6, 7}; // right edge
-Line(7) = {7, 8}; // top edge
-Line(8) = {8, 5}; // left edge
-
-// Define the surfaces
-// Outer square boundary
-Line Loop(1) = {1, 2, 3, 4};
-
-// Inner square boundary (hole in the outer region)
-Line Loop(2) = {5, 6, 7, 8};
-
-// Define the surface areas
-// Outer region (annular region between squares)
-Plane Surface(1) = {1, 2}; // Outer loop minus inner loop (creates hole)
-
-// Inner region (solid inner square)
-Plane Surface(2) = {2}; // Inner loop only
-
-// Assign physical groups for materials
-Physical Surface(1) = {1}; // Outer material (annular region)
-Physical Surface(2) = {2}; // Inner material (solid square)
-
-// Physical lines for boundary conditions
-// Outer square boundary edges
-Physical Line(1) = {1}; // outer bottom
-Physical Line(2) = {2}; // outer right
-Physical Line(3) = {3}; // outer top
-Physical Line(4) = {4}; // outer left
-
-// Inner square boundary edges
-Physical Line(5) = {5}; // inner bottom
-Physical Line(6) = {6}; // inner right
-Physical Line(7) = {7}; // inner top
-Physical Line(8) = {8}; // inner left
-
-// Mesh control for quality
-Mesh.OptimizeNetgen = 1;
-Mesh.Optimize = 1;
-Mesh.ElementOrder = 1;
-Mesh.RecombineAll = 0; // Keep triangular elements (don't recombine to quads)

data/square_in_square.msh

@@ -1,50 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-13
-1 -1 -1 0
-2 1 -1 0
-3 1 1 0
-4 -1 1 0
-5 -0.25 -0.25 0
-6 0.25 -0.25 0
-7 0.25 0.25 0
-8 -0.25 0.25 0
-9 -2.752797989558076e-12 -1 0
-10 1 -2.752797989558076e-12 0
-11 2.752797989558076e-12 1 0
-12 -1 2.752797989558076e-12 0
-13 0 0 0
-$EndNodes
-$Elements
-28
-1 1 2 1 1 1 9
-2 1 2 1 1 9 2
-3 1 2 2 2 2 10
-4 1 2 2 2 10 3
-5 1 2 3 3 3 11
-6 1 2 3 3 11 4
-7 1 2 4 4 4 12
-8 1 2 4 4 12 1
-9 1 2 5 5 5 6
-10 1 2 6 6 6 7
-11 1 2 7 7 7 8
-12 1 2 8 8 8 5
-13 2 2 1 1 6 5 9
-14 2 2 1 1 5 8 12
-15 2 2 1 1 7 6 10
-16 2 2 1 1 8 7 11
-17 2 2 1 1 9 5 1
-18 2 2 1 1 5 12 1
-19 2 2 1 1 6 9 2
-20 2 2 1 1 10 6 2
-21 2 2 1 1 7 10 3
-22 2 2 1 1 11 7 3
-23 2 2 1 1 8 11 4
-24 2 2 1 1 8 4 12
-25 2 2 2 2 5 6 13
-26 2 2 2 2 8 5 13
-27 2 2 2 2 6 7 13
-28 2 2 2 2 7 8 13
-$EndElements

data/tinyzoo-2d.mesh

@@ -1,38 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-
-dimension
-2
-
-elements
-2
-1 3 0 1 4 3
-1 2 1 2 4
-
-boundary
-5
-1 1 0 1
-1 1 1 2
-1 1 2 4
-1 1 4 3
-1 1 3 0
-
-vertices
-5
-2
-0 0
-1 0
-2 0
-0 1
-1 1

doc/CodeDocumentation.conf.in

@@ -1083,8 +1083,7 @@ EXCLUDE_PATTERNS       =
 # ANamespace::AClass, ANamespace::*Test
 
 EXCLUDE_SYMBOLS        = mfem::internal \
-                         mfem::kernels::internal \
-                         mfem::future::detail
+                         mfem::kernels::internal
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include

doc/CodeDocumentation.dox

@@ -201,7 +201,6 @@ namespace mfem {
  * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
  * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
  * - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
- * - <a class="el" href="nurbs__surface_8cpp_source.html">NURBS Surface</a>: interpolate a 3D Surface in a NURBS Patch
  *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -246,9 +245,6 @@ namespace mfem {
  * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
  * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
  * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
- * - <a class="el" href="reflector_8cpp_source.html">Reflector Miniapp</a>: reflect a mesh about a plane
- * - <a class="el" href="ref321_8cpp_source.html">3:1 Refinement Miniapp</a>: perform 3:1 anisotropic mesh refinements
- * - <a class="el" href="pref321_8cpp_source.html">3:1 Refinement Miniapp</a>: parallel 3:1 anisotropic mesh refinements
  *
  * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
  */

fem/CMakeLists.txt

@@ -57,8 +57,6 @@ set(SRCS
   integ/lininteg_domain_grad.cpp
   integ/lininteg_domain_vectorfe.cpp
   integ/nonlininteg_vecconvection_pa.cpp
-  integ/nonlininteg_vecconvection_pa_diag.cpp
-  integ/nonlininteg_vecconvection_pa_grad.cpp
   integ/nonlininteg_vecconvection_mf.cpp
   coefficient.cpp
   complex_fem.cpp
@@ -206,11 +204,7 @@ set(HDRS
   integ/bilininteg_mass_kernels.hpp
   integ/bilininteg_mass_pa_simplices.hpp
   integ/bilininteg_vecdiffusion_pa.hpp
-  integ/bilininteg_vecdiv_pa.hpp
   integ/bilininteg_vecmass_pa.hpp
-  integ/nonlininteg_vecconvection_pa.hpp
-  integ/nonlininteg_vecconvection_pa_diag.hpp
-  integ/nonlininteg_vecconvection_pa_grad.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp

fem/bilinearform.cpp

@@ -1255,31 +1255,6 @@ void BilinearForm::Mult(const Vector &x, Vector &y) const
    }
 }
 
-void BilinearForm::AddMult(const Vector &x, Vector &y, const real_t a) const
-{
-   if (ext)
-   {
-      ext->AddMult(x, y, a);
-   }
-   else
-   {
-      mat->AddMult(x, y, a);
-   }
-}
-
-void BilinearForm::AddMultTranspose(const Vector &x, Vector &y,
-                                    const real_t a) const
-{
-   if (ext)
-   {
-      ext->AddMultTranspose(x, y, a);
-   }
-   else
-   {
-      mat->AddMultTranspose(x, y, a);
-   }
-}
-
 void BilinearForm::MultTranspose(const Vector & x, Vector & y) const
 {
    if (ext)

fem/bilinearform.hpp

@@ -307,8 +307,8 @@ public:
    { mat->Mult(x, y); mat_e->AddMult(x, y); }
 
    /// Add the matrix vector multiple to a vector:  $ y += a M x $
-   void AddMult(const Vector &x, Vector &y,
-                const real_t a = 1.0) const override;
+   void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const override
+   { mat -> AddMult (x, y, a); }
 
    /** @brief Add the original uneliminated matrix vector multiple to a vector.
        The original matrix is $ M + Me $ so we have:
@@ -318,7 +318,8 @@ public:
 
    /// Add the matrix transpose vector multiplication:  $ y += a M^T x $
    void AddMultTranspose(const Vector & x, Vector & y,
-                         const real_t a = 1.0) const override;
+                         const real_t a = 1.0) const override
+   { mat->AddMultTranspose(x, y, a); }
 
    /** @brief Add the original uneliminated matrix transpose vector
        multiple to a vector. The original matrix is $ M + M_e $

fem/bilinearform_ext.cpp

@@ -1997,11 +1997,7 @@ void PADiscreteLinearOperatorExtension::Assemble()
    }
    else
    {
-      const L2ElementRestriction* l2_elem_restrict =
-         dynamic_cast<const L2ElementRestriction*>(elem_restrict_test);
-      MFEM_VERIFY(l2_elem_restrict,
-                  "A real ElementRestriction is required in this setting!");
-      test_multiplicity = 1.0;
+      mfem_error("A real ElementRestriction is required in this setting!");
    }
 
    auto tm = test_multiplicity.ReadWrite();
@@ -2040,13 +2036,7 @@ void PADiscreteLinearOperatorExtension::AddMult(
    }
    else
    {
-      const L2ElementRestriction* l2_elem_restrict =
-         dynamic_cast<const L2ElementRestriction*>(elem_restrict_test);
-      MFEM_VERIFY(l2_elem_restrict,
-                  "In this setting you need a real ElementRestriction!");
-      tempY.SetSize(y.Size());
-      l2_elem_restrict->MultTranspose(localTest, tempY);
-      y += tempY;
+      mfem_error("In this setting you need a real ElementRestriction!");
    }
 }
 

fem/coefficient.hpp

@@ -1055,8 +1055,7 @@ public:
 
 typedef VectorCoefficient DiagonalMatrixCoefficient;
 
-/** Base class for matrix-valued coefficients that optionally depend on time
-    and space. */
+/// Base class for Matrix Coefficients that optionally depend on time and space.
 class MatrixCoefficient
 {
 protected:
@@ -1103,9 +1102,6 @@ public:
    /// the quadrature points. The matrix will be transposed or not according to
    /// the boolean argument @a transpose.
    ///
-   /// The stored entries use the same row/column convention as `Eval()`,
-   /// unless `transpose == true`, in which case `K^T` is stored instead.
-   ///
    /// The @a vdim of the QuadratureFunction should be equal to the height times
    /// the width of the matrix.
    virtual void Project(QuadratureFunction &qf, bool transpose=false);

fem/complex_fem.hpp

@@ -392,9 +392,6 @@ private:
    bool RealInteg();
    bool ImagInteg();
 
-   void BuildComplexOperator(OperatorHandle &A_r, OperatorHandle &A_i,
-                             OperatorHandle &A) const;
-
 public:
    SesquilinearForm(FiniteElementSpace *fes,
                     ComplexOperator::Convention
@@ -508,186 +505,6 @@ public:
    virtual ~SesquilinearForm();
 };
 
-/** Class for a mixed sesquilinear form
-
-    A mixed sesquilinear form is a generalization of a mixed bilinear form to
-    complex-valued fields. Mixed sesquilinear forms are linear in the second
-    argument but the first argument involves a complex conjugate in the sense
-    that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the MixedSesquilinearForm either the real or
-    imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class MixedSesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   MixedBilinearForm * mblfr;
-   MixedBilinearForm * mblfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are not
-      empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   MixedSesquilinearForm(
-      FiniteElementSpace * trial_fes,
-      FiniteElementSpace * test_fes,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a MixedSesquilinearForm on the given trial and test
-       FiniteElementSpaces, using the same integrators as the
-       MixedBilinearForms @a bfr and @a bfi.
-
-       The FiniteElementSpace pointers are not owned by the newly constructed
-       object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed MixedSesquilinearForm. */
-   MixedSesquilinearForm(
-      FiniteElementSpace * trial_fes,
-      FiniteElementSpace * test_fes,
-      MixedBilinearForm * bfr,
-      MixedBilinearForm * bfi,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
-
-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::LEGACY (default)
-       - AssemblyLevel::FULL
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      mblfr->SetAssemblyLevel(assembly_level);
-      mblfi->SetAssemblyLevel(assembly_level);
-   }
-
-   MixedBilinearForm & real() { return *mblfr; }
-   MixedBilinearForm & imag() { return *mblfi; }
-   const MixedBilinearForm & real() const { return *mblfr; }
-   const MixedBilinearForm & imag() const { return *mblfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag,
-                            Array<int> & elem_marker);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag,
-                              Array<int> & bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag,
-                             Array<int> & bdr_marker);
-
-   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
-
-       This type of integrator assembles terms over all faces of the mesh using
-       the face FE from the trial space and the two adjacent volume FEs from
-       the test space. */
-   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                               BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag,
-                                  Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Updates the internal mixed forms with the new finite element space.
-   virtual void Update();
-
-   /** @brief Return a ComplexSparseMatrix wrapping the local (L-dof) real
-       and imaginary matrices of the form.
-
-       The returned wrapper has to be deleted by the caller, but it does not
-       own the wrapped real and imaginary matrices, which remain owned by
-       this form. */
-   ComplexSparseMatrix *AssembleComplexSparseMatrix();
-
-   /// Return the trial FE space associated with the MixedSesquilinearForm.
-   FiniteElementSpace *TrialFESpace() { return mblfr->TrialFESpace(); }
-
-   /// Read-only access to the associated trial FiniteElementSpace.
-   const FiniteElementSpace *TrialFESpace() const { return mblfr->TrialFESpace(); }
-
-   /// Return the test FE space associated with the MixedSesquilinearForm.
-   FiniteElementSpace *TestFESpace() { return mblfr->TestFESpace(); }
-
-   /// Read-only access to the associated test FiniteElementSpace.
-   const FiniteElementSpace *TestFESpace() const { return mblfr->TestFESpace(); }
-
-
-   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    Vector & x,
-                                    Vector & b,
-                                    OperatorHandle & A,
-                                    Vector & X,
-                                    Vector & B);
-
-   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    OperatorHandle & A);
-
-   virtual ~MixedSesquilinearForm();
-};
-
 #ifdef MFEM_USE_MPI
 
 /// Class for parallel complex-valued grid function - real + imaginary part
@@ -989,12 +806,6 @@ private:
    bool RealInteg();
    bool ImagInteg();
 
-   void SetImaginaryEssentialDiagonalToZero(
-      const Array<int> &ess_tdof_list, OperatorHandle &A);
-
-   void BuildComplexOperator(OperatorHandle &A_r, OperatorHandle &A_i,
-                             OperatorHandle &A) const;
-
 public:
    ParSesquilinearForm(ParFiniteElementSpace *pf,
                        ComplexOperator::Convention
@@ -1110,169 +921,6 @@ public:
    virtual ~ParSesquilinearForm();
 };
 
-/** Class for a parallel mixed sesquilinear form
-
-    A mixed sesquilinear form is a generalization of a mixed bilinear form to
-    complex-valued fields. Mixed sesquilinear forms are linear in the second
-    argument but the first argument involves a complex conjugate in the sense
-    that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the ParMixedSesquilinearForm either the real
-    or imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class ParMixedSesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   ParMixedBilinearForm * pmblfr;
-   ParMixedBilinearForm * pmblfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are
-    not empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   ParMixedSesquilinearForm(
-      ParFiniteElementSpace * trial_fes,
-      ParFiniteElementSpace * test_fes,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a ParMixedSesquilinearForm on the given trial and test
-       ParFiniteElementSpaces, using the same integrators as the
-       ParMixedBilinearForms @a pbfr and @a pbfi.
-
-       The ParFiniteElementSpace pointers are not owned by the newly
-       constructed object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed ParMixedSesquilinearForm. */
-   ParMixedSesquilinearForm(
-      ParFiniteElementSpace * trial_fes,
-      ParFiniteElementSpace * test_fes,
-      ParMixedBilinearForm * pbfr,
-      ParMixedBilinearForm * pbfi,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
-
-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::LEGACY (default)
-       - AssemblyLevel::FULL
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      pmblfr->SetAssemblyLevel(assembly_level);
-      pmblfi->SetAssemblyLevel(assembly_level);
-   }
-
-   ParMixedBilinearForm & real() { return *pmblfr; }
-   ParMixedBilinearForm & imag() { return *pmblfi; }
-   const ParMixedBilinearForm & real() const { return *pmblfr; }
-   const ParMixedBilinearForm & imag() const { return *pmblfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag,
-                            Array<int> & elem_marker);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag,
-                              Array<int> & bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag,
-                             Array<int> & bdr_marker);
-
-   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
-
-      This type of integrator assembles terms over all faces of the mesh using
-      the face FE from the trial space and the two adjacent volume FEs from
-      the test space. */
-   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                               BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag,
-                                  Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Updates the internal mixed forms with the new finite element space.
-   virtual void Update();
-
-   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
-   /** The returned matrix has to be deleted by the caller. */
-   ComplexHypreParMatrix * ParallelAssemble();
-
-   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    Vector & x,
-                                    Vector & b,
-                                    OperatorHandle & A,
-                                    Vector & X,
-                                    Vector & B);
-
-   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    OperatorHandle & A);
-
-   virtual ~ParMixedSesquilinearForm();
-};
-
 #endif // MFEM_USE_MPI
 
 }

fem/datacollection.cpp

@@ -38,24 +38,9 @@ int DataCollection::create_directory(const std::string &dir_name,
    // create directories recursively
    const char path_delim = '/';
    std::string::size_type pos = 0;
-   int err_flag = 0;
+   int err_flag;
 #ifdef MFEM_USE_MPI
    const ParMesh *pmesh = dynamic_cast<const ParMesh*>(mesh);
-   // In addition to the global root, let the lowest rank on each shared-memory
-   // node create the directory too, so that node-local (non-shared) filesystems
-   // get it on every node rather than only where the global root lives. On a
-   // shared filesystem the extra mkdir() hits EEXIST and is tolerated below.
-   bool node_root = true;
-   if (pmesh)
-   {
-      MPI_Comm node_comm;
-      MPI_Comm_split_type(pmesh->GetComm(), MPI_COMM_TYPE_SHARED, myid,
-                          MPI_INFO_NULL, &node_comm);
-      int node_rank;
-      MPI_Comm_rank(node_comm, &node_rank);
-      node_root = (node_rank == 0);
-      MPI_Comm_free(&node_comm);
-   }
 #endif
 
    do
@@ -67,7 +52,7 @@ int DataCollection::create_directory(const std::string &dir_name,
       err_flag = mkdir(subdir.c_str(), 0777);
       err_flag = (err_flag && (errno != EEXIST)) ? 1 : 0;
 #else
-      if (node_root || pmesh == NULL)
+      if (myid == 0 || pmesh == NULL)
       {
          err_flag = mkdir(subdir.c_str(), 0777);
          err_flag = (err_flag && (errno != EEXIST)) ? 1 : 0;
@@ -79,8 +64,7 @@ int DataCollection::create_directory(const std::string &dir_name,
 #ifdef MFEM_USE_MPI
    if (pmesh)
    {
-      MPI_Allreduce(MPI_IN_PLACE, &err_flag, 1, MPI_INT, MPI_MAX,
-                    pmesh->GetComm());
+      MPI_Bcast(&err_flag, 1, MPI_INT, 0, pmesh->GetComm());
    }
 #endif
 
@@ -825,7 +809,7 @@ ParaViewDataCollectionBase::ParaViewDataCollectionBase(
 
 void ParaViewDataCollectionBase::SetLevelsOfDetail(int levels_of_detail_)
 {
-   levels_of_detail = std::max(levels_of_detail_, 1);
+   levels_of_detail = levels_of_detail_;
 }
 
 void ParaViewDataCollectionBase::SetHighOrderOutput(bool high_order_output_)

fem/datacollection.hpp

@@ -25,21 +25,35 @@
 namespace mfem
 {
 
-/// Lightweight adaptor over an std::map from strings to pointer to T
-template<typename T>
-class NamedFieldsMap
+/// Lightweight adaptor over an std::map from type K to type to V
+template<typename K, typename V,
+         typename = typename std::enable_if<std::is_default_constructible<V>::value>::type>
+class GenericFieldMap
 {
+private:
+   static constexpr bool ValueIsPointer = std::is_pointer<V>::value;
+
 public:
-   typedef std::map<std::string, T*> MapType;
+   typedef std::map<K, V> MapType;
    typedef typename MapType::iterator iterator;
    typedef typename MapType::const_iterator const_iterator;
 
-   /// Register field @a field with name @a fname
-   /** Replace existing field associated with @a fname (and optionally
-       delete associated pointer if @a own_data is true) */
-   void Register(const std::string& fname, T* field, bool own_data)
+   /// Register field @a field with name @a key
+   /// Only enabled if the template parameter V is not a pointer
+   template<typename = std::enable_if<!ValueIsPointer, bool>>
+   void Register(const K& key, V field)
    {
-      T*& ref = field_map[fname];
+      field_map[key] = field;
+   }
+
+   /// Register field @a field with name @a key
+   /** Replace existing field associated with @a key (and optionally
+       delete associated pointer if @a own_data is true).
+       Only enabled if the template parameter V is a pointer*/
+   template<typename = std::enable_if<ValueIsPointer, bool>>
+   void Register(const K& key, V field, bool own_data)
+   {
+      V& ref = field_map[key];
       if (own_data)
       {
          delete ref; // if newly allocated -> ref is null -> OK
@@ -47,23 +61,40 @@ public:
       ref = field;
    }
 
-   /// Unregister association between field @a field and name @a fname
-   /** Optionally delete associated pointer if @a own_data is true */
-   void Deregister(const std::string& fname, bool own_data)
+   /// Unregister association between field @a field and name @a key
+   /// Only enabled if the template parameter V is not a pointer
+   template<typename = std::enable_if<!ValueIsPointer, bool>>
+   void Deregister(const K& key)
    {
-      iterator it = field_map.find(fname);
+      iterator it = field_map.find(key);
+      if ( it != field_map.end() )
+      {
+         field_map.erase(it);
+      }
+   }
+
+   /// Unregister association between field @a field and name @a key
+   /** Optionally delete associated pointer if @a own_data is true.
+       Only enabled if the template parameter V is a pointer */
+   template<typename = std::enable_if<ValueIsPointer, bool>>
+   void Deregister(const K& key, bool own_data)
+   {
+      iterator it = field_map.find(key);
       if ( it != field_map.end() )
       {
          if (own_data)
          {
             delete it->second;
+            it->second = nullptr;
          }
          field_map.erase(it);
       }
    }
 
    /// Clear all associations between names and fields
-   /** Delete associated pointers when @a own_data is true */
+   /** Delete associated pointers when @a own_data is true.
+       Only enabled if the template parameter V is a pointer */
+   template<typename = std::enable_if<ValueIsPointer, bool>>
    void DeleteData(bool own_data)
    {
       for (iterator it = field_map.begin(); it != field_map.end(); ++it)
@@ -76,22 +107,37 @@ public:
       }
    }
 
-   /// Predicate to check if a field is associated with name @a fname
-   bool Has(const std::string& fname) const
+   /// Predicate to check if a field is associated with name @a key
+   bool Has(const K& key) const
    {
-      return field_map.find(fname) != field_map.end();
+      return field_map.find(key) != field_map.end();
    }
 
-   /// Get a pointer to the field associated with name @a fname
-   /** @return Pointer to field associated with @a fname or NULL */
-   T* Get(const std::string& fname) const
+   /// Get a pointer to the field associated with name @a key
+   /** @return Field associated with @a key or NULL,
+       if value is pointer and key not found */
+   V Get(const K& key) const
    {
-      const_iterator it = field_map.find(fname);
-      return it != field_map.end() ? it->second : NULL;
+      const_iterator it = field_map.find(key);
+      if (it != field_map.end())
+      {
+         return it->second;
+      }
+      else
+      {
+         if constexpr (ValueIsPointer)
+         {
+            return nullptr;
+         }
+         else
+         {
+            return V(); // Return default-constructed value for non-pointer types
+         }
+      }
    }
 
    /// Returns a const reference to the underlying map
-   const MapType& GetMap() const { return field_map; }
+   const MapType &GetMap() const { return field_map; }
 
    /// Returns the number of registered fields
    int NumFields() const { return field_map.size(); }
@@ -106,21 +152,24 @@ public:
    /// Returns an end const iterator to the registered fields
    const_iterator end() const { return field_map.end(); }
 
-   /// Returns an iterator to the field @a fname
-   iterator find(const std::string& fname)
-   { return field_map.find(fname); }
+   /// Returns an iterator to the field @a key
+   iterator find(const K& key)
+   { return field_map.find(key); }
 
-   /// Returns a const iterator to the field @a fname
-   const_iterator find(const std::string& fname) const
-   { return field_map.find(fname); }
+   /// Returns a const iterator to the field @a key
+   const_iterator find(const K& key) const
+   { return field_map.find(key); }
 
-   /// Clears the map of registered fields without reclaiming memory
+   /// Clears the map of registered fields
    void clear() { field_map.clear(); }
 
 protected:
    MapType field_map;
 };
 
+/// Lightweight adaptor over an std::map from strings to pointer to T
+template<typename T>
+using NamedFieldsMap = GenericFieldMap<std::string, T*>;
 
 /** A class for collecting finite element data that is part of the same
     simulation. Currently, this class groups together grid functions (fields),

fem/dfem/doperator.cpp

@@ -51,52 +51,4 @@ DifferentiableOperator::DifferentiableOperator(
    }
 }
 
-
-void FDJacobian::Mult(const Vector &v, Vector &y) const
-{
-   // See [1] for choice of eps.
-   //
-   // [1] Woodward, C.S., Gardner, D.J. and Evans, K.J., 2015. On the use of
-   // finite difference matrix-vector products in Newton-Krylov solvers for
-   // implicit climate dynamics with spectral elements. Procedia Computer
-   // Science, 51, pp.2036-2045.
-   real_t eps;
-   if (fixed_eps > 0.0)
-   {
-      eps = fixed_eps;
-   }
-   else
-   {
-      const real_t vnorm_local = v.Norml2();
-      real_t vnorm;
-      MPI_Allreduce(&vnorm_local, &vnorm, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
-                    MPI_COMM_WORLD);
-      eps = lambda * (lambda + xnorm / vnorm);
-   }
-
-   // x + eps * v
-   {
-      const auto d_v = v.Read();
-      const auto d_x = x.Read();
-      auto d_xpev = xpev.Write();
-      mfem::forall(x.Size(), [=] MFEM_HOST_DEVICE (int i)
-      {
-         d_xpev[i] = d_x[i] + eps * d_v[i];
-      });
-   }
-
-   // y = f(x + eps * v)
-   op.Mult(xpev, y);
-
-   // y = (f(x + eps * v) - f(x)) / eps
-   {
-      const auto d_f = f.Read();
-      auto d_y = y.ReadWrite();
-      mfem::forall(f.Size(), [=] MFEM_HOST_DEVICE (int i)
-      {
-         d_y[i] = (d_y[i] - d_f[i]) / eps;
-      });
-   }
-}
-
 #endif // MFEM_USE_MPI

fem/dfem/doperator.hpp

@@ -697,18 +697,17 @@ void DifferentiableOperator::AddIntegrator(
 
    // The explicit captures are necessary to avoid dependency on
    // the specific instance of this class (this pointer).
-   restriction_callback = [element_dof_ordering,
-                           solutions_ = this->solutions,
-                           parameters_ = this->parameters]
-   (std::vector<Vector> &sol,
-    const std::vector<Vector> &par,
-    std::vector<Vector> &f)
+   restriction_callback =
+      [=, solutions = this->solutions, parameters = this->parameters]
+      (std::vector<Vector> &sol,
+       const std::vector<Vector> &par,
+       std::vector<Vector> &f)
    {
-      restriction<entity_t>(solutions_, sol, f,
+      restriction<entity_t>(solutions, sol, f,
                             element_dof_ordering);
-      restriction<entity_t>(parameters_, par, f,
+      restriction<entity_t>(parameters, par, f,
                             element_dof_ordering,
-                            solutions_.size());
+                            solutions.size());
    };
 
    prolongation_transpose = get_prolongation_transpose(
@@ -836,19 +835,19 @@ void DifferentiableOperator::AddIntegrator(
 
          // capture by ref:
          &restriction_cb = this->restriction_callback,
-         &fields_e_ = this->fields_e,
-         &residual_e_ = this->residual_e,
-         &output_restriction_transpose_ = this->output_restriction_transpose
+         &fields_e = this->fields_e,
+         &residual_e = this->residual_e,
+         &output_restriction_transpose = this->output_restriction_transpose
       ]
       (std::vector<Vector> &sol, const std::vector<Vector> &par, Vector &res)
       mutable // mutable: needed to modify 'shmem_cache'
    {
-      restriction_cb(sol, par, fields_e_);
+      restriction_cb(sol, par, fields_e);
 
-      residual_e_ = 0.0;
-      auto ye = Reshape(residual_e_.ReadWrite(), test_vdim, num_test_dof, num_entities);
+      residual_e = 0.0;
+      auto ye = Reshape(residual_e.ReadWrite(), test_vdim, num_test_dof, num_entities);
 
-      auto wrapped_fields_e = wrap_fields(fields_e_,
+      auto wrapped_fields_e = wrap_fields(fields_e,
                                           action_shmem_info.field_sizes,
                                           num_entities);
 
@@ -879,7 +878,7 @@ void DifferentiableOperator::AddIntegrator(
             y, fhat, output_fop, output_dtq_shmem[0],
             scratch_shmem, dimension, use_sum_factorization);
       }, num_entities, thread_blocks, action_shmem_info.total_size, shmem_cache.ReadWrite());
-      output_restriction_transpose_(residual_e_, res);
+      output_restriction_transpose(residual_e, res);
    });
 
    // Without this compile-time check, some valid instantiations of this method
@@ -1194,7 +1193,7 @@ void DifferentiableOperator::AddIntegrator(
 
                // capture by ref:
                &qpdc_mem = derivative_qp_caches_ref,
-               &fields_ = fields_ref
+               &fields = fields_ref
             ](std::vector<Vector> &f_e, SparseMatrix *&A) mutable
          {
             auto wrapped_fields_e = wrap_fields(f_e, shmem_info.field_sizes,
@@ -1242,14 +1241,14 @@ void DifferentiableOperator::AddIntegrator(
             {
                if (input_is_dependent[s])
                {
-                  trial_field = &fields_[input_to_field[s]];
+                  trial_field = &fields[input_to_field[s]];
                }
             }
 
             auto trial_fes = *std::get_if<const ParFiniteElementSpace *>
                              (&trial_field->data);
             auto test_fes = *std::get_if<const ParFiniteElementSpace *>
-                            (&fields_[output_to_field[0]].data);
+                            (&fields[output_to_field[0]].data);
 
             A = new SparseMatrix(test_fes->GetVSize(), trial_fes->GetVSize());
 
@@ -1335,7 +1334,7 @@ void DifferentiableOperator::AddIntegrator(
                input_to_field,
                output_to_field,
                &spmatcb = assemble_derivative_sparsematrix_callbacks_ref,
-               &fields_ = fields_ref
+               &fields = fields_ref
             ](std::vector<Vector> &f_e, HypreParMatrix *&A) mutable
          {
             SparseMatrix *spmat = nullptr;
@@ -1367,14 +1366,14 @@ void DifferentiableOperator::AddIntegrator(
             {
                if (input_is_dependent[s])
                {
-                  trial_field = &fields_[input_to_field[s]];
+                  trial_field = &fields[input_to_field[s]];
                }
             }
 
             auto trial_fes = *std::get_if<const ParFiniteElementSpace *>
                              (&trial_field->data);
             auto test_fes = *std::get_if<const ParFiniteElementSpace *>
-                            (&fields_[output_to_field[0]].data);
+                            (&fields[output_to_field[0]].data);
 
             if (same_test_and_trial)
             {

fem/dfem/util.hpp

@@ -597,7 +597,7 @@ struct ThreadBlocks
    int z = 1;
 };
 
-#if defined(MFEM_USE_CUDA_OR_HIP_LANG)
+#if defined(MFEM_USE_CUDA_OR_HIP)
 template <typename func_t>
 __global__ void forall_kernel_shmem(func_t f, int n)
 {
@@ -617,11 +617,10 @@ void forall(func_t f,
             int num_shmem = 0,
             real_t *shmem = nullptr)
 {
-   internal::RequireKernelCompilation();
-
-#if defined(MFEM_USE_CUDA_OR_HIP_LANG)
-   if (Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
+   if (Device::Allows(Backend::CUDA_MASK) ||
+       Device::Allows(Backend::HIP_MASK))
    {
+#if defined(MFEM_USE_CUDA_OR_HIP)
       // int gridsize = (N + Z - 1) / Z;
       int num_bytes = num_shmem * sizeof(decltype(shmem));
       dim3 block_size(blocks.x, blocks.y, blocks.z);
@@ -632,10 +631,9 @@ void forall(func_t f,
       MFEM_GPU_CHECK(hipGetLastError());
 #endif
       MFEM_DEVICE_SYNC;
-      return;
-   }
 #endif
-   if (Device::Allows(Backend::CPU_MASK))
+   }
+   else if (Device::Allows(Backend::CPU_MASK))
    {
       MFEM_ASSERT(!((bool)num_shmem != (bool)shmem),
                   "Backend::CPU needs a pre-allocated shared memory block");
@@ -673,7 +671,66 @@ public:
                     MPI_COMM_WORLD);
    }
 
-   void Mult(const Vector &v, Vector &y) const override;
+   Operator& GetGradient(const Vector &x0) const override
+   {
+      x = x0;
+      f.UseDevice(x.UseDevice());
+      xpev.UseDevice(x.UseDevice());
+
+      op.Mult(x, f);
+      const real_t xnorm_local = x.Norml2();
+      MPI_Allreduce(&xnorm_local, &xnorm, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                    MPI_COMM_WORLD);
+
+      return const_cast<FDJacobian&>(*this);
+   }
+
+   void Mult(const Vector &v, Vector &y) const override
+   {
+      // See [1] for choice of eps.
+      //
+      // [1] Woodward, C.S., Gardner, D.J. and Evans, K.J., 2015. On the use of
+      // finite difference matrix-vector products in Newton-Krylov solvers for
+      // implicit climate dynamics with spectral elements. Procedia Computer
+      // Science, 51, pp.2036-2045.
+      real_t eps;
+      if (fixed_eps > 0.0)
+      {
+         eps = fixed_eps;
+      }
+      else
+      {
+         const real_t vnorm_local = v.Norml2();
+         real_t vnorm;
+         MPI_Allreduce(&vnorm_local, &vnorm, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                       MPI_COMM_WORLD);
+         eps = lambda * (lambda + xnorm / vnorm);
+      }
+
+      // x + eps * v
+      {
+         const auto d_v = v.Read();
+         const auto d_x = x.Read();
+         auto d_xpev = xpev.Write();
+         mfem::forall(x.Size(), [=] MFEM_HOST_DEVICE (int i)
+         {
+            d_xpev[i] = d_x[i] + eps * d_v[i];
+         });
+      }
+
+      // y = f(x + eps * v)
+      op.Mult(xpev, y);
+
+      // y = (f(x + eps * v) - f(x)) / eps
+      {
+         const auto d_f = f.Read();
+         auto d_y = y.ReadWrite();
+         mfem::forall(f.Size(), [=] MFEM_HOST_DEVICE (int i)
+         {
+            d_y[i] = (d_y[i] - d_f[i]) / eps;
+         });
+      }
+   }
 
    virtual MemoryClass GetMemoryClass() const override
    {
@@ -682,11 +739,11 @@ public:
 
 private:
    const Operator &op;
-   Vector x, f;
+   mutable Vector x, f;
    mutable Vector xpev;
    real_t lambda = 1.0e-6;
    real_t fixed_eps;
-   real_t xnorm;
+   mutable real_t xnorm;
 };
 
 /// @brief Find the index of a field descriptor in a vector of field descriptors.

fem/fe/fe_base.cpp

@@ -1316,14 +1316,13 @@ void VectorFiniteElement::Project_RT(
    }
 }
 
-void VectorFiniteElement::ProjectCurl2D_RT(
+void VectorFiniteElement::ProjectGrad_RT(
    const real_t *nk, const Array<int> &d2n, const FiniteElement &fe,
    ElementTransformation &Trans, DenseMatrix &grad) const
 {
-   // 2D "ProjectCurl_RT"
    if (dim != 2)
    {
-      mfem_error("VectorFiniteElement::ProjectCurl2D_RT works only in 2D!");
+      mfem_error("VectorFiniteElement::ProjectGrad_RT works only in 2D!");
    }
 
    DenseMatrix dshape(fe.GetDof(), fe.GetDim());
@@ -1334,8 +1333,8 @@ void VectorFiniteElement::ProjectCurl2D_RT(
    for (int k = 0; k < dof; k++)
    {
       fe.CalcDShape(Nodes.IntPoint(k), dshape);
-      tk[0] = -nk[d2n[k]*dim+1];
-      tk[1] = nk[d2n[k]*dim];
+      tk[0] = nk[d2n[k]*dim+1];
+      tk[1] = -nk[d2n[k]*dim];
       dshape.Mult(tk, grad_k);
       for (int j = 0; j < grad_k.Size(); j++)
       {
@@ -1382,7 +1381,7 @@ void VectorFiniteElement::ProjectCurl_ND(
    }
 }
 
-void VectorFiniteElement::ProjectCurl3D_RT(
+void VectorFiniteElement::ProjectCurl_RT(
    const real_t *nk, const Array<int> &d2n, const FiniteElement &fe,
    ElementTransformation &Trans, DenseMatrix &curl) const
 {

fem/fe/fe_base.hpp

@@ -957,11 +957,10 @@ protected:
                    const FiniteElement &fe, ElementTransformation &Trans,
                    DenseMatrix &I) const;
 
-   // Input is a scalar representing the Z (out of plane) component, Output is
-   // the X-Y (in-plane) RT curl
-   void ProjectCurl2D_RT(const real_t *nk, const Array<int> &d2n,
-                         const FiniteElement &fe, ElementTransformation &Trans,
-                         DenseMatrix &grad) const;
+   // rotated gradient in 2D
+   void ProjectGrad_RT(const real_t *nk, const Array<int> &d2n,
+                       const FiniteElement &fe, ElementTransformation &Trans,
+                       DenseMatrix &grad) const;
 
    // Compute the curl as a discrete operator from ND FE (fe) to ND FE (this).
    // The natural FE for the range is RT, so this is an approximation.
@@ -969,9 +968,9 @@ protected:
                        const FiniteElement &fe, ElementTransformation &Trans,
                        DenseMatrix &curl) const;
 
-   void ProjectCurl3D_RT(const real_t *nk, const Array<int> &d2n,
-                         const FiniteElement &fe, ElementTransformation &Trans,
-                         DenseMatrix &curl) const;
+   void ProjectCurl_RT(const real_t *nk, const Array<int> &d2n,
+                       const FiniteElement &fe, ElementTransformation &Trans,
+                       DenseMatrix &curl) const;
 
    /** @brief Project a vector coefficient onto the ND basis functions
        @param tk    Edge tangent vectors for this element type
@@ -1447,8 +1446,6 @@ public:
                                 dof2quad_array_open);
    }
 
-   const Poly_1D::Basis &GetOpenBasis1D() const { return obasis1d; }
-
    virtual ~VectorTensorFiniteElement();
 };
 

fem/fe/fe_fixed_order.hpp

@@ -307,12 +307,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (6) */
+       them in the vector shape of dimension Dof (4) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (6 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;
@@ -336,12 +336,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (5) */
+       them in the vector shape of dimension Dof (4) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (5 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;

fem/fe/fe_h1.cpp

@@ -1757,45 +1757,22 @@ H1_BergotPyramidElement::H1_BergotPyramidElement(const int p, const int btype)
       real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       real_t z = ip.z;
 
-      if (std::abs(z - 1.0) < apex_tol)
-      {
-         // Compute the limit of the basis functions as z->1 with x and y on the
-         // line between the center of the base and the apex
-         o = 0;
-         for (int i = 0; i <= p; i++)
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               for (int k = 0; k <= p - maxij; k++)
-                  if (i == 0 && j == 0)
-                  {
-                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
-                  }
-                  else
-                  {
-                     T(o++, m) = 0.;
-                  }
-            }
-      }
-      else
-      {
-         poly1d.CalcLegendre(p, x, shape_x.GetData());
-         poly1d.CalcLegendre(p, y, shape_y.GetData());
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-         o = 0;
-         for (int i = 0; i <= p; i++)
+      o = 0;
+      for (int i = 0; i <= p; i++)
+      {
+         for (int j = 0; j <= p; j++)
          {
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                                z, 1.0, shape_z);
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
 
-               for (int k = 0; k <= p - maxij; k++)
-               {
-                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                              pow(1.0 - ip.z, maxij);
-               }
+            for (int k = 0; k <= p - maxij; k++)
+            {
+               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                           pow(1.0 - ip.z, maxij);
             }
          }
       }
@@ -1816,44 +1793,25 @@ void H1_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    Vector u(dof);
 #endif
 
-   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   const real_t z = ip.z;
+   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the basis functions as z->1 with x and y on the
-      // line between the center of the base and the apex
-      u = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-               if (i == 0 && j == 0)
-               {
-                  u(o) = ((k + 3.) * k + 2.) / 2.;
-               }
-         }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
+   poly1d.CalcLegendre(p, x, shape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData());
+
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z);
+         for (int k = 0; k <= p - maxij; k++)
+            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                     pow(1.0 - ip.z, maxij);
+      }
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
-            for (int k = 0; k <= p - maxij; k++)
-               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                        pow(1.0 - ip.z, maxij);
-         }
-   }
    Ti.Mult(u, shape);
 }
 
@@ -1872,68 +1830,37 @@ void H1_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    Vector dshape_z(order+1);
    Vector dshape_z_dt(order+1);
 #endif
-   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   const real_t z = ip.z;
+   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the gradients of the basis functions as
-      // z->1 with x and y on the line between the center of the base and the
-      // apex
-      du = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
+   poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z, dshape_z, dshape_z_dt);
+
+         for (int k = 0; k <= p - maxij; k++, o++)
          {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               if (i == 0 && j == 0)
-               {
-                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
-               }
-               else if (i == 1 && j == 0)
-               {
-                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-               else if (i == 0 && j == 1)
-               {
-                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-            }
+            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                      pow(1.0 - ip.z, maxij) +
+                      (ip.x * dshape_x(i) * shape_y(j) +
+                       ip.y * shape_x(i) * dshape_y(j)) *
+                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                      maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
          }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+      }
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0,
-                                             shape_z, dshape_z, dshape_z_dt);
-
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                         pow(1.0 - ip.z, maxij) +
-                         (ip.x * dshape_x(i) * shape_y(j) +
-                          ip.y * shape_x(i) * dshape_y(j)) *
-                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
-            }
-         }
-   }
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_h1.hpp

@@ -208,8 +208,6 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
-   static constexpr real_t apex_tol = 1e-8;
-
 public:
    H1_BergotPyramidElement(const int p,
                            const int btype = BasisType::GaussLobatto);

fem/fe/fe_l2.cpp

@@ -1106,16 +1106,9 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
          {
             const real_t wik = op[i] + op[k] + op[p-i-k];
             const real_t w = wik * wjk * op[p-k];
-            if (std::abs(w) < apex_tol)
-            {
-               Nodes.IntPoint(o++).Set3(0.,0.,1.);
-            }
-            else
-            {
-               Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
-                                        op[j] * (op[i] + op[p-i-k]) / w,
-                                        op[k] * op[p-k] / w);
-            }
+            Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
+                                     op[j] * (op[j] + op[p-j-k]) / w,
+                                     op[k] * op[p-k] / w);
          }
       }
 
@@ -1132,45 +1125,22 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
       const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       const real_t z = ip.z;
 
-      if (std::abs(z - 1.0) < apex_tol)
-      {
-         // Compute the limit of the basis functions as z->1 with x and y on the
-         // line between the center of the base and the apex
-         o = 0;
-         for (int i = 0; i <= p; i++)
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               for (int k = 0; k <= p - maxij; k++)
-                  if (i == 0 && j == 0)
-                  {
-                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
-                  }
-                  else
-                  {
-                     T(o++, m) = 0.;
-                  }
-            }
-      }
-      else
-      {
-         poly1d.CalcLegendre(p, x, shape_x.GetData());
-         poly1d.CalcLegendre(p, y, shape_y.GetData());
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-         o = 0;
-         for (int i = 0; i <= p; i++)
+      o = 0;
+      for (int i = 0; i <= p; i++)
+      {
+         for (int j = 0; j <= p; j++)
          {
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                                z, 1.0, shape_z);
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
 
-               for (int k = 0; k <= p - maxij; k++)
-               {
-                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                              pow(1.0 - ip.z, maxij);
-               }
+            for (int k = 0; k <= p - maxij; k++)
+            {
+               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                           pow(1.0 - ip.z, maxij);
             }
          }
       }
@@ -1195,41 +1165,26 @@ void L2_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the basis functions as z->1 with x and y on the
-      // line between the center of the base and the apex
-      u = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-               if (i == 0 && j == 0)
-               {
-                  u(o) = ((k + 3.) * k + 2.) / 2.;
-               }
-         }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
+   poly1d.CalcLegendre(p, x, shape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+   {
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z);
 
-            for (int k = 0; k <= p - maxij; k++)
-               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                        pow(1.0 - ip.z, maxij);
+         for (int k = 0; k <= p - maxij; k++)
+         {
+            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                     pow(1.0 - ip.z, maxij);
          }
+      }
    }
+
    Ti.Mult(u, shape);
 }
 
@@ -1253,64 +1208,35 @@ void L2_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the gradients of the basis functions as
-      // z->1 with x and y on the line between the center of the base and the
-      // apex
-      du = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               if (i == 0 && j == 0)
-               {
-                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
-               }
-               else if (i == 1 && j == 0)
-               {
-                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-               else if (i == 0 && j == 1)
-               {
-                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-            }
-         }
-   }
-   else
-   {
-      Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-      Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+   Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+   Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0,
-                                             shape_z, dshape_z, dshape_z_dt);
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+   {
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z, dshape_z, dshape_z_dt);
 
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                         pow(1.0 - ip.z, maxij) +
-                         (ip.x * dshape_x(i) * shape_y(j) +
-                          ip.y * shape_x(i) * dshape_y(j)) *
-                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
-            }
+         for (int k = 0; k <= p - maxij; k++, o++)
+         {
+            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                      pow(1.0 - ip.z, maxij) +
+                      (ip.x * dshape_x(i) * shape_y(j) +
+                       ip.y * shape_x(i) * dshape_y(j)) *
+                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                      ((maxij > 0) ? (maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                                      pow(1.0 - ip.z, maxij - 1)) : 0.0);
          }
+      }
    }
+
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_l2.hpp

@@ -225,8 +225,6 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
-   static constexpr real_t apex_tol = 1e-8;
-
 public:
    /// Construct the L2_PyramidElement of order @a p and BasisType @a btype
    L2_BergotPyramidElement(const int p,

fem/fe/fe_nd.cpp

@@ -1282,49 +1282,12 @@ ND_SegmentElement::ND_SegmentElement(const int p, const int ob_type)
    }
 }
 
-void ND_SegmentElement::CalcShape(const IntegrationPoint &ip,
-                                  Vector &shape) const
-{
-   if (obasis1d.IsIntegratedType()) { obasis1d.ScaleIntegrated(false); }
-   obasis1d.Eval(ip.x, shape);
-}
-
 void ND_SegmentElement::CalcVShape(const IntegrationPoint &ip,
                                    DenseMatrix &shape) const
 {
    Vector vshape(shape.Data(), dof);
 
-   CalcShape(ip, vshape);
-}
-
-void ND_SegmentElement::ProjectIntegrated(VectorCoefficient &vc,
-                                          ElementTransformation &Trans,
-                                          Vector &dofs) const
-{
-   MFEM_ASSERT(obasis1d.IsIntegratedType(), "Not integrated type");
-   real_t vk[Geometry::MaxDim];
-   Vector xk(vk, vc.GetVDim());
-
-   const real_t *cp = poly1d.ClosedPoints(dof, BasisType::GaussLobatto);
-   const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, dof);
-   IntegrationPoint ip;
-
-   for (int i = 0; i < dof; i++)
-   {
-      const real_t h = cp[i+1] - cp[i];
-      real_t val = 0.0;
-
-      for (int q = 0; q < ir.GetNPoints(); q++)
-      {
-         const IntegrationPoint &ip1d = ir.IntPoint(q);
-         ip.x = cp[i] + h*ip1d.x;
-         Trans.SetIntPoint(&ip);
-         vc.Eval(xk, Trans, ip);
-         val += ip1d.weight*Trans.Jacobian().InnerProduct(tk, vk);
-      }
-
-      dofs(i) = val*h;
-   }
+   obasis1d.Eval(ip.x, vshape);
 }
 
 const real_t ND_WedgeElement::tk[15] =

fem/fe/fe_nd.hpp

@@ -303,7 +303,8 @@ public:
    /** @brief Construct the ND_SegmentElement of order @a p and open
        BasisType @a ob_type */
    ND_SegmentElement(const int p, const int ob_type = BasisType::GaussLegendre);
-   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
+   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override
+   { obasis1d.Eval(ip.x, shape); }
    void CalcVShape(const IntegrationPoint &ip,
                    DenseMatrix &shape) const override;
    void CalcVShape(ElementTransformation &Trans,
@@ -324,10 +325,7 @@ public:
    using FiniteElement::Project;
    void Project(VectorCoefficient &vc,
                 ElementTransformation &Trans, Vector &dofs) const override
-   {
-      if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
-      else { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-   }
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    void ProjectMatrixCoefficient(MatrixCoefficient &mc,
                                  ElementTransformation &T,
                                  Vector &dofs) const override
@@ -340,11 +338,6 @@ public:
                     ElementTransformation &Trans,
                     DenseMatrix &grad) const override
    { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
-
-protected:
-   void ProjectIntegrated(VectorCoefficient &vc,
-                          ElementTransformation &Trans,
-                          Vector &dofs) const;
 };
 
 class ND_WedgeElement : public VectorFiniteElement

fem/fe/fe_pyramid.hpp

@@ -17,12 +17,6 @@
 namespace mfem
 {
 
-struct ScalarPyramid
-{
-   // Default basis type for H1 and L2 pyramids
-   static inline int DefaultType = 1; // Bergot(0) or Fuentes(1)
-};
-
 /** Base class for arbitrary order basis functions on pyramid-shaped elements
 
   This base class provides a common class to store temporary vectors,

fem/fe/fe_rt.hpp

@@ -73,11 +73,16 @@ public:
    void Project(const FiniteElement &fe, ElementTransformation &Trans,
                 DenseMatrix &I) const override
    { Project_RT(nk, dof2nk, fe, Trans, I); }
+   // Gradient + rotation = Curl: H1 -> H(div)
+   void ProjectGrad(const FiniteElement &fe,
+                    ElementTransformation &Trans,
+                    DenseMatrix &grad) const override
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
    // Curl = Gradient + rotation: H1 -> H(div)
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl2D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
 
    void GetFaceMap(const int face_id, Array<int> &face_map) const override;
 
@@ -143,7 +148,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
 
    /// @brief Return the mapping from lexicographically ordered face DOFs to
    /// lexicographically ordered element DOFs corresponding to local face
@@ -205,11 +210,16 @@ public:
    void Project(const FiniteElement &fe, ElementTransformation &Trans,
                 DenseMatrix &I) const override
    { Project_RT(nk, dof2nk, fe, Trans, I); }
+   // Gradient + rotation = Curl: H1 -> H(div)
+   void ProjectGrad(const FiniteElement &fe,
+                    ElementTransformation &Trans,
+                    DenseMatrix &grad) const override
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, grad); }
    // Curl = Gradient + rotation: H1 -> H(div)
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl2D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectGrad_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 
@@ -264,7 +274,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 class RT_WedgeElement : public VectorFiniteElement
@@ -322,7 +332,7 @@ public:
    void ProjectCurl(const FiniteElement &fe,
                     ElementTransformation &Trans,
                     DenseMatrix &curl) const override
-   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
 };
 
 /** Arbitrary order H(Div) basis functions defined on pyramid-shaped elements
@@ -418,7 +428,7 @@ public:
    virtual void ProjectCurl(const FiniteElement &fe,
                             ElementTransformation &Trans,
                             DenseMatrix &curl) const
-   { ProjectCurl3D_RT(nk, dof2nk, fe, Trans, curl); }
+   { ProjectCurl_RT(nk, dof2nk, fe, Trans, curl); }
 
    void CalcRawVShape(const IntegrationPoint &ip,
                       DenseMatrix &shape) const;

fem/fe_coll.cpp

@@ -228,19 +228,7 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1_", 3))
    {
-      // Parse pyramid basis type if included in the name
-      const char *pyr = strstr(name, "Pyr");
-      if (pyr == NULL)
-      {
-         // Use default pyramid type elements
-         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
-      }
-      else
-      {
-         // Use specific pyramid type elements
-         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3),
-                                   BasisType::GaussLobatto, atoi(pyr + 3));
-      }
+      fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
    }
    else if (!strncmp(name, "H1Pos_Trace_", 12))
    {
@@ -257,44 +245,26 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1@", 3))
    {
-      // Parse pyramid basis type if included in the name
-      const char *pyr = strstr(name, "Pyr");
-      if (pyr == NULL)
-      {
-         // Use default pyramid type elements
-         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                   BasisType::GetType(name[3]));
-      }
-      else
-      {
-         // Use specific pyramid type elements
-         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                   BasisType::GetType(name[3]),
-                                   atoi(pyr + 3));
-      }
+      fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
+                                BasisType::GetType(name[3]));
    }
-   else if (!strncmp(name, "L2", 2))
+   else if (!strncmp(name, "L2_T", 4))
+      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
+                                atoi(name + 4));
+   else if (!strncmp(name, "L2_", 3))
    {
-      // Parse Map Type
-      const int mtype = strstr(name, "Int") == NULL ?
-                        FiniteElement::VALUE : FiniteElement::INTEGRAL;
-
-      // Parse the base order
-      const int p = atoi(strstr(name, "_P") + 2);
-
-      // Parse the mesh dimension
-      const int dim = atoi(strstr(name, "D") - 1);
-
-      // Parse basis type if specified
-      const char *t = strstr(name, "_T");
-      const int btype = t == NULL ? BasisType::GaussLegendre : atoi(t + 2);
-
-      // Parse the pyramid type if specified
-      const char *pyr = strstr(name, "Pyr");
-      const int ptype = pyr == NULL ? 1 : atoi(pyr + 3);
-
-      // Create collection
-      fec = new L2_FECollection(p, dim, btype, mtype, ptype);
+      fec = new L2_FECollection(atoi(name + 7), atoi(name + 3));
+   }
+   else if (!strncmp(name, "L2Int_T", 7))
+   {
+      fec = new L2_FECollection(atoi(name + 13), atoi(name + 9),
+                                atoi(name + 7), FiniteElement::INTEGRAL);
+   }
+   else if (!strncmp(name, "L2Int_", 6))
+   {
+      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
+                                BasisType::GaussLegendre,
+                                FiniteElement::INTEGRAL);
    }
    else if (!strncmp(name, "RT_Trace_", 9))
    {
@@ -1739,10 +1709,9 @@ const int *RT1_3DFECollection::DofOrderForOrientation(Geometry::Type GeomType,
 
 
 H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
-                                 const int pyr_type)
+                                 const int pyrtype)
    : FiniteElementCollection(p)
    , dim(dim)
-   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 1, "H1_FECollection requires order >= 1.");
    MFEM_VERIFY(dim >= 0 && dim <= 3, "H1_FECollection requires 0 <= dim <= 3.");
@@ -1755,14 +1724,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
    {
       case BasisType::GaussLobatto:
       {
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
-         }
-         else
-         {
-            snprintf(h1_name, 32, "H1_%dD_P%d_Pyr%d", dim, p, pyr_type);
-         }
+         snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
          break;
       }
       case BasisType::Positive:
@@ -1948,11 +1910,11 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
          H1_dof[Geometry::TETRAHEDRON] = (TriDof*pm3)/3;
          H1_dof[Geometry::CUBE] = QuadDof*pm1;
          H1_dof[Geometry::PRISM] = TriDof*pm1;
-         if (pyr_type == 0 || b_type == BasisType::Positive)
+         if (pyrtype == 0 || b_type == BasisType::Positive)
          {
             H1_dof[Geometry::PYRAMID] = pm2*pm1*(2*p-3)/6; // Bergot (JSC)
          }
-         else if (pyr_type == 1)
+         else if (pyrtype == 1)
          {
             H1_dof[Geometry::PYRAMID] = pm1*pm1*pm1; // Fuentes
          }
@@ -1973,15 +1935,13 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
                new H1_TetrahedronElement(p, btype);
             H1_Elements[Geometry::CUBE] = new H1_HexahedronElement(p, btype);
             H1_Elements[Geometry::PRISM] = new H1_WedgeElement(p, btype);
-            if (pyr_type == 0)
+            if (pyrtype == 0)
             {
-               H1_Elements[Geometry::PYRAMID] =
-                  new H1_BergotPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] = new H1_BergotPyramidElement(p, btype);
             }
             else
             {
-               H1_Elements[Geometry::PYRAMID] =
-                  new H1_FuentesPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] = new H1_FuentesPyramidElement(p, btype);
             }
          }
 
@@ -2188,7 +2148,6 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    : FiniteElementCollection(p)
    , dim(dim)
    , m_type(map_type)
-   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 0, "L2_FECollection requires order >= 0.");
 
@@ -2204,25 +2163,10 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    switch (btype)
    {
       case BasisType::GaussLegendre:
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
-         }
-         else
-         {
-            snprintf(d_name, 32, "%s_%dD_P%d_Pyr%d", prefix, dim, p, pyr_type);
-         }
+         snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
          break;
       default:
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
-         }
-         else
-         {
-            snprintf(d_name, 32, "%s_T%d_%dD_P%d_Pyr%d",
-                     prefix, btype, dim, p, pyr_type);
-         }
+         snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
    }
 
    for (int g = 0; g < Geometry::NumGeom; g++)
@@ -2341,13 +2285,11 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
          L2_Elements[Geometry::PRISM] = new L2_WedgeElement(p, btype);
          if (pyr_type == 0)
          {
-            L2_Elements[Geometry::PYRAMID] =
-               new L2_BergotPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] = new L2_BergotPyramidElement(p, btype);
          }
          else
          {
-            L2_Elements[Geometry::PYRAMID] =
-               new L2_FuentesPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] = new L2_FuentesPyramidElement(p, btype);
          }
       }
 

fem/fe_coll.hpp

@@ -100,10 +100,6 @@ public:
       return FiniteElementForGeometry(GeomType);
    }
 
-   /** @brief Returns a collection of the trace elements.
-
-       @note The collection is owned by the caller and is NOT deleted in the
-       destructor. */
    virtual FiniteElementCollection *GetTraceCollection() const;
 
    virtual ~FiniteElementCollection();
@@ -290,7 +286,7 @@ protected:
 class H1_FECollection : public FiniteElementCollection
 {
 protected:
-   int dim, b_type, p_type;
+   int dim, b_type;
    char h1_name[32];
    FiniteElement *H1_Elements[Geometry::NumGeom];
    int H1_dof[Geometry::NumGeom];
@@ -299,7 +295,7 @@ protected:
 public:
    explicit H1_FECollection(const int p, const int dim = 3,
                             const int btype = BasisType::GaussLobatto,
-                            const int pyr_type = ScalarPyramid::DefaultType);
+                            const int pyrtype = 1);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -324,7 +320,7 @@ public:
    const int *GetDofMap(Geometry::Type GeomType, int p) const;
 
    FiniteElementCollection *Clone(int p) const override
-   { return new H1_FECollection(p, dim, b_type, p_type); }
+   { return new H1_FECollection(p, dim, b_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }
@@ -371,7 +367,6 @@ private:
    int dim;
    int b_type; // BasisType
    int m_type; // map type
-   int p_type; // Pyramid type (0 -> Bergot, 1 -> Fuentes)
    char d_name[32];
    ScalarFiniteElement *L2_Elements[Geometry::NumGeom];
    ScalarFiniteElement *Tr_Elements[Geometry::NumGeom];
@@ -384,7 +379,7 @@ public:
    L2_FECollection(const int p, const int dim,
                    const int btype = BasisType::GaussLegendre,
                    const int map_type = FiniteElement::VALUE,
-                   const int pyr_type = ScalarPyramid::DefaultType);
+                   const int pyrtype = 1);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -414,7 +409,7 @@ public:
    int GetBasisType() const { return b_type; }
 
    FiniteElementCollection *Clone(int p) const override
-   { return new L2_FECollection(p, dim, b_type, m_type, p_type); }
+   { return new L2_FECollection(p, dim, b_type, m_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }

fem/fespace.cpp

@@ -22,8 +22,6 @@
 #include <algorithm>
 #include <cmath>
 #include <cstdarg>
-#include <unordered_map>
-#include <unordered_set>
 
 using namespace std;
 
@@ -4529,210 +4527,6 @@ void FiniteElementSpace
    }
 }
 
-void FiniteElementSpace::GetBoundaryLoopEdgeDofs(
-   const Array<int> &boundary_element_indices,
-   Array<int> &boundary_edge_dofs,
-   Array<int> *dof_edges,
-   Array<int> *dof_boundary_elements) const
-{
-   MFEM_VERIFY(mesh->Dimension() >= 2,
-               "GetBoundaryLoopEdgeDofs requires 2D or 3D meshes to find edge objects");
-
-   boundary_edge_dofs.SetSize(0);
-   if (dof_edges) { dof_edges->SetSize(0); }
-   if (dof_boundary_elements) { dof_boundary_elements->SetSize(0); }
-
-   // A DOF that appears in exactly one selected boundary element lies on the
-   // bounding loop; one appearing in two or more is interior to the boundary
-   // region and is dropped. Count occurrences of each DOF (using scratch maps,
-   // exposed only as parallel-indexed Array<int> below) and record, on first
-   // sight, the local edge and boundary element carrying it.
-   //
-   // The count is over GetEdgeDofs, which returns endpoint vertex DOFs as well
-   // as edge-interior DOFs (relevant for collections such as ND_R2D that carry
-   // vertex DOFs). Edge-interior DOFs occur once per edge, so the count mainly
-   // resolves vertex DOFs: a vertex shared by several elements is interior and
-   // dropped, while a genuine loop-corner (open-curve endpoint) vertex is kept.
-   // This is why we count GetEdgeDofs rather than collecting GetEdgeInteriorDofs,
-   // which would omit the endpoint vertex DOFs the method is documented to keep.
-   // The 3D removal criterion (any edge in two or more faces) matches the
-   // parallel version rather than a parity toggle.
-   std::unordered_map<int, int> dof_count, dof_edge, dof_belem;
-   Array<int> edge_dofs, edges, edge_orientations;
-
-   const int dim = mesh->Dimension();
-   for (int i = 0; i < boundary_element_indices.Size(); ++i)
-   {
-      const int boundary_element_idx = boundary_element_indices[i];
-      std::unordered_set<int> boundary_element_dofs;
-
-      if (dim == 3)
-      {
-         // Boundary elements are 2D faces; extract their 1D edges.
-         int face_index, face_orientation;
-         mesh->GetBdrElementFace(boundary_element_idx, &face_index,
-                                 &face_orientation);
-         mesh->GetFaceEdges(face_index, edges, edge_orientations);
-      }
-      else
-      {
-         // Boundary elements are 1D segments, each being a single edge.
-         mesh->GetBdrElementEdges(boundary_element_idx, edges, edge_orientations);
-         MFEM_VERIFY(edges.Size() == 1,
-                     "2D boundary element should have exactly one edge");
-      }
-
-      for (int j = 0; j < edges.Size(); ++j)
-      {
-         GetEdgeDofs(edges[j], edge_dofs);
-         for (int k = 0; k < edge_dofs.Size(); ++k)
-         {
-            const int dof = edge_dofs[k];
-            // Count each DOF once per boundary element and record metadata the
-            // first time it is seen, so H1 DOFs shared by multiple edges of the
-            // same element are not double counted.
-            if (boundary_element_dofs.insert(dof).second &&
-                dof_count[dof]++ == 0)
-            {
-               dof_edge[dof] = edges[j];
-               dof_belem[dof] = boundary_element_idx;
-            }
-         }
-      }
-   }
-
-   // Emit the DOFs seen in exactly one selected boundary element, in a
-   // deterministic (increasing DOF index) order shared by all output arrays.
-   std::vector<int> kept;
-   kept.reserve(dof_count.size());
-   for (const auto &[dof, count] : dof_count)
-   {
-      if (count == 1) { kept.push_back(dof); }
-   }
-   std::sort(kept.begin(), kept.end());
-
-   boundary_edge_dofs.Reserve(static_cast<int>(kept.size()));
-   if (dof_edges) { dof_edges->Reserve(static_cast<int>(kept.size())); }
-   if (dof_boundary_elements)
-   {
-      dof_boundary_elements->Reserve(static_cast<int>(kept.size()));
-   }
-   for (int dof : kept)
-   {
-      boundary_edge_dofs.Append(dof);
-      if (dof_edges) { dof_edges->Append(dof_edge[dof]); }
-      if (dof_boundary_elements) { dof_boundary_elements->Append(dof_belem[dof]); }
-   }
-}
-
-void FiniteElementSpace::GetBoundaryElementsByAttribute(
-   const Array<int> &bdr_attrs,
-   std::vector<Array<int>> &attr_to_elements)
-{
-   // One (initially empty) list of boundary elements per requested attribute,
-   // indexed to match bdr_attrs.
-   attr_to_elements.assign(bdr_attrs.Size(), Array<int>());
-
-   // Map attribute value -> position in bdr_attrs for quick lookup.
-   std::unordered_map<int, int> attr_to_index;
-   for (int i = 0; i < bdr_attrs.Size(); ++i)
-   {
-      attr_to_index[bdr_attrs[i]] = i;
-   }
-
-   // Bucket boundary elements by their attribute.
-   for (int i = 0; i < mesh->GetNBE(); ++i)
-   {
-      int attr = mesh->GetBdrElement(i)->GetAttribute();
-      auto it = attr_to_index.find(attr);
-      if (it != attr_to_index.end())
-      {
-         attr_to_elements[it->second].Append(i);
-      }
-   }
-}
-
-void FiniteElementSpace::GetBoundaryElementsByAttribute(int bdr_attr,
-                                                        Array<int> &boundary_elements)
-{
-   boundary_elements.SetSize(0);
-
-   for (int i = 0; i < mesh->GetNBE(); ++i)
-   {
-      if (mesh->GetBdrElement(i)->GetAttribute() == bdr_attr)
-      {
-         boundary_elements.Append(i);
-      }
-   }
-}
-
-void FiniteElementSpace::ComputeLoopEdgeOrientations(
-   const Array<int> &dof_edges,
-   const Array<int> &dof_boundary_elements,
-   const Vector &loop_normal,
-   Array<int> &dof_orientations) const
-{
-   MFEM_VERIFY(dof_edges.Size() == dof_boundary_elements.Size(),
-               "dof_edges and dof_boundary_elements must be parallel-indexed");
-
-   const int ndof = dof_edges.Size();
-   dof_orientations.SetSize(ndof);
-
-   Array<int> edge_verts, bdr_elem_verts;
-   Vector edge_vec(3), to_edge_vec(3), cross_product(3);
-   for (int i = 0; i < ndof; i++)
-   {
-      const int edge_id = dof_edges[i];
-      const int bdr_elem_idx = dof_boundary_elements[i];
-
-      // Get edge vertices
-      mesh->GetEdgeVertices(edge_id, edge_verts);
-
-      const real_t *v0 = mesh->GetVertex(edge_verts[0]);
-      const real_t *v1 = mesh->GetVertex(edge_verts[1]);
-
-      // Get boundary element vertices
-      mesh->GetBdrElement(bdr_elem_idx)->GetVertices(bdr_elem_verts);
-
-      // Find the third vertex (not part of the edge)
-      int third_vertex = -1;
-      for (int j = 0; j < bdr_elem_verts.Size(); j++)
-      {
-         int v = bdr_elem_verts[j];
-         if (v != edge_verts[0] && v != edge_verts[1])
-         {
-            third_vertex = v;
-            break;
-         }
-      }
-
-      if (third_vertex == -1)
-      {
-         MFEM_ABORT("Boundary element " << bdr_elem_idx << " has only 2 vertices, "
-                    "but 3D boundary elements must have at least 3 vertices");
-      }
-
-      const real_t *v2 = mesh->GetVertex(third_vertex);
-
-      // Edge vector
-      for (int j = 0; j < 3; j++) { edge_vec[j] = v1[j] - v0[j]; }
-
-      // Vector from third vertex to edge (use edge midpoint)
-      for (int j = 0; j < 3; j++)
-      {
-         real_t edge_midpoint = (v0[j] + v1[j]) * 0.5;
-         to_edge_vec[j] = edge_midpoint - v2[j];
-      }
-
-      // Cross product: to_edge × edge
-      to_edge_vec.cross3D(edge_vec, cross_product);
-
-      // Check alignment with loop normal
-      real_t dot_product = cross_product * loop_normal;
-      dof_orientations[i] = (dot_product > 0) ? 1 : -1;
-   }
-}
-
 FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)
 {
    string buff;

fem/fespace.hpp

@@ -22,7 +22,6 @@
 #include "restriction.hpp"
 #include <iostream>
 #include <unordered_map>
-#include <vector>
 
 namespace mfem
 {
@@ -1390,80 +1389,6 @@ public:
    virtual void GetExteriorTrueDofs(Array<int> &exterior_dofs,
                                     int component = -1) const;
 
-   /** @brief Extract the edge degrees of freedom of a boundary "loop".
-
-       Here a "loop" is the set of boundary edges bounding the region covered by
-       @a boundary_element_indices: in 3D the outer edges of a patch of boundary
-       faces, in 2D the boundary segments themselves. An edge that is shared by
-       two (or more) of the selected boundary elements is interior to that region
-       rather than on its bounding loop, so its DOFs are excluded from the result.
-       This exclusion of interior DOFs is the defining feature of the method.
-
-       The three output arrays share a single indexing: for each valid index @a i,
-       @a dof_edges[i] and @a dof_boundary_elements[i] describe the DOF
-       @a boundary_edge_dofs[i].
-
-       @param[in]  boundary_element_indices Boundary element indices spanning a
-                   boundary surface (3D) or curve (2D).
-       @param[out] boundary_edge_dofs Local DOF indices on the boundary loop.
-       @param[out] dof_edges Optional; local edge index carrying each DOF.
-       @param[out] dof_boundary_elements Optional; a boundary element containing
-                   each DOF.
-
-       @note In 3D the edge DOFs are extracted from the 1D edges of the 2D
-       boundary faces; in 2D they come directly from the 1D boundary segments, so
-       @a dof_edges then holds the boundary element (segment) edge indices.
-       @note This method uses GetEdgeDofs internally, which returns both vertex and
-       edge DOFs. Standard Nédélec elements (ND_FECollection) have no vertex DOFs,
-       so only genuine edge DOFs appear. Collections that carry vertex DOFs (e.g.
-       ND_R2D_FECollection) additionally contribute the vertex DOFs at loop
-       endpoints.
-       @note This is the serial version. For parallel meshes, use the parallel
-       version in ParFiniteElementSpace which handles processor boundaries
-       correctly.
-       @note Requires a 2D or 3D mesh to identify edge objects. The method will
-       assert if called on 1D meshes.
-       @note Only supports conforming meshes; non-conforming meshes are not
-       supported. */
-   void GetBoundaryLoopEdgeDofs(const Array<int> &boundary_element_indices,
-                                Array<int> &boundary_edge_dofs,
-                                Array<int> *dof_edges = nullptr,
-                                Array<int> *dof_boundary_elements = nullptr) const;
-
-   /** @brief Get boundary elements grouped by attribute.
-
-       For each attribute in @a bdr_attrs, collect the indices of all boundary
-       elements carrying that attribute. The result is indexed to match
-       @a bdr_attrs: @a attr_to_elements[i] holds the boundary elements with
-       attribute @a bdr_attrs[i]. */
-   void GetBoundaryElementsByAttribute(
-      const Array<int> &bdr_attrs,
-      std::vector<Array<int>> &attr_to_elements);
-
-   /** @brief Get all boundary elements with a specific attribute. */
-   void GetBoundaryElementsByAttribute(int bdr_attr,
-                                       Array<int> &boundary_elements);
-
-   /** @brief Compute edge orientations relative to a boundary loop direction.
-
-       For each boundary-loop DOF described by @a dof_edges and
-       @a dof_boundary_elements (see GetBoundaryLoopEdgeDofs), determine whether
-       the carrying edge is
-       traversed in the direction consistent with @a loop_normal, following the
-       right-hand rule. Intended for 3D meshes.
-
-       @param[in]  dof_edges Local edge index of each DOF (parallel-indexed with
-                   the boundary_edge_dofs output of GetBoundaryLoopEdgeDofs).
-       @param[in]  dof_boundary_elements A boundary element containing each DOF,
-                   using the same indexing as @a dof_edges.
-       @param[in]  loop_normal Normal vector defining the loop orientation.
-       @param[out] dof_orientations Orientation (+1 or -1) for each DOF, using the
-                   same indexing as @a dof_edges. */
-   void ComputeLoopEdgeOrientations(const Array<int> &dof_edges,
-                                    const Array<int> &dof_boundary_elements,
-                                    const Vector &loop_normal,
-                                    Array<int> &dof_orientations) const;
-
    /// Convert a Boolean marker array to a list containing all marked indices.
    static void MarkerToList(const Array<int> &marker, Array<int> &list);
 

fem/gslib.cpp

@@ -556,7 +556,7 @@ void obboxsurf_calc_3(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, n);
    lag(I0, work, n, 1, 0);
 
-   for (int ie = 0; (unsigned)ie < nel; ie++,x+=n2,y+=n2,z+=n2)
+   for (int ie = 0; ie < nel; ie++,x+=n2,y+=n2,z+=n2)
    {
       struct gslib::dbl_range ab[3];
       struct gslib::dbl_range tb[3];
@@ -780,7 +780,7 @@ void obboxedge_calc_2(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, nr);
    lag(I0r, work, nr,1, 0);
 
-   for (int ie = 0; (unsigned)ie < nel; ie++,x+=nr,y+=nr)
+   for (int ie = 0; ie < nel; ie++,x+=nr,y+=nr)
    {
       double x0[2], A[4];
       struct gslib::dbl_range ab[2], tb[2];
@@ -892,7 +892,7 @@ void obboxedge_calc_3(Vector &bb,
    gslib::lagrange_fun *const lag = gslib::gll_lag_setup(work, nr);
    lag(I0r, work, nr, 1, 0);
 
-   for (int ie = 0; (unsigned)ie < nel; ie++,x+=nr,y+=nr,z+=nr)
+   for (int ie = 0; ie < nel; ie++,x+=nr,y+=nr,z+=nr)
    {
       double x0[3], A[9], Ai[9];
       struct gslib::dbl_range ab[3], tb[3];
@@ -4518,7 +4518,7 @@ Mesh* FindPointsGSLIB::GetBoundingBoxMesh(int type)
    int eidx = 0;
    if (myid == save_rank)
    {
-      for (int p = 0; (unsigned)p < gsl_comm->np; p++)
+      for (int p = 0; p < gsl_comm->np; p++)
       {
          if (static_cast<unsigned int>(p) != save_rank)
          {

fem/integ/bilininteg_convection_pa.cpp

@@ -178,8 +178,6 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    // Assumes tensor-product elements
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
-   MFEM_VERIFY(el.GetMapType() == FiniteElement::VALUE,
-               "Only value map type currently supported");
    ElementTransformation &Trans = *mesh->GetTypicalElementTransformation();
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Trans);
    if (DeviceCanUseCeed())

fem/integ/bilininteg_elasticity_kernels.cpp

@@ -91,15 +91,15 @@ void ElasticityAddMultPA(const int dim, const int nDofs,
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag)
+                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
    switch (dim)
    {
       case 2:
-         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, ir, diag);
+         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, QVec, diag);
          break;
       case 3:
-         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, ir, diag);
+         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, QVec, diag);
          break;
       default:
          MFEM_ABORT("Only dimensions 2 and 3 supported.");

fem/integ/bilininteg_elasticity_kernels.hpp

@@ -38,6 +38,7 @@
 #include "../../linalg/vector.hpp"
 #include "../../linalg/tensor.hpp"
 #include "../quadinterpolator.hpp"
+#include "../bilininteg.hpp"
 #include "../coefficient.hpp"
 #include "../qfunction.hpp"
 
@@ -132,12 +133,12 @@ void ElasticityAssembleEA(const int dim, const int i_block, const int j_block,
 /// @param[in] mu Quadrature function for second Lame param.
 /// @param[in] geom Geometric factors corresponding to fespace.
 /// @param[in] maps DofToQuad maps for one element (assume elements all same).
-/// @param[in] ir Integration rule.
+/// @param QVec Scratch Q-Vector. nQuad x dim x dim x dim x dim x numEls.
 /// @param[out] diag diagonal of A. nDofs x dim x numEls.
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag);
+                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag);
 
 /// Templated implementation of ElasticityAddMultPA.
 template<int dim, int i_block = -1, int j_block = -1>
@@ -279,67 +280,77 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
 template<int dim>
 void ElasticityAssembleDiagonalPA_(const int nDofs,
                                    const CoefficientVector &lambda,
-                                   const CoefficientVector &mu,
-                                   const GeometricFactors &geom,
-                                   const DofToQuad &maps,
-                                   const IntegrationRule &ir,
-                                   Vector &diag)
+                                   const CoefficientVector &mu, const GeometricFactors &geom,
+                                   const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
+   using future::tensor;
+   using future::make_tensor;
    using future::det;
    using future::inv;
-   using future::make_tensor;
-   using future::tensor;
 
    // Assuming all elements are the same
+   const auto &ir = QVec.GetIntRule(0);
    static constexpr int d = dim;
    const int numPoints = ir.GetNPoints();
-   const int numEls = lambda.Size() / numPoints;
-
+   const int numEls = lambda.Size()/numPoints;
    const auto lamDev = Reshape(lambda.Read(), numPoints, numEls);
    const auto muDev = Reshape(mu.Read(), numPoints, numEls);
    const auto J = Reshape(geom.J.Read(), numPoints, d, d, numEls);
+   auto Q = Reshape(QVec.ReadWrite(), numPoints, d,d, d, numEls);
    const real_t *ipWeights = ir.GetWeights().Read();
-   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
-   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
+   mfem::forall_2D(numEls, numPoints,1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(p, x,numPoints)
+      {
+         auto invJ = inv(make_tensor<d, d>(
+         [&](int i, int j) { return J(p, i, j, e); }));
+         const real_t w = ipWeights[p] /det(invJ);
+         for (int n = 0; n < d; n++)
+         {
+            for (int m = 0; m < d; m++)
+            {
+               for (int q = 0; q < d; q++)
+               {
+                  // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                  // this contraction could be made slightly cheaper using Voigt
+                  // notation, but repeated entries are summed for simplicity.
+                  real_t contraction = 0.;
+                  for (int a = 0; a < d; a++)
+                  {
+                     for (int b = 0; b < d; b++)
+                     {
+                        contraction += ((a == q)*invJ(m,b) + (b==q)*invJ(m,a))*((a == q)
+                                                                                *invJ(n, b) + (b==q)*invJ(n,a));
+                     }
+                  }
+                  // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
+                  // contraction = 4*sym(grad(u))sym(grad(v))
+                  Q(p,m,n,q,e) = w*(lamDev(p, e)*invJ(m,q)*invJ(n,q)
+                                    + 0.5*muDev(p, e)*contraction);
+               }
+            }
+         }
+      }
+   });
 
+   // Reduce quadrature function to an E-Vector
+   const auto QRead = Reshape(QVec.Read(), numPoints, d, d, d, numEls);
+   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
+   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
    mfem::forall_2D(numEls, d, nDofs, [=] MFEM_HOST_DEVICE (int e)
    {
-      MFEM_FOREACH_THREAD_DIRECT(i, y, nDofs)
+      MFEM_FOREACH_THREAD(i, y, nDofs)
       {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, d)
+         MFEM_FOREACH_THREAD(q, x, d)
          {
-            real_t sum = 0.0;
-            for (int p = 0; p < numPoints; p++)
+            real_t sum = 0.;
+            for (int n = 0; n < d; n++)
             {
-               const auto invJ = inv(make_tensor<d, d>([&](int r, int c)
+               for (int m = 0; m < d; m++)
                {
-                  return J(p, r, c, e);
-               }));
-               const real_t w = ipWeights[p] / det(invJ);
-
-               for (int n = 0; n < d; n++)
-               {
-                  for (int m = 0; m < d; m++)
+                  for (int p = 0; p < numPoints; p++ )
                   {
-                     // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-                     // this contraction could be made slightly cheaper using Voigt
-                     // notation, but repeated entries are summed for simplicity.
-                     real_t contraction = 0.0;
-                     for (int a = 0; a < d; a++)
-                     {
-                        for (int b = 0; b < d; b++)
-                        {
-                           contraction +=
-                              ((a == q) * invJ(m, b) + (b == q) * invJ(m, a)) *
-                              ((a == q) * invJ(n, b) + (b == q) * invJ(n, a));
-                        }
-                     }
-                     // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
-                     // contraction = 4*sym(grad(u))sym(grad(v))
-                     const real_t Q =
-                        w * (lamDev(p, e) * invJ(m, q) * invJ(n, q)
-                             + 0.5 * muDev(p, e) * contraction);
-                     sum += Q * G(p, m, i) * G(p, n, i);
+                     sum += QRead(p,m,n,q,e)*G(p,m,i)*G(p,n,i);
                   }
                }
             }

fem/integ/bilininteg_elasticity_pa.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 #include "bilininteg_elasticity_kernels.hpp"
 
@@ -58,8 +59,9 @@ void ElasticityIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
 void ElasticityIntegrator::AssembleDiagonalPA(Vector &diag)
 {
+   q_vec->SetVDim(vdim*vdim*vdim*vdim);
    internal::ElasticityAssembleDiagonalPA(vdim, ndofs, *lambda_quad, *mu_quad,
-                                          *geom, *maps, *IntRule, diag);
+                                          *geom, *maps, *q_vec, diag);
 }
 
 void ElasticityIntegrator::AddMultPA(const Vector &x, Vector &y) const

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -147,16 +147,18 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHcurlMassApply2D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int D1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D)
+void PAHcurlMassApply2D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y)
 {
-   MFEM_ASSERT(D1D == TestD1D,
-               "Trial and Test space must have the same number of dofs");
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(bc.Read(), Q1D, D1D);
    auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
@@ -275,16 +277,18 @@ void PAHcurlMassApply2D(const int NE, const bool symmetric,
    }); // end of element loop
 }
 
-void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int D1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D)
+void PAHcurlMassApply3D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
@@ -785,23 +789,6 @@ void PAHcurlL2Setup2D(const int Q1D,
    });
 }
 
-void PAHcurlL2IntSetup2D(const int Q1D, const int NE, const Array<real_t> &w,
-                         Vector &coeff, const Vector &detJ, Vector &op)
-{
-   const int NQ = Q1D*Q1D;
-   auto W = w.Read();
-   auto C = Reshape(coeff.Read(), NQ, NE);
-   auto J = Reshape(detJ.Read(), NQ, NE);
-   auto y = Reshape(op.Write(), NQ, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         y(q,e) = W[q] * C(q,e) / J(q,e);
-      }
-   });
-}
-
 void PAHcurlL2Setup3D(const int NQ,
                       const int coeffDim,
                       const int NE,

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -181,312 +181,228 @@ inline void SmemPAHcurlMassAssembleDiagonal3D(const int d1d,
 }
 
 // PA H(curl) Mass Apply 2D kernel
-void PAHcurlMassApply2D(const int NE, const bool symmetric,
-                        const bool scalar_coeff, const Array<real_t> &bo,
-                        const Array<real_t> &bc, const Array<real_t> &bot,
-                        const Array<real_t> &bct, const Vector &pa_data,
-                        const Vector &x, Vector &y, const int TrialD1D,
-                        const int TestD1D, const int Q1D);
+void PAHcurlMassApply2D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
 
 // PA H(curl) Mass Apply 3D kernel
-void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int TrialD1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D);
+void PAHcurlMassApply3D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
 
 // Shared memory PA H(curl) Mass Apply 3D kernel
-template <int T_D1D = 0, int T_Q1D = 0, int TBATCH = 0, bool ACCUMULATE = true>
-inline void SmemPAHcurlMassApply3D(
-   const int NE, const bool symmetric, [[maybe_unused]] const bool scalar_coeff,
-   const Array<real_t> &bo, const Array<real_t> &bc,
-   [[maybe_unused]] const Array<real_t> &bot,
-   [[maybe_unused]] const Array<real_t> &bct, const Vector &pa_data,
-   const Vector &x, Vector &y, const int d1d = 0,
-   [[maybe_unused]] const int test_d1d = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHcurlMassApply3D(const int d1d,
+                                   const int q1d,
+                                   const int NE,
+                                   const bool symmetric,
+                                   const Array<real_t> &bo,
+                                   const Array<real_t> &bc,
+                                   const Array<real_t> &bot,
+                                   const Array<real_t> &bct,
+                                   const Vector &pa_data,
+                                   const Vector &x,
+                                   Vector &y)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
    MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: d1d > HCURL_MAX_D1D");
    MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
                "Error: q1d > HCURL_MAX_Q1D");
-   MFEM_ASSERT(Q1D >= D1D, "Expected Q1D >= D1D");
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
    const int dataSize = symmetric ? 6 : 9;
 
-   // assume trial space == test space
-   auto Bo = bo.Read();
-   auto Bc = bc.Read();
-   auto op =
-      Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
-   auto X_ = Reshape(x.Read(), 3 * (D1D - 1) * D1D * D1D, NE);
-   auto y_ = y.ReadWrite();
+   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
+   auto X = Reshape(x.Read(), 3*(D1D-1)*D1D*D1D, NE);
+   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
 
-   constexpr int MD_ = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-   constexpr int MQ_ = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-   constexpr int MDQ_ = std::max(MD_, MQ_);
-   constexpr int MB_ = TBATCH ? TBATCH : 1;
-
-   mfem::forall_2D_batch<MDQ_ * MDQ_ * MDQ_ * MB_>(
-      NE, MDQ_ * MDQ_ * MDQ_, 1, MB_, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-      constexpr int nbz = TBATCH ? TBATCH : 1;
-      int tidz = MFEM_THREAD_ID(z);
-#else
-      constexpr int nbz = 1;
-      constexpr int tidz = 0;
-#endif
-
       constexpr int VDIM = 3;
       constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
       constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-      constexpr int MDQ = std::max(MD1D, MQ1D);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
 
-      // nvcc limit work-around: can't have Y_ be captured first in
-      // if constexpr, so capture y_ and construct Y_ locally
-      // only works on GPU
-      auto Y = Reshape(y_, VDIM * (D1D - 1) * D1D * D1D, NE);
+      MFEM_SHARED real_t sBo[MQ1D][MD1D];
+      MFEM_SHARED real_t sBc[MQ1D][MD1D];
 
-      MFEM_SHARED real_t sBo[MDQ * (MD1D - 1)];
-      MFEM_SHARED real_t sBc[MDQ * MD1D];
-      auto BO = Reshape(sBo, Q1D, D1D - 1);
-      auto BC = Reshape(sBc, Q1D, D1D);
+      real_t op9[9];
+      MFEM_SHARED real_t sop[9*MQ1D*MQ1D];
+      MFEM_SHARED real_t mass[MQ1D][MQ1D][3];
 
-      MFEM_SHARED real_t sX[nbz * VDIM * (MD1D - 1) * MD1D * MD1D];
-      MFEM_SHARED real_t sm0[nbz * VDIM * MDQ * MDQ * MDQ];
-      MFEM_SHARED real_t sm1[nbz * VDIM * MDQ * MDQ * MDQ];
+      MFEM_SHARED real_t sX[MD1D][MD1D][MD1D];
 
-      real_t(*X)[nbz][(MD1D - 1) * MD1D * MD1D] =
-         (real_t(*)[nbz][(MD1D - 1) * MD1D * MD1D])(sX);
-      // shapes of buffers always use MQ1D to mitigate shared memory bank
-      // conflicts
-      real_t(*DDQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-      real_t(*DQQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
-      real_t(*QQQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-      real_t(*QQD)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
-      real_t(*QDD)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-
-      // load dofs into smem
-      const int offset = (D1D - 1) * D1D * D1D;
-      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
-         for (int dim = 0; dim < VDIM; ++dim)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
-            X[dim][tidz][ix] = X_(ix + dim * offset, e);
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               for (int i=0; i<dataSize; ++i)
+               {
+                  op9[i] = op(qx,qy,qz,i,e);
+               }
+            }
          }
       }
-      // load basis functions data
+
+      const int tidx = MFEM_THREAD_ID(x);
+      const int tidy = MFEM_THREAD_ID(y);
+      const int tidz = MFEM_THREAD_ID(z);
+
       if (tidz == 0)
       {
-         MFEM_FOREACH_THREAD_DIRECT(ix, x, D1D * Q1D) { sBc[ix] = Bc[ix]; }
-         MFEM_FOREACH_THREAD_DIRECT(ix, x, (D1D - 1) * Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            sBo[ix] = Bo[ix];
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               sBc[q][d] = Bc(q,d);
+               if (d < D1D-1)
+               {
+                  sBo[q][d] = Bo(q,d);
+               }
+            }
          }
       }
+      MFEM_SYNC_THREAD;
 
-      for (int dim0 = 0; dim0 < VDIM; ++dim0)
+      for (int qz=0; qz < Q1D; ++qz)
       {
-         MFEM_SYNC_THREAD;
-         // sum factor to QQQ = Q_{dim0,dim1} B X_{dim1}
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         int osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
          {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
 
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, Q1D, D1Dy, D1Dz,
-                                                 Q1D, Q1D, Q1D)
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
             {
-               real_t u = 0;
-               for (int dx = 0; dx < D1Dx; ++dx)
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
                {
-                  real_t b;
-                  if (dim1 == 0)
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
                   {
-                     b = BO(qx, dx);
+                     sX[dz][dy][dx] = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
                   }
-                  else
-                  {
-                     b = BC(qx, dx);
-                  }
-                  u += X[dim1][tidz][dx + (dy + dz * D1Dy) * D1Dx] * b;
-               }
-               DDQ[dim1][tidz][dz][dy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
-         {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, Q1D, Q1D, D1Dz,
-                                                 Q1D, Q1D, Q1D)
-            {
-               real_t u = 0;
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  real_t b;
-                  if (dim1 == 1)
-                  {
-                     b = BO(qy, dy);
-                  }
-                  else
-                  {
-                     b = BC(qy, dy);
-                  }
-                  u += DDQ[dim1][tidz][dz][dy][qx] * b;
-               }
-               DQQ[dim1][tidz][dz][qy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
-         {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            // const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
-            MFEM_FOREACH_THREAD_DIRECT_3D(qx, qy, qz, x, Q1D, Q1D, Q1D)
-            {
-               real_t u = 0;
-               for (int dz = 0; dz < D1Dz; ++dz)
-               {
-                  real_t b;
-                  if (dim1 == 2)
-                  {
-                     b = BO(qz, dz);
-                  }
-                  else
-                  {
-                     b = BC(qz, dz);
-                  }
-                  u += DQQ[dim1][tidz][dz][qy][qx] * b;
-               }
-               // pa_data is row major
-               int idx;
-               if (symmetric)
-               {
-                  int row;
-                  int col;
-                  if (dim0 > dim1)
-                  {
-                     row = dim1;
-                     col = dim0;
-                  }
-                  else
-                  {
-                     row = dim0;
-                     col = dim1;
-                  }
-                  idx = col + VDIM * row - row * (row + 1) / 2;
-               }
-               else
-               {
-                  idx = dim0 * VDIM + dim1;
-               }
-               QQQ[dim1][tidz][qz][qy][qx] = op(qx, qy, qz, idx, e) * u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         // sum factor back to Y
-         // Assume bot and bct == bo^t and bc^t respectively (i.e. test ==
-         // trial functions), skip loading them again.
-         {
-            const int D1Dz = (dim0 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim0 == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (dim0 == 0) ? D1D - 1 : D1D;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, D1Dz, Q1D, Q1D,
-                                                 Q1D, Q1D, Q1D)
-            {
-               for (int dim1 = 0; dim1 < VDIM; ++dim1)
-               {
-                  real_t u = 0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     real_t b = 0;
-                     if (dim0 == 2)
-                     {
-                        b = BO(qz, dz);
-                     }
-                     else
-                     {
-                        b = BC(qz, dz);
-                     }
-                     u += QQQ[dim1][tidz][qz][qy][qx] * b;
-                  }
-                  QQD[dim1][tidz][qy][qx][dz] = u;
-               }
-            }
-            MFEM_SYNC_THREAD;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, D1Dy, D1Dz, Q1D,
-                                                 Q1D, Q1D, Q1D)
-            {
-               for (int dim1 = 0; dim1 < VDIM; ++dim1)
-               {
-                  real_t u = 0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     real_t b;
-                     if (dim0 == 1)
-                     {
-                        b = BO(qy, dy);
-                     }
-                     else
-                     {
-                        b = BC(qy, dy);
-                     }
-                     u += QQD[dim1][tidz][qy][qx][dz] * b;
-                  }
-                  QDD[dim1][tidz][qx][dz][dy] = u;
                }
             }
             MFEM_SYNC_THREAD;
 
-            MFEM_FOREACH_THREAD_DIRECT_3D(dx, dy, dz, x, D1Dx, D1Dy, D1Dz)
+            if (tidz == qz)
             {
-               int ix = dx + D1Dx * (dy + D1Dy * dz);
-               real_t u = 0;
-               for (int qx = 0; qx < Q1D; ++qx)
+               for (int i=0; i<dataSize; ++i)
                {
-                  real_t b;
-                  if (dim0 == 0)
-                  {
-                     b = BO(qx, dx);
-                  }
-                  else
-                  {
-                     b = BC(qx, dx);
-                  }
-                  for (int dim1 = 0; dim1 < VDIM; ++dim1)
-                  {
-                     u += QDD[dim1][tidz][qx][dz][dy] * b;
-                  }
+                  sop[i + (dataSize*tidx) + (dataSize*Q1D*tidy)] = op9[i];
                }
-               if constexpr (ACCUMULATE)
+
+               MFEM_FOREACH_THREAD(qy,y,Q1D)
                {
-                  Y(ix + dim0 * offset, e) += u;
-               }
-               else
+                  MFEM_FOREACH_THREAD(qx,x,Q1D)
+                  {
+                     real_t u = 0.0;
+
+                     for (int dz = 0; dz < D1Dz; ++dz)
+                     {
+                        const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
+                        for (int dy = 0; dy < D1Dy; ++dy)
+                        {
+                           const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
+                           for (int dx = 0; dx < D1Dx; ++dx)
+                           {
+                              const real_t t = sX[dz][dy][dx];
+                              const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
+                              u += t * wx * wy * wz;
+                           }
+                        }
+                     }
+
+                     mass[qy][qx][c] = u;
+                  } // qx
+               } // qy
+            } // tidz == qz
+
+            osc += D1Dx * D1Dy * D1Dz;
+            MFEM_SYNC_THREAD;
+         } // c
+
+         MFEM_SYNC_THREAD;  // Sync mass[qy][qx][d] and sop
+
+         osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
+         {
+            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
+
+            real_t dxyz = 0.0;
+
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
+            {
+               const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
+
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
                {
-                  Y(ix + dim0 * offset, e) = u;
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
+                  {
+                     for (int qy = 0; qy < Q1D; ++qy)
+                     {
+                        const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
+                        for (int qx = 0; qx < Q1D; ++qx)
+                        {
+                           const int os = (dataSize*qx) + (dataSize*Q1D*qy);
+                           const int id1 = os + ((c == 0) ? 0 : ((c == 1) ? (symmetric ? 1 : 3) :
+                                                                 (symmetric ? 2 : 6))); // O11, O21, O31
+                           const int id2 = os + ((c == 0) ? 1 : ((c == 1) ? (symmetric ? 3 : 4) :
+                                                                 (symmetric ? 4 : 7))); // O12, O22, O32
+                           const int id3 = os + ((c == 0) ? 2 : ((c == 1) ? (symmetric ? 4 : 5) :
+                                                                 (symmetric ? 5 : 8))); // O13, O23, O33
+
+                           const real_t m_c = (sop[id1] * mass[qy][qx][0]) + (sop[id2] * mass[qy][qx][1]) +
+                                              (sop[id3] * mass[qy][qx][2]);
+
+                           const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
+                           dxyz += m_c * wx * wy * wz;
+                        }
+                     }
+                  }
                }
             }
-         }
-      }
+
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
+            {
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
+               {
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
+                  {
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) += dxyz;
+                  }
+               }
+            }
+
+            osc += D1Dx * D1Dy * D1Dz;
+         } // c loop
+      } // qz
    }); // end of element loop
 }
 
@@ -1889,17 +1805,13 @@ inline void SmemPACurlCurlApply3D(const int d1d,
    ForallWrap<3>(true, NE, device_kernel, host_kernel, Q1D, Q1D, Q1D);
 }
 
-// PA H(curl)-L2 value Assemble 2D kernel
+// PA H(curl)-L2 Assemble 2D kernel
 void PAHcurlL2Setup2D(const int Q1D,
                       const int NE,
                       const Array<real_t> &w,
                       Vector &coeff,
                       Vector &op);
 
-// PA H(curl)-L2 integral Assemble 2D kernel
-void PAHcurlL2IntSetup2D(const int Q1D, const int NE, const Array<real_t> &w,
-                         Vector &coeff, const Vector &detJ, Vector &op);
-
 // PA H(curl)-L2 Assemble 3D kernel
 void PAHcurlL2Setup3D(const int NQ,
                       const int coeffDim,

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -62,30 +62,6 @@ void PAHcurlHdivMassApply2D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
-/// H(curl) test, H(div) trial
-inline void
-PAHcurlHdivMassApply2D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false,
-                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
-/// H(div) test, H(curl) trial
-inline void
-PAHdivHcurlMassApply2D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true,
-                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
 // PA H(curl)-H(div) Mass Apply 3D kernel
 void PAHcurlHdivMassApply3D(const int D1D,
                             const int D1Dtest,
@@ -102,30 +78,6 @@ void PAHcurlHdivMassApply3D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
-/// H(curl) test, H(div) trial
-inline void
-PAHcurlHdivMassApply3D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false, false, Bo_,
-                          Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
-/// H(div) test, H(curl) trial
-inline void
-PAHdivHcurlMassApply3D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true, false, Bo_,
-                          Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
 // PA H(curl)-H(div) Curl Apply 3D kernel
 template<int T_D1D = 0, int T_D1D_TEST = 0, int T_Q1D = 0>
 inline void PAHcurlHdivApply3D(const int d1d,
@@ -864,656 +816,8 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
    }); // end of element loop
 }
 
-namespace curlinterp
-{
-constexpr int NBZ3D(int ndof_o, int nquad_o, int mdq)
-{
-   if (ndof_o <= 0 || nquad_o <= 0)
-   {
-      return 1;
-   }
-   int ndof_c = ndof_o + 1;
-   int nquad_c = nquad_o + 1;
-   // z dimension is capped at 64 on nvidia and amd gpus
-   int tmp =
-      std::min((128 + mdq * mdq * (mdq - 1) - 1) / (mdq * mdq * (mdq - 1)), 64);
-   int smem_req =
-      sizeof(mfem::real_t) *
-      ((3 * ndof_c * ndof_c * ndof_o + 2 * 2 * mdq * mdq * mdq) * tmp +
-       ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
-   // assume GPU has at least 48k shared memory
-   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
-}
-}
-
-template <int T_NDOF_O, int T_NQUAD_O>
-void CurlInterpolatorApply3DSmem(const int ne, const int ndof_o,
-                                 const int nquad_o, const Vector &pa,
-                                 const Vector &x_, Vector &y_)
-{
-   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-   constexpr int mnq_o =
-      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
-   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
-   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
-   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
-   int mnq = std::max(ndof_o + 1, nquad_o + 1);
-   auto pa_data = pa.Read();
-   auto x_d = x_.Read();
-   auto y_d = y_.ReadWrite();
-   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
-      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MND_O =
-         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-      constexpr int MNQ_O =
-         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
-      int tidz = MFEM_THREAD_ID(z);
-      // Make mnq a local variable since capturing would result in different
-      // captures between host/device versions, and spuriously fails
-      int mnq = std::max(ndof_o + 1, nquad_o + 1);
-#else
-      constexpr int nbz = 1;
-      constexpr int tidz = 0;
-#endif
-      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
-      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
-      const int NDOF_C = NDOF_O + 1;
-      const int NQUAD_C = NQUAD_O + 1;
-      MFEM_SHARED real_t
-      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
-      auto X_ = Reshape(x_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
-      auto Y = Reshape(y_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
-      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
-      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
-      auto Boo =
-         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
-      MFEM_SHARED real_t X[3][nbz][MND_O * (MND_O + 1) * (MND_O + 1)];
-      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
-      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
-
-      // shapes of buffers always use MNDQ to mitigate shared memory bank
-      // conflicts
-      real_t(*DDQ)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      real_t(*DQQ)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
-      real_t(*QQQ)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      const int offset = NDOF_O * NDOF_C * NDOF_C;
-      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
-      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
-      {
-         for (int dim = 0; dim < 3; ++dim)
-         {
-            X[dim][tidz][ix] = X_(ix + dim * offset, e);
-         }
-      }
-      // load basis functions data
-      if (tidz == 0)
-      {
-         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
-         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
-      }
-      MFEM_SYNC_THREAD;
-
-      // x: Vz Bcc Gco Boo - Vy Bcc Boo Gco
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Bcc(qx, dx);
-         }
-         DDQ[0][tidz][dz][dy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Bcc(qx, dx);
-         }
-         DDQ[1][tidz][dz][dy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[0][tidz][dz][dy][qx] * Gco(qy, dy);
-         }
-         DQQ[0][tidz][dz][qy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_O; ++dy)
-         {
-            u += DDQ[1][tidz][dz][dy][qx] * Boo(qy, dy);
-         }
-         DQQ[1][tidz][dz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_O; ++dz)
-         {
-            u += DQQ[0][tidz][dz][qy][qx] * Boo(qz, dz);
-         }
-         QQQ[0][tidz][qz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[1][tidz][dz][qy][qx] * Gco(qz, dz);
-         }
-         Y(qx + (qy + qz * NQUAD_O) * NQUAD_C, e) =
-            QQQ[0][tidz][qz][qy][qx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // y: Vx Boo Bcc Gco - Vz Gco Bcc Boo
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_O; ++dx)
-         {
-            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
-         }
-         DDQ[0][tidz][dz][dy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Gco(qx, dx);
-         }
-         DDQ[1][tidz][dz][dy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[0][tidz][dz][dy][qx] * Bcc(qy, dy);
-         }
-         DQQ[0][tidz][dz][qy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
-                                           NDOF_O, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[1][tidz][dz][dy][qx] * Bcc(qy, dy);
-         }
-         DQQ[1][tidz][dz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[0][tidz][dz][qy][qx] * Gco(qz, dz);
-         }
-         QQQ[0][tidz][qz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_O; ++dz)
-         {
-            u += DQQ[1][tidz][dz][qy][qx] * Boo(qz, dz);
-         }
-         Y(qx + (qy + qz * NQUAD_C) * NQUAD_O + offsetq, e) =
-            QQQ[0][tidz][qz][qy][qx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // z: Vy Gco Boo Bcc - Vx Boo Gco Bcc
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Gco(qx, dx);
-         }
-         DDQ[0][tidz][dz][dy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_O; ++dx)
-         {
-            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
-         }
-         DDQ[1][tidz][dz][dy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_O; ++dy)
-         {
-            u += DDQ[0][tidz][dz][dy][qx] * Boo(qy, dy);
-         }
-         DQQ[0][tidz][dz][qy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[1][tidz][dz][dy][qx] * Gco(qy, dy);
-         }
-         DQQ[1][tidz][dz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[0][tidz][dz][qy][qx] * Bcc(qz, dz);
-         }
-         QQQ[0][tidz][qz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[1][tidz][dz][qy][qx] * Bcc(qz, dz);
-         }
-         Y(qx + (qy + qz * NQUAD_O) * NQUAD_O + 2 * offsetq, e) =
-            QQQ[0][tidz][qz][qy][qx] - u;
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-template <int T_NDOF_O, int T_NQUAD_O>
-void CurlInterpolatorTApply3DSmem(const int ne, const int ndof_o,
-                                  const int nquad_o, const Vector &pa,
-                                  const Vector &x_, Vector &y_)
-{
-   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-   constexpr int mnq_o =
-      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
-   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
-   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
-   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
-   int mnq = std::max(ndof_o + 1, nquad_o + 1);
-   auto pa_data = pa.Read();
-   auto x_d = x_.Read();
-   auto y_d = y_.ReadWrite();
-   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
-      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MND_O =
-         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-      constexpr int MNQ_O =
-         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
-      int tidz = MFEM_THREAD_ID(z);
-      // Make mnq a local variable since capturing would result in different
-      // captures between host/device versions, and spuriously fails
-      int mnq = std::max(ndof_o + 1, nquad_o + 1);
-#else
-      constexpr int nbz = 1;
-      constexpr int tidz = 0;
-#endif
-      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
-      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
-      const int NDOF_C = NDOF_O + 1;
-      const int NQUAD_C = NQUAD_O + 1;
-      MFEM_SHARED real_t
-      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
-      auto X_ = Reshape(x_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
-      auto Y = Reshape(y_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
-      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
-      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
-      auto Boo =
-         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
-      MFEM_SHARED real_t X[3][nbz][MNQ_O * MNQ_O * (MNQ_O + 1)];
-      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
-      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
-
-      // shapes of buffers always use MNDQ to mitigate shared memory bank
-      // conflicts
-      real_t(*QQD)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      real_t(*QDD)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
-      real_t(*DDD)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      const int offset = NDOF_O * NDOF_C * NDOF_C;
-      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
-      MFEM_FOREACH_THREAD_DIRECT(ix, x, offsetq)
-      {
-         for (int dim = 0; dim < 3; ++dim)
-         {
-            X[dim][tidz][ix] = X_(ix + dim * offsetq, e);
-         }
-      }
-      // load basis functions data
-      if (tidz == 0)
-      {
-         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
-         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
-      }
-      MFEM_SYNC_THREAD;
-
-      // x: Vy Boo Bcc Gco - Vz Boo Gco Bcc
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Gco(qz, dz);
-         }
-         QQD[0][tidz][qy][qx][dz] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_C; ++qz)
-         {
-            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
-         }
-         QQD[1][tidz][qy][qx][dz] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_C; ++qy)
-         {
-            u += QQD[0][tidz][qy][qx][dz] * Bcc(qy, dy);
-         }
-         QDD[0][tidz][qx][dz][dy] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[1][tidz][qy][qx][dz] * Gco(qy, dy);
-         }
-         QDD[1][tidz][qx][dz][dy] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[0][tidz][qx][dz][dy] * Boo(qx, dx);
-         }
-         DDD[0][tidz][dz][dy][dx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[1][tidz][qx][dz][dy] * Boo(qx, dx);
-         }
-         Y(dx + (dy + dz * NDOF_C) * NDOF_O, e) =
-            DDD[0][tidz][dz][dy][dx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // y: Vz Gco Boo Bcc - Vx Bcc Boo Gco
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_C; ++qz)
-         {
-            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
-         }
-         QQD[0][tidz][qy][qx][dz] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Gco(qz, dz);
-         }
-         QQD[1][tidz][qy][qx][dz] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[0][tidz][qy][qx][dz] * Boo(qy, dy);
-         }
-         QDD[0][tidz][qx][dz][dy] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
-                                           NQUAD_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[1][tidz][qy][qx][dz] * Boo(qy, dy);
-         }
-         QDD[1][tidz][qx][dz][dy] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[0][tidz][qx][dz][dy] * Gco(qx, dx);
-         }
-         DDD[0][tidz][dz][dy][dx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_C; ++qx)
-         {
-            u += QDD[1][tidz][qx][dz][dy] * Bcc(qx, dx);
-         }
-         Y(dx + (dy + dz * NDOF_O) * NDOF_C + offset, e) =
-            DDD[0][tidz][dz][dy][dx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // z: Vx Bcc Gco Boo - Vy Gco Bcc Boo
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Boo(qz, dz);
-         }
-         QQD[0][tidz][qy][qx][dz] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Boo(qz, dz);
-         }
-         QQD[1][tidz][qy][qx][dz] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[0][tidz][qy][qx][dz] * Gco(qy, dy);
-         }
-         QDD[0][tidz][qx][dz][dy] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_C; ++qy)
-         {
-            u += QQD[1][tidz][qy][qx][dz] * Bcc(qy, dy);
-         }
-         QDD[1][tidz][qx][dz][dy] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_C; ++qx)
-         {
-            u += QDD[0][tidz][qx][dz][dy] * Bcc(qx, dx);
-         }
-         DDD[0][tidz][dz][dy][dx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[1][tidz][qx][dz][dy] * Gco(qx, dx);
-         }
-         Y(dx + (dy + dz * NDOF_C) * NDOF_C + 2 * offset, e) =
-            DDD[0][tidz][dz][dy][dx] - u;
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
 } // namespace internal
 
-template <int DIM, int NDOF_O, int NQUAD_O>
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyPAKernels::Kernel()
-{
-   if constexpr (DIM == 3)
-   {
-      return internal::CurlInterpolatorApply3DSmem<NDOF_O, NQUAD_O>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
-template <int DIM, int NDOF_O, int NQUAD_O>
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyTPAKernels::Kernel()
-{
-   if constexpr (DIM == 3)
-   {
-      return internal::CurlInterpolatorTApply3DSmem<NDOF_O, NQUAD_O>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
 } // namespace mfem
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_hdiv_kernels.cpp

@@ -294,14 +294,61 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int TestD1D, const int Q1D)
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<real_t> &Bo,
+                     const Array<real_t> &Bc,
+                     const Array<real_t> &Bot,
+                     const Array<real_t> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y)
+{
+   const int id = (D1D << 4) | Q1D;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x22: return SmemPAHdivMassApply2D<2,2>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x33: return SmemPAHdivMassApply2D<3,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x44: return SmemPAHdivMassApply2D<4,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x55: return SmemPAHdivMassApply2D<5,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply2D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x23: return SmemPAHdivMassApply3D<2,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x34: return SmemPAHdivMassApply3D<3,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x45: return SmemPAHdivMassApply3D<4,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x56: return SmemPAHdivMassApply3D<5,6>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x67: return SmemPAHdivMassApply3D<6,7>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x78: return SmemPAHdivMassApply3D<7,8>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply3D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+}
+
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
@@ -421,14 +468,18 @@ void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
    }); // end of element loop
 }
 
-void PAHdivMassApply3D(const int NE, const bool symmetric, const bool,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int TestD1D, const int Q1D)
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
                "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,

fem/integ/bilininteg_hdiv_kernels.hpp

@@ -66,29 +66,58 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
                                   const Vector &op_,
                                   Vector &diag_);
 
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<real_t> &Bo,
+                     const Array<real_t> &Bc,
+                     const Array<real_t> &Bot,
+                     const Array<real_t> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y);
+
 // PA H(div) Mass Apply 2D kernel
-void PAHdivMassApply2D(const int NE, const bool symmetric,
-                       const bool scalar_coeff, const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_, const Vector &op_,
-                       const Vector &x_, Vector &y_, const int D1D,
-                       const int TestD1D, const int Q1D);
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
 
 // PA H(div) Mass Apply 3D kernel
-void PAHdivMassApply3D(const int NE, const bool symmetric,
-                       const bool scalar_coeff, const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_, const Vector &op_,
-                       const Vector &x_, Vector &y_, const int D1D,
-                       const int TestD1D, const int Q1D);
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
 
 // Shared memory PA H(div) Mass Apply 2D kernel
-template <int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHdivMassApply2D(
-   const int NE, const bool symmetric, const bool, const Array<real_t> &Bo_,
-   const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-   const Array<real_t> &Bct_, const Vector &op_, const Vector &x_, Vector &y_,
-   const int d1d = 0, const int = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply2D(const int NE,
+                                  const bool symmetric,
+                                  const Array<real_t> &Bo_,
+                                  const Array<real_t> &Bc_,
+                                  const Array<real_t> &Bot_,
+                                  const Array<real_t> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);
@@ -251,13 +280,18 @@ inline void SmemPAHdivMassApply2D(
 }
 
 // Shared memory PA H(div) Mass Apply 3D kernel
-template <int T_D1D = 0, int T_Q1D = 0>
-inline void
-SmemPAHdivMassApply3D(const int NE, const bool symmetric, const bool,
-                      const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                      const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                      const Vector &op_, const Vector &x_, Vector &y_,
-                      const int d1d = 0, const int = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply3D(const int NE,
+                                  const bool symmetric,
+                                  const Array<real_t> &Bo_,
+                                  const Array<real_t> &Bc_,
+                                  const Array<real_t> &Bot_,
+                                  const Array<real_t> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);

fem/integ/bilininteg_interp_pa.cpp

@@ -14,218 +14,9 @@
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 
-#include "bilininteg_hcurlhdiv_kernels.hpp"
-
 namespace mfem
 {
 
-namespace
-{
-
-void PAHcurlApplyCurl2D(const int c_dofs1D,
-                        const int o_dofs1D,
-                        const int NE,
-                        const Array<real_t> &Bo_,
-                        const Array<real_t> &Gc_,
-                        const Vector &x_,
-                        Vector &y_)
-{
-   auto Bo = Reshape(Bo_.Read(), o_dofs1D, o_dofs1D);
-   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
-   auto X = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), o_dofs1D, o_dofs1D, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int iy = 0; iy < c_dofs1D; ++iy)
-      {
-         for (int ix = 0; ix < o_dofs1D; ++ix)
-         {
-            const real_t xv = X(ix + iy * o_dofs1D, e);
-            for (int oy = 0; oy < o_dofs1D; ++oy)
-            {
-               const real_t gy = Gc(oy, iy);
-               for (int ox = 0; ox < o_dofs1D; ++ox)
-               {
-                  Y(ox, oy, e) -= Bo(ox, ix) * gy * xv;
-               }
-            }
-         }
-      }
-
-      const int y_nd = c_dofs1D * o_dofs1D;
-      for (int iy = 0; iy < o_dofs1D; ++iy)
-      {
-         for (int ix = 0; ix < c_dofs1D; ++ix)
-         {
-            const real_t xv = X(y_nd + ix + iy * c_dofs1D, e);
-            for (int oy = 0; oy < o_dofs1D; ++oy)
-            {
-               const real_t by = Bo(oy, iy);
-               for (int ox = 0; ox < o_dofs1D; ++ox)
-               {
-                  Y(ox, oy, e) += Gc(ox, ix) * by * xv;
-               }
-            }
-         }
-      }
-   });
-}
-
-void PAHcurlApplyCurlTranspose2D(const int c_dofs1D,
-                                 const int o_dofs1D,
-                                 const int NE,
-                                 const Array<real_t> &Bo_,
-                                 const Array<real_t> &Gc_,
-                                 const Vector &x_,
-                                 Vector &y_)
-{
-   auto Bo = Reshape(Bo_.Read(), o_dofs1D, o_dofs1D);
-   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
-   auto X = Reshape(x_.Read(), o_dofs1D, o_dofs1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int dy = 0; dy < c_dofs1D; ++dy)
-      {
-         for (int dx = 0; dx < o_dofs1D; ++dx)
-         {
-            real_t sum = 0.0;
-            for (int oy = 0; oy < o_dofs1D; ++oy)
-            {
-               const real_t gy = Gc(oy, dy);
-               for (int ox = 0; ox < o_dofs1D; ++ox)
-               {
-                  sum -= Bo(ox, dx) * gy * X(ox, oy, e);
-               }
-            }
-            Y(dx + dy * o_dofs1D, e) += sum;
-         }
-      }
-
-      const int y_nd = c_dofs1D * o_dofs1D;
-      for (int dy = 0; dy < o_dofs1D; ++dy)
-      {
-         for (int dx = 0; dx < c_dofs1D; ++dx)
-         {
-            real_t sum = 0.0;
-            for (int oy = 0; oy < o_dofs1D; ++oy)
-            {
-               const real_t by = Bo(oy, dy);
-               for (int ox = 0; ox < o_dofs1D; ++ox)
-               {
-                  sum += Gc(ox, dx) * by * X(ox, oy, e);
-               }
-            }
-            Y(y_nd + dx + dy * c_dofs1D, e) += sum;
-         }
-      }
-   });
-}
-
-void PAHdivApplyCurl2D(const int c_dofs1D,
-                       const int o_dofs1D,
-                       const int NE,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Gc_,
-                       const Vector &x_,
-                       Vector &y_)
-{
-   auto Bc = Reshape(Bc_.Read(), c_dofs1D, c_dofs1D);
-   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
-   auto X = Reshape(x_.Read(), c_dofs1D, c_dofs1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int iy = 0; iy < c_dofs1D; ++iy)
-      {
-         for (int ix = 0; ix < c_dofs1D; ++ix)
-         {
-            const real_t xv = X(ix, iy, e);
-            for (int oy = 0; oy < o_dofs1D; ++oy)
-            {
-               const real_t gy = Gc(oy, iy);
-               for (int ox = 0; ox < c_dofs1D; ++ox)
-               {
-                  Y(ox + oy * c_dofs1D, e) += Bc(ox, ix) * gy * xv;
-               }
-            }
-         }
-      }
-
-      const int y_nd = c_dofs1D * o_dofs1D;
-      for (int iy = 0; iy < c_dofs1D; ++iy)
-      {
-         for (int ix = 0; ix < c_dofs1D; ++ix)
-         {
-            const real_t xv = X(ix, iy, e);
-            for (int oy = 0; oy < c_dofs1D; ++oy)
-            {
-               const real_t by = Bc(oy, iy);
-               for (int ox = 0; ox < o_dofs1D; ++ox)
-               {
-                  Y(y_nd + ox + oy * o_dofs1D, e) -= Gc(ox, ix) * by * xv;
-               }
-            }
-         }
-      }
-   });
-}
-
-void PAHdivApplyCurlTranspose2D(const int c_dofs1D,
-                                const int o_dofs1D,
-                                const int NE,
-                                const Array<real_t> &Bc_,
-                                const Array<real_t> &Gc_,
-                                const Vector &x_,
-                                Vector &y_)
-{
-   auto Bc = Reshape(Bc_.Read(), c_dofs1D, c_dofs1D);
-   auto Gc = Reshape(Gc_.Read(), o_dofs1D, c_dofs1D);
-   auto X = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), c_dofs1D, c_dofs1D, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int dy = 0; dy < o_dofs1D; ++dy)
-      {
-         for (int dx = 0; dx < c_dofs1D; ++dx)
-         {
-            const real_t xv = X(dx + dy * c_dofs1D, e);
-            for (int iy = 0; iy < c_dofs1D; ++iy)
-            {
-               const real_t gy = Gc(dy, iy);
-               for (int ix = 0; ix < c_dofs1D; ++ix)
-               {
-                  Y(ix, iy, e) += Bc(dx, ix) * gy * xv;
-               }
-            }
-         }
-      }
-
-      const int y_nd = c_dofs1D * o_dofs1D;
-      for (int dy = 0; dy < c_dofs1D; ++dy)
-      {
-         for (int dx = 0; dx < o_dofs1D; ++dx)
-         {
-            const real_t xv = X(y_nd + dx + dy * o_dofs1D, e);
-            for (int iy = 0; iy < c_dofs1D; ++iy)
-            {
-               const real_t by = Bc(dy, iy);
-               for (int ix = 0; ix < c_dofs1D; ++ix)
-               {
-                  Y(ix, iy, e) -= Gc(dx, ix) * by * xv;
-               }
-            }
-         }
-      }
-   });
-}
-
-}
-
 // Apply to x corresponding to DOFs in H^1 (domain) the (topological) gradient
 // to get a dof in H(curl) (range). You can think of the range as the "test" space
 // and the domain as the "trial" space, but there's no integration.
@@ -2159,266 +1950,4 @@ void IdentityInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
    }
 }
 
-void CurlInterpolator::AssemblePA(const FiniteElementSpace &dom_fes,
-                                  const FiniteElementSpace &ran_fes)
-{
-   Mesh *mesh = dom_fes.GetMesh();
-   dim = mesh->Dimension();
-   ne = dom_fes.GetNE();
-   pa_mode_2d = 0;
-   MFEM_VERIFY(ne == ran_fes.GetNE(),
-               "Different meshes for domain and range spaces");
-
-   if (dim == 2)
-   {
-      pa_data.SetSize(0);
-      const FiniteElement *dom_fel = dom_fes.GetTypicalFE();
-      const FiniteElement *ran_fel = ran_fes.GetTypicalFE();
-      const bool hcurl_to_scalar =
-         dynamic_cast<const VectorTensorFiniteElement*>(dom_fel) != NULL &&
-         dom_fel->GetDerivType() == FiniteElement::CURL &&
-         dynamic_cast<const TensorBasisElement*>(ran_fel) != NULL &&
-         ran_fel->GetRangeType() == FiniteElement::SCALAR;
-      const bool scalar_to_hdiv =
-         dynamic_cast<const TensorBasisElement*>(dom_fel) != NULL &&
-         dom_fel->GetRangeType() == FiniteElement::SCALAR &&
-         dynamic_cast<const VectorTensorFiniteElement*>(ran_fel) != NULL &&
-         ran_fel->GetDerivType() == FiniteElement::DIV;
-
-      MFEM_VERIFY(hcurl_to_scalar || scalar_to_hdiv,
-                  "2D CurlInterpolator PA supports H(curl)->scalar and scalar->H(div) only.");
-
-      int closed_basis_type = -1;
-      int open_basis_type = -1;
-      if (hcurl_to_scalar)
-      {
-         const auto *trial_fec = dynamic_cast<const ND_FECollection*>(dom_fes.FEColl());
-         const auto *range_fec = dynamic_cast<const L2_FECollection*>(ran_fes.FEColl());
-         MFEM_VERIFY(trial_fec != NULL, "H(curl) domain must use ND_FECollection.");
-         MFEM_VERIFY(range_fec != NULL, "Scalar range must use L2_FECollection.");
-         MFEM_VERIFY(ran_fel->GetMapType() == FiniteElement::INTEGRAL,
-                     "2D H(curl)->scalar CurlInterpolator PA supports integral-map scalar range spaces only.");
-         closed_basis_type = trial_fec->GetClosedBasisType();
-         open_basis_type = trial_fec->GetOpenBasisType();
-         MFEM_VERIFY(range_fec->GetBasisType() == open_basis_type,
-                     "Domain/range open basis types do not match.");
-         pa_mode_2d = 1;
-      }
-      else
-      {
-         const auto *trial_fec = dynamic_cast<const H1_FECollection*>(dom_fes.FEColl());
-         const auto *range_fec = dynamic_cast<const RT_FECollection*>(ran_fes.FEColl());
-         MFEM_VERIFY(trial_fec != NULL, "Scalar domain must use H1_FECollection.");
-         MFEM_VERIFY(range_fec != NULL, "H(div) range must use RT_FECollection.");
-         closed_basis_type = trial_fec->GetBasisType();
-         open_basis_type = range_fec->GetOpenBasisType();
-         MFEM_VERIFY(range_fec->GetClosedBasisType() == closed_basis_type,
-                     "Domain/range closed basis types do not match.");
-         pa_mode_2d = 2;
-      }
-
-      const int order = hcurl_to_scalar
-                        ? dynamic_cast<const VectorTensorFiniteElement*>(dom_fel)->GetOrder()
-                        : dynamic_cast<const NodalTensorFiniteElement*>(dom_fel)->GetOrder();
-      c_dofs1D = order + 1;
-      o_dofs1D = order;
-
-      closed_dofquad_fe.reset(new H1_SegmentElement(order, closed_basis_type));
-      open_dofquad_fe.reset(new L2_SegmentElement(order - 1, open_basis_type));
-
-      mfem::QuadratureFunctions1D qf1d;
-      mfem::IntegrationRule closed_ir;
-      closed_ir.SetSize(c_dofs1D);
-      qf1d.GaussLobatto(c_dofs1D, &closed_ir);
-
-      mfem::IntegrationRule open_ir;
-      open_ir.SetSize(o_dofs1D);
-      qf1d.GaussLegendre(o_dofs1D, &open_ir);
-
-      maps_C_C = &closed_dofquad_fe->GetDofToQuad(closed_ir, DofToQuad::TENSOR);
-      maps_O_C = &closed_dofquad_fe->GetDofToQuad(open_ir, DofToQuad::TENSOR);
-      maps_O_O = &open_dofquad_fe->GetDofToQuad(open_ir, DofToQuad::TENSOR);
-
-      MFEM_VERIFY(maps_C_C->ndof == c_dofs1D && maps_C_C->nqpt == c_dofs1D, "");
-      MFEM_VERIFY(maps_O_C->ndof == c_dofs1D && maps_O_C->nqpt == o_dofs1D, "");
-      MFEM_VERIFY(maps_O_O->ndof == o_dofs1D && maps_O_O->nqpt == o_dofs1D, "");
-      return;
-   }
-
-   closed_dofquad_fe.reset();
-   open_dofquad_fe.reset();
-   maps_C_C = nullptr;
-   maps_O_C = nullptr;
-   maps_O_O = nullptr;
-
-   const VectorTensorFiniteElement *dom_el =
-      dynamic_cast<const VectorTensorFiniteElement *>(dom_fes.GetTypicalFE());
-   const VectorTensorFiniteElement *ran_el =
-      dynamic_cast<const VectorTensorFiniteElement *>(ran_fes.GetTypicalFE());
-   MFEM_VERIFY(dom_el != NULL, "Only VectorTensorFiniteElement is supported!");
-   MFEM_VERIFY(ran_el != NULL, "Only VectorTensorFiniteElement is supported!");
-   MFEM_VERIFY(dom_el->GetDerivType() == FiniteElement::CURL,
-               "Domain space must be H(curl)");
-   MFEM_VERIFY(ran_el->GetDerivType() == FiniteElement::DIV,
-               "Range space must be H(div)");
-
-   const int dims = dom_el->GetDim();
-   MFEM_VERIFY(dims == 3, "");
-
-   ndof_o = dom_el->GetOrder();
-   int ndof_c = ndof_o + 1;
-   nquad_o = ran_el->GetOrder();
-   int nquad_c = nquad_o + 1;
-
-   // extract the tensor product range dof locations
-   std::vector<real_t> qc(nquad_c);
-   std::vector<real_t> qo(nquad_o);
-   {
-      const IntegrationRule &ran_nodes = ran_el->GetNodes();
-      const Array<int> &quad_map = ran_el->GetDofMap();
-      for (int i = 0; i < nquad_c; ++i)
-      {
-         int idx = UnsignIndex(quad_map[i]);
-         qc[i] = ran_nodes.IntPoint(idx).x;
-      }
-      int offset = ndof_c * ndof_o * ndof_o;
-      for (int i = 0; i < nquad_o; ++i)
-      {
-         int idx = UnsignIndex(quad_map[i + offset]);
-         qo[i] = ran_nodes.IntPoint(idx).x;
-      }
-   }
-
-   // evaluate closed/open 1D basis (and their derivatives) at closed and
-   // open quads
-   // storage order: GCO, BCC, BOO
-   pa_data.SetSize(ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
-   auto ptr = pa_data.HostWrite();
-   auto &cbasis1d = dom_el->GetBasis1D();
-   auto &obasis1d = dom_el->GetOpenBasis1D();
-   Vector b, g;
-   b.SetSize(ndof_c);
-   g.SetSize(ndof_c);
-   for (int j = 0; j < nquad_o; ++j)
-   {
-      cbasis1d.Eval(qo[j], b, g);
-      for (int i = 0; i < ndof_c; ++i)
-      {
-         ptr[j + i * nquad_o] = g[i];
-      }
-   }
-   ptr += nquad_o * ndof_c;
-
-   for (int j = 0; j < nquad_c; ++j)
-   {
-      cbasis1d.Eval(qc[j], b);
-      for (int i = 0; i < ndof_c; ++i)
-      {
-         ptr[j + i * nquad_c] = b[i];
-      }
-   }
-   ptr += ndof_c * nquad_c;
-
-   b.SetSize(ndof_o);
-   for (int j = 0; j < nquad_o; ++j)
-   {
-      obasis1d.Eval(qo[j], b);
-      for (int i = 0; i < ndof_o; ++i)
-      {
-         ptr[j + i * nquad_o] = b[i];
-      }
-   }
-}
-
-CurlInterpolator::Kernels::Kernels()
-{
-   CurlInterpolator::AddSpecialization<3, 1, 1>();
-   CurlInterpolator::AddSpecialization<3, 2, 2>();
-   CurlInterpolator::AddSpecialization<3, 3, 3>();
-   CurlInterpolator::AddSpecialization<3, 4, 4>();
-   CurlInterpolator::AddSpecialization<3, 5, 5>();
-}
-
-CurlInterpolator::CurlInterpolator() { static Kernels kernels{}; }
-
-void CurlInterpolator::AddMultPA(const Vector &x, Vector &y) const
-{
-   if (dim == 2)
-   {
-      MFEM_VERIFY(maps_C_C != nullptr && maps_O_C != nullptr,
-                  "2D CurlInterpolator PA data is not assembled.");
-      if (pa_mode_2d == 1)
-      {
-         MFEM_VERIFY(maps_O_O != nullptr,
-                     "2D CurlInterpolator scalar curl map is not assembled.");
-         PAHcurlApplyCurl2D(c_dofs1D, o_dofs1D, ne, maps_O_O->B, maps_O_C->G,
-                            x, y);
-      }
-      else if (pa_mode_2d == 2)
-      {
-         PAHdivApplyCurl2D(c_dofs1D, o_dofs1D, ne, maps_C_C->B, maps_O_C->G,
-                           x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported 2D CurlInterpolator mode.");
-      }
-      return;
-   }
-
-   ApplyPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x, y);
-}
-
-void CurlInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
-{
-   if (dim == 2)
-   {
-      MFEM_VERIFY(maps_C_C != nullptr && maps_O_C != nullptr,
-                  "2D CurlInterpolator PA data is not assembled.");
-      if (pa_mode_2d == 1)
-      {
-         MFEM_VERIFY(maps_O_O != nullptr,
-                     "2D CurlInterpolator scalar curl map is not assembled.");
-         PAHcurlApplyCurlTranspose2D(c_dofs1D, o_dofs1D, ne, maps_O_O->B,
-                                     maps_O_C->G, x, y);
-      }
-      else if (pa_mode_2d == 2)
-      {
-         PAHdivApplyCurlTranspose2D(c_dofs1D, o_dofs1D, ne, maps_C_C->B,
-                                    maps_O_C->G, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported 2D CurlInterpolator mode.");
-      }
-      return;
-   }
-
-   ApplyTPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x, y);
-}
-
-/// \cond DO_NOT_DOCUMENT
-
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyPAKernels::Fallback(int DIM, int, int)
-{
-   if (DIM == 3)
-   {
-      return internal::CurlInterpolatorApply3DSmem<0, 0>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyTPAKernels::Fallback(int DIM, int, int)
-{
-   if (DIM == 3)
-   {
-      return internal::CurlInterpolatorTApply3DSmem<0, 0>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
 } // namespace mfem

fem/integ/bilininteg_vecdiv_pa.hpp

@@ -1,365 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/array.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../linalg/vector.hpp"
-#include "../bilininteg.hpp"
-#include "../kernels.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-// Shared memory PA Divergence Apply 2D kernel
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApply2D(const int NE,
-                                    const Array<real_t> &b_,
-                                    const Array<real_t> &g_,
-                                    const Array<real_t> &bt_,
-                                    const Vector &q_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
-   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, 2, NE);
-   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, 1, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
-      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
-
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs2d(e, TR_D1D, X, g0);
-      kernels::internal::Grad2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            r0[0][qy][qx] =
-               g1[0][0][qy][qx] * Q(qx, qy, 0, 0, e) +
-               g1[0][1][qy][qx] * Q(qx, qy, 1, 0, e) +
-               g1[1][0][qy][qx] * Q(qx, qy, 0, 1, e) +
-               g1[1][1][qy][qx] * Q(qx, qy, 1, 1, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TE_D1D, Q1D, Bt, sB);
-      kernels::internal::EvalTranspose2d(TE_D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs2d(e, TE_D1D, r1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 2D kernel transpose
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApplyTranspose2D(const int NE,
-                                             const Array<real_t> &bt,
-                                             const Array<real_t> &gt,
-                                             const Array<real_t> &b,
-                                             const Vector &q_,
-                                             const Vector &x_,
-                                             Vector &y_,
-                                             const int tr_d1d = 0,
-                                             const int te_d1d = 0,
-                                             const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
-   const auto X  = Reshape(x_.Read(), TE_D1D, TE_D1D, 1, NE);
-   auto Y  = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, 2, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
-      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
-      kernels::internal::LoadDofs2d(e, TE_D1D, X, r0);
-      kernels::internal::Eval2d(TE_D1D, Q1D, smem, sB, r0, r1);
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            g0[0][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 0, e);
-            g0[0][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 0, e);
-            g0[1][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 1, e);
-            g0[1][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 1, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
-      kernels::internal::GradTranspose2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-      kernels::internal::WriteDofs2d(e, TR_D1D, g1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 3D kernel transpose
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApplyTranspose3D(const int NE,
-                                             const Array<real_t> &bt,
-                                             const Array<real_t> &gt,
-                                             const Array<real_t> &b,
-                                             const Vector &q_,
-                                             const Vector &x_,
-                                             Vector &y_,
-                                             int tr_d1d = 0,
-                                             int te_d1d = 0,
-                                             int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
-   const auto X = Reshape(x_.Read(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
-   auto Y = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
-      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
-      kernels::internal::LoadDofs3d(e, TE_D1D, X, r0);
-      kernels::internal::Eval3d(TE_D1D, Q1D, smem, sB, r0, r1);
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const auto r = r1[0][qz][qy][qx];
-               g0[0][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 0, e);
-               g0[0][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 0, e);
-               g0[0][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 0, e);
-
-               g0[1][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 1, e);
-               g0[1][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 1, e);
-               g0[1][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 1, e);
-
-               g0[2][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 2, e);
-               g0[2][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 2, e);
-               g0[2][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 2, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
-      kernels::internal::GradTranspose3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-      kernels::internal::WriteDofs3d(e, TR_D1D, g1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 3D kernel
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApply3D(const int NE,
-                                    const Array<real_t> &b_,
-                                    const Array<real_t> &g_,
-                                    const Array<real_t> &bt_,
-                                    const Vector &q_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3,3, NE);
-   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
-   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
-      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
-
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs3d(e, TR_D1D, X, g0);
-      kernels::internal::Grad3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               r0[0][qz][qy][qx] =
-                  // c = 0
-                  g1[0][0][qz][qy][qx] * Q(qx, qy, qz, 0, 0, e) +
-                  g1[0][1][qz][qy][qx] * Q(qx, qy, qz, 1, 0, e) +
-                  g1[0][2][qz][qy][qx] * Q(qx, qy, qz, 2, 0, e) +
-                  // c = 1
-                  g1[1][0][qz][qy][qx] * Q(qx, qy, qz, 0, 1, e) +
-                  g1[1][1][qz][qy][qx] * Q(qx, qy, qz, 1, 1, e) +
-                  g1[1][2][qz][qy][qx] * Q(qx, qy, qz, 2, 1, e) +
-                  // c = 2
-                  g1[2][0][qz][qy][qx] * Q(qx, qy, qz, 0, 2, e) +
-                  g1[2][1][qz][qy][qx] * Q(qx, qy, qz, 1, 2, e) +
-                  g1[2][2][qz][qy][qx] * Q(qx, qy, qz, 2, 2, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1, true>(TE_D1D, Q1D, Bt, sB);
-      kernels::internal::EvalTranspose3d(TE_D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs3d(e, TE_D1D, r1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
-VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Kernel()
-{
-   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPADivergenceApply2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADivergenceApply3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Fallback
-(int dim, int tr_d1d, int te_d1d, int q1d)
-{
-   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
-   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPADivergenceApply2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPADivergenceApply3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Kernel()
-{
-   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPADivergenceApplyTranspose2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADivergenceApplyTranspose3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Fallback
-(int dim, int tr_d1d, int te_d1d, int q1d)
-{
-   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
-   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPADivergenceApplyTranspose2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPADivergenceApplyTranspose3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/bilininteg_vecmass_pa.cpp

@@ -22,8 +22,6 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
-   MFEM_VERIFY(el.GetMapType() == FiniteElement::VALUE,
-               "Only value map type supported");
    ElementTransformation &Trans = *mesh->GetTypicalElementTransformation();
    const auto *ir = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, Trans);
 
@@ -207,40 +205,157 @@ void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 
 }
 
+template <const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal2D(const int NE,
+                                           const Array<real_t> &b,
+                                           const Vector &pa_data, Vector &diag,
+                                           const int d1d = 0, const int q1d = 0)
+{
+   constexpr int VDIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const auto B = Reshape(b.Read(), Q1D, D1D);
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, VDIM, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t temp[max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            temp[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               temp[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            real_t temp1 = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               temp1 += B(qx, dx) * B(qx, dx) * temp[qx][dy];
+            }
+            Y(dx, dy, 0, e) = temp1;
+            Y(dx, dy, 1, e) = temp1;
+         }
+      }
+   });
+}
+
+template <const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal3D(const int NE,
+                                           const Array<real_t> &B_,
+                                           const Vector &pa_data, Vector &diag,
+                                           const int d1d = 0, const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const auto B = Reshape(B_.Read(), Q1D, D1D);
+   MFEM_VERIFY(pa_data.Size() == Q1D * Q1D * Q1D * NE, "pa_data size error");
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t temp[max_Q1D][max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               temp[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  temp[qx][qy][dz] +=
+                     B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      real_t temp2[max_Q1D][max_D1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               temp2[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  temp2[qx][dy][dz] +=
+                     B(qy, dy) * B(qy, dy) * temp[qx][qy][dz];
+               }
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               real_t temp3 = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  temp3 += B(qx, dx) * B(qx, dx) * temp2[qx][dy][dz];
+               }
+               Y(dx, dy, dz, 0, e) = temp3;
+               Y(dx, dy, dz, 1, e) = temp3;
+               Y(dx, dy, dz, 2, e) = temp3;
+            }
+         }
+      }
+   });
+}
+
+static void PAVectorMassAssembleDiagonal(const int dim, const int D1D,
+                                         const int Q1D, const int NE,
+                                         const Array<real_t> &B,
+                                         const Vector &pa_data,
+                                         Vector &diag)
+{
+   if (dim == 2)
+   {
+      return PAVectorMassAssembleDiagonal2D(NE, B, pa_data, diag, D1D, Q1D);
+   }
+   else if (dim == 3)
+   {
+      return PAVectorMassAssembleDiagonal3D(NE, B, pa_data, diag, D1D, Q1D);
+   }
+   MFEM_ABORT("Dimension not implemented.");
+}
+
 void VectorMassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   if (DeviceCanUseCeed()) { return ceedOp->GetDiagonal(diag); }
-
-   MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
-   MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
-
-   // Add the VectorMassAssembleDiagonalPA specializations
-   static const auto vector_mass_assemble_diagonal_kernel_specializations =
-      ( // 2D
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 2>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 3>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 4>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 5>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 6>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 7>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 8>::Add(),
-         // 3D
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 2>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 3>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 4>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 5>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 6>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 7>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 8>::Add(),
-         true);
-   MFEM_CONTRACT_VAR(vector_mass_assemble_diagonal_kernel_specializations);
-
-   VectorMassAssembleDiagonalPA::Run(dim, quad1D, // templated arguments
-                                     ne, dofs1D, quad1D,
-                                     maps->B.Read(),
-                                     pa_data.Read(),
-                                     diag.ReadWrite());
-
+   if (DeviceCanUseCeed()) { ceedOp->GetDiagonal(diag); }
+   else
+   {
+      MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
+      MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
+      PAVectorMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
+   }
 }
 
 } // namespace mfem

fem/integ/bilininteg_vecmass_pa.hpp

@@ -176,146 +176,8 @@ void SmemPAVectorMassApply3D(const int NE,
    });
 }
 
-template <int T_Q1D = 0, int T_MDQ = 16>
-static void SmemPAVectorMassAssembleDiagonal2D(const int ne,
-                                               const int d1d,
-                                               const int q1d,
-                                               const real_t *b_r,
-                                               const real_t *d_r,
-                                               real_t *y_rw)
-{
-   constexpr int VDIM = 2;
-
-   const int D1D = d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
-
-   const auto B = Reshape(b_r, Q1D, D1D);
-   const auto D = Reshape(d_r, Q1D, Q1D, ne);
-   auto Y = Reshape(y_rw, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(
-      ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
-
-      MFEM_SHARED real_t sm[MQ1][MQ1];
-
-      MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            real_t u = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               u += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-            sm[qx][dy] = u;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-         {
-            real_t u = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               u += B(qx, dx) * B(qx, dx) * sm[qx][dy];
-            }
-            Y(dx, dy, 0, e) += u;
-            Y(dx, dy, 1, e) += u;
-         }
-      }
-   });
-}
-
-// T_MDQ <= 10 so the Q1D^3 thread block stays within the 1024/block GPU limit
-template <int T_Q1D = 0, int T_MDQ = 10>
-static void SmemPAVectorMassAssembleDiagonal3D(const int ne,
-                                               const int d1d,
-                                               const int q1d,
-                                               const real_t *b_r,
-                                               const real_t *d_r,
-                                               real_t *y_rw)
-{
-   constexpr int VDIM = 3;
-
-   const int D1D = d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
-
-   const auto B = Reshape(b_r, Q1D, D1D);
-   const auto D = Reshape(d_r, Q1D, Q1D, Q1D, ne);
-   auto Y = Reshape(y_rw, D1D, D1D, D1D, VDIM, ne);
-
-   mfem::forall_3D<T_Q1D*T_Q1D*T_Q1D>(
-      ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
-
-      MFEM_SHARED real_t sm[2][MQ1][MQ1][MQ1];
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               real_t u = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-               sm[0][dz][qy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += B(qy, dy) * B(qy, dy) * sm[0][dz][qy][qx];
-               }
-               sm[1][dz][dy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += B(qx, dx) * B(qx, dx) * sm[1][dz][dy][qx];
-               }
-               Y(dx, dy, dz, 0, e) += u;
-               Y(dx, dy, dz, 1, e) += u;
-               Y(dx, dy, dz, 2, e) += u;
-            }
-         }
-      }
-   });
-}
-
 } // namespace internal
 
-// AddMultPA kernels
 template<int DIM, int T_D1D, int T_Q1D>
 VectorMassIntegrator::VectorMassAddMultPAType
 VectorMassIntegrator::VectorMassAddMultPA::Kernel()
@@ -332,7 +194,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Kernel()
 }
 
 inline VectorMassIntegrator::VectorMassAddMultPAType
-VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
+VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int d1d, int q1d)
 {
    if (dim == 2)
    {
@@ -342,37 +204,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
    {
       return internal::SmemPAVectorMassApply3D;
    }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-// DiagonalPA kernels
-template<int DIM, int T_Q1D>
-VectorMassIntegrator::VectorMassAssembleDiagonalPAType
-VectorMassIntegrator::VectorMassAssembleDiagonalPA::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal2D<T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal3D<T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorMassIntegrator::VectorMassAssembleDiagonalPAType
-VectorMassIntegrator::VectorMassAssembleDiagonalPA::Fallback(int dim, int)
-{
-   if (dim == 2)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
+   else { MFEM_ABORT("Unsupported kernel"); }
 }
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_vectorfemass_kernels.hpp

@@ -1,113 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
-#define MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
-
-#include "../../config/config.hpp"
-#include "../bilininteg.hpp"
-
-#include "bilininteg_diffusion_kernels.hpp"
-#include "bilininteg_hcurl_kernels.hpp"
-#include "bilininteg_hdiv_kernels.hpp"
-#include "bilininteg_hcurlhdiv_kernels.hpp"
-
-namespace mfem
-{
-/// \cond DO_NOT_DOCUMENT
-namespace internal
-{
-namespace hcurlmass
-{
-constexpr int NBZ3D(int d1d, int q1d)
-{
-   if (d1d <= 1 || q1d <= 0)
-   {
-      return 1;
-   }
-   // assume q1d >= d1d
-   // z dimension is capped at 64 on nvidia and amd gpus
-   int tmp = std::min((128 + q1d * q1d * q1d - 1) / (q1d * q1d * q1d), 64);
-   int smem_req =
-      sizeof(mfem::real_t) *
-      (3 * ((d1d - 1) * d1d * d1d + 2 * q1d * q1d * q1d) * tmp +
-       q1d * (d1d - 1) + q1d * d1d);
-   // assume GPU has at least 48k shared memory
-   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
-}
-} // namespace hcurlmass
-} // namespace internal
-
-template <FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
-          int DIM, int TrialD1D, int TestD1D, int Q1D>
-VectorFEMassIntegrator::ApplyKernelType
-VectorFEMassIntegrator::ApplyPAKernels::Kernel()
-{
-   constexpr bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
-   constexpr bool trial_div = (TrialType == mfem::FiniteElement::DIV);
-   constexpr bool test_curl = (TestType == mfem::FiniteElement::CURL);
-   constexpr bool test_div = (TestType == mfem::FiniteElement::DIV);
-
-   if constexpr (DIM == 3)
-   {
-      if constexpr (trial_curl && test_curl)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            // assume TrialD1D == TestD1D
-            return internal::SmemPAHcurlMassApply3D<
-                   TrialD1D, Q1D, internal::hcurlmass::NBZ3D(TrialD1D, Q1D)>;
-         }
-         else
-         {
-            return internal::PAHcurlMassApply3D;
-         }
-      }
-      else if constexpr (trial_div && test_div)
-      {
-         // assumes TrialD1D == TestD1D
-         return internal::SmemPAHdivMassApply3D<TrialD1D, Q1D>;
-      }
-      else if constexpr (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply3D;
-      }
-      else if constexpr (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply3D;
-      }
-   }
-   else if constexpr (DIM == 2) // 2D
-   {
-      if constexpr (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply2D;
-      }
-      else if constexpr (trial_div && test_div)
-      {
-         // assumes TrialD1D == TestD1D
-         return internal::SmemPAHdivMassApply2D<TrialD1D, Q1D>;
-      }
-      else if constexpr (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply2D;
-      }
-      else if constexpr (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply2D;
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-/// \endcond DO_NOT_DOCUMENT
-}
-
-#endif

fem/integ/bilininteg_vectorfemass_pa.cpp

@@ -10,123 +10,15 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
-#include "bilininteg_vectorfemass_kernels.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "bilininteg_diffusion_kernels.hpp"
+#include "bilininteg_hcurl_kernels.hpp"
+#include "bilininteg_hdiv_kernels.hpp"
+#include "bilininteg_hcurlhdiv_kernels.hpp"
 
 namespace mfem
 {
-/// \cond DO_NOT_DOCUMENT
-VectorFEMassIntegrator::ApplyKernelType
-VectorFEMassIntegrator::ApplyPAKernels::Fallback(
-   FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
-   int dim, int, int, int)
-{
-   const bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
-   const bool trial_div = (TrialType == mfem::FiniteElement::DIV);
-   const bool test_curl = (TestType == mfem::FiniteElement::CURL);
-   const bool test_div = (TestType == mfem::FiniteElement::DIV);
-
-   if (dim == 3)
-   {
-      if (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply3D;
-      }
-      else if (trial_div && test_div)
-      {
-         return internal::PAHdivMassApply3D;
-      }
-      else if (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply3D;
-      }
-      else if (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply3D;
-      }
-   }
-   else if (dim == 2) // 2D
-   {
-      if (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply2D;
-      }
-      else if (trial_div && test_div)
-      {
-         return internal::PAHdivMassApply2D;
-      }
-      else if (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply2D;
-      }
-      else if (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply2D;
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-/// \endcond DO_NOT_DOCUMENT
-
-VectorFEMassIntegrator::Kernels::Kernels()
-{
-   // h(curl), h(curl)
-   // Q = P + 1 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 6>();
-   // Q = P + 2 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 6>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 7>();
-   // Q = P + 4 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 6>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 7>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 8>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 9>();
-   // h(div), h(div)
-   // Q = P (2D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 2, 2, 2>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 3, 3, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 4, 4, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 5, 5, 5>();
-
-   // Q = P + 1 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 2, 2, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 3, 3, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 4, 4, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 5, 5, 6>();
-}
-
-void VectorFEMassIntegrator::Init(Coefficient *q, DiagonalMatrixCoefficient *dq,
-                                  MatrixCoefficient *mq)
-{
-   static Kernels kernels{};
-   Q = q;
-   DQ = dq;
-   MQ = mq;
-}
 
 void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
@@ -175,8 +67,8 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
 
-   trial_fetype = static_cast<FiniteElement::DerivType>(trial_el->GetDerivType());
-   test_fetype = static_cast<FiniteElement::DerivType>(test_el->GetDerivType());
+   trial_fetype = trial_el->GetDerivType();
+   test_fetype = test_el->GetDerivType();
 
    const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
    const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
@@ -323,34 +215,225 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
 
 void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   const bool scalar_coeff = !(DQ || MQ);
-   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
-                       quad1D, ne, symmetric, scalar_coeff, mapsO->B, mapsC->B,
-                       mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y, dofs1D,
-                       dofs1Dtest, quad1D);
+   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
+   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
+   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
+   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
+
+   if (dim == 3)
+   {
+      if (trial_curl && test_curl)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            const int ID = (dofs1D << 4) | quad1D;
+            switch (ID)
+            {
+               case 0x23:
+                  return internal::SmemPAHcurlMassApply3D<2,3>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x34:
+                  return internal::SmemPAHcurlMassApply3D<3,4>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x45:
+                  return internal::SmemPAHcurlMassApply3D<4,5>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x56:
+                  return internal::SmemPAHcurlMassApply3D<5,6>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               default:
+                  return internal::SmemPAHcurlMassApply3D(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+            }
+         }
+         else
+         {
+            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                         mapsO->Bt, mapsC->Bt, pa_data, x, y);
+         }
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                   mapsO->Bt, mapsC->Bt, pa_data, x, y);
+      }
+      else if (trial_curl && test_div)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          true, false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                          mapsCtest->Bt, pa_data, x, y);
+      }
+      else if (trial_div && test_curl)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          false, false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                          mapsCtest->Bt, pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
+   else // 2D
+   {
+      if (trial_curl && test_curl)
+      {
+         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                      mapsO->Bt, mapsC->Bt, pa_data, x, y);
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                   mapsO->Bt,
+                                   mapsC->Bt, pa_data, x, y);
+      }
+      else if ((trial_curl && test_div) || (trial_div && test_curl))
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          trial_curl, false, mapsO->B, mapsC->B,
+                                          mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
 }
 
 void VectorFEMassIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
 {
-   const bool scalar_coeff = !(DQ || MQ);
+   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
+   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
+   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
+   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
 
    Vector abs_pa_data(pa_data);
    abs_pa_data.Abs();
 
    Array<real_t> absBo(mapsO->B);
    Array<real_t> absBc(mapsC->B);
+   Array<real_t> absBto(mapsO->Bt);
+   Array<real_t> absBtc(mapsC->Bt);
    Array<real_t> absBto_t(mapsOtest->Bt);
    Array<real_t> absBtc_t(mapsCtest->Bt);
 
    absBo.Abs();
    absBc.Abs();
+   absBto.Abs();
+   absBtc.Abs();
    absBto_t.Abs();
    absBtc_t.Abs();
 
-   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
-                       quad1D, ne, symmetric, scalar_coeff, absBo, absBc,
-                       absBto_t, absBtc_t, abs_pa_data, x, y, dofs1D,
-                       dofs1Dtest, quad1D);
+   if (dim == 3)
+   {
+      if (trial_curl && test_curl)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            const int ID = (dofs1D << 4) | quad1D;
+            switch (ID)
+            {
+               case 0x23:
+                  return internal::SmemPAHcurlMassApply3D<2,3>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x34:
+                  return internal::SmemPAHcurlMassApply3D<3,4>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x45:
+                  return internal::SmemPAHcurlMassApply3D<4,5>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x56:
+                  return internal::SmemPAHcurlMassApply3D<5,6>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               default:
+                  return internal::SmemPAHcurlMassApply3D(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+            }
+         }
+         else
+         {
+            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric,
+                                         absBo, absBc, absBto, absBtc,
+                                         abs_pa_data, x, y);
+         }
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric,
+                                   absBo, absBc, absBto, absBtc,
+                                   abs_pa_data, x, y);
+      }
+      else if (trial_curl && test_div)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, true, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else if (trial_div && test_curl)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, false, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
+   else // 2D
+   {
+      if (trial_curl && test_curl)
+      {
+         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric,
+                                      absBo, absBc, absBto, absBtc,
+                                      abs_pa_data, x, y);
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric,
+                                   absBo, absBc, absBto, absBtc,
+                                   abs_pa_data, x, y);
+      }
+      else if ((trial_curl && test_div) || (trial_div && test_curl))
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, trial_curl, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
 }
 
 void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,

fem/integ/nonlininteg_vecconvection_pa.cpp

@@ -9,51 +9,21 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
+#include "../../general/forall.hpp"
+#include "../nonlininteg.hpp"
 #include "../ceed/integrators/nlconvection/nlconvection.hpp"
-#include "./nonlininteg_vecconvection_pa.hpp" // IWYU pragma: keep
-#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
-#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
 
 namespace mfem
 {
 
-VectorConvectionNLFIntegrator::Kernels::Kernels()
-{
-   // 2D
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 2>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 3>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 8>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 6, 8>();
-   // 3D
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 3>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 8>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 8>();
-}
-
 void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES,
                "PA Only supports Ordering::byNODES!");
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
-   ElementTransformation &Tr = *mesh->GetTypicalElementTransformation();
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Tr);
-
+   ElementTransformation &T = *mesh->GetTypicalElementTransformation();
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, T);
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
@@ -69,124 +39,769 @@ void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
       }
       return;
    }
-
-   ne = mesh->GetNE();
-   nq = ir->GetNPoints();
    dim = mesh->Dimension();
-   MFEM_VERIFY(dim == 2 || dim == 3, "Dimension not supported");
-
-   const MemoryType mt = pa_mt == MemoryType::DEFAULT
-                         ? Device::GetDeviceMemoryType()
-                         : pa_mt;
-   pa_adj.SetSize(ne * nq * dim * dim, mt);
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
+   ne = fes.GetMesh()->GetNE();
+   nq = ir->GetNPoints();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   d1d = maps->ndof;
-   q1d = maps->nqpt;
-
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
-
-   const int nq1d = q1d * q1d * (dim==3 ? q1d : 1);
-   MFEM_VERIFY(coeff.Size() == 1 || coeff.Size() == nq1d*ne, "Invalid coeff");
-   MFEM_VERIFY(ir->GetWeights().Size() == nq1d, "Invalid weights size");
-
-   const auto w_r = ir->GetWeights().Read();
-   const bool const_coeff = coeff.Size() == 1;
-
+   pa_data.SetSize(ne * nq * dim * dim, Device::GetMemoryType());
+   real_t COEFF = 1.0;
+   if (Q)
+   {
+      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient *>(Q);
+      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
+      COEFF = cQ->constant;
+   }
+   const int NE = ne;
+   const int NQ = nq;
+   auto W = ir->GetWeights().Read();
+   if (dim == 1)
+   {
+      MFEM_ABORT("dim==1 not supported!");
+   }
    if (dim == 2)
    {
-      const int Q1D = q1d;
-      constexpr int VDIM = 2, DIM = 2;
-      const auto W = Reshape(w_r, Q1D, Q1D);
-      const auto C = const_coeff ?
-                     Reshape(coeff.Read(), 1, 1, 1) :
-                     Reshape(coeff.Read(), Q1D, Q1D, ne);
-      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, VDIM, DIM, ne);
-      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, ne);
-
-      mfem::forall_2D(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+      auto J = Reshape(geom->J.Read(), NQ, 2, 2, NE);
+      auto G = Reshape(pa_data.Write(), NQ, 2, 2, NE);
+      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
       {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         for (int q = 0; q < NQ; ++q)
          {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const real_t J11 = J(qx, qy, 0, 0, e), J12 = J(qx, qy, 0, 1, e);
-               const real_t J21 = J(qx, qy, 1, 0, e), J22 = J(qx, qy, 1, 1, e);
-               // adj(J)
-               const real_t A11 = +J22, A12 = -J12;
-               const real_t A21 = -J21, A22 = +J11;
-               // Store w * coeff * adj(J)
-               const real_t w = W(qx, qy);
-               const real_t c = const_coeff ? C(0, 0, 0) : C(qx, qy, e);
-               A(0, 0, qx, qy, e) = w * c * A11;
-               A(1, 0, qx, qy, e) = w * c * A12;
-               A(0, 1, qx, qy, e) = w * c * A21;
-               A(1, 1, qx, qy, e) = w * c * A22;
-            }
+            const real_t J11 = J(q, 0, 0, e);
+            const real_t J12 = J(q, 0, 1, e);
+            const real_t J21 = J(q, 1, 0, e);
+            const real_t J22 = J(q, 1, 1, e);
+            // Store wq * Q * adj(J)
+            G(q, 0, 0, e) = W[q] * COEFF * J22;  // 1,1
+            G(q, 0, 1, e) = W[q] * COEFF * -J12; // 1,2
+            G(q, 1, 0, e) = W[q] * COEFF * -J21; // 2,1
+            G(q, 1, 1, e) = W[q] * COEFF * J11;  // 2,2
          }
       });
    }
-   else if (dim == 3)
+   if (dim == 3)
    {
-      const int Q1D = q1d;
-      constexpr int VDIM = 3, DIM = 3;
-      const auto W = Reshape(w_r, Q1D, Q1D, Q1D);
-      const auto C = const_coeff ?
-                     Reshape(coeff.Read(), 1, 1, 1, 1) :
-                     Reshape(coeff.Read(), Q1D, Q1D, Q1D, ne);
-      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, Q1D, VDIM, DIM, ne);
-      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, Q1D, ne);
-
-      mfem::forall_3D(ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+      auto J = Reshape(geom->J.Read(), NQ, 3, 3, NE);
+      auto G = Reshape(pa_data.Write(), NQ, 3, 3, NE);
+      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
       {
-         MFEM_FOREACH_THREAD_DIRECT(qz, z, Q1D)
+         for (int q = 0; q < NQ; ++q)
          {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            const real_t J11 = J(q, 0, 0, e);
+            const real_t J21 = J(q, 1, 0, e);
+            const real_t J31 = J(q, 2, 0, e);
+            const real_t J12 = J(q, 0, 1, e);
+            const real_t J22 = J(q, 1, 1, e);
+            const real_t J32 = J(q, 2, 1, e);
+            const real_t J13 = J(q, 0, 2, e);
+            const real_t J23 = J(q, 1, 2, e);
+            const real_t J33 = J(q, 2, 2, e);
+            const real_t cw = W[q] * COEFF;
+            // adj(J)
+            const real_t A11 = (J22 * J33) - (J23 * J32);
+            const real_t A12 = (J32 * J13) - (J12 * J33);
+            const real_t A13 = (J12 * J23) - (J22 * J13);
+            const real_t A21 = (J31 * J23) - (J21 * J33);
+            const real_t A22 = (J11 * J33) - (J13 * J31);
+            const real_t A23 = (J21 * J13) - (J11 * J23);
+            const real_t A31 = (J21 * J32) - (J31 * J22);
+            const real_t A32 = (J31 * J12) - (J11 * J32);
+            const real_t A33 = (J11 * J22) - (J12 * J21);
+            // Store wq * Q * adj(J)
+            G(q, 0, 0, e) = cw * A11; // 1,1
+            G(q, 0, 1, e) = cw * A12; // 1,2
+            G(q, 0, 2, e) = cw * A13; // 1,3
+            G(q, 1, 0, e) = cw * A21; // 2,1
+            G(q, 1, 1, e) = cw * A22; // 2,2
+            G(q, 1, 2, e) = cw * A23; // 2,3
+            G(q, 2, 0, e) = cw * A31; // 3,1
+            G(q, 2, 1, e) = cw * A32; // 3,2
+            G(q, 2, 2, e) = cw * A33; // 3,3
+         }
+      });
+   }
+}
+
+// PA Convection NL 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAConvectionNLApply2D(const int NE,
+                                  const Array<real_t> &b,
+                                  const Array<real_t> &g,
+                                  const Array<real_t> &bt,
+                                  const Vector &q_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Q = Reshape(q_.Read(), Q1D * Q1D, 2, 2, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, 2, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t data[max_Q1D][max_Q1D][2];
+      real_t grad0[max_Q1D][max_Q1D][2];
+      real_t grad1[max_Q1D][max_Q1D][2];
+      real_t Z[max_Q1D][max_Q1D][2];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            data[qy][qx][0] = 0.0;
+            data[qy][qx][1] = 0.0;
+            grad0[qy][qx][0] = 0.0;
+            grad0[qy][qx][1] = 0.0;
+            grad1[qy][qx][0] = 0.0;
+            grad1[qy][qx][1] = 0.0;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         real_t dataX[max_Q1D][2];
+         real_t gradX0[max_Q1D][2];
+         real_t gradX1[max_Q1D][2];
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            dataX[qx][0] = 0.0;
+            dataX[qx][1] = 0.0;
+            gradX0[qx][0] = 0.0;
+            gradX0[qx][1] = 0.0;
+            gradX1[qx][0] = 0.0;
+            gradX1[qx][1] = 0.0;
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const real_t s0 = x(dx, dy, 0, e);
+            const real_t s1 = x(dx, dy, 1, e);
+            for (int qx = 0; qx < Q1D; ++qx)
             {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               const real_t Bx = B(qx, dx);
+               const real_t Gx = G(qx, dx);
+               dataX[qx][0] += s0 * Bx;
+               dataX[qx][1] += s1 * Bx;
+               gradX0[qx][0] += s0 * Gx;
+               gradX0[qx][1] += s0 * Bx;
+               gradX1[qx][0] += s1 * Gx;
+               gradX1[qx][1] += s1 * Bx;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const real_t By = B(qy, dy);
+            const real_t Gy = G(qy, dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               data[qy][qx][0] += dataX[qx][0] * By;
+               data[qy][qx][1] += dataX[qx][1] * By;
+               grad0[qy][qx][0] += gradX0[qx][0] * By;
+               grad0[qy][qx][1] += gradX0[qx][1] * Gy;
+               grad1[qy][qx][0] += gradX1[qx][0] * By;
+               grad1[qy][qx][1] += gradX1[qx][1] * Gy;
+            }
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const int q = qx + qy * Q1D;
+            const real_t u1 = data[qy][qx][0];
+            const real_t u2 = data[qy][qx][1];
+            const real_t grad00 = grad0[qy][qx][0];
+            const real_t grad01 = grad0[qy][qx][1];
+            const real_t grad10 = grad1[qy][qx][0];
+            const real_t grad11 = grad1[qy][qx][1];
+            const real_t Dxu1 = grad00 * Q(q, 0, 0, e) + grad01 * Q(q, 1, 0, e);
+            const real_t Dyu1 = grad00 * Q(q, 0, 1, e) + grad01 * Q(q, 1, 1, e);
+            const real_t Dxu2 = grad10 * Q(q, 0, 0, e) + grad11 * Q(q, 1, 0, e);
+            const real_t Dyu2 = grad10 * Q(q, 0, 1, e) + grad11 * Q(q, 1, 1, e);
+            Z[qy][qx][0] = u1 * Dxu1 + u2 * Dyu1;
+            Z[qy][qx][1] = u1 * Dxu2 + u2 * Dyu2;
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         real_t Y[max_D1D][2];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            Y[dx][0] = 0.0;
+            Y[dx][1] = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const real_t Btx = Bt(dx, qx);
+               Y[dx][0] += Btx * Z[qy][qx][0];
+               Y[dx][1] += Btx * Z[qy][qx][1];
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const real_t Bty = Bt(dy, qy);
+               y(dx, dy, 0, e) += Bty * Y[dx][0];
+               y(dx, dy, 1, e) += Bty * Y[dx][1];
+            }
+         }
+      }
+   });
+}
+
+// PA Convection NL 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAConvectionNLApply3D(const int NE,
+                                  const Array<real_t> &b,
+                                  const Array<real_t> &g,
+                                  const Array<real_t> &bt,
+                                  const Vector &q_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Q = Reshape(q_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int VDIM = 3;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t data[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad0[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad1[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad2[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t Z[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               data[qz][qy][qx][0] = 0.0;
+               data[qz][qy][qx][1] = 0.0;
+               data[qz][qy][qx][2] = 0.0;
+
+               grad0[qz][qy][qx][0] = 0.0;
+               grad0[qz][qy][qx][1] = 0.0;
+               grad0[qz][qy][qx][2] = 0.0;
+
+               grad1[qz][qy][qx][0] = 0.0;
+               grad1[qz][qy][qx][1] = 0.0;
+               grad1[qz][qy][qx][2] = 0.0;
+
+               grad2[qz][qy][qx][0] = 0.0;
+               grad2[qz][qy][qx][1] = 0.0;
+               grad2[qz][qy][qx][2] = 0.0;
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         real_t dataXY[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY0[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY1[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY2[max_Q1D][max_Q1D][VDIM];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dataXY[qy][qx][0] = 0.0;
+               dataXY[qy][qx][1] = 0.0;
+               dataXY[qy][qx][2] = 0.0;
+
+               gradXY0[qy][qx][0] = 0.0;
+               gradXY0[qy][qx][1] = 0.0;
+               gradXY0[qy][qx][2] = 0.0;
+
+               gradXY1[qy][qx][0] = 0.0;
+               gradXY1[qy][qx][1] = 0.0;
+               gradXY1[qy][qx][2] = 0.0;
+
+               gradXY2[qy][qx][0] = 0.0;
+               gradXY2[qy][qx][1] = 0.0;
+               gradXY2[qy][qx][2] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            real_t dataX[max_Q1D][VDIM];
+            real_t gradX0[max_Q1D][VDIM];
+            real_t gradX1[max_Q1D][VDIM];
+            real_t gradX2[max_Q1D][VDIM];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dataX[qx][0] = 0.0;
+               dataX[qx][1] = 0.0;
+               dataX[qx][2] = 0.0;
+
+               gradX0[qx][0] = 0.0;
+               gradX0[qx][1] = 0.0;
+               gradX0[qx][2] = 0.0;
+
+               gradX1[qx][0] = 0.0;
+               gradX1[qx][1] = 0.0;
+               gradX1[qx][2] = 0.0;
+
+               gradX2[qx][0] = 0.0;
+               gradX2[qx][1] = 0.0;
+               gradX2[qx][2] = 0.0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const real_t s0 = x(dx, dy, dz, 0, e);
+               const real_t s1 = x(dx, dy, dz, 1, e);
+               const real_t s2 = x(dx, dy, dz, 2, e);
+               for (int qx = 0; qx < Q1D; ++qx)
                {
-                  const real_t J11 = J(qx, qy, qz, 0, 0, e),
-                               J12 = J(qx, qy, qz, 0, 1, e),
-                               J13 = J(qx, qy, qz, 0, 2, e);
-                  const real_t J21 = J(qx, qy, qz, 1, 0, e),
-                               J22 = J(qx, qy, qz, 1, 1, e),
-                               J23 = J(qx, qy, qz, 1, 2, e);
-                  const real_t J31 = J(qx, qy, qz, 2, 0, e),
-                               J32 = J(qx, qy, qz, 2, 1, e),
-                               J33 = J(qx, qy, qz, 2, 2, e);
-                  const real_t c =
-                     const_coeff ? C(0, 0, 0, 0) : C(qx, qy, qz, e);
-                  const real_t cw = W(qx, qy, qz) * c;
-                  // adj(J)
-                  const real_t A11 = (J22 * J33) - (J23 * J32);
-                  const real_t A12 = (J32 * J13) - (J12 * J33);
-                  const real_t A13 = (J12 * J23) - (J22 * J13);
-                  const real_t A21 = (J31 * J23) - (J21 * J33);
-                  const real_t A22 = (J11 * J33) - (J13 * J31);
-                  const real_t A23 = (J21 * J13) - (J11 * J23);
-                  const real_t A31 = (J21 * J32) - (J31 * J22);
-                  const real_t A32 = (J31 * J12) - (J11 * J32);
-                  const real_t A33 = (J11 * J22) - (J12 * J21);
-                  // Store wq * coeff * adj(J)
-                  A(0, 0, qx, qy, qz, e) = cw * A11;
-                  A(1, 0, qx, qy, qz, e) = cw * A12;
-                  A(2, 0, qx, qy, qz, e) = cw * A13;
-                  A(0, 1, qx, qy, qz, e) = cw * A21;
-                  A(1, 1, qx, qy, qz, e) = cw * A22;
-                  A(2, 1, qx, qy, qz, e) = cw * A23;
-                  A(0, 2, qx, qy, qz, e) = cw * A31;
-                  A(1, 2, qx, qy, qz, e) = cw * A32;
-                  A(2, 2, qx, qy, qz, e) = cw * A33;
+                  const real_t Bx = B(qx, dx);
+                  const real_t Gx = G(qx, dx);
+
+                  dataX[qx][0] += s0 * Bx;
+                  dataX[qx][1] += s1 * Bx;
+                  dataX[qx][2] += s2 * Bx;
+
+                  gradX0[qx][0] += s0 * Gx;
+                  gradX0[qx][1] += s0 * Bx;
+                  gradX0[qx][2] += s0 * Bx;
+
+                  gradX1[qx][0] += s1 * Gx;
+                  gradX1[qx][1] += s1 * Bx;
+                  gradX1[qx][2] += s1 * Bx;
+
+                  gradX2[qx][0] += s2 * Gx;
+                  gradX2[qx][1] += s2 * Bx;
+                  gradX2[qx][2] += s2 * Bx;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const real_t By = B(qy, dy);
+               const real_t Gy = G(qy, dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  dataXY[qy][qx][0] += dataX[qx][0] * By;
+                  dataXY[qy][qx][1] += dataX[qx][1] * By;
+                  dataXY[qy][qx][2] += dataX[qx][2] * By;
+
+                  gradXY0[qy][qx][0] += gradX0[qx][0] * By;
+                  gradXY0[qy][qx][1] += gradX0[qx][1] * Gy;
+                  gradXY0[qy][qx][2] += gradX0[qx][2] * By;
+
+                  gradXY1[qy][qx][0] += gradX1[qx][0] * By;
+                  gradXY1[qy][qx][1] += gradX1[qx][1] * Gy;
+                  gradXY1[qy][qx][2] += gradX1[qx][2] * By;
+
+                  gradXY2[qy][qx][0] += gradX2[qx][0] * By;
+                  gradXY2[qy][qx][1] += gradX2[qx][1] * Gy;
+                  gradXY2[qy][qx][2] += gradX2[qx][2] * By;
                }
             }
          }
-      });
-   }
-   else
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            const real_t Bz = B(qz, dz);
+            const real_t Gz = G(qz, dz);
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  data[qz][qy][qx][0] += dataXY[qy][qx][0] * Bz;
+                  data[qz][qy][qx][1] += dataXY[qy][qx][1] * Bz;
+                  data[qz][qy][qx][2] += dataXY[qy][qx][2] * Bz;
+
+                  grad0[qz][qy][qx][0] += gradXY0[qy][qx][0] * Bz;
+                  grad0[qz][qy][qx][1] += gradXY0[qy][qx][1] * Bz;
+                  grad0[qz][qy][qx][2] += gradXY0[qy][qx][2] * Gz;
+
+                  grad1[qz][qy][qx][0] += gradXY1[qy][qx][0] * Bz;
+                  grad1[qz][qy][qx][1] += gradXY1[qy][qx][1] * Bz;
+                  grad1[qz][qy][qx][2] += gradXY1[qy][qx][2] * Gz;
+
+                  grad2[qz][qy][qx][0] += gradXY2[qy][qx][0] * Bz;
+                  grad2[qz][qy][qx][1] += gradXY2[qy][qx][1] * Bz;
+                  grad2[qz][qy][qx][2] += gradXY2[qy][qx][2] * Gz;
+               }
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const int q = qx + Q1D * (qy + qz * Q1D);
+
+               const real_t u1 = data[qz][qy][qx][0];
+               const real_t u2 = data[qz][qy][qx][1];
+               const real_t u3 = data[qz][qy][qx][2];
+
+               const real_t grad00 = grad0[qz][qy][qx][0];
+               const real_t grad01 = grad0[qz][qy][qx][1];
+               const real_t grad02 = grad0[qz][qy][qx][2];
+
+               const real_t grad10 = grad1[qz][qy][qx][0];
+               const real_t grad11 = grad1[qz][qy][qx][1];
+               const real_t grad12 = grad1[qz][qy][qx][2];
+
+               const real_t grad20 = grad2[qz][qy][qx][0];
+               const real_t grad21 = grad2[qz][qy][qx][1];
+               const real_t grad22 = grad2[qz][qy][qx][2];
+
+               const real_t Dxu1 = grad00 * Q(q, 0, 0, e)
+                                   + grad01 * Q(q, 1, 0, e)
+                                   + grad02 * Q(q, 2, 0, e);
+               const real_t Dyu1 = grad00 * Q(q, 0, 1, e)
+                                   + grad01 * Q(q, 1, 1, e)
+                                   + grad02 * Q(q, 2, 1, e);
+               const real_t Dzu1 = grad00 * Q(q, 0, 2, e)
+                                   + grad01 * Q(q, 1, 2, e)
+                                   + grad02 * Q(q, 2, 2, e);
+
+               const real_t Dxu2 = grad10 * Q(q, 0, 0, e)
+                                   + grad11 * Q(q, 1, 0, e)
+                                   + grad12 * Q(q, 2, 0, e);
+               const real_t Dyu2 = grad10 * Q(q, 0, 1, e)
+                                   + grad11 * Q(q, 1, 1, e)
+                                   + grad12 * Q(q, 2, 1, e);
+               const real_t Dzu2 = grad10 * Q(q, 0, 2, e)
+                                   + grad11 * Q(q, 1, 2, e)
+                                   + grad12 * Q(q, 2, 2, e);
+
+               const real_t Dxu3 = grad20 * Q(q, 0, 0, e)
+                                   + grad21 * Q(q, 1, 0, e)
+                                   + grad22 * Q(q, 2, 0, e);
+               const real_t Dyu3 = grad20 * Q(q, 0, 1, e)
+                                   + grad21 * Q(q, 1, 1, e)
+                                   + grad22 * Q(q, 2, 1, e);
+               const real_t Dzu3 = grad20 * Q(q, 0, 2, e)
+                                   + grad21 * Q(q, 1, 2, e)
+                                   + grad22 * Q(q, 2, 2, e);
+
+               Z[qz][qy][qx][0] = u1 * Dxu1 + u2 * Dyu1 + u3 * Dzu1;
+               Z[qz][qy][qx][1] = u1 * Dxu2 + u2 * Dyu2 + u3 * Dzu2;
+               Z[qz][qy][qx][2] = u1 * Dxu3 + u2 * Dyu3 + u3 * Dzu3;
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         real_t opXY[max_D1D][max_D1D][VDIM];
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               opXY[dy][dx][0] = 0.0;
+               opXY[dy][dx][1] = 0.0;
+               opXY[dy][dx][2] = 0.0;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            real_t opX[max_D1D][VDIM];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               opX[dx][0] = 0.0;
+               opX[dx][1] = 0.0;
+               opX[dx][2] = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const real_t Btx = Bt(dx, qx);
+                  opX[dx][0] += Btx * Z[qz][qy][qx][0];
+                  opX[dx][1] += Btx * Z[qz][qy][qx][1];
+                  opX[dx][2] += Btx * Z[qz][qy][qx][2];
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t Bty = Bt(dy, qy);
+                  opXY[dy][dx][0] += Bty * opX[dx][0];
+                  opXY[dy][dx][1] += Bty * opX[dx][1];
+                  opXY[dy][dx][2] += Bty * opX[dx][2];
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t Btz = Bt(dz, qz);
+                  y(dx, dy, dz, 0, e) += Btz * opXY[dy][dx][0];
+                  y(dx, dy, dz, 1, e) += Btz * opXY[dy][dx][1];
+                  y(dx, dy, dz, 2, e) += Btz * opXY[dy][dx][2];
+               }
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, int T_MAX_D1D = 0, int T_MAX_Q1D = 0>
+static void SmemPAConvectionNLApply3D(const int NE,
+                                      const Array<real_t> &b_,
+                                      const Array<real_t> &g_,
+                                      const Vector &d_,
+                                      const Vector &x_,
+                                      Vector &y_,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto g = Reshape(g_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-      MFEM_ABORT("dim " << dim << " not supported!");
-   }
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
+      MFEM_SHARED real_t BG[2][MQ1 * MD1];
+      real_t(*B)[MD1] = (real_t(*)[MD1])(BG + 0);
+      real_t(*G)[MD1] = (real_t(*)[MD1])(BG + 1);
+      real_t(*Bt)[MQ1] = (real_t(*)[MQ1])(BG + 0);
+      MFEM_SHARED real_t U[2][MQ1][MQ1][MQ1];
+      MFEM_SHARED real_t sm0[3][MQ1 * MQ1 * MQ1];
+      MFEM_SHARED real_t sm1[3][MQ1 * MQ1 * MQ1];
+      real_t(*DDQ0)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 0);
+      real_t(*DDQ1)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 1);
+      real_t(*X)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 2);
+      real_t(*DQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 0);
+      real_t(*DQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 1);
+      real_t(*DQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 2);
+      real_t(*QQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 0);
+      real_t(*QQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 1);
+      real_t(*QQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 2);
+      real_t(*QQD0)[MQ1][MD1] = (real_t(*)[MQ1][MD1])(sm1 + 0);
+      real_t(*QDD0)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 0);
+      MFEM_SHARED real_t Z[MQ1][MQ1][MQ1];
+
+      for (int cy = 0; cy < VDIM; ++cy)
+      {
+         if (tidz == 0)
+         {
+            MFEM_FOREACH_THREAD(q, x, Q1D)
+            {
+               MFEM_FOREACH_THREAD(d, y, D1D)
+               {
+                  B[q][d] = b(q, d);
+                  G[q][d] = g(q, d);
+               }
+            }
+         }
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx, x, Q1D) { Z[qz][qy][qx] = 0.0; }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         for (int c = 0; c < VDIM; ++c)
+         {
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(dy, y, D1D)
+               {
+                  MFEM_FOREACH_THREAD(dx, x, D1D)
+                  {
+                     X[dz][dy][dx] = x(dx, dy, dz, cy, e);
+                     U[0][dz][dy][dx] = x(dx, dy, dz, c, e);
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(dy, y, D1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t z = 0.0;
+                     for (int dx = 0; dx < D1D; ++dx)
+                     {
+                        const real_t coord = X[dz][dy][dx];
+                        const real_t value = U[0][dz][dy][dx];
+                        u += coord * B[qx][dx];
+                        v += coord * G[qx][dx];
+                        z += value * B[qx][dx];
+                     }
+                     DDQ0[dz][dy][qx] = u;
+                     DDQ1[dz][dy][qx] = v;
+                     U[1][dz][dy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t w = 0.0;
+                     real_t z = 0.0;
+                     for (int dy = 0; dy < D1D; ++dy)
+                     {
+                        u += DDQ1[dz][dy][qx] * B[qy][dy];
+                        v += DDQ0[dz][dy][qx] * G[qy][dy];
+                        w += DDQ0[dz][dy][qx] * B[qy][dy];
+                        z += U[1][dz][dy][qx] * B[qy][dy];
+                     }
+                     DQQ0[dz][qy][qx] = u;
+                     DQQ1[dz][qy][qx] = v;
+                     DQQ2[dz][qy][qx] = w;
+                     U[0][dz][qy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(qz, z, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t w = 0.0;
+                     real_t z = 0.0;
+                     for (int dz = 0; dz < D1D; ++dz)
+                     {
+                        u += DQQ0[dz][qy][qx] * B[qz][dz];
+                        v += DQQ1[dz][qy][qx] * B[qz][dz];
+                        w += DQQ2[dz][qy][qx] * G[qz][dz];
+                        z += U[0][dz][qy][qx] * B[qz][dz];
+                     }
+                     QQQ0[qz][qy][qx] = u;
+                     QQQ1[qz][qy][qx] = v;
+                     QQQ2[qz][qy][qx] = w;
+                     U[1][qz][qy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(qz, z, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     const int q = qx + (qy + qz * Q1D) * Q1D;
+                     const real_t z = U[1][qz][qy][qx];
+                     const real_t gX = QQQ0[qz][qy][qx];
+                     const real_t gY = QQQ1[qz][qy][qx];
+                     const real_t gZ = QQQ2[qz][qy][qx];
+                     const real_t d = gX * D(q, 0, c, e) + gY * D(q, 1, c, e)
+                                      + gZ * D(q, 2, c, e);
+                     Z[qz][qy][qx] += z * d;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+         } // for each conv component
+         if (tidz == 0)
+         {
+            MFEM_FOREACH_THREAD(d, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(q, x, Q1D) { Bt[d][q] = b(q, d); }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += Z[qz][qy][qx] * Bt[dx][qx];
+                  }
+                  QQD0[qz][qy][dx] = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += QQD0[qz][qy][dx] * Bt[dy][qy];
+                  }
+                  QDD0[qz][dy][dx] = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz, z, D1D)
+         {
+            MFEM_FOREACH_THREAD(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += QDD0[qz][dy][dx] * Bt[dz][qz];
+                  }
+                  Y(dx, dy, dz, cy, e) += u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
 }
 
 void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
@@ -197,13 +812,26 @@ void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
    else
    {
-      AddMultPAKernels::Run(dim, d1d, q1d, ne,
-                            maps->B.Read(),
-                            maps->G.Read(),
-                            pa_adj.Read(),
-                            x.Read(),
-                            y.ReadWrite(),
-                            d1d, q1d);
+      const int NE = ne;
+      const int D1D = maps->ndof;
+      const int Q1D = maps->nqpt;
+      const Vector &QV = pa_data;
+      const Array<real_t> &B = maps->B;
+      const Array<real_t> &G = maps->G;
+      const Array<real_t> &Bt = maps->Bt;
+      if (dim == 2)
+      {
+         return PAConvectionNLApply2D(NE, B, G, Bt, QV, x, y, D1D, Q1D);
+      }
+      if (dim == 3)
+      {
+         constexpr int T_MAX_D1D = 8;
+         constexpr int T_MAX_Q1D = 8;
+         MFEM_VERIFY(D1D <= T_MAX_D1D && Q1D <= T_MAX_Q1D, "Not yet implemented!");
+         return SmemPAConvectionNLApply3D<0, 0, T_MAX_D1D, T_MAX_Q1D>
+                (NE, B, G, QV, x, y, D1D, Q1D);
+      }
+      MFEM_ABORT("Not yet implemented!");
    }
 }
 

fem/integ/nonlininteg_vecconvection_pa.hpp

@@ -1,209 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-// PA Convection NL 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLApply2D(const int NE,
-                                      const real_t *b,
-                                      const real_t *g,
-                                      const real_t *a,
-                                      const real_t *x,
-                                      real_t *y,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
-   const auto X = Reshape(x, D1D, D1D, VDIM, NE);
-   auto Y = Reshape(y, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
-      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::v_regs2d_t<VDIM, MQ1> s0, s1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, X, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
-      kernels::internal::LoadDofs2d(e, D1D, X, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            const future::tensor<real_t, 2> U =
-            {
-               r1[0][qy][qx], r1[1][qy][qx]
-            };
-            const future::tensor<real_t, 2,2> gradU = {{
-                  {g1[0][0][qy][qx], g1[1][0][qy][qx]},
-                  {g1[0][1][qy][qx], g1[1][1][qy][qx]},
-               }
-            };
-            const future::tensor<real_t, 2,2> Q = {{
-                  {A(0,0,qx,qy,e), A(1,0,qx,qy,e)},
-                  {A(0,1,qx,qy,e), A(1,1,qx,qy,e)},
-               }
-            };
-            const future::tensor<real_t, 2> conv = transpose(gradU) * (Q * U);
-            s0[0][qy][qx] = conv[0];
-            s0[1][qy][qx] = conv[1];
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, s0, s1);
-      kernels::internal::WriteDofs2d(e, D1D, s1, Y);
-   });
-}
-
-// PA Convection NL 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLApply3D(const int NE,
-                                      const real_t *b,
-                                      const real_t *g,
-                                      const real_t *a,
-                                      const real_t *x,
-                                      real_t *y,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto X = Reshape(x, D1D, D1D, D1D, VDIM, NE);
-   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::v_regs3d_t<VDIM, MQ1> s0, s1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, X, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
-      kernels::internal::LoadDofs3d(e, D1D, X, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const future::tensor<real_t, 3> U =
-               {
-                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, 3,3> gradU = {{
-                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                  }
-               };
-               const future::tensor<real_t, 3,3> Q = {{
-                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                  }
-               };
-               const future::tensor<real_t, 3> conv = transpose(gradU) * (Q * U);
-               s0[0][qz][qy][qx] = conv[0];
-               s0[1][qz][qy][qx] = conv[1];
-               s0[2][qz][qy][qx] = conv[2];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, s0, s1);
-      kernels::internal::WriteDofs3d(e, D1D, s1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int DIM, int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultPAType
-VectorConvectionNLFIntegrator::AddMultPAKernels::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPAConvectionNLApply2D<T_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPAConvectionNLApply3D<T_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorConvectionNLFIntegrator::AddMultPAType
-VectorConvectionNLFIntegrator::AddMultPAKernels::Fallback
-(int dim, int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPAConvectionNLApply2D<>;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPAConvectionNLApply3D<>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.cpp

@@ -1,50 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../ceed/interface/util.hpp"
-#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
-
-namespace mfem
-{
-
-void VectorConvectionNLFIntegrator::AssembleGradDiagonalPA(Vector &de) const
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   if (dim == 2)
-   {
-      GradDiagPA2D::Run(d1d, q1d, ne,
-                        maps->B.Read(),
-                        maps->G.Read(),
-                        pa_adj.Read(),
-                        pa_u.Read(),
-                        de.ReadWrite(),
-                        d1d, q1d);
-   }
-   else if (dim == 3)
-   {
-      GradDiagPA3D::Run(d1d, q1d, ne,
-                        maps->B.Read(),
-                        maps->G.Read(),
-                        pa_adj.Read(),
-                        pa_u.Read(),
-                        de.ReadWrite(),
-                        d1d, q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-   }
-}
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.hpp

@@ -1,302 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradDiagonal2D(const int NE,
-                                             const real_t *b,
-                                             const real_t *g,
-                                             const real_t *a,
-                                             const real_t *u,
-                                             real_t *de,
-                                             const int d1d,
-                                             const int q1d)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
-   const auto U = Reshape(u, D1D, D1D, VDIM, NE);
-   auto D = Reshape(de, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t sM[3][MQ1][MQ1], sQ[3][MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, U, r0);
-      kernels::internal::Eval2d(D1D, Q1D, sM[0], sB, r0, r1);
-
-      kernels::internal::LoadDofs2d(e, D1D, U, g0);
-      kernels::internal::Grad2d(D1D, Q1D, sM[0], sB, sG, g0, g1);
-
-      for (int v = 0; v < VDIM; ++v)
-      {
-         future::tensor<real_t, VDIM> e_v = {};
-         e_v[v] = real_t(1);
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               const future::tensor<real_t, VDIM> u_val =
-               {
-                  r1[0][qy][qx], r1[1][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> Q_adj =
-               {
-                  {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
-                     { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
-                  }
-               };
-               const future::tensor<real_t, VDIM, DIM> grad_U =
-               {
-                  {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
-                     { g1[0][1][qy][qx], g1[1][1][qy][qx] }
-                  }
-               };
-               const auto one = Q_adj * u_val;
-               const auto two = transpose(grad_U) * (Q_adj * e_v);
-               sQ[0][qx][qy] = one[0];
-               sQ[1][qx][qy] = one[1];
-               sQ[2][qx][qy] = two[v];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               real_t s[3] = {};
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const real_t By = sB[dy][qy], Gy = sG[dy][qy];
-                  s[0] += By * By * sQ[0][qx][qy];
-                  s[1] += Gy * By * sQ[1][qx][qy];
-                  s[2] += By * By * sQ[2][qx][qy];
-               }
-               sM[0][qx][dy] = s[0];
-               sM[1][qx][dy] = s[1];
-               sM[2][qx][dy] = s[2];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-            {
-               real_t d = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
-                  d += Gx * Bx * sM[0][qx][dy] +
-                       Bx * Bx * sM[1][qx][dy] +
-                       Bx * Bx * sM[2][qx][dy];
-               }
-               D(dx, dy, v, e) += d;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradDiagonal3D(const int NE,
-                                             const real_t *b,
-                                             const real_t *g,
-                                             const real_t *a,
-                                             const real_t *u,
-                                             real_t *de,
-                                             const int d1d,
-                                             const int q1d)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, NE);
-   auto D = Reshape(de, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t sM[4][MQ1][MQ1], sQ[4][MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, U, r0);
-      kernels::internal::Eval3d(D1D, Q1D, sM[0], sB, r0, r1);
-
-      kernels::internal::LoadDofs3d(e, D1D, U, g0);
-      kernels::internal::Grad3d(D1D, Q1D, sM[0], sB, sG, g0, g1);
-
-      for (int v = 0; v < VDIM; ++v)
-      {
-         future::tensor<real_t, VDIM> e_v = {};
-         e_v[v] = real_t(1);
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  real_t s[4] = {};
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     const future::tensor<real_t, VDIM> u_val =
-                     {
-                        r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-                     };
-                     const future::tensor<real_t, VDIM, DIM> Q_adj = {{
-                           {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                           {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                           {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                        }
-                     };
-                     const future::tensor<real_t, VDIM, DIM> grad_U = {{
-                           {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                           {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                           {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                        }
-                     };
-                     const auto one = Q_adj * u_val;
-                     const auto two = transpose(grad_U) * (Q_adj * e_v);
-
-                     const real_t Bz = sB[dz][qz], Gz = sG[dz][qz];
-                     s[0] += one[0] * Bz * Bz;
-                     s[1] += one[1] * Bz * Bz;
-                     s[2] += one[2] * Bz * Gz;
-                     s[3] += two[v] * Bz * Bz;
-                  }
-                  sQ[0][qx][qy] = s[0];
-                  sQ[1][qx][qy] = s[1];
-                  sQ[2][qx][qy] = s[2];
-                  sQ[3][qx][qy] = s[3];
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  real_t s[4] = {};
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     const real_t By = sB[dy][qy], Gy = sG[dy][qy];
-                     s[0] += By * By * sQ[0][qx][qy];
-                     s[1] += Gy * By * sQ[1][qx][qy];
-                     s[2] += By * By * sQ[2][qx][qy];
-                     s[3] += By * By * sQ[3][qx][qy];
-                  }
-                  sM[0][dy][qx] = s[0];
-                  sM[1][dy][qx] = s[1];
-                  sM[2][dy][qx] = s[2];
-                  sM[3][dy][qx] = s[3];
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-               {
-                  real_t d = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
-                     d += Gx * Bx * sM[0][dy][qx];
-                     d += Bx * Bx * sM[1][dy][qx];
-                     d += Bx * Bx * sM[2][dy][qx];
-                     d += Bx * Bx * sM[3][dy][qx];
-                  }
-                  D(dx, dy, dz, v, e) += d;
-               }
-            }
-            MFEM_SYNC_THREAD;
-         }
-      }
-   });
-}
-
-} // namespace internal
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA2D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradDiagonal2D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA2D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradDiagonal2D<>;
-}
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA3D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradDiagonal3D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA3D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradDiagonal3D<>;
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.cpp

@@ -1,64 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../ceed/interface/util.hpp"
-#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
-
-namespace mfem
-{
-
-void VectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &u, const FiniteElementSpace &fes)
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   this->pa_u = u;
-   AssemblePA(fes);
-}
-
-void VectorConvectionNLFIntegrator::AddMultGradPA(const Vector &x,
-                                                  Vector &y) const
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   if (dim == 2)
-   {
-      AddMultGradPA2D::Run(d1d, q1d, ne,
-                           maps->B.Read(),
-                           maps->G.Read(),
-                           pa_adj.Read(),
-                           pa_u.Read(),
-                           x.Read(),
-                           y.ReadWrite(),
-                           d1d, q1d);
-   }
-   else if (dim == 3)
-   {
-      AddMultGradPA3D::Run(d1d, q1d, ne,
-                           maps->B.Read(),
-                           maps->G.Read(),
-                           pa_adj.Read(),
-                           pa_u.Read(),
-                           x.Read(),
-                           y.ReadWrite(),
-                           d1d, q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-   }
-}
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.hpp

@@ -1,257 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradApply2D(const int ne,
-                                          const real_t *b,
-                                          const real_t *g,
-                                          const real_t *a,
-                                          const real_t *u,
-                                          const real_t *du,
-                                          real_t *y,
-                                          const int d1d,
-                                          const int q1d)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, ne);
-   const auto U = Reshape(u, D1D, D1D, VDIM, ne);
-   const auto dU = Reshape(du, D1D, D1D, VDIM, ne);
-   auto Y = Reshape(y, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1, g2;
-      kernels::internal::v_regs2d_t<DIM, MQ1> r0, r1, r2;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, dU, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
-
-      kernels::internal::LoadDofs2d(e, D1D, U, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r2); // u value
-
-      kernels::internal::LoadDofs2d(e, D1D, dU, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // δu value
-
-      kernels::internal::LoadDofs2d(e, D1D, U, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            // First part of the Jacobian: u·∇δu
-            const future::tensor<real_t, DIM> u_val =
-            {
-               r2[0][qy][qx], r2[1][qy][qx]
-            };
-            const future::tensor<real_t, VDIM, DIM> Q_adj =
-            {
-               {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
-                  { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
-               }
-            };
-            const future::tensor<real_t, VDIM, DIM> grad_dU =
-            {
-               {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
-                  { g1[0][1][qy][qx], g1[1][1][qy][qx] }
-               }
-            };
-            const auto one = transpose(grad_dU) * (Q_adj * u_val);
-
-            // Second part of the Jacobian: δu·∇u
-            const future::tensor<real_t, DIM> du_val =
-            {
-               r1[0][qy][qx], r1[1][qy][qx]
-            };
-            const future::tensor<real_t, VDIM, DIM> grad_U =
-            {
-               {  { g2[0][0][qy][qx], g2[1][0][qy][qx] },
-                  { g2[0][1][qy][qx], g2[1][1][qy][qx] }
-               }
-            };
-            const auto two = transpose(grad_U) * (Q_adj * du_val);
-
-            // u⋅∇δu + δu⋅∇u
-            r0[0][qy][qx] = one[0] + two[0];
-            r0[1][qy][qx] = one[1] + two[1];
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs2d(e, D1D, r1, Y);
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradApply3D(const int ne,
-                                          const real_t *b,
-                                          const real_t *g,
-                                          const real_t *a,
-                                          const real_t *u,
-                                          const real_t *du,
-                                          real_t *y,
-                                          const int d1d,
-                                          const int q1d)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, ne);
-   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, ne);
-   const auto dU = Reshape(du, D1D, D1D, D1D, VDIM, ne);
-   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1, r2;
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1, g2;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, dU, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
-
-      kernels::internal::LoadDofs3d(e, D1D, U, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r2); // u value
-
-      kernels::internal::LoadDofs3d(e, D1D, dU, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // δu value
-
-      kernels::internal::LoadDofs3d(e, D1D, U, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               // First part of the Jacobian: u·∇δu
-               const future::tensor<real_t, DIM> u_val =
-               {
-                  r2[0][qz][qy][qx],
-                  r2[1][qz][qy][qx],
-                  r2[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> Q_adj = {{
-                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                  }
-               };
-               const future::tensor<real_t, DIM, DIM> grad_dU = {{
-                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                  }
-               };
-               const auto one = transpose(grad_dU) * (Q_adj * u_val);
-
-               // Second part of the Jacobian: δu·∇u
-               const future::tensor<real_t, DIM> du_val =
-               {
-                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> grad_U = {{
-                     {g2[0][0][qz][qy][qx], g2[1][0][qz][qy][qx], g2[2][0][qz][qy][qx]},
-                     {g2[0][1][qz][qy][qx], g2[1][1][qz][qy][qx], g2[2][1][qz][qy][qx]},
-                     {g2[0][2][qz][qy][qx], g2[1][2][qz][qy][qx], g2[2][2][qz][qy][qx]}
-                  }
-               };
-               const auto two = transpose(grad_U) * (Q_adj * du_val);
-
-               // u⋅∇δu + δu⋅∇u
-               r0[0][qz][qy][qx] = one[0] + two[0];
-               r0[1][qz][qy][qx] = one[1] + two[1];
-               r0[2][qz][qy][qx] = one[2] + two[2];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs3d(e, D1D, r1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA2D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradApply2D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA2D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradApply2D<>;
-}
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA3D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradApply3D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA3D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradApply3D<>;
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/intrules.cpp

@@ -542,10 +542,7 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
          return;
    }
 
-#ifdef MFEM_USE_MPFR
-   MFEM_WARNING("MPFR implementation of Gauss-Jacobi quadrature not implemented yet. Falling "
-                "back to double precision implementation...");
-#endif
+#ifndef MFEM_USE_MPFR
 
    const int n = np;
    // common constants for Jacobi polynomials
@@ -614,6 +611,13 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
                                                 ab + 1) / ((1.0 - xi*xi)*pp*pp) / pow(2, ab);
       // map nodes and weights to the interval [0,1]
    }
+
+#else // MFEM_USE_MPFR is defined
+
+   MFEM_ABORT("MPFR implementation of Gauss-Jacobi quadrature not defined yet");
+
+#endif // MFEM_USE_MPFR
+
 }
 
 

fem/kernel_dispatch.hpp

@@ -61,7 +61,7 @@ namespace mfem
 #define MFEM_REGISTER_KERNELS_1(KernelName, KernelType, Params)                \
    MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, (), Params)
 
-// Version of MFEM_REGISTER_KERNELS with optional (non-dispatch)
+// Version of MFEM_REGISTER_KERNELS without any optional (non-dispatch)
 // parameters (e.g. NBZ).
 #define MFEM_REGISTER_KERNELS_2(KernelName, KernelType, Params, OptParams)     \
    MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, OptParams,           \

fem/kernels.hpp

@@ -83,7 +83,7 @@ constexpr int SetMaxOf(int n) { return NextMultipleOf<4>(n); }
 #endif // CUDA/HIP && DEVICE_COMPILE
 
 /// Load 2D matrix into shared memory
-template <int MQ1, bool TRANSPOSE = false>
+template <int MQ1>
 inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d,
                                         const real_t *M, real_t (*N)[MQ1])
 {
@@ -91,14 +91,7 @@ inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d,
    {
       MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
       {
-         if constexpr (TRANSPOSE)
-         {
-            N[dy][qx] = M[qx * d1d + dy];
-         }
-         else
-         {
-            N[dy][qx] = M[dy * q1d + qx];
-         }
+         N[dy][qx] = M[dy * q1d + qx];
       }
    }
    MFEM_SYNC_THREAD;

fem/lor/lor_ams.cpp

@@ -94,10 +94,10 @@ void BatchedLOR_AMS::Form2DEdgeToVertex_RT(Array<int> &edge2vert)
          const int iv0 = ix + iy*op1;
          const int iv1 = ix1 + iy1*op1;
 
-         // 2D curl (dy, -dx), so flip the sign for the second
-         // component (c == 1).
-         e2v(0, iedge) = (c == 0) ? iv0 : iv1;
-         e2v(1, iedge) = (c == 0) ? iv1 : iv0;
+         // Rotated gradient in 2D (-dy, dx), so flip the sign for the first
+         // component (c == 0).
+         e2v(0, iedge) = (c == 1) ? iv0 : iv1;
+         e2v(1, iedge) = (c == 1) ? iv1 : iv0;
       }
    }
 }

fem/lor/lor_batched.cpp

@@ -142,6 +142,8 @@ static MFEM_HOST_DEVICE int GetAndIncrementNnzIndex(const int i_L, int* I)
 
 int BatchedLORAssembly::FillI(SparseMatrix &A) const
 {
+   static constexpr int Max = 16;
+
    const int nvdof = fes_ho.GetVSize();
 
    const int ndof_per_el = fes_ho.GetTypicalFE()->GetDof();
@@ -163,8 +165,6 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
    const auto K = dof_glob2loc_offsets_.Read();
    const auto map = Reshape(sparse_mapping.Read(), nnz_per_row, ndof_per_el);
 
-   Array<int> ij_elts(dof_glob2loc_.Size() * 2);
-   auto d_ij_elts = Reshape(ij_elts.Write(), dof_glob2loc_.Size(), 2);
 
    auto I = A.WriteI();
 
@@ -176,10 +176,10 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
       const int sii = el_dof_lex(ii_el, iel_ho);
       const int ii = (sii >= 0) ? sii : -1 -sii;
       // Get number and list of elements containing this DOF
+      int i_elts[Max];
       const int i_offset = K[ii];
       const int i_next_offset = K[ii+1];
       const int i_ne = i_next_offset - i_offset;
-      int *i_elts = &d_ij_elts(i_offset, 0);
       for (int e_i = 0; e_i < i_ne; ++e_i)
       {
          const int si_E = dof_glob2loc[i_offset+e_i]; // signed
@@ -202,7 +202,7 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
          }
          else // assembly required
          {
-            int *j_elts = &d_ij_elts(j_offset, 1);
+            int j_elts[Max];
             for (int e_j = 0; e_j < j_ne; ++e_j)
             {
                const int sj_E = dof_glob2loc[j_offset+e_j]; // signed
@@ -269,8 +269,7 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
       mfem::forall(nvdof + 1, [=] MFEM_HOST_DEVICE (int i) { I[i] = I2[i]; });
    }
 
-   Array<int> ij_B_el(dof_glob2loc_.Size() * 4);
-   auto d_ij_B_el = Reshape(ij_B_el.Write(), dof_glob2loc_.Size(), 4);
+   static constexpr int Max = 16;
 
    mfem::forall(ndof_per_el*nel_ho, [=] MFEM_HOST_DEVICE (int i)
    {
@@ -280,13 +279,11 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
       const int sii = el_dof_lex(ii_el, iel_ho); // signed
       const int ii = (sii >= 0) ? sii : -1 - sii;
       // Get number and list of elements containing this DOF
+      int i_elts[Max];
+      int i_B[Max];
       const int i_offset = K[ii];
       const int i_next_offset = K[ii+1];
       const int i_ne = i_next_offset - i_offset;
-
-      int *i_elts = &d_ij_B_el(i_offset, 0);
-      int *i_B = &d_ij_B_el(i_offset, 1);
-
       for (int e_i = 0; e_i < i_ne; ++e_i)
       {
          const int si_E = dof_glob2loc[i_offset+e_i]; // signed
@@ -315,8 +312,8 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
          }
          else // assembly required
          {
-            int *j_elts = &d_ij_B_el(j_offset, 2);
-            int *j_B = &d_ij_B_el(j_offset, 3);
+            int j_elts[Max];
+            int j_B[Max];
             for (int e_j = 0; e_j < j_ne; ++e_j)
             {
                const int sj_E = dof_glob2loc[j_offset+e_j]; // signed

fem/nonlinearform.hpp

@@ -224,6 +224,9 @@ public:
    /** @see GetGradient(const Vector &) */
    Operator &GetGradient(const Vector &x, bool finalize) const;
 
+   /// Suppress a warning about hiding overloaded virtual function.
+   using Operator::GetGradient;
+
    /// Update the NonlinearForm to propagate updates of the associated FE space.
    /** After calling this method, the essential boundary conditions need to be
        set again. */

fem/nonlinearform_ext.cpp

@@ -100,17 +100,6 @@ PANonlinearFormExtension::Gradient::Gradient(const PANonlinearFormExtension &e):
 
 void PANonlinearFormExtension::Gradient::AssembleGrad(const Vector &g)
 {
-   if (DeviceCanUseCeed())
-   {
-      for (int i = 0; i < ext.dnfi.Size(); ++i)
-      {
-         MFEM_VERIFY(dynamic_cast<VectorConvectionNLFIntegrator *>
-                     (ext.dnfi[i]) == nullptr,
-                     "VectorConvectionNLFIntegrator PA gradients are not supported "
-                     "with the libCEED backend");
-      }
-   }
-
    ext.elemR->Mult(g, ext.xe);
    for (int i = 0; i < ext.dnfi.Size(); ++i)
    {

fem/nonlininteg.cpp

@@ -954,74 +954,4 @@ void SkewSymmetricVectorConvectionNLFIntegrator::AssembleElementGrad(
    }
 }
 
-void ConvectiveVectorConvectionNLFIntegrator::AssemblePA(
-   const FiniteElementSpace &)
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &, const FiniteElementSpace &)
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AddMultPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AddMultGradPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
-   Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssemblePA(
-   const FiniteElementSpace &)
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &, const FiniteElementSpace &)
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AddMultPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AddMultGradPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
-   Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
 }

fem/nonlininteg.hpp

@@ -18,7 +18,6 @@
 #include "fespace.hpp"
 #include "ceed/interface/operator.hpp"
 #include "integrator.hpp"
-#include "kernel_dispatch.hpp"
 
 namespace mfem
 {
@@ -385,17 +384,15 @@ private:
    DenseMatrix dshape, dshapex, EF, gradEF, ELV, elmat_comp;
    Vector shape;
    // PA extension
-   int dim, ne, nq, d1d, q1d;
-   Vector pa_adj, pa_u;
+   Vector pa_data;
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq;
 
 public:
-   struct Kernels { Kernels(); };
+   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { }
 
-   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { static Kernels kernels; }
-
-   VectorConvectionNLFIntegrator() { static Kernels kernels; }
+   VectorConvectionNLFIntegrator() = default;
 
    static const IntegrationRule &GetRule(const FiniteElement &fe,
                                          const ElementTransformation &T);
@@ -414,56 +411,13 @@ public:
 
    void AssemblePA(const FiniteElementSpace &fes) override;
 
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
+   void AssembleMF(const FiniteElementSpace &fes) override;
 
    void AddMultPA(const Vector &x, Vector &y) const override;
 
-   using AddMultPAType =
-      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-              const real_t *x, real_t *y,
-              const int d1d, const int q1d);
-   MFEM_REGISTER_KERNELS(AddMultPAKernels, AddMultPAType, (int, int, int));
-
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-
-   using AddMultGradPAType =
-      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-              const real_t *u, const real_t *x, real_t *y,
-              const int d1d, const int q1d);
-
-   MFEM_REGISTER_KERNELS(AddMultGradPA2D, AddMultGradPAType, (int, int));
-   MFEM_REGISTER_KERNELS(AddMultGradPA3D, AddMultGradPAType, (int, int));
-
-   void AssembleGradDiagonalPA(Vector &) const override;
-
-   using GradDiagPAType =
-      void (*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-               const real_t *u, real_t *y,
-               const int d1d, const int q1d);
-
-   MFEM_REGISTER_KERNELS(GradDiagPA2D, GradDiagPAType, (int, int));
-   MFEM_REGISTER_KERNELS(GradDiagPA3D, GradDiagPAType, (int, int));
-
-   template <int DIM, int D1D, int Q1D>
-   static void AddSpecialization()
-   {
-      AddMultPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      if constexpr (DIM == 2)
-      {
-         AddMultGradPA2D::Specialization<D1D, Q1D>::Add();
-         GradDiagPA2D::Specialization<D1D, Q1D>::Add();
-      }
-      else if constexpr (DIM == 3)
-      {
-         AddMultGradPA3D::Specialization<D1D, Q1D>::Add();
-         GradDiagPA3D::Specialization<D1D, Q1D>::Add();
-      }
-   }
-
-   void AssembleMF(const FiniteElementSpace &fes) override;
-
    void AddMultMF(const Vector &x, Vector &y) const override;
 
+
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
       const FiniteElement& trial_fe,
@@ -476,8 +430,7 @@ protected:
 
 
 /** This class is used to assemble the convective form of the nonlinear term
-    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$.
-    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
+    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$ */
 class ConvectiveVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -495,20 +448,12 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
-
-   using NonlinearFormIntegrator::AssemblePA;
-   void AssemblePA(const FiniteElementSpace &fes) override;
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
-   void AddMultPA(const Vector &x, Vector &y) const override;
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 
 /** This class is used to assemble the skew-symmetric form of the nonlinear term
     arising in the Navier-Stokes equations
-    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$.
-    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
+    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$ */
 class SkewSymmetricVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -526,13 +471,6 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
-
-   using NonlinearFormIntegrator::AssemblePA;
-   void AssemblePA(const FiniteElementSpace &fes) override;
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
-   void AddMultPA(const Vector &x, Vector &y) const override;
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 }

fem/pfespace.cpp

@@ -26,8 +26,6 @@
 
 #include <limits>
 #include <list>
-#include <unordered_map>
-#include <unordered_set>
 
 namespace mfem
 {
@@ -1287,342 +1285,6 @@ void ParFiniteElementSpace::GetExteriorVDofs(Array<int> &ext_dofs,
    Synchronize(ext_dofs);
 }
 
-void ParFiniteElementSpace::GetBoundaryLoopEdgeDofs(
-   const Array<int> &boundary_element_indices,
-   Array<int> &ess_tdof_list,
-   Array<int> &boundary_edge_dofs_out,
-   Array<int> *ldof_marker,
-   Array<int> *dof_edges,
-   Array<int> *dof_boundary_elements,
-   Array<int> *ess_edge_list)
-{
-   MFEM_VERIFY(!pmesh->Nonconforming(),
-               "GetBoundaryLoopEdgeDofs does not support nonconforming meshes");
-   MFEM_VERIFY(pmesh->Dimension() >= 2,
-               "GetBoundaryLoopEdgeDofs requires 2D or 3D meshes to find 1D edge objects");
-
-   // Call the serial version, then rebuild scratch maps/set from the returned
-   // arrays for the O(1) lookups the parallel reconciliation below needs.
-   Array<int> loc_dofs, loc_edges, loc_belems;
-   FiniteElementSpace::GetBoundaryLoopEdgeDofs(boundary_element_indices, loc_dofs,
-                                               &loc_edges, &loc_belems);
-
-   std::unordered_set<int> boundary_edge_dofs;
-   std::unordered_map<int, int> dof_to_edge_map;
-   std::unordered_map<int, int> dof_to_boundary_element;
-   boundary_edge_dofs.reserve(loc_dofs.Size());
-   dof_to_edge_map.reserve(loc_dofs.Size());
-   dof_to_boundary_element.reserve(loc_dofs.Size());
-   for (int i = 0; i < loc_dofs.Size(); i++)
-   {
-      const int dof = loc_dofs[i];
-      boundary_edge_dofs.insert(dof);
-      dof_to_edge_map[dof] = loc_edges[i];
-      dof_to_boundary_element[dof] = loc_belems[i];
-   }
-
-   // Parallel processing: Build edge sharing lookup table
-   std::unordered_map<int, int> edge_to_group_size;
-   int num_groups = pmesh->GetNGroups();
-
-   int total_shared_edges = 0;
-   for (int group = 1; group < num_groups; group++)
-   {
-      total_shared_edges += pmesh->GroupNEdges(group);
-   }
-   edge_to_group_size.reserve(total_shared_edges);
-
-   for (int group = 1; group < num_groups; group++)
-   {
-      int group_size = pmesh->gtopo.GetGroupSize(group);
-      int num_edges_in_group = pmesh->GroupNEdges(group);
-
-      for (int i = 0; i < num_edges_in_group; i++)
-      {
-         edge_to_group_size.emplace(pmesh->GroupEdge(group, i), group_size);
-      }
-   }
-
-   // Get global indices
-   Array<HYPRE_BigInt> global_edge_indices;
-   pmesh->GetGlobalEdgeIndices(global_edge_indices);
-
-   // Handle dimension-specific boundary element relationships
-   Array<HYPRE_BigInt> global_face_indices;
-   std::unordered_map<int, int> boundary_element_to_companion;
-   std::unordered_set<int> dofs_to_remove;
-
-   const int dim = pmesh->Dimension();
-   if (dim == 3)
-   {
-      // In 3D: boundary elements are faces, we track which face each boundary element is
-      pmesh->GetGlobalFaceIndices(global_face_indices);
-      for (int boundary_element_idx : boundary_element_indices)
-      {
-         int face_index, face_orientation;
-         pmesh->GetBdrElementFace(boundary_element_idx, &face_index, &face_orientation);
-         boundary_element_to_companion[boundary_element_idx] = face_index;
-      }
-
-      std::vector<HYPRE_BigInt> local_data;
-      local_data.reserve(boundary_edge_dofs.size() * 2);
-
-      std::unordered_set<int> processed_edges;
-      processed_edges.reserve(boundary_edge_dofs.size());
-
-      for (const auto& [dof, local_edge] : dof_to_edge_map)
-      {
-         // Skip if already processed this edge
-         if (!processed_edges.insert(local_edge).second) { continue; }
-
-         // Check if edge is shared (fast lookup)
-         auto it = edge_to_group_size.find(local_edge);
-         if (it != edge_to_group_size.end() && it->second > 1)
-         {
-            // Get boundary element and companion index directly from pre-computed map
-            int boundary_element_idx = dof_to_boundary_element[dof];
-            int companion_index = boundary_element_to_companion[boundary_element_idx];
-
-            // Store edge-face pair for 3D artificial boundary detection
-            local_data.push_back(global_edge_indices[local_edge]);
-            local_data.push_back(global_face_indices[companion_index]);
-         }
-      }
-
-      // MPI communication for 3D artificial boundary detection
-      int num_procs = pmesh->GetNRanks();
-      int local_size = local_data.size();
-
-      std::vector<int> mpi_arrays(num_procs * 4);
-      int* all_sizes = mpi_arrays.data();
-      int* displs = all_sizes + num_procs;
-      int* byte_sizes = displs + num_procs;
-      int* byte_displs = byte_sizes + num_procs;
-
-      MPI_Allgather(&local_size, 1, MPI_INT, all_sizes, 1, MPI_INT, pmesh->GetComm());
-
-      int total_size = 0;
-      constexpr int hypre_size = sizeof(HYPRE_BigInt);
-      for (int i = 0; i < num_procs; i++)
-      {
-         displs[i] = total_size;
-         byte_displs[i] = total_size * hypre_size;
-         total_size += all_sizes[i];
-         byte_sizes[i] = all_sizes[i] * hypre_size;
-      }
-
-      if (total_size > 0)
-      {
-         std::vector<HYPRE_BigInt> all_data(total_size);
-         MPI_Allgatherv(local_data.data(), local_size * hypre_size, MPI_BYTE,
-                        all_data.data(), byte_sizes, byte_displs, MPI_BYTE, pmesh->GetComm());
-
-         // Build global-to-local edge mapping
-         std::unordered_map<HYPRE_BigInt, int> global_to_local_edge;
-         global_to_local_edge.reserve(global_edge_indices.Size());
-         for (int i = 0; i < global_edge_indices.Size(); ++i)
-         {
-            global_to_local_edge[global_edge_indices[i]] = i;
-         }
-
-         // Process collected data to find edges in multiple faces (artificial boundaries)
-         std::unordered_map<HYPRE_BigInt, std::unordered_set<HYPRE_BigInt>>edge_to_faces;
-         edge_to_faces.reserve(total_size / 2);
-
-         for (size_t i = 0; i < all_data.size(); i += 2)
-         {
-            edge_to_faces[all_data[i]].insert(all_data[i + 1]);
-         }
-
-         // Mark DOFs from artificial edges for removal
-         dofs_to_remove.reserve(local_data.size() / 4);
-
-         for (size_t i = 0; i < local_data.size(); i += 2)
-         {
-            HYPRE_BigInt global_edge_id = local_data[i];
-
-            // If this edge appears in 2+ distinct faces, it's artificial
-            if (edge_to_faces[global_edge_id].size() >= 2)
-            {
-               int local_edge = global_to_local_edge[global_edge_id];
-               Array<int> local_edge_dofs;
-               GetEdgeDofs(local_edge, local_edge_dofs);
-
-               // Mark boundary DOFs of this edge for removal
-               for (int k = 0; k < local_edge_dofs.Size(); ++k)
-               {
-                  int dof = local_edge_dofs[k];
-                  if (boundary_edge_dofs.count(dof))
-                  {
-                     dofs_to_remove.insert(dof);
-                  }
-               }
-            }
-         }
-      }
-   }
-   else if (dim == 2)
-   {
-      // In 2D the boundary elements are themselves the edges, so there are no
-      // artificial boundary edges to detect. However, for collections with
-      // vertex DOFs (e.g. ND_R2D), a vertex shared by two boundary segments is
-      // interior to the boundary curve and must be dropped. The serial code
-      // does this by erasing a DOF on its second occurrence, which only sees
-      // the occurrences local to this rank. When the two segments meeting at a
-      // vertex live on different ranks, each rank sees a single occurrence and
-      // wrongly keeps the DOF. Reconcile the occurrence parity across each
-      // sharing group: membership in boundary_edge_dofs is the local parity,
-      // and the parities sum (mod 2) to the global occurrence parity.
-      Array<int> boundary_dof_count(GetVSize());
-      boundary_dof_count = 0;
-      for (const int dof : boundary_edge_dofs)
-      {
-         boundary_dof_count[dof] = 1;
-      }
-
-      // implement allreduce(+) as reduce(+) + broadcast
-      gcomm->Reduce<int>(boundary_dof_count, GroupCommunicator::Sum);
-      gcomm->Bcast(boundary_dof_count);
-
-      for (const int dof : boundary_edge_dofs)
-      {
-         if (boundary_dof_count[dof] % 2 == 0)
-         {
-            dofs_to_remove.insert(dof);
-         }
-      }
-   }
-
-   // Remove artificial DOFs
-   for (int dof : dofs_to_remove)
-   {
-      boundary_edge_dofs.erase(dof);
-      dof_to_edge_map.erase(dof);
-      dof_to_boundary_element.erase(dof);
-   }
-
-   // Convert to true DOFs and output
-   ess_tdof_list.SetSize(0);
-   ess_tdof_list.Reserve(boundary_edge_dofs.size());
-   if (ess_edge_list)
-   {
-      // Reset as well, so that it stays in correspondence with ess_tdof_list
-      // when the same output array is reused across calls.
-      ess_edge_list->SetSize(0);
-      ess_edge_list->Reserve(boundary_edge_dofs.size());
-   }
-   // Marker of the boundary edge DOFs. Always computed locally because the
-   // parallel reconciliation below needs it; only copied to the caller's output
-   // if requested (see the ldof_marker parameter).
-   Array<int> local_ldof_marker(GetVSize());
-   local_ldof_marker = 0;
-
-   for (int dof : boundary_edge_dofs)
-   {
-      local_ldof_marker[dof] = 1; // Mark all boundary edge dofs
-   }
-
-   // Make sure that a selected shared DOF is marked on every rank of its
-   // sharing group, including ranks holding none of the selected boundary
-   // elements. Only the group master owns the corresponding true DOF, so
-   // without this the true DOF would be emitted by no rank at all: the
-   // non-master ranks get -1 from GetLocalTDofNumber(), while the master may
-   // not have selected the DOF locally.
-   Synchronize(local_ldof_marker);
-
-   // A DOF marked only through the synchronization above has no local
-   // dof_to_edge_map entry, but the shared edge carrying it is still present in
-   // the local mesh. Build the missing DOF -> edge entries from the shared
-   // edges of the groups, so that ess_edge_list stays in correspondence with
-   // ess_tdof_list. Note that a vertex DOF is not associated with a unique
-   // edge, so it is only resolved when it is an interior DOF of an edge.
-   std::unordered_map<int, int> shared_dof_to_edge;
-   Array<int> shared_edge_dofs;
-   for (int group = 1; group < num_groups; group++)
-   {
-      const int num_edges_in_group = pmesh->GroupNEdges(group);
-      for (int i = 0; i < num_edges_in_group; i++)
-      {
-         const int edge = pmesh->GroupEdge(group, i);
-         GetEdgeInteriorDofs(edge, shared_edge_dofs);
-         for (int k = 0; k < shared_edge_dofs.Size(); k++)
-         {
-            shared_dof_to_edge.emplace(shared_edge_dofs[k], edge);
-         }
-      }
-   }
-
-   // Build parallel arrays for DOFs and corresponding edges
-   std::vector<std::pair<int, int>> tdof_edge_pairs;
-   tdof_edge_pairs.reserve(boundary_edge_dofs.size());
-
-   for (int dof = 0; dof < local_ldof_marker.Size(); dof++)
-   {
-      if (!local_ldof_marker[dof]) { continue; }
-
-      const int tdof = GetLocalTDofNumber(dof);
-      if (tdof < 0) { continue; } // tdof == -1 means not owned by this rank
-
-      int edge = -1;
-      auto it = dof_to_edge_map.find(dof);
-      if (it != dof_to_edge_map.end())
-      {
-         edge = it->second;
-      }
-      else
-      {
-         auto shared_it = shared_dof_to_edge.find(dof);
-         if (shared_it != shared_dof_to_edge.end())
-         {
-            edge = shared_it->second;
-         }
-      }
-      tdof_edge_pairs.push_back({tdof, edge});
-   }
-
-   // Sort by true DOF index to maintain consistent ordering
-   std::sort(tdof_edge_pairs.begin(), tdof_edge_pairs.end());
-
-   // Extract sorted true DOFs and edges
-   for (const auto& pair : tdof_edge_pairs)
-   {
-      ess_tdof_list.Append(pair.first);
-      if (ess_edge_list)
-      {
-         ess_edge_list->Append(pair.second);
-      }
-   }
-
-   // Emit the local boundary-loop DOFs in a deterministic (increasing DOF
-   // index) order shared by all output arrays.
-   std::vector<int> kept(boundary_edge_dofs.begin(), boundary_edge_dofs.end());
-   std::sort(kept.begin(), kept.end());
-
-   boundary_edge_dofs_out.SetSize(0);
-   boundary_edge_dofs_out.Reserve(static_cast<int>(kept.size()));
-   if (dof_edges)
-   {
-      dof_edges->SetSize(0);
-      dof_edges->Reserve(static_cast<int>(kept.size()));
-   }
-   if (dof_boundary_elements)
-   {
-      dof_boundary_elements->SetSize(0);
-      dof_boundary_elements->Reserve(static_cast<int>(kept.size()));
-   }
-   for (int dof : kept)
-   {
-      boundary_edge_dofs_out.Append(dof);
-      if (dof_edges) { dof_edges->Append(dof_to_edge_map[dof]); }
-      if (dof_boundary_elements)
-      {
-         dof_boundary_elements->Append(dof_to_boundary_element[dof]);
-      }
-   }
-
-   if (ldof_marker) { ldof_marker->Swap(local_ldof_marker); }
-}
-
 void ParFiniteElementSpace::GetExteriorTrueDofs(Array<int> &ext_tdof_list,
                                                 int component) const
 {

fem/pfespace.hpp

@@ -460,41 +460,6 @@ public:
    void GetExteriorTrueDofs(Array<int> &ext_tdof_list,
                             int component = -1) const override;
 
-   /** @brief Extract the edge degrees of freedom of a boundary "loop" on a
-       parallel mesh (see the serial FiniteElementSpace::GetBoundaryLoopEdgeDofs
-       for the definition of a loop). This version removes the artificial
-       boundary edges that appear at processor boundaries, so the selected DOFs
-       are independent of the mesh partitioning.
-
-       As in the serial version, the @a boundary_edge_dofs_out, @a dof_edges and
-       @a dof_boundary_elements outputs share a single indexing describing the
-       same local DOF at each position.
-
-       Requirements:
-       - Mesh must be conforming (no hanging nodes)
-       - Mesh dimension must be >= 2
-       @param[in]  boundary_element_indices Array of boundary element indices.
-       @param[out] ess_tdof_list Essential true DOF indices, sorted ascending.
-       @param[out] boundary_edge_dofs_out Local boundary-loop DOF indices.
-       @param[out] ldof_marker Optional; marker of the boundary edge DOFs,
-                   derivable from @a boundary_edge_dofs_out via ListToMarker().
-       @param[out] dof_edges Optional; local edge index of each DOF.
-       @param[out] dof_boundary_elements Optional; a boundary element containing
-                   each DOF.
-       @param[out] ess_edge_list Optional array of edge indices, in one-to-one
-                                 correspondence with @a ess_tdof_list. An entry
-                                 is -1 when the true DOF is owned by this rank
-                                 but no local edge can be associated with it,
-                                 which can happen for a shared vertex DOF whose
-                                 boundary elements are all on other ranks. */
-   void GetBoundaryLoopEdgeDofs(const Array<int> &boundary_element_indices,
-                                Array<int> &ess_tdof_list,
-                                Array<int> &boundary_edge_dofs_out,
-                                Array<int> *ldof_marker = nullptr,
-                                Array<int> *dof_edges = nullptr,
-                                Array<int> *dof_boundary_elements = nullptr,
-                                Array<int> *ess_edge_list = nullptr);
-
    /** If the given ldof is owned by the current processor, return its local
        tdof number, otherwise return -1 */
    int GetLocalTDofNumber(int ldof) const;

fem/pgridfunc.cpp

@@ -22,20 +22,10 @@ using namespace std;
 namespace mfem
 {
 
-ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
-                                 bool preserve)
+ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf)
 {
    fes = pfes = pf;
    SetDataAndSize(gf->GetData(), gf->Size());
-
-   if (pfes->HaveDofSigns())
-   {
-      MFEM_VERIFY(!preserve, "Differing sign conventions for the serial and "
-                  "parallel grid functions will prevent preserving the serial "
-                  "GridFunctions in this context.");
-
-      pfes->ApplyDofSigns(HostReadWrite());
-   }
 }
 
 ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, HypreParVector *tv)

fem/pgridfunc.hpp

@@ -100,12 +100,8 @@ public:
    /// Construct a ParGridFunction using a GridFunction as external data.
    /** The parallel space @a *pf and the space used by @a *gf should match. The
        data from @a *gf is used as the local data of the ParGridFunction on each
-       processor. The ParGridFunction does not assume ownership of the data.
-       The boolean, @a preserve, indicates that the data stored in @a *gf should
-       remain unchanged. An error will occur if @a preserve is true and
-       construction of a valid ParGridFunction requires the data to change. */
-   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
-                   bool preserve = true);
+       processor. The ParGridFunction does not assume ownership of the data. */
+   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf);
 
    /** @brief Creates grid function on (all) dofs from a given vector on the
        true dofs, i.e. P tv. */

fem/qinterp/det.cpp

@@ -21,23 +21,24 @@ namespace quadrature_interpolator
 
 void InitDetKernels()
 {
+   using k = QuadratureInterpolator::DetKernels;
    // 2D
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,2>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,3>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,3,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,3,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,4,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,4,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,5,6>();
+   k::Specialization<2,2,2,2>::Add();
+   k::Specialization<2,2,2,3>::Add();
+   k::Specialization<2,2,2,4>::Add();
+   k::Specialization<2,2,2,6>::Add();
+   k::Specialization<2,2,3,4>::Add();
+   k::Specialization<2,2,3,6>::Add();
+   k::Specialization<2,2,4,4>::Add();
+   k::Specialization<2,2,4,6>::Add();
+   k::Specialization<2,2,5,6>::Add();
    // 3D
-   QuadratureInterpolator::AddDetSpecializations<3,3,2,4>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,3>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,5>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,6>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,4,6>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,4>();
+   k::Specialization<3,3,2,4>::Add();
+   k::Specialization<3,3,3,3>::Add();
+   k::Specialization<3,3,3,5>::Add();
+   k::Specialization<3,3,3,6>::Add();
+   k::Specialization<3,3,4,6>::Add();
+   k::Specialization<3,3,3,4>::Add();
 }
 
 } // namespace quadrature_interpolator
@@ -46,8 +47,8 @@ void InitDetKernels()
 /// @cond Suppress_Doxygen_warnings
 
 QuadratureInterpolator::DetKernelType
-QuadratureInterpolator::DetKernels::Fallback(int DIM, int SDIM, int D1D,
-                                             int Q1D)
+QuadratureInterpolator::DetKernels::Fallback(
+   int DIM, int SDIM, int D1D, int Q1D)
 {
    if (DIM == 1)
    {

fem/qinterp/eval.hpp

@@ -30,18 +30,23 @@ namespace internal
 namespace quadrature_interpolator
 {
 
-template <QVectorLayout Q_LAYOUT, bool Integral>
-static void ImplValues1D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim,
-                         const int d1d, const int q1d)
+template<QVectorLayout Q_LAYOUT>
+static void Values1D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim,
+                     const int d1d,
+                     const int q1d)
 {
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   const auto b = Reshape(b_, q1d, d1d);
+   const auto x = Reshape(x_, d1d, vdim, NE);
+   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
+            Reshape(y_, q1d, vdim, NE):
+            Reshape(y_, vdim, q1d, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto b = Reshape(b_, q1d, d1d);
-      const auto x = Reshape(x_, d1d, vdim, NE);
-      const auto detJ = Reshape(detJ_, q1d, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES ? Reshape(y_, q1d, vdim, NE)
-               : Reshape(y_, vdim, q1d, NE);
       for (int c = 0; c < vdim; c++)
       {
          for (int q = 0; q < q1d; q++)
@@ -51,36 +56,24 @@ static void ImplValues1D(const int NE, const real_t *b_, const real_t *detJ_,
             {
                u += b(q, d) * x(d, c, e);
             }
-            if constexpr (Integral)
-            {
-               u /= detJ(q, e);
-            }
-            if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-            {
-               y(c, q, e) = u;
-            }
-            if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-            {
-               y(q, c, e) = u;
-            }
+            if (Q_LAYOUT == QVectorLayout::byVDIM)  { y(c, q, e) = u; }
+            if (Q_LAYOUT == QVectorLayout::byNODES) { y(q, c, e) = u; }
          }
       }
    });
 }
 
-template <QVectorLayout Q_LAYOUT>
-static void Values1D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim, const int d1d, const int q1d)
-{
-   ImplValues1D<Q_LAYOUT, false>(NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
 // Template compute kernel for Values in 2D: tensor product version.
-template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
-          int T_Q1D = 0, int T_NBZ = 1>
-static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim = 0,
-                         const int d1d = 0, const int q1d = 0)
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int T_NBZ = 1>
+static void Values2D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim = 0,
+                     const int d1d = 0,
+                     const int q1d = 0)
 {
    static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
 
@@ -89,14 +82,13 @@ static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
+            Reshape(y_, Q1D, Q1D, VDIM, NE):
+            Reshape(y_, VDIM, Q1D, Q1D, NE);
 
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
-      const auto detJ = Reshape(detJ_, Q1D, Q1D, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES
-               ? Reshape(y_, Q1D, Q1D, VDIM, NE)
-               : Reshape(y_, VDIM, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -118,33 +110,16 @@ static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
 
       for (int c = 0; c < VDIM; c++)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx, x, D1D)
-            {
-               DD(dx, dy) = x(dx, dy, c, e);
-            }
-         }
-         MFEM_SYNC_THREAD;
+         kernels::internal::LoadX(e,D1D,c,x,DD);
          kernels::internal::EvalX(D1D,Q1D,B,DD,DQ);
          kernels::internal::EvalY(D1D,Q1D,B,DQ,QQ);
          MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
             MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               real_t u = QQ(qx, qy);
-               if constexpr (Integral)
-               {
-                  u /= detJ(qx, qy, e);
-               }
-               if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-               {
-                  y(c, qx, qy, e) = u;
-               }
-               if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-               {
-                  y(qx, qy, c, e) = u;
-               }
+               real_t u = QQ(qx,qy);
+               if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,e) = u; }
+               if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,c,e) = u; }
             }
          }
          MFEM_SYNC_THREAD;
@@ -152,37 +127,29 @@ static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
    });
 }
 
-// Template compute kernel for Values in 2D: tensor product version.
-template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-          int T_NBZ = 1>
-static void Values2D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim = 0, const int d1d = 0,
-                     const int q1d = 0)
-{
-   return ImplValues2D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D, T_NBZ>(
-             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
 // Template compute kernel for Values in 3D: tensor product version.
-template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
-          int T_Q1D = 0>
-static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim = 0,
-                         const int d1d = 0, const int q1d = 0)
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
+static void Values3D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim = 0,
+                     const int d1d = 0,
+                     const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
+            Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE):
+            Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
 
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
-      const auto detJ = Reshape(detJ_, Q1D, Q1D, Q1D, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES
-               ? Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE)
-               : Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -204,17 +171,7 @@ static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
 
       for (int c = 0; c < VDIM; c++)
       {
-         MFEM_FOREACH_THREAD(dz, z, D1D)
-         {
-            MFEM_FOREACH_THREAD(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD(dx, x, D1D)
-               {
-                  DDD(dx, dy, dz) = x(dx, dy, dz, c, e);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
+         kernels::internal::LoadX(e,D1D,c,x,DDD);
          kernels::internal::EvalX(D1D,Q1D,B,DDD,DDQ);
          kernels::internal::EvalY(D1D,Q1D,B,DDQ,DQQ);
          kernels::internal::EvalZ(D1D,Q1D,B,DQQ,QQQ);
@@ -224,19 +181,9 @@ static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
             {
                MFEM_FOREACH_THREAD(qx,x,Q1D)
                {
-                  real_t u = QQQ(qz,qy,qx);
-                  if constexpr (Integral)
-                  {
-                     u /= detJ(qx, qy, qz, e);
-                  }
-                  if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     y(c, qx, qy, qz, e) = u;
-                  }
-                  if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-                  {
-                     y(qx, qy, qz, c, e) = u;
-                  }
+                  const real_t u = QQQ(qz,qy,qx);
+                  if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,qz,e) = u; }
+                  if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,qz,c,e) = u; }
                }
             }
          }
@@ -245,431 +192,14 @@ static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
    });
 }
 
-// Template compute kernel for Values in 3D: tensor product version.
-template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
-static void Values3D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim = 0, const int d1d = 0,
-                     const int q1d = 0)
-{
-   return ImplValues3D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D>(
-             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
-template <bool Integral>
-void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-                const real_t *detJ, const GeometricFactors *geom,
-                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                Vector &q_der, Vector &q_det, const int eval_flags);
-
-inline void Eval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der, Vector &q_det,
-                   const int eval_flags)
-{
-   ImplEval1D<false>(NE, vdim, q_layout, nullptr, geom, maps, e_vec, q_val,
-                     q_der, q_det, eval_flags);
-}
-
-// Template compute kernel for 2D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
-static void ImplEval2D(const int NE, const int vdim,
-                       const QVectorLayout q_layout, const real_t *detJ_,
-                       const GeometricFactors *geom, const DofToQuad &maps,
-                       const Vector &e_vec, Vector &q_val, Vector &q_der,
-                       Vector &q_det, const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
-
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
-   if constexpr(Integral)
-   {
-      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
-                                  QI::DETERMINANTS)),
-                  "Integral FE does not support computing derivatives");
-   }
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
-   const auto E_ = e_vec.Read();
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 2, NE):
-              Reshape(q_der.Write(), VDIM, 2, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const auto E = Reshape(E_, ND, VDIM, NE);
-      const auto detJ = Reshape(detJ_, NQ, NE);
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND2D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM2D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c + d * VDIM] = E(d, c, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++)
-            {
-               ed[c] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  ed[c] += b * s_E[c + d * VDIM];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if constexpr (Integral)
-               {
-                  ed[c] /= detJ(q, e);
-               }
-               if (q_layout == QVectorLayout::byVDIM)
-               {
-                  val(c, q, e) = ed[c];
-               }
-               if (q_layout == QVectorLayout::byNODES)
-               {
-                  val(q, c, e) = ed[c];
-               }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM2D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM2D*2];
-            for (int i = 0; i < 2*VDIM; i++)
-            {
-               D[i] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[4], Jinv[4];
-               Jloc[0] = J(q,0,0,e);
-               Jloc[1] = J(q,1,0,e);
-               Jloc[2] = J(q,0,1,e);
-               Jloc[3] = J(q,1,1,e);
-               kernels::CalcInverse<2>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v;
-                  const real_t JiV = Jinv[2]*u + Jinv[3]*v;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                  }
-               }
-            }
-            if (eval_flags & QI::DETERMINANTS)
-            {
-               if (VDIM == 2)
-               {
-                  det(q, e) = kernels::Det<2>(D);
-               }
-               else
-               {
-                  DeviceTensor<2> j(D, 3, 2);
-                  const real_t dE = j(0,0)*j(0,0) + j(1,0)*j(1,0) + j(2,0)*j(2,0);
-                  const real_t dF = j(0,0)*j(0,1) + j(1,0)*j(1,1) + j(2,0)*j(2,1);
-                  const real_t dG = j(0,1)*j(0,1) + j(1,1)*j(1,1) + j(2,1)*j(2,1);
-                  det(q,e) = std::sqrt(dE*dG - dF*dF);
-               }
-            }
-         }
-      }
-   });
-}
-
-// Template compute kernel for 2D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval2D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der,
-                   Vector &q_det, const int eval_flags)
-{
-   ImplEval2D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
-                                         maps, e_vec, q_val, q_der, q_det,
-                                         eval_flags);
-}
-
-// Template compute kernel for 3D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
-static void ImplEval3D(const int NE, const int vdim,
-                       const QVectorLayout q_layout, const real_t *detJ_,
-                       const GeometricFactors *geom, const DofToQuad &maps,
-                       const Vector &e_vec, Vector &q_val, Vector &q_der,
-                       Vector &q_det, const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
-
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 3, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
-   MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
-   if constexpr(Integral)
-   {
-      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
-                                  QI::DETERMINANTS)),
-                  "Integral FE does not support computing derivatives");
-   }
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
-   auto E_ = e_vec.Read();
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 3, NE):
-              Reshape(q_der.Write(), VDIM, 3, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const auto E = Reshape(E_, ND, VDIM, NE);
-      const auto detJ = Reshape(detJ_, NQ, NE);
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND3D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM3D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c + d * VDIM] = E(d, c, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++)
-            {
-               ed[c] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  ed[c] += b * s_E[c + d * VDIM];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if constexpr (Integral)
-               {
-                  ed[c] /= detJ(q, e);
-               }
-               if (q_layout == QVectorLayout::byVDIM)
-               {
-                  val(c, q, e) = ed[c];
-               }
-               if (q_layout == QVectorLayout::byNODES)
-               {
-                  val(q, c, e) = ed[c];
-               }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM3D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM3D*3];
-            for (int i = 0; i < 3*VDIM; i++)
-            {
-               D[i] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               const real_t wz = G(q,2,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-                  D[c+VDIM*2] += s_e * wz;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                     der(c,2,q,e) = D[c+VDIM*2];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                     der(q,c,2,e) = D[c+VDIM*2];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[9], Jinv[9];
-               for (int col = 0; col < 3; col++)
-               {
-                  for (int row = 0; row < 3; row++)
-                  {
-                     Jloc[row+3*col] = J(q,row,col,e);
-                  }
-               }
-               kernels::CalcInverse<3>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t w = D[c+VDIM*2];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
-                  const real_t JiV = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
-                  const real_t JiW = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                     der(c,2,q,e) = JiW;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                     der(q,c,2,e) = JiW;
-                  }
-               }
-            }
-            if (VDIM == 3 && (eval_flags & QI::DETERMINANTS))
-            {
-               // The check (VDIM == 3) should eliminate this block when VDIM is
-               // known at compile time and (VDIM != 3).
-               det(q,e) = kernels::Det<3>(D);
-            }
-         }
-      }
-   });
-}
-
-// Template compute kernel for 3D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval3D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der,
-                   Vector &q_det, const int eval_flags)
-{
-   ImplEval3D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
-                                         maps, e_vec, q_val, q_der, q_det,
-                                         eval_flags);
-}
-
 } // namespace quadrature_interpolator
 
 } // namespace internal
 
 /// @cond Suppress_Doxygen_warnings
 
-template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
-QuadratureInterpolator::IntTensorEvalKernelType
-QuadratureInterpolator::IntTensorEvalKernels::Kernel()
-{
-   if constexpr (DIM == 1) { return internal::quadrature_interpolator::ImplValues1D<Q_LAYOUT, true>; }
-   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::ImplValues2D<Q_LAYOUT, true, VDIM, D1D, Q1D, NBZ>; }
-   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::ImplValues3D<Q_LAYOUT, true, VDIM, D1D, Q1D>; }
-   MFEM_ABORT("");
-}
-
-template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
+template<int DIM, QVectorLayout Q_LAYOUT,
+         int VDIM, int D1D, int Q1D, int NBZ>
 QuadratureInterpolator::TensorEvalKernelType
 QuadratureInterpolator::TensorEvalKernels::Kernel()
 {
@@ -679,28 +209,6 @@ QuadratureInterpolator::TensorEvalKernels::Kernel()
    MFEM_ABORT("");
 }
 
-template <int DIM, int VDIM, int ND, int NQ>
-QuadratureInterpolator::IntEvalKernelType
-QuadratureInterpolator::IntEvalKernels::Kernel()
-{
-   using namespace internal::quadrature_interpolator;
-   if constexpr (DIM == 1) { return ImplEval1D<true>; }
-   else if constexpr (DIM == 2) { return ImplEval2D<true,VDIM,ND,NQ>; }
-   else if constexpr (DIM == 3) { return ImplEval3D<true,VDIM,ND,NQ>; }
-   MFEM_ABORT("");
-}
-
-template <int DIM, int VDIM, int ND, int NQ>
-QuadratureInterpolator::EvalKernelType
-QuadratureInterpolator::EvalKernels::Kernel()
-{
-   using namespace internal::quadrature_interpolator;
-   if constexpr (DIM == 1) { return Eval1D; }
-   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
-   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
-   MFEM_ABORT("");
-}
-
 /// @endcond
 
 } // namespace mfem

fem/qinterp/eval_by_nodes.cpp

@@ -21,105 +21,63 @@ namespace quadrature_interpolator
 
 void InitEvalByNodesKernels()
 {
+   using k = QuadratureInterpolator::TensorEvalKernels;
+
    // 2D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 4, 4, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,2>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,5>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 7, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,5>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,6>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,7>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 5, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,5,6>::Opt<1>::Add();
 
    // 3D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,8>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 3, 4, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 6, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 6, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 7, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,7>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,8>::Opt<1>::Add();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/eval_by_vdim.cpp

@@ -21,59 +21,36 @@ namespace quadrature_interpolator
 
 void InitEvalByVDimKernels()
 {
+   using k = QuadratureInterpolator::TensorEvalKernels;
    // 2D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 2, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 3, 6, 4>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 4, 8, 2>();
+   k::Specialization<2,QVectorLayout::byVDIM,1,2,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,1,3,6>::Opt<4>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,1,4,8>::Opt<2>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 2, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 3, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 3, 6, 4>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 4, 6, 2>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 4, 8, 2>();
+   k::Specialization<2,QVectorLayout::byVDIM,2,2,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,3,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,3,6>::Opt<4>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,4,6>::Opt<2>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,4,8>::Opt<2>::Add();
    // 3D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 4, 8, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,1,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,1,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,1,4,8>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,8>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 5, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 6, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 7, 7, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 8, 8, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 9, 9, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,3,2,2>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,5,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,6,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,7,7>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,8,8>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,9,9>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 4, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,4>::Opt<1>::Add();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/grad.hpp

@@ -268,9 +268,8 @@ static void Derivatives3D(const int NE,
       DeviceMatrix B(BG[0], D1D, Q1D);
       DeviceMatrix G(BG[1], D1D, Q1D);
 
-      constexpr int MDQ = MD1 > MQ1 ? MD1 : MQ1;
-      MFEM_SHARED real_t sm0[3][MD1*MD1*MDQ];
-      MFEM_SHARED real_t sm1[3][MD1*MQ1*MQ1];
+      MFEM_SHARED real_t sm0[3][MQ1*MQ1*MQ1];
+      MFEM_SHARED real_t sm1[3][MQ1*MQ1*MQ1];
       DeviceTensor<3> X(sm0[2], D1D, D1D, D1D);
       DeviceTensor<3> DDQ0(sm0[0], D1D, D1D, Q1D);
       DeviceTensor<3> DDQ1(sm0[1], D1D, D1D, Q1D);

fem/qinterp/grad_by_nodes.cpp

@@ -22,71 +22,74 @@ namespace quadrature_interpolator
 template <bool P>
 void InitGradByNodesKernels()
 {
+   using k = QuadratureInterpolator::GradKernels;
    constexpr auto L = QVectorLayout::byNODES;
    // 2D
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,3,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,4,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,3,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,4,16>();
+   k::Specialization<2,L,P,1,3,3>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,3,4>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,4,3>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,4,4>::template Opt<16>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,2,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,3,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,5,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,6,2>();
+   k::Specialization<2,L,P,2,2,2>::template Opt<16>::Add();
+   k::Specialization<2,L,P,2,2,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,2,4>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,2,5>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,2,6>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,3,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,6,2>();
+   k::Specialization<2,L,P,2,3,3>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,4,3>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,3,6>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,4,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,5,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,6,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,7,2>();
+   k::Specialization<2,L,P,2,4,4>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,5>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,7>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,5,6,2>();
+   k::Specialization<2,L,P,2,5,6>::template Opt<2>::Add();
    // 3D
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,2,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,3>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,8>();
+   k::Specialization<3,L,P,1,2,4>::Add();
+   k::Specialization<3,L,P,1,3,3>::Add();
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,3,6>::Add();
+   k::Specialization<3,L,P,1,4,4>::Add();
+   k::Specialization<3,L,P,1,4,8>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,3>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,5>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,6>();
+   k::Specialization<3,L,P,3,2,3>::Add();
+   k::Specialization<3,L,P,3,2,4>::Add();
+   k::Specialization<3,L,P,3,2,5>::Add();
+   k::Specialization<3,L,P,3,2,6>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,3>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,5>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,7>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,8>();
+   k::Specialization<3,L,P,3,3,3>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,3,5>::Add();
+   k::Specialization<3,L,P,3,3,6>::Add();
+   k::Specialization<3,L,P,3,4,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,4,7>::Add();
+   k::Specialization<3,L,P,3,4,8>::Add();
+
+   using k2 = QuadratureInterpolator::CollocatedGradKernels;
 
    // 2D
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,3,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,4,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,3,4>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,4,2>();
+   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
+   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,4>();
+   k2::Specialization<3,L,P,1,2>::Add();
+   k2::Specialization<3,L,P,1,3>::Add();
+   k2::Specialization<3,L,P,1,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,4>();
+   k2::Specialization<3,L,P,2,2>::Add();
+   k2::Specialization<3,L,P,2,3>::Add();
+   k2::Specialization<3,L,P,2,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,4>();
+   k2::Specialization<3,L,P,3,2>::Add();
+   k2::Specialization<3,L,P,3,3>::Add();
+   k2::Specialization<3,L,P,3,4>::Add();
 }
 
 template void InitGradByNodesKernels<true>();

fem/qinterp/grad_by_vdim.cpp

@@ -22,45 +22,47 @@ namespace quadrature_interpolator
 template <bool P>
 void InitGradByVDimKernels()
 {
+   using k = QuadratureInterpolator::GradKernels;
    constexpr auto L = QVectorLayout::byVDIM;
    // 2D
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,4,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,6,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,5,8,2>();
+   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,3,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,4,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,6,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,5,8,2>();
+   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
    // 3D
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,5,8>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,5,8>();
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,4,6>::Add();
+   k::Specialization<3,L,P,1,5,8>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,5,8>::Add();
 
+   using k2 = QuadratureInterpolator::CollocatedGradKernels;
    // 2D
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,3,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,4,16>();
+   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,3,4>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,4,2>();
+   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
+   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
 
    // 3D
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,4>();
+   k2::Specialization<3,L,P,1,2>::Add();
+   k2::Specialization<3,L,P,1,3>::Add();
+   k2::Specialization<3,L,P,1,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,4>();
+   k2::Specialization<3,L,P,2,2>::Add();
+   k2::Specialization<3,L,P,2,3>::Add();
+   k2::Specialization<3,L,P,2,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,4>();
+   k2::Specialization<3,L,P,3,2>::Add();
+   k2::Specialization<3,L,P,3,3>::Add();
+   k2::Specialization<3,L,P,3,4>::Add();
 }
 
 template void InitGradByVDimKernels<true>();

fem/quadinterpolator.cpp

@@ -69,9 +69,8 @@ QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
 
    d_buffer.UseDevice(true);
    if (fespace->GetNE() == 0) { return; }
-   MFEM_VERIFY(
-      SupportsFESpace(fes),
-      "Only elements with MapType VALUE, INTEGRAL, or H_DIV are supported!");
+   MFEM_VERIFY(SupportsFESpace(fes),
+               "Only elements with MapType VALUE and H_DIV are supported!");
 }
 
 QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
@@ -85,9 +84,8 @@ QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
 {
    d_buffer.UseDevice(true);
    if (fespace->GetNE() == 0) { return; }
-   MFEM_VERIFY(
-      SupportsFESpace(fes),
-      "Only elements with MapType VALUE, INTEGRAL, or H_DIV are supported!");
+   MFEM_VERIFY(SupportsFESpace(fes),
+               "Only elements with MapType VALUE and H_DIV are supported!");
 }
 
 bool QuadratureInterpolator::SupportsFESpace(const FiniteElementSpace &fespace)
@@ -95,9 +93,9 @@ bool QuadratureInterpolator::SupportsFESpace(const FiniteElementSpace &fespace)
    const FiniteElement *fe = fespace.GetTypicalFE();
    const Mesh &mesh = *fespace.GetMesh();
    return (fe->GetMapType() == FiniteElement::MapType::VALUE ||
-           fe->GetMapType() == FiniteElement::MapType::INTEGRAL ||
-           fe->GetMapType() == FiniteElement::MapType::H_DIV) &&
-          (!fespace.IsVariableOrder()) && (!mesh.IsMixedMesh());
+           fe->GetMapType() == FiniteElement::MapType::H_DIV)
+          && (!fespace.IsVariableOrder())
+          && (!mesh.IsMixedMesh());
 }
 
 namespace internal
@@ -110,11 +108,16 @@ namespace quadrature_interpolator
 // * non-tensor product version,
 // * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
 // * assumes 'maps.mode == FULL'.
-template <bool Integral>
-void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-                const real_t *detJ_, const GeometricFactors *geom,
-                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                Vector &q_der, Vector &q_det, const int eval_flags)
+static void Eval1D(const int NE,
+                   const int vdim,
+                   const QVectorLayout q_layout,
+                   const GeometricFactors *geom,
+                   const DofToQuad &maps,
+                   const Vector &e_vec,
+                   Vector &q_val,
+                   Vector &q_der,
+                   Vector &q_det,
+                   const int eval_flags)
 {
    using QI = QuadratureInterpolator;
 
@@ -123,16 +126,13 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
    MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
    MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 1, "");
    MFEM_VERIFY(vdim == 1 || !(eval_flags & QI::DETERMINANTS), "");
-   if constexpr(Integral)
-   {
-      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
-                                  QI::DETERMINANTS)),
-                  "Integral FE does not support computing derivatives");
-   }
-   const auto B_ = maps.B.Read();
-   const auto G_ = maps.G.Read();
+   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
+               "'geom' must be given (non-null) only when evaluating physical"
+               " derivatives");
+   const auto B = Reshape(maps.B.Read(), nq, nd);
+   const auto G = Reshape(maps.G.Read(), nq, nd);
    const auto J = Reshape(geom ? geom->J.Read() : nullptr, nq, NE);
-   const auto E_ = e_vec.Read();
+   const auto E = Reshape(e_vec.Read(), nd, vdim, NE);
    auto val = q_layout == QVectorLayout::byNODES ?
               Reshape(q_val.Write(), nq, vdim, NE):
               Reshape(q_val.Write(), vdim, nq, NE);
@@ -140,12 +140,8 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
               Reshape(q_der.Write(), nq, vdim, NE):
               Reshape(q_der.Write(), vdim, nq, NE);
    auto det = Reshape(q_det.Write(), nq, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto B = Reshape(B_, nq, nd);
-      const auto G = Reshape(G_, nq, nd);
-      const auto E = Reshape(E_, nd, vdim, NE);
-      const auto detJ = Reshape(detJ_, nq, NE);
       for (int q = 0; q < nq; ++q)
       {
          if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
@@ -155,20 +151,10 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
                real_t q_val = 0.0;
                for (int d = 0; d < nd; ++d)
                {
-                  q_val += B(q, d) * E(d, c, e);
-               }
-               if constexpr (Integral)
-               {
-                  q_val /= detJ(q, e);
-               }
-               if (q_layout == QVectorLayout::byVDIM)
-               {
-                  val(c, q, e) = q_val;
-               }
-               if (q_layout == QVectorLayout::byNODES)
-               {
-                  val(q, c, e) = q_val;
+                  q_val += B(q,d)*E(d,c,e);
                }
+               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = q_val; }
+               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = q_val; }
             }
          }
          if ((eval_flags & QI::DERIVATIVES) ||
@@ -180,7 +166,7 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
                real_t q_d = 0.0;
                for (int d = 0; d < nd; ++d)
                {
-                  q_d += G(q, d) * E(d, c, e);
+                  q_d += G(q,d)*E(d,c,e);
                }
                if (eval_flags & QI::PHYSICAL_DERIVATIVES)
                {
@@ -188,14 +174,8 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
                }
                if (eval_flags & QI::DERIVATIVES || eval_flags & QI::PHYSICAL_DERIVATIVES)
                {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c, q, e) = q_d;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q, c, e) = q_d;
-                  }
+                  if (q_layout == QVectorLayout::byVDIM) { der(c,q,e) = q_d; }
+                  if (q_layout == QVectorLayout::byNODES) { der(q,c,e) = q_d; }
                }
                if (vdim == 1 && (eval_flags & QI::DETERMINANTS))
                {
@@ -207,17 +187,317 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
    });
 }
 
-template void
-ImplEval1D<true>(const int NE, const int vdim, const QVectorLayout q_layout,
-                 const real_t *detJ, const GeometricFactors *geom,
-                 const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                 Vector &q_der, Vector &q_det, const int eval_flags);
+// Template compute kernel for 2D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template<const int T_VDIM, const int T_ND, const int T_NQ>
+static void Eval2D(const int NE,
+                   const int vdim,
+                   const QVectorLayout q_layout,
+                   const GeometricFactors *geom,
+                   const DofToQuad &maps,
+                   const Vector &e_vec,
+                   Vector &q_val,
+                   Vector &q_der,
+                   Vector &q_det,
+                   const int eval_flags)
+{
+   using QI = QuadratureInterpolator;
 
-template void
-ImplEval1D<false>(const int NE, const int vdim, const QVectorLayout q_layout,
-                  const real_t *detJ, const GeometricFactors *geom,
-                  const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                  Vector &q_der, Vector &q_det, const int eval_flags);
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+   const int ND = T_ND ? T_ND : nd;
+   const int NQ = T_NQ ? T_NQ : nq;
+   const int NMAX = NQ > ND ? NQ : ND;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
+   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
+   MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
+   MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
+   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
+               "'geom' must be given (non-null) only when evaluating physical"
+               " derivatives");
+   const auto B = Reshape(maps.B.Read(), NQ, ND);
+   const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
+   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
+   const auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ, VDIM, NE):
+              Reshape(q_val.Write(), VDIM, NQ, NE);
+   auto der = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_der.Write(), NQ, VDIM, 2, NE):
+              Reshape(q_der.Write(), VDIM, 2, NQ, NE);
+   auto det = Reshape(q_det.Write(), NQ, NE);
+   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int ND = T_ND ? T_ND : nd;
+      const int NQ = T_NQ ? T_NQ : nq;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND2D;
+      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM2D;
+      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
+      MFEM_FOREACH_THREAD(d, x, ND)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            s_E[c+d*VDIM] = E(d,c,e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(q, x, NQ)
+      {
+         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
+         {
+            real_t ed[max_VDIM];
+            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t b = B(q,d);
+               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = ed[c]; }
+               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = ed[c]; }
+            }
+         }
+         if ((eval_flags & QI::DERIVATIVES) ||
+             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
+             (eval_flags & QI::DETERMINANTS))
+         {
+            // use MAX_VDIM2D to avoid "subscript out of range" warnings
+            real_t D[QI::MAX_VDIM2D*2];
+            for (int i = 0; i < 2*VDIM; i++) { D[i] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t wx = G(q,0,d);
+               const real_t wy = G(q,1,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  real_t s_e = s_E[c+d*VDIM];
+                  D[c+VDIM*0] += s_e * wx;
+                  D[c+VDIM*1] += s_e * wy;
+               }
+            }
+            if (eval_flags & QI::DERIVATIVES)
+            {
+               for (int c = 0; c < VDIM; c++)
+               {
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = D[c+VDIM*0];
+                     der(c,1,q,e) = D[c+VDIM*1];
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = D[c+VDIM*0];
+                     der(q,c,1,e) = D[c+VDIM*1];
+                  }
+               }
+            }
+            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
+            {
+               real_t Jloc[4], Jinv[4];
+               Jloc[0] = J(q,0,0,e);
+               Jloc[1] = J(q,1,0,e);
+               Jloc[2] = J(q,0,1,e);
+               Jloc[3] = J(q,1,1,e);
+               kernels::CalcInverse<2>(Jloc, Jinv);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  const real_t u = D[c+VDIM*0];
+                  const real_t v = D[c+VDIM*1];
+                  const real_t JiU = Jinv[0]*u + Jinv[1]*v;
+                  const real_t JiV = Jinv[2]*u + Jinv[3]*v;
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = JiU;
+                     der(c,1,q,e) = JiV;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = JiU;
+                     der(q,c,1,e) = JiV;
+                  }
+               }
+            }
+            if (eval_flags & QI::DETERMINANTS)
+            {
+               if (VDIM == 2) { det(q,e) = kernels::Det<2>(D); }
+               else
+               {
+                  DeviceTensor<2> j(D, 3, 2);
+                  const double E = j(0,0)*j(0,0) + j(1,0)*j(1,0) + j(2,0)*j(2,0);
+                  const double F = j(0,0)*j(0,1) + j(1,0)*j(1,1) + j(2,0)*j(2,1);
+                  const double G = j(0,1)*j(0,1) + j(1,1)*j(1,1) + j(2,1)*j(2,1);
+                  det(q,e) = std::sqrt(E*G - F*F);
+               }
+            }
+         }
+      }
+   });
+}
+
+// Template compute kernel for 3D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template<const int T_VDIM, const int T_ND, const int T_NQ>
+static void Eval3D(const int NE,
+                   const int vdim,
+                   const QVectorLayout q_layout,
+                   const GeometricFactors *geom,
+                   const DofToQuad &maps,
+                   const Vector &e_vec,
+                   Vector &q_val,
+                   Vector &q_der,
+                   Vector &q_det,
+                   const int eval_flags)
+{
+   using QI = QuadratureInterpolator;
+
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+   const int ND = T_ND ? T_ND : nd;
+   const int NQ = T_NQ ? T_NQ : nq;
+   const int NMAX = NQ > ND ? NQ : ND;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
+   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 3, "");
+   MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
+   MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
+   MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
+   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
+               "'geom' must be given (non-null) only when evaluating physical"
+               " derivatives");
+   const auto B = Reshape(maps.B.Read(), NQ, ND);
+   const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
+   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
+   const auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ, VDIM, NE):
+              Reshape(q_val.Write(), VDIM, NQ, NE);
+   auto der = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_der.Write(), NQ, VDIM, 3, NE):
+              Reshape(q_der.Write(), VDIM, 3, NQ, NE);
+   auto det = Reshape(q_det.Write(), NQ, NE);
+   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int ND = T_ND ? T_ND : nd;
+      const int NQ = T_NQ ? T_NQ : nq;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND3D;
+      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM3D;
+      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
+      MFEM_FOREACH_THREAD(d, x, ND)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            s_E[c+d*VDIM] = E(d,c,e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(q, x, NQ)
+      {
+         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
+         {
+            real_t ed[max_VDIM];
+            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t b = B(q,d);
+               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = ed[c]; }
+               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = ed[c]; }
+            }
+         }
+         if ((eval_flags & QI::DERIVATIVES) ||
+             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
+             (eval_flags & QI::DETERMINANTS))
+         {
+            // use MAX_VDIM3D to avoid "subscript out of range" warnings
+            real_t D[QI::MAX_VDIM3D*3];
+            for (int i = 0; i < 3*VDIM; i++) { D[i] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t wx = G(q,0,d);
+               const real_t wy = G(q,1,d);
+               const real_t wz = G(q,2,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  real_t s_e = s_E[c+d*VDIM];
+                  D[c+VDIM*0] += s_e * wx;
+                  D[c+VDIM*1] += s_e * wy;
+                  D[c+VDIM*2] += s_e * wz;
+               }
+            }
+            if (eval_flags & QI::DERIVATIVES)
+            {
+               for (int c = 0; c < VDIM; c++)
+               {
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = D[c+VDIM*0];
+                     der(c,1,q,e) = D[c+VDIM*1];
+                     der(c,2,q,e) = D[c+VDIM*2];
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = D[c+VDIM*0];
+                     der(q,c,1,e) = D[c+VDIM*1];
+                     der(q,c,2,e) = D[c+VDIM*2];
+                  }
+               }
+            }
+            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
+            {
+               real_t Jloc[9], Jinv[9];
+               for (int col = 0; col < 3; col++)
+               {
+                  for (int row = 0; row < 3; row++)
+                  {
+                     Jloc[row+3*col] = J(q,row,col,e);
+                  }
+               }
+               kernels::CalcInverse<3>(Jloc, Jinv);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  const real_t u = D[c+VDIM*0];
+                  const real_t v = D[c+VDIM*1];
+                  const real_t w = D[c+VDIM*2];
+                  const real_t JiU = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
+                  const real_t JiV = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
+                  const real_t JiW = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = JiU;
+                     der(c,1,q,e) = JiV;
+                     der(c,2,q,e) = JiW;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = JiU;
+                     der(q,c,1,e) = JiV;
+                     der(q,c,2,e) = JiW;
+                  }
+               }
+            }
+            if (VDIM == 3 && (eval_flags & QI::DETERMINANTS))
+            {
+               // The check (VDIM == 3) should eliminate this block when VDIM is
+               // known at compile time and (VDIM != 3).
+               det(q,e) = kernels::Det<3>(D);
+            }
+         }
+      }
+   });
+}
 
 } // namespace quadrature_interpolator
 
@@ -255,20 +535,10 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    const int nd = maps.ndof;
    const int nq = maps.nqpt;
    const GeometricFactors *geom = nullptr;
+   if (eval_flags & PHYSICAL_DERIVATIVES)
    {
-      int jac_factors = 0;
-      if (eval_flags & PHYSICAL_DERIVATIVES)
-      {
-         jac_factors = GeometricFactors::JACOBIANS;
-      }
-      if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-      {
-         jac_factors |= GeometricFactors::DETERMINANTS;
-      }
-      if (jac_factors)
-      {
-         geom = fespace->GetMesh()->GetGeometricFactors(*ir, jac_factors);
-      }
+      const int jacobians = GeometricFactors::JACOBIANS;
+      geom = fespace->GetMesh()->GetGeometricFactors(*ir, jacobians);
    }
 
    MFEM_ASSERT(!(eval_flags & DETERMINANTS) || dim == vdim ||
@@ -282,61 +552,29 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    {
       if (eval_flags & (VALUES | PHYSICAL_VALUES))
       {
-         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-         {
-            IntTensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne,
-                                      maps.B.Read(), geom->detJ.Read(),
-                                      e_vec.Read(), q_val.Write(), vdim, nd, nq);
-         }
-         else
-         {
-            TensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne,
-                                   maps.B.Read(), e_vec.Read(), q_val.Write(),
-                                   vdim, nd, nq);
-         }
+         TensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne, maps.B.Read(),
+                                e_vec.Read(), q_val.Write(), vdim, nd, nq);
       }
       if (eval_flags & (DERIVATIVES | PHYSICAL_DERIVATIVES))
       {
          const bool phys = (eval_flags & PHYSICAL_DERIVATIVES);
          const real_t *J = phys ? geom->J.Read() : nullptr;
          const int s_dim = phys ? sdim : dim;
-         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-         {
-            MFEM_ABORT("");
-         }
-         else
-         {
-            GradKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
-                             maps.B.Read(), maps.G.Read(), J, e_vec.Read(),
-                             q_der.Write(), s_dim, vdim, nd, nq);
-         }
+         GradKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
+                          maps.B.Read(), maps.G.Read(), J, e_vec.Read(),
+                          q_der.Write(), s_dim, vdim, nd, nq);
       }
       if (eval_flags & DETERMINANTS)
       {
-         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-         {
-            MFEM_ABORT("");
-         }
-         else
-         {
-            DetKernels::Run(dim, vdim, nd, nq, ne, maps.B.Read(), maps.G.Read(),
-                            e_vec.Read(), q_det.Write(), nd, nq, &d_buffer);
-         }
+         DetKernels::Run(dim, vdim, nd, nq, ne, maps.B.Read(),
+                         maps.G.Read(), e_vec.Read(), q_det.Write(), nd,
+                         nq, &d_buffer);
       }
    }
    else // use_tensor_eval == false
    {
-      if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-      {
-         IntEvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne, vdim,
-                             q_layout, geom->detJ.Read(), geom, maps, e_vec,
-                             q_val, q_der, q_det, eval_flags);
-      }
-      else
-      {
-         EvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne, vdim, q_layout,
-                          geom, maps, e_vec, q_val, q_der, q_det, eval_flags);
-      }
+      EvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne,vdim, q_layout,
+                       geom, maps, e_vec, q_val, q_der, q_det, eval_flags);
    }
 }
 
@@ -462,41 +700,22 @@ namespace
 
 using namespace internal::quadrature_interpolator;
 
-template <QVectorLayout Q_LAYOUT> auto IntFallbackTensorEvalKernel(int DIM)
+using EvalKernel = QuadratureInterpolator::EvalKernelType;
+using TensorEvalKernel = QuadratureInterpolator::TensorEvalKernelType;
+using GradKernel = QuadratureInterpolator::GradKernelType;
+using CollocatedGradKernel = QuadratureInterpolator::CollocatedGradKernelType;
+
+template <QVectorLayout Q_LAYOUT>
+TensorEvalKernel FallbackTensorEvalKernel(int DIM)
 {
-   if (DIM == 1)
-   {
-      return ImplValues1D<Q_LAYOUT, true>;
-   }
-   else if (DIM == 2)
-   {
-      return ImplValues2D<Q_LAYOUT, true>;
-   }
-   else if (DIM == 3)
-   {
-      return ImplValues3D<Q_LAYOUT, true>;
-   }
-   MFEM_ABORT("");
+   if (DIM == 1) { return Values1D<Q_LAYOUT>; }
+   else if (DIM == 2) { return Values2D<Q_LAYOUT>; }
+   else if (DIM == 3) { return Values3D<Q_LAYOUT>; }
+   else { MFEM_ABORT(""); }
 }
 
-template <QVectorLayout Q_LAYOUT> auto FallbackTensorEvalKernel(int DIM)
-{
-   if (DIM == 1)
-   {
-      return Values1D<Q_LAYOUT>;
-   }
-   else if (DIM == 2)
-   {
-      return Values2D<Q_LAYOUT>;
-   }
-   else if (DIM == 3)
-   {
-      return Values3D<Q_LAYOUT>;
-   }
-   MFEM_ABORT("");
-}
-
-template <QVectorLayout Q_LAYOUT, bool GRAD_PHYS> auto GetGradKernel(int DIM)
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
+GradKernel GetGradKernel(int DIM)
 {
    if (DIM == 1) { return Derivatives1D<Q_LAYOUT, GRAD_PHYS>; }
    else if (DIM == 2) { return Derivatives2D<Q_LAYOUT, GRAD_PHYS>; }
@@ -504,185 +723,79 @@ template <QVectorLayout Q_LAYOUT, bool GRAD_PHYS> auto GetGradKernel(int DIM)
    else { MFEM_ABORT(""); }
 }
 
-template <QVectorLayout Q_LAYOUT> auto GetGradKernel(int DIM, bool GRAD_PHYS)
+
+template<QVectorLayout Q_LAYOUT>
+GradKernel GetGradKernel(int DIM, bool GRAD_PHYS)
 {
    if (GRAD_PHYS) { return GetGradKernel<Q_LAYOUT, true>(DIM); }
    else { return GetGradKernel<Q_LAYOUT, false>(DIM); }
 }
 
 template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
-auto GetCollocatedGradKernel(int DIM)
+CollocatedGradKernel GetCollocatedGradKernel(int DIM)
 {
-   if (DIM == 1)
-   {
-      return CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>;
-   }
-   else if (DIM == 2)
-   {
-      return CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS>;
-   }
-   else if (DIM == 3)
-   {
-      return CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS>;
-   }
-   MFEM_ABORT("");
+   if (DIM == 1) { return CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 2) { return CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 3) { return CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS>; }
+   else { MFEM_ABORT(""); }
 }
 
-template <QVectorLayout Q_LAYOUT>
-auto GetCollocatedGradKernel(int DIM, bool GRAD_PHYS)
+template<QVectorLayout Q_LAYOUT>
+CollocatedGradKernel GetCollocatedGradKernel(int DIM, bool GRAD_PHYS)
 {
    if (GRAD_PHYS) { return GetCollocatedGradKernel<Q_LAYOUT, true>(DIM); }
    else { return GetCollocatedGradKernel<Q_LAYOUT, false>(DIM); }
 }
-
-auto GetCollocatedGradKernel(int DIM, bool GRAD_PHYS, QVectorLayout Q_LAYOUT)
-{
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      return GetCollocatedGradKernel<QVectorLayout::byNODES>(
-                DIM, GRAD_PHYS);
-   }
-   else
-   {
-      return GetCollocatedGradKernel<QVectorLayout::byVDIM>(
-                DIM, GRAD_PHYS);
-   }
-}
 } // namespace
 
-template <int DIM, bool Integral>
-auto GetEvalKernelVDimFallback(int VDIM)
+template <int DIM, int VDIM, int ND, int NQ>
+EvalKernel QuadratureInterpolator::EvalKernels::Kernel()
 {
-   if constexpr (Integral)
-   {
-      using EvalKernels = QuadratureInterpolator::IntEvalKernels;
-      if (VDIM == 1)
-      {
-         return EvalKernels::Kernel<DIM, 1, 0, 0>();
-      }
-      else if (VDIM == 2)
-      {
-         return EvalKernels::Kernel<DIM, 2, 0, 0>();
-      }
-      else if (VDIM == 3)
-      {
-         return EvalKernels::Kernel<DIM, 3, 0, 0>();
-      }
-   }
-   if constexpr (!Integral)
-   {
-      using EvalKernels = QuadratureInterpolator::EvalKernels;
-      if (VDIM == 1)
-      {
-         return EvalKernels::Kernel<DIM, 1, 0, 0>();
-      }
-      else if (VDIM == 2)
-      {
-         return EvalKernels::Kernel<DIM, 2, 0, 0>();
-      }
-      else if (VDIM == 3)
-      {
-         return EvalKernels::Kernel<DIM, 3, 0, 0>();
-      }
-   }
+   using namespace internal::quadrature_interpolator;
+   if constexpr (DIM == 1) { return Eval1D; }
+   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
+   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
    MFEM_ABORT("");
 }
 
-template auto GetEvalKernelVDimFallback<1, true>(int VDIM);
-template auto GetEvalKernelVDimFallback<1, false>(int VDIM);
-template auto GetEvalKernelVDimFallback<2, true>(int VDIM);
-template auto GetEvalKernelVDimFallback<2, false>(int VDIM);
-template auto GetEvalKernelVDimFallback<3, true>(int VDIM);
-template auto GetEvalKernelVDimFallback<3, false>(int VDIM);
-
-QuadratureInterpolator::IntEvalKernelType
-QuadratureInterpolator::IntEvalKernels::Fallback(int DIM, int VDIM, int ND,
-                                                 int NQ)
+template <int DIM>
+EvalKernel GetEvalKernelVDimFallback(int VDIM)
 {
-   if (DIM == 1)
-   {
-      return GetEvalKernelVDimFallback<1, true>(VDIM);
-   }
-   else if (DIM == 2)
-   {
-      return GetEvalKernelVDimFallback<2, true>(VDIM);
-   }
-   else if (DIM == 3)
-   {
-      return GetEvalKernelVDimFallback<3, true>(VDIM);
-   }
-   else
-   {
-      MFEM_ABORT("");
-   }
+   using EvalKernels = QuadratureInterpolator::EvalKernels;
+   if (VDIM == 1) { return EvalKernels::Kernel<DIM,1,0,0>(); }
+   else if (VDIM == 2) { return EvalKernels::Kernel<DIM,2,0,0>(); }
+   else if (VDIM == 3) { return EvalKernels::Kernel<DIM,3,0,0>(); }
+   else { MFEM_ABORT(""); }
 }
 
-QuadratureInterpolator::EvalKernelType
-QuadratureInterpolator::EvalKernels::Fallback(int DIM, int VDIM, int ND, int NQ)
+EvalKernel QuadratureInterpolator::EvalKernels::Fallback(
+   int DIM, int VDIM, int ND, int NQ)
 {
-   if (DIM == 1)
-   {
-      return GetEvalKernelVDimFallback<1, false>(VDIM);
-   }
-   else if (DIM == 2)
-   {
-      return GetEvalKernelVDimFallback<2, false>(VDIM);
-   }
-   else if (DIM == 3)
-   {
-      return GetEvalKernelVDimFallback<3, false>(VDIM);
-   }
-   else
-   {
-      MFEM_ABORT("");
-   }
+   if (DIM == 1) { return GetEvalKernelVDimFallback<1>(VDIM); }
+   else if (DIM == 2) { return GetEvalKernelVDimFallback<2>(VDIM); }
+   else if (DIM == 3) { return GetEvalKernelVDimFallback<3>(VDIM); }
+   else { MFEM_ABORT(""); }
 }
 
-QuadratureInterpolator::IntTensorEvalKernelType
-QuadratureInterpolator::IntTensorEvalKernels::Fallback(int DIM,
-                                                       QVectorLayout Q_LAYOUT,
-                                                       int, int, int)
+TensorEvalKernel QuadratureInterpolator::TensorEvalKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, int, int, int)
 {
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      return IntFallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM);
-   }
-   else
-   {
-      return IntFallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM);
-   }
+   if (Q_LAYOUT == QVectorLayout::byNODES) { return FallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM); }
+   else { return FallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM); }
 }
 
-QuadratureInterpolator::TensorEvalKernelType
-QuadratureInterpolator::TensorEvalKernels::Fallback(int DIM,
-                                                    QVectorLayout Q_LAYOUT, int,
-                                                    int, int)
-{
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      return FallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM);
-   }
-   else
-   {
-      return FallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM);
-   }
-}
-
-QuadratureInterpolator::GradKernelType
-QuadratureInterpolator::GradKernels::Fallback(int DIM, QVectorLayout Q_LAYOUT,
-                                              bool GRAD_PHYS, int, int, int)
+GradKernel QuadratureInterpolator::GradKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int, int)
 {
    if (Q_LAYOUT == QVectorLayout::byNODES) { return GetGradKernel<QVectorLayout::byNODES>(DIM, GRAD_PHYS); }
    else { return GetGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
 }
 
-QuadratureInterpolator::CollocatedGradKernelType
-QuadratureInterpolator::CollocatedGradKernels::Fallback(int DIM,
-                                                        QVectorLayout Q_LAYOUT,
-                                                        bool GRAD_PHYS, int,
-                                                        int)
+CollocatedGradKernel QuadratureInterpolator::CollocatedGradKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int)
 {
-   return GetCollocatedGradKernel(DIM, GRAD_PHYS, Q_LAYOUT);
+   if (Q_LAYOUT == QVectorLayout::byNODES) { return GetCollocatedGradKernel<QVectorLayout::byNODES>(DIM, GRAD_PHYS); }
+   else { return GetCollocatedGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
 }
 
 /// @endcond
@@ -693,97 +806,98 @@ namespace quadrature_interpolator
 {
 void InitEvalKernels()
 {
+   using k = QuadratureInterpolator::EvalKernels;
    // 2D, VDIM = 1
-   QuadratureInterpolator::AddEvalSpecializations<2,1,1,1>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,1,4>();
+   k::Specialization<2,1,1,1>::Add();
+   k::Specialization<2,1,1,4>::Add();
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<2,1,4,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,4,9>();
+   k::Specialization<2,1,4,4>::Add();
+   k::Specialization<2,1,4,9>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<2,1,9,9>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,9,16>();
+   k::Specialization<2,1,9,9>::Add();
+   k::Specialization<2,1,9,16>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<2,1,16,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,16,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,16,36>();
+   k::Specialization<2,1,16,16>::Add();
+   k::Specialization<2,1,16,25>::Add();
+   k::Specialization<2,1,16,36>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,36>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,49>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,64>();
+   k::Specialization<2,1,25,25>::Add();
+   k::Specialization<2,1,25,36>::Add();
+   k::Specialization<2,1,25,49>::Add();
+   k::Specialization<2,1,25,64>::Add();
 
    // 3D, VDIM = 1
    // Q0
-   QuadratureInterpolator::AddEvalSpecializations<3,1,1,1>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,1,8>();
+   k::Specialization<3,1,1,1>::Add();
+   k::Specialization<3,1,1,8>::Add();
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<3,1,8,8>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,8,27>();
+   k::Specialization<3,1,8,8>::Add();
+   k::Specialization<3,1,8,27>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<3,1,27,27>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,27,64>();
+   k::Specialization<3,1,27,27>::Add();
+   k::Specialization<3,1,27,64>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<3,1,64,64>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,64,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,64,216>();
+   k::Specialization<3,1,64,64>::Add();
+   k::Specialization<3,1,64,125>::Add();
+   k::Specialization<3,1,64,216>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<3,1,125,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,125,216>();
+   k::Specialization<3,1,125,125>::Add();
+   k::Specialization<3,1,125,216>::Add();
 
    // 2D, VDIM = 3
    // Q0
-   QuadratureInterpolator::AddEvalSpecializations<2,3,1,1>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,1,4>();
+   k::Specialization<2,3,1,1>::Add();
+   k::Specialization<2,3,1,4>::Add();
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<2,3,4,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,4,9>();
+   k::Specialization<2,3,4,4>::Add();
+   k::Specialization<2,3,4,9>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,9>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,25>();
+   k::Specialization<2,3,9,4>::Add();
+   k::Specialization<2,3,9,9>::Add();
+   k::Specialization<2,3,9,16>::Add();
+   k::Specialization<2,3,9,25>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<2,3,16,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,16,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,16,36>();
+   k::Specialization<2,3,16,16>::Add();
+   k::Specialization<2,3,16,25>::Add();
+   k::Specialization<2,3,16,36>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,36>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,49>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,64>();
+   k::Specialization<2,3,25,25>::Add();
+   k::Specialization<2,3,25,36>::Add();
+   k::Specialization<2,3,25,49>::Add();
+   k::Specialization<2,3,25,64>::Add();
 
    // 2D, VDIM = 2
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<2,2,4,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,4,9>();
+   k::Specialization<2,2,4,4>::Add();
+   k::Specialization<2,2,4,9>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<2,2,9,9>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,9,16>();
+   k::Specialization<2,2,9,9>::Add();
+   k::Specialization<2,2,9,16>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<2,2,16,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,16,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,16,36>();
+   k::Specialization<2,2,16,16>::Add();
+   k::Specialization<2,2,16,25>::Add();
+   k::Specialization<2,2,16,36>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,36>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,49>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,64>();
+   k::Specialization<2,2,25,25>::Add();
+   k::Specialization<2,2,25,36>::Add();
+   k::Specialization<2,2,25,49>::Add();
+   k::Specialization<2,2,25,64>::Add();
 
    // 3D, VDIM = 3
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<3,3,8,8>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,8,27>();
+   k::Specialization<3,3,8,8>::Add();
+   k::Specialization<3,3,8,27>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<3,3,27,27>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,27,64>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,27,125>();
+   k::Specialization<3,3,27,27>::Add();
+   k::Specialization<3,3,27,64>::Add();
+   k::Specialization<3,3,27,125>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<3,3,64,64>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,64,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,64,216>();
+   k::Specialization<3,3,64,64>::Add();
+   k::Specialization<3,3,64,125>::Add();
+   k::Specialization<3,3,64,216>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<3,3,125,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,125,216>();
+   k::Specialization<3,3,125,125>::Add();
+   k::Specialization<3,3,125,216>::Add();
 }
 
 } // namespace quadrature_Interpolator

fem/quadinterpolator.hpp

@@ -117,10 +117,6 @@ public:
        FiniteElementSpace is a vector space) and their determinants are computed
        and stored in @a q_det.
 
-       For Integral spaces, the flags VALUES requests the computation of the
-       scalar field values. The result is stored in @a q_val. Derivative types
-       are not supported.
-
        For H(div)-conforming spaces, the flags VALUES / PHYSICAL_VALUES request
        the computation of the vector field values in reference or physical
        space, respectively. The flag PHYSICAL_MAGNITUDES requests the
@@ -163,49 +159,26 @@ public:
    /// QuadratureInterpolator.
    static bool SupportsFESpace(const FiniteElementSpace &fespace);
 
-   // value map types
-   using TensorEvalKernelType = void (*)(const int ne, const real_t *B,
-                                         const real_t *e_vec, real_t *q_val,
-                                         const int vdim, const int nd,
-                                         const int nq);
-   using GradKernelType = void (*)(const int ne, const real_t *B,
-                                   const real_t *G, const real_t *J,
-                                   const real_t *e_vec, real_t *q_der,
-                                   const int s_dim, const int v_dim,
-                                   const int nd, const int nq);
-   using CollocatedGradKernelType = void (*)(const int ne, const real_t *G,
-                                             const real_t *J,
-                                             const real_t *e_vec, real_t *q_der,
-                                             const int sdim, const int vdim,
-                                             const int d1d);
-   using DetKernelType = void (*)(const int NE, const real_t *B,
-                                  const real_t *G, const real_t *e_vec,
-                                  real_t *q_det, const int nd, const int nq,
-                                  Vector *d_buffer);
-   using EvalKernelType = void (*)(const int NE, const int vdim,
-                                   const QVectorLayout q_layout,
-                                   const GeometricFactors *geom,
-                                   const DofToQuad &maps, const Vector &e_vec,
-                                   Vector &q_val, Vector &q_der, Vector &q_det,
-                                   const int eval_flags);
-
-   // integral map types
-   using IntTensorEvalKernelType = void (*)(const int ne, const real_t *B,
-                                            const real_t *detJ,
-                                            const real_t *e_vec, real_t *q_val,
-                                            const int vdim, const int nd,
-                                            const int nq);
-   using IntEvalKernelType =
-      void (*)(const int NE, const int vdim, const QVectorLayout q_layout,
-               const real_t *detJ, const GeometricFactors *geom,
-               const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-               Vector &q_der, Vector &q_det, const int eval_flags);
-
+   using TensorEvalKernelType = void(*)(const int, const real_t *, const real_t *,
+                                        real_t *, const int, const int, const int);
+   using GradKernelType = void(*)(const int, const real_t *, const real_t *,
+                                  const real_t *, const real_t *, real_t *,
+                                  const int, const int, const int, const int);
+   using CollocatedGradKernelType = void(*)(const int, const real_t *,
+                                            const real_t *, const real_t *,
+                                            real_t *, const int, const int,
+                                            const int);
+   using DetKernelType = void(*)(const int NE, const real_t *, const real_t *,
+                                 const real_t *, real_t *, const int, const int,
+                                 Vector *);
+   using EvalKernelType = void(*)(const int, const int, const QVectorLayout,
+                                  const GeometricFactors *, const DofToQuad &,
+                                  const Vector &, Vector &, Vector &, Vector &,
+                                  const int);
    using TensorEvalHDivKernelType =
       void(*)(const int, const real_t *, const real_t *, const real_t *,
               const real_t *, real_t *, const int, const int);
 
-   // value-type mapping
    MFEM_REGISTER_KERNELS(TensorEvalKernels, TensorEvalKernelType,
                          (int, QVectorLayout, int, int, int), (int));
    MFEM_REGISTER_KERNELS(GradKernels, GradKernelType,
@@ -214,84 +187,8 @@ public:
    MFEM_REGISTER_KERNELS(EvalKernels, EvalKernelType, (int, int, int, int));
    MFEM_REGISTER_KERNELS(CollocatedGradKernels, CollocatedGradKernelType,
                          (int, QVectorLayout, bool, int, int), (int));
-
-   // integral-type mapping
-   MFEM_REGISTER_KERNELS(IntTensorEvalKernels, IntTensorEvalKernelType,
-                         (int, QVectorLayout, int, int, int), (int));
-   MFEM_REGISTER_KERNELS(IntEvalKernels, IntEvalKernelType, (int, int, int, int));
-
    MFEM_REGISTER_KERNELS(TensorEvalHDivKernels, TensorEvalHDivKernelType,
                          (int, QVectorLayout, unsigned, int, int));
-
-   /// Adds specializations for TensorEvalKernels
-   template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D,
-             int NBZ = 0>
-   static void AddTensorEvalSpecializations()
-   {
-      if constexpr (NBZ)
-      {
-         IntTensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                              Q1D>::template Opt<NBZ>::Add();
-         TensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                           Q1D>::template Opt<NBZ>::Add();
-      }
-      else if constexpr (NBZ == 0)
-      {
-         IntTensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                              Q1D>::Add();
-         TensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                           Q1D>::Add();
-      }
-   }
-
-   /// Adds specializations for EvalKernels
-   template <int DIM, int VDIM, int ND, int NQ>
-   static void AddEvalSpecializations()
-   {
-      IntEvalKernels::Specialization<DIM, VDIM, ND, NQ>::Add();
-      EvalKernels::Specialization<DIM, VDIM, ND, NQ>::Add();
-   }
-
-   /// Adds specializations for GradKernels
-   template <int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
-             int Q1D, int NBZ = 0>
-   static void AddGradSpecializations()
-   {
-      if constexpr (NBZ)
-      {
-         GradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM, D1D,
-                     Q1D>::template Opt<NBZ>::Add();
-      }
-      else if constexpr (NBZ == 0)
-      {
-         GradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM, D1D,
-                     Q1D>::Add();
-      }
-   }
-
-   /// Adds specializations for CollocatedGradKernels
-   template <int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
-             int NBZ = 0>
-   static void AddCollocatedGradSpecializations()
-   {
-      if constexpr (NBZ)
-      {
-         CollocatedGradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM,
-                               D1D>::template Opt<NBZ>::Add();
-      }
-      else if constexpr (NBZ == 0)
-      {
-         CollocatedGradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM,
-                               D1D>::Add();
-      }
-   }
-
-   /// Adds specializations for DetKernels
-   template <int DIM, int SDIM, int D1D, int Q1D>
-   static void AddDetSpecializations()
-   {
-      DetKernels::Specialization<DIM, SDIM, D1D, Q1D>::Add();
-   }
 };
 
 }

fem/transfer.cpp

@@ -231,11 +231,9 @@ const Operator &InterpolationGridTransfer::BackwardOperator()
 
 L2ProjectionGridTransfer::L2Projection::L2Projection(
    const FiniteElementSpace &fes_ho_, const FiniteElementSpace &fes_lor_,
-   CoefficientWithOrder coeff_ho_, CoefficientWithOrder coeff_lor_,
    MemoryType d_mt_)
    : Operator(fes_lor_.GetVSize(), fes_ho_.GetVSize()),
-     fes_ho(fes_ho_), fes_lor(fes_lor_), coeff_ho(coeff_ho_),
-     coeff_lor(coeff_lor_), d_mt(d_mt_)
+     fes_ho(fes_ho_), fes_lor(fes_lor_), d_mt(d_mt_)
 { }
 
 void L2ProjectionGridTransfer::L2Projection::BuildHo2Lor(
@@ -265,13 +263,12 @@ void L2ProjectionGridTransfer::L2Projection::ElemMixedMass(
    IntegrationPointTransformation& ip_tr,
    DenseMatrix& M_mixed_el) const
 {
-   int order = fe_lor.GetOrder() + fe_ho.GetOrder() + tr_lor->OrderW() +
-               coeff_ho.order;
-   const IntegrationRule &ir = IntRules.Get(geom, order);
+   int order = fe_lor.GetOrder() + fe_ho.GetOrder() + tr_lor->OrderW();
+   const IntegrationRule* ir = &IntRules.Get(geom, order);
    M_mixed_el = 0.0;
-   for (int i = 0; i < ir.GetNPoints(); i++)
+   for (int i = 0; i < ir->GetNPoints(); i++)
    {
-      const IntegrationPoint& ip_lor = ir.IntPoint(i);
+      const IntegrationPoint& ip_lor = ir->IntPoint(i);
       IntegrationPoint ip_ho;
       ip_tr.Transform(ip_lor, ip_ho);
       Vector shape_lor(fe_lor.GetDof());
@@ -287,23 +284,23 @@ void L2ProjectionGridTransfer::L2Projection::ElemMixedMass(
       {
          w *= tr_lor->Weight();
       }
-      if (coeff_ho)
-      {
-         w *= coeff_ho.coeff->Eval(*tr_ho, ip_ho);
-      }
       shape_lor *= w;
       AddMultVWt(shape_lor, shape_ho, M_mixed_el);
    }
 }
 
-void L2ProjectionGridTransfer::L2Projection::ElemMixedEvaluation(
-   Geometry::Type geom, const FiniteElement& fe_ho, const FiniteElement& fe_lor,
-   IntegrationPointTransformation& ip_tr, const IntegrationRule& ir,
+void L2ProjectionGridTransfer::L2Projection::ElemMixedMass(
+   Geometry::Type geom, const FiniteElement& fe_ho,
+   const FiniteElement& fe_lor, ElementTransformation* el_tr,
+   IntegrationPointTransformation& ip_tr,
    DenseMatrix& B_L, DenseMatrix& B_H) const
 {
-   for (int i = 0; i < ir.GetNPoints(); i++)
+   int order = fe_lor.GetOrder() + fe_ho.GetOrder() + el_tr->OrderW();
+   const IntegrationRule* ir = &IntRules.Get(geom, order);
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
    {
-      const IntegrationPoint& ip_lor = ir.IntPoint(i);
+      const IntegrationPoint& ip_lor = ir->IntPoint(i);
       IntegrationPoint ip_ho;
 
       // maps integration point ip_lor -> ip_ho
@@ -323,6 +320,7 @@ void L2ProjectionGridTransfer::L2Projection::ElemMixedEvaluation(
          B_H(i, j) = shape_ho(j);
       }
    }
+
 }
 
 void L2ProjectionGridTransfer::L2Projection::MixedMassEA(
@@ -330,11 +328,10 @@ void L2ProjectionGridTransfer::L2Projection::MixedMassEA(
    const FiniteElementSpace& fes_lor_ea,
    Vector &M_LH, MemoryType d_mt_)
 {
-   Mesh &mesh_ho = *fes_ho_ea.GetMesh();
-   Mesh &mesh_lor = *fes_lor_ea.GetMesh();
-
-   const int nel_ho = mesh_ho.GetNE();
-   const int nel_lor = mesh_lor.GetNE();
+   Mesh* mesh_ho = fes_ho_ea.GetMesh();
+   Mesh* mesh_lor = fes_lor_ea.GetMesh();
+   int nel_ho = mesh_ho->GetNE();
+   int nel_lor = mesh_lor->GetNE();
 
    if (nel_ho == 0)
    {
@@ -342,11 +339,11 @@ void L2ProjectionGridTransfer::L2Projection::MixedMassEA(
       return;
    }
 
-   const CoarseFineTransformations& cf_tr = mesh_lor.GetRefinementTransforms();
+   const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();
 
    int nref_max = 0;
    Array<Geometry::Type> geoms;
-   mesh_ho.GetGeometries(mesh_ho.Dimension(), geoms);
+   mesh_ho->GetGeometries(mesh_ho->Dimension(), geoms);
    for (int ig = 0; ig < geoms.Size(); ++ig)
    {
       Geometry::Type geom = geoms[ig];
@@ -363,226 +360,130 @@ void L2ProjectionGridTransfer::L2Projection::MixedMassEA(
    {
       // Assume all HO elements are LOR in the same way
       const int iho = 0;
-      Array<int> lor_els;
-      ho2lor.GetRow(iho, lor_els);
-      const int nref = ho2lor.RowSize(iho);
-      MFEM_VERIFY(nel_ho*nref == nel_lor, "we expect nel_ho*nref == nel_lor");
-
-      Geometry::Type geom = mesh_ho.GetElementBaseGeometry(iho);
-
-      emb_tr.SetIdentityTransformation(geom);
-      const DenseTensor &pmats = cf_tr.point_matrices[geom];
-
-      const FiniteElement &fe_ho = *fes_ho_ea.GetFE(iho);
-      const FiniteElement &fe_lor = *fes_lor_ea.GetFE(lor_els[0]);
-
-      // Allocate space for DenseTensors
-      ElementTransformation &el_tr = *mesh_lor.GetTypicalElementTransformation();
-      const int order = fe_lor.GetOrder() + fe_ho.GetOrder() + el_tr.OrderW()
-                        + coeff_ho.order;
-      const IntegrationRule &ir_ea = IntRules.Get(geom, order);
-      const int qPts = ir_ea.GetNPoints();
-
-      // Containers for the basis functions sampled at quadrature points
-      B_L.SetSize(qPts, fe_lor.GetDof(), nref, d_mt);
-      B_H.SetSize(qPts, fe_ho.GetDof(), nref, d_mt);
-      D.SetSize(qPts, nref, nel_ho, d_mt);
-
-      const GeometricFactors *geo_facts =
-         mesh_lor.GetGeometricFactors(ir_ea, GeometricFactors::DETERMINANTS);
-
-      Vector coeff_vec(qPts*nel_lor);
-      coeff_vec.UseDevice(true);
-
-      const int dim = mesh_ho.Dimension();
-      const int nq1d = (int)floor(pow(ir_ea.Size(), 1.0/dim) + 0.5);
-      const int nref_1d = (int)floor(pow(nref, 1.0/dim) + 0.5);
-
-      if (!coeff_ho)
       {
-         coeff_vec = 1.0;
-      }
-      else if (UsesTensorBasis(fes_ho) &&
-               nq1d*nref_1d <= DeviceDofQuadLimits::Get().MAX_Q1D)
-      {
-         // Fast coefficient evaluation for tensor-product case. We create a
-         // "composite" quadrature rule in the high-order element that is the
-         // union of the quadrature rules within each of the low-order-refined
-         // subelements.
-         //
-         // NOTE: if the integration rule order is high and there are many LOR
-         // subelements, this can create a very big quadrature rule. That is
-         // why we need to check that we do not exceed MAX_Q1D. If we do, then
-         // we fall back on the slower "legacy" evaluation.
+         Array<int> lor_els;
+         ho2lor.GetRow(iho, lor_els);
+         int nref = ho2lor.RowSize(iho);
 
-         // Construct the composite rule as a tensor-product of the 1D LOR rule.
-         IntegrationRule ir_ho = [&]()
+         Geometry::Type geom = mesh_ho->GetElementBaseGeometry(iho);
+         const FiniteElement &fe_ho = *fes_ho_ea.GetFE(iho);
+         const FiniteElement &fe_lor = *fes_lor_ea.GetFE(lor_els[0]);
+
+         // Allocate space for DenseTensors
+         ElementTransformation *el_tr = fes_lor_ea.GetElementTransformation(0);
+         int order = fe_lor.GetOrder() + fe_ho.GetOrder() + el_tr->OrderW();
+         const IntegrationRule* ir_ea = &IntRules.Get(geom, order);
+         int qPts = ir_ea->GetNPoints();
+
+         // Containers for the basis functions sampled at quadrature points
+         B_L.SetSize(qPts, fe_lor.GetDof(), nref, d_mt);
+         B_H.SetSize(qPts, fe_ho.GetDof(), nref, d_mt);
+         D.SetSize(qPts, nref, nel_ho, d_mt);
+
+         const GeometricFactors *geo_facts =
+            mesh_lor->GetGeometricFactors(*ir_ea, GeometricFactors::DETERMINANTS);
+
+         MFEM_ASSERT(nel_ho*nref == nel_lor, "we expect nel_ho*nref == nel_lor");
+
+         // Setup data at quadrature points
+         // TODO add support for user coefficient
+         const auto W = Reshape(ir_ea->GetWeights().Read(), qPts);
+         const auto J = Reshape(geo_facts->detJ.Read(), qPts, nel_lor);
+         const auto d_D = Reshape(D.Write(), qPts, nref, nel_ho);
+
+         mfem::forall(qPts * nref * nel_ho, [=] MFEM_HOST_DEVICE (int tid)
          {
-            IntegrationRule ir_ho_1d(nq1d * nref_1d);
-            for (int iref = 0; iref < nref_1d; ++iref)
-            {
-               const real_t a = pmats(cf_tr.embeddings[iref].matrix)(0,0);
-               const real_t b = pmats(cf_tr.embeddings[iref].matrix)(0,1);
-               for (int iq = 0; iq < nq1d; ++iq)
-               {
-                  ir_ho_1d[iq + iref*nq1d].x = a + ir_ea[iq].x*(b - a);
-               }
-            }
-            if (dim == 1) { return ir_ho_1d; }
-            else if (dim == 2) { return IntegrationRule(ir_ho_1d, ir_ho_1d); }
-            else { return IntegrationRule(ir_ho_1d, ir_ho_1d, ir_ho_1d); }
-         }();
+            const int q    = tid % qPts;
+            const int iref = (tid / qPts) % nref;
+            const int iho  = (tid / (qPts * nref)) % nel_ho;
 
-         // Project the high-order coefficient on the high-order composite rule.
-         QuadratureSpace qs(mesh_ho, ir_ho);
-         CoefficientVector coeff_vec_ho(*coeff_ho.coeff, qs);
+            const int lo_el_id = iref + nref*iho;
+            const real_t detJ = J(q, lo_el_id);
 
-         // Permute the coefficient values to the expected LOR ordering.
-         const int nq_ho = ir_ho.Size();
-         const auto d_Q_ho = Reshape(coeff_vec_ho.Read(), nq_ho, nel_ho);
-         const auto d_Q = Reshape(coeff_vec.Write(), qPts, nel_lor);
+            d_D(q, iref, iho) = W(q) * detJ;
 
-         mfem::forall(nq_ho * nel_ho, [=] MFEM_HOST_DEVICE (int ii)
-         {
-            const int e_ho = ii / nq_ho;
-            const int iq_ho = ii % nq_ho;
-
-            int iq_tensor = iq_ho;
-            int iq_lor = 0;
-            int iref = 0;
-            int iq_stride = 1;
-            int iref_stride = 1;
-            const int nq_ho_1d = nq1d*nref_1d;
-            for (int d = 0; d < dim; ++d)
-            {
-               const int iq_ho_1d = iq_tensor % nq_ho_1d;
-               iq_tensor /= nq_ho_1d;
-
-               iq_lor += (iq_ho_1d % nq1d)*iq_stride;
-               iref += (iq_ho_1d / nq1d)*iref_stride;
-               iq_stride *= nq1d;
-               iref_stride *= nref_1d;
-            }
-            const int e_lor = iref + e_ho*nref;
-
-            d_Q(iq_lor, e_lor) = d_Q_ho(iq_ho, e_ho);
          });
-      }
-      else
-      {
-         // Legacy/fallback coefficient evaluation for non-tensor-product cases
-         // or when the number of quadrature points is too large for the device
-         // kernels.
-         IntegrationPoint ip_ho;
-         for (int e_ho = 0; e_ho < nel_ho; ++e_ho)
+
+         emb_tr.SetIdentityTransformation(geom);
+         const DenseTensor &pmats = cf_tr.point_matrices[geom];
+
+         // Collect the basis functions
+         for (int iref = 0; iref < nref; ++iref)
          {
-            ElementTransformation &ho_tr = *mesh_ho.GetElementTransformation(e_ho);
-            for (int iref = 0; iref < nref; ++iref)
-            {
-               const int e_lor = iref + e_ho*nref;
-               emb_tr.SetPointMat(pmats(cf_tr.embeddings[e_lor].matrix));
+            int ilor = lor_els[iref];
+            // Now assemble the block-row of the mixed mass matrix associated
+            // with integrating HO functions against LOR functions on the LOR
+            // sub-element.
 
-               for (int iq = 0; iq < qPts; ++iq)
-               {
-                  const IntegrationPoint &ip_lor = ir_ea[iq];
-                  ip_tr.Transform(ip_lor, ip_ho);
-                  ho_tr.SetIntPoint(&ip_ho);
-                  coeff_vec[iq + e_lor*qPts] = coeff_ho.coeff->Eval(ho_tr, ip_ho);
-               }
-            }
-         }
+            // Create the transformation that embeds the fine low-order element
+            // within the coarse high-order element in reference space
+            emb_tr.SetPointMat(pmats(cf_tr.embeddings[ilor].matrix));
+
+            DenseMatrix &b_lo = B_L(ilor);
+            DenseMatrix &b_ho = B_H(ilor);
+
+            ElemMixedMass(geom, fe_ho, fe_lor, el_tr, ip_tr, b_lo, b_ho);
+
+         } // loop over subcells of ho element
+         // end of quadrature point setup
       }
 
-      // Setup data at quadrature points
-      const auto W = Reshape(ir_ea.GetWeights().Read(), qPts);
-      const auto J = Reshape(geo_facts->detJ.Read(), qPts, nel_lor);
-      const auto d_D = Reshape(D.Write(), qPts, nref, nel_ho);
-      const auto d_Q = Reshape(coeff_vec.Read(), qPts, nel_lor);
-
-      mfem::forall(qPts * nref * nel_ho, [=] MFEM_HOST_DEVICE (int tid)
-      {
-         const int q    = tid % qPts;
-         const int iref = (tid / qPts) % nref;
-         const int iho  = (tid / (qPts * nref)) % nel_ho;
-
-         const int lo_el_id = iref + nref*iho;
-         const real_t detJ = J(q, lo_el_id);
-
-         d_D(q, iref, iho) = W(q) * d_Q(q, lo_el_id) * detJ;
-      });
-
-      // Collect the basis functions
-      for (int iref = 0; iref < nref; ++iref)
-      {
-         int ilor = lor_els[iref];
-         // Now assemble the block-row of the mixed mass matrix associated
-         // with integrating HO functions against LOR functions on the LOR
-         // sub-element.
-
-         // Create the transformation that embeds the fine low-order element
-         // within the coarse high-order element in reference space
-         emb_tr.SetPointMat(pmats(cf_tr.embeddings[ilor].matrix));
-
-         DenseMatrix &b_lo = B_L(ilor);
-         DenseMatrix &b_ho = B_H(ilor);
-
-         ElemMixedEvaluation(geom, fe_ho, fe_lor, ip_tr, ir_ea, b_lo, b_ho);
-      } // loop over subcells of ho element
-      // end of quadrature point setup
    } // completed setup of basis function and quadrature point
 
    // Assemble mixed mass matrix
-   int iho = 0;
-   Array<int> lor_els;
-   ho2lor.GetRow(iho, lor_els);
-   int nref = ho2lor.RowSize(iho);
-
-   const FiniteElement &fe_ho = *fes_ho_ea.GetFE(iho);
-   const FiniteElement &fe_lor = *fes_lor_ea.GetFE(lor_els[0]);
-   const int ndof_ho = fe_ho.GetDof();
-   const int ndof_lor = fe_lor.GetDof();
-
-   const int qPts = D.SizeI();
-
-   M_LH.SetSize(ndof_lor*ndof_ho*nref*nel_ho, d_mt);
-
-   // Rows x columns
-   // Recall MFEM is column major
-   // rows x columns is inverted  - matrix is ndof_lor x ndof_ho
-   auto v_M_LH = Reshape(M_LH.Write(), ndof_lor, ndof_ho, nref,
-                         nel_ho);
-
-   const int fe_ho_ndof = fe_ho.GetDof();
-   const int fe_lor_ndof = fe_lor.GetDof();
-
-   auto d_B_L = Reshape(B_L.Read(), qPts, fe_lor_ndof, nref);
-   auto d_B_H = Reshape(B_H.Read(), qPts, fe_ho_ndof, nref);
-   auto d_D   = Reshape(D.Read(), qPts, nref, nel_ho);
-
-   mfem::forall(fe_ho_ndof*nref*nel_ho, [=] MFEM_HOST_DEVICE (int idx)
    {
-      const int bh = idx % fe_ho_ndof;
-      const int iref = (idx / fe_ho_ndof) % nref;
-      const int iho = idx / fe_ho_ndof / nref;
-      // (B_lo_dofs x Q) x (Q x B_ho_dofs)
-      for (int bl = 0; bl < fe_lor_ndof; ++bl)
+      int iho = 0;
+      Array<int> lor_els;
+      ho2lor.GetRow(iho, lor_els);
+      int nref = ho2lor.RowSize(iho);
+
+      const FiniteElement &fe_ho = *fes_ho_ea.GetFE(iho);
+      const FiniteElement &fe_lor = *fes_lor_ea.GetFE(lor_els[0]);
+      const int ndof_ho = fe_ho.GetDof();
+      const int ndof_lor = fe_lor.GetDof();
+
+      const int qPts = D.SizeI();
+
+      M_LH.SetSize(ndof_lor*ndof_ho*nref*nel_ho, d_mt);
+
+      // Rows x columns
+      // Recall MFEM is column major
+      // rows x columns is inverted  - matrix is ndof_lor x ndof_ho
+      auto v_M_LH = Reshape(M_LH.Write(), ndof_lor, ndof_ho, nref,
+                            nel_ho);
+
+      const int fe_ho_ndof = fe_ho.GetDof();
+      const int fe_lor_ndof = fe_lor.GetDof();
+
+      auto d_B_L = Reshape(B_L.Read(), qPts, fe_lor_ndof, nref);
+      auto d_B_H = Reshape(B_H.Read(), qPts, fe_ho_ndof, nref);
+      auto d_D   = Reshape(D.Read(), qPts, nref, nel_ho);
+
+      mfem::forall(fe_ho_ndof*nref*nel_ho, [=] MFEM_HOST_DEVICE (int idx)
       {
-         real_t dot = 0.0;
-         for (int qi=0; qi<qPts; ++qi)
+         const int bh = idx % fe_ho_ndof;
+         const int iref = (idx / fe_ho_ndof) % nref;
+         const int iho = idx / fe_ho_ndof / nref;
+         // (B_lo_dofs x Q) x (Q x B_ho_dofs)
+         for (int bl = 0; bl < fe_lor_ndof; ++bl)
          {
-            dot += d_B_L(qi, bl, iref) *  d_D(qi, iref, iho) * d_B_H(qi, bh, iref);
+            real_t dot = 0.0;
+            for (int qi=0; qi<qPts; ++qi)
+            {
+               dot += d_B_L(qi, bl, iref) *  d_D(qi, iref, iho) * d_B_H(qi, bh, iref);
+            }
+            // column major storage
+            v_M_LH(bl, bh, iref, iho) = dot;
          }
-         // column major storage
-         v_M_LH(bl, bh, iref, iho) = dot;
-      }
-   });
+      });
+   } // end of mixed assembly mass matrix
 }
 
 L2ProjectionGridTransfer::L2ProjectionL2Space::L2ProjectionL2Space
 (const FiniteElementSpace &fes_ho_, const FiniteElementSpace &fes_lor_,
- CoefficientWithOrder coeff_ho_, CoefficientWithOrder coeff_lor_,
  const bool use_ea_, MemoryType d_mt_)
-   : L2Projection(fes_ho_, fes_lor_, coeff_ho_, coeff_lor_, d_mt_), use_ea(use_ea_)
+   : L2Projection(fes_ho_, fes_lor_, d_mt_),
+     use_ea(use_ea_)
 {
    if (use_ea)
    {
@@ -658,11 +559,7 @@ L2ProjectionGridTransfer::L2ProjectionL2Space::L2ProjectionL2Space
       DenseMatrix Minv_lor(ndof_lor*nref, ndof_lor*nref);
       DenseMatrix M_mixed(ndof_lor*nref, ndof_ho);
 
-      MassIntegrator mi = [&]()
-      {
-         return coeff_lor ? MassIntegrator(*coeff_lor.coeff) : MassIntegrator();
-      }();
-
+      MassIntegrator mi;
       DenseMatrix M_lor_el(ndof_lor, ndof_lor);
       DenseMatrixInverse Minv_lor_el(&M_lor_el);
       DenseMatrix M_lor(ndof_lor*nref, ndof_lor*nref);
@@ -680,10 +577,6 @@ L2ProjectionGridTransfer::L2ProjectionL2Space::L2ProjectionL2Space
          // Assemble the low-order refined mass matrix and invert locally
          int ilor = lor_els[iref];
          ElementTransformation *tr_lor = fes_lor.GetElementTransformation(ilor);
-
-         const int order = 2*fe_lor.GetOrder() + tr_lor->OrderW() + coeff_lor.order;
-         mi.SetIntegrationRule(IntRules.Get(geom, order));
-
          mi.AssembleElementMatrix(fe_lor, *tr_lor, M_lor_el);
          M_lor.CopyMN(M_lor_el, iref*ndof_lor, iref*ndof_lor);
          Minv_lor_el.Factor();
@@ -775,22 +668,25 @@ void L2ProjectionGridTransfer::L2ProjectionL2Space::EAL2ProjectionL2Space()
    // Need to compute M_L
    // Note: Using user-inputted M_LH IntegrationRule ir
    // (higher order than needed) in order to re-use coeff
-   MassIntegrator mi = [&]()
-   {
-      return coeff_lor ? MassIntegrator(*coeff_lor.coeff) : MassIntegrator();
-   }();
-
-   const int order = 2*fes_lor.GetMaxElementOrder()
-                     + mesh_lor->GetTypicalElementTransformation()->OrderW()
-                     + coeff_lor.order;
-   mi.SetIntegrationRule(
-      IntRules.Get(mesh_lor->GetTypicalElementGeometry(), order));
+   MassIntegrator mi;
 
    Vector M_ea_lor;
-   const int ndof_lor = fes_lor.GetTypicalFE()->GetDof();
-   const int ndof_ho = fes_ho.GetTypicalFE()->GetDof();
-   const int nref = ho2lor.RowSize(0);
-   M_ea_lor.SetSize(ndof_lor*ndof_lor*nel_lor, d_mt);
+   int ndof_lor;
+   int ndof_ho;
+   int nref;
+   {
+      int iho = 0;
+      Array<int> lor_els;
+      ho2lor.GetRow(iho, lor_els);
+      nref = ho2lor.RowSize(iho);
+
+      const FiniteElement &fe_ho = *fes_ho.GetFE(iho);
+      const FiniteElement &fe_lor = *fes_lor.GetFE(lor_els[0]);
+      ndof_ho = fe_ho.GetDof();
+      ndof_lor = fe_lor.GetDof();
+
+      M_ea_lor.SetSize(ndof_lor*ndof_lor*nel_lor, d_mt);
+   }
 
    const bool add = false;
    mi.AssembleEA(fes_lor, M_ea_lor, add);
@@ -1136,9 +1032,8 @@ void L2ProjectionGridTransfer::L2ProjectionL2Space::EAProlongateTranspose(
 
 L2ProjectionGridTransfer::L2ProjectionH1Space::L2ProjectionH1Space(
    const FiniteElementSpace& fes_ho_, const FiniteElementSpace& fes_lor_,
-   CoefficientWithOrder coeff_ho_, CoefficientWithOrder coeff_lor_,
    const bool use_ea_, MemoryType d_mt_)
-   : L2Projection(fes_ho_, fes_lor_, coeff_ho_, coeff_lor_, d_mt_),
+   : L2Projection(fes_ho_, fes_lor_, d_mt_),
      use_ea(use_ea_)
 {
 
@@ -1197,9 +1092,8 @@ L2ProjectionGridTransfer::L2ProjectionH1Space::L2ProjectionH1Space(
 
 L2ProjectionGridTransfer::L2ProjectionH1Space::L2ProjectionH1Space(
    const ParFiniteElementSpace& pfes_ho, const ParFiniteElementSpace& pfes_lor,
-   CoefficientWithOrder coeff_ho_, CoefficientWithOrder coeff_lor_,
    const bool use_ea_, MemoryType d_mt_)
-   : L2Projection(pfes_ho, pfes_lor, coeff_ho_, coeff_lor_, d_mt_),
+   : L2Projection(pfes_ho, pfes_lor, d_mt_),
      use_ea(use_ea_), pcg(pfes_ho.GetComm())
 {
 
@@ -1271,12 +1165,12 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::SetupPCG()
 
 void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
 {
-   Mesh &mesh_ho = *fes_ho.GetMesh();
-   Mesh &mesh_lor = *fes_lor.GetMesh();
-   const int nel_ho = mesh_ho.GetNE();
-   const int nel_lor = mesh_lor.GetNE();
-   const int ndof_ho = fes_ho.GetNDofs();
-   const int ndof_lor = fes_lor.GetNDofs();
+   Mesh* mesh_ho = fes_ho.GetMesh();
+   Mesh* mesh_lor = fes_lor.GetMesh();
+   int nel_ho = mesh_ho->GetNE();
+   int nel_lor = mesh_lor->GetNE();
+   int ndof_ho = fes_ho.GetNDofs();
+   int ndof_lor = fes_lor.GetNDofs();
 
    // If the local mesh is empty, skip all computations
    if (nel_ho == 0)
@@ -1284,11 +1178,11 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
       return;
    }
 
-   const CoarseFineTransformations& cf_tr = mesh_lor.GetRefinementTransforms();
+   const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();
 
    int nref_max = 0;
    Array<Geometry::Type> geoms;
-   mesh_ho.GetGeometries(mesh_ho.Dimension(), geoms);
+   mesh_ho->GetGeometries(mesh_ho->Dimension(), geoms);
    for (int ig = 0; ig < geoms.Size(); ++ig)
    {
       Geometry::Type geom = geoms[ig];
@@ -1311,8 +1205,7 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
 
    BilinearForm Mho(fes_ho_scalar.get());
    Mho.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   Mho.AddDomainIntegrator(coeff_ho ? new MassIntegrator(*coeff_ho.coeff)
-                           : new MassIntegrator);
+   Mho.AddDomainIntegrator(new MassIntegrator);
    Mho.Assemble();
 
    // Processor local lumped Mass
@@ -1322,16 +1215,7 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
 
    BilinearForm Mlor(fes_lor_scalar.get());
    Mlor.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   {
-      MassIntegrator *mi = coeff_lor ? new MassIntegrator(*coeff_lor.coeff)
-                           : new MassIntegrator;
-      const int order = 2*fes_lor.GetMaxElementOrder()
-                        + mesh_lor.GetTypicalElementTransformation()->OrderW()
-                        + coeff_lor.order;
-      mi->SetIntegrationRule(
-         IntRules.Get(mesh_lor.GetTypicalElementGeometry(), order));
-      Mlor.AddDomainIntegrator(mi);
-   }
+   Mlor.AddDomainIntegrator(new MassIntegrator);
    Mlor.Assemble();
 
    Vector ones_lor(Mlor.Width()); ones_lor = 1.0;
@@ -1344,14 +1228,15 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
    MixedMassEA(fes_ho, fes_lor, M_LH_ea, d_mt);
 
    // Set ownership
-   M_LH.reset(new H1SpaceMixedMassOperator(fes_ho_scalar.get(),
-                                           fes_lor_scalar.get(),
-                                           &ho2lor,
-                                           &M_LH_ea));
+   M_LH_local_op = new H1SpaceMixedMassOperator(fes_ho_scalar.get(),
+                                                fes_lor_scalar.get(),
+                                                &ho2lor,
+                                                &M_LH_ea);
 
    ML_inv_vea.reset(new H1SpaceLumpedMassOperator(fes_ho_scalar.get(),
                                                   fes_lor_scalar.get(),
                                                   ML_inv_ea));
+   M_LH.reset(M_LH_local_op);
    R.reset(new ProductOperator(ML_inv_vea.get(), M_LH.get(), false,
                                false));
 
@@ -1368,18 +1253,18 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space()
 void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space
 (const ParFiniteElementSpace& pfes_ho, const ParFiniteElementSpace& pfes_lor)
 {
-   Mesh &mesh_ho = *pfes_ho.GetParMesh();
-   Mesh &mesh_lor = *pfes_lor.GetParMesh();
-   int nel_ho = mesh_ho.GetNE();
-   int nel_lor = mesh_lor.GetNE();
+   Mesh* mesh_ho = pfes_ho.GetParMesh();
+   Mesh* mesh_lor = pfes_lor.GetParMesh();
+   int nel_ho = mesh_ho->GetNE();
+   int nel_lor = mesh_lor->GetNE();
    int ndof_ho = pfes_ho.GetNDofs();
    int ndof_lor = pfes_lor.GetNDofs();
 
-   const CoarseFineTransformations& cf_tr = mesh_lor.GetRefinementTransforms();
+   const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();
 
    int nref_max = 0;
    Array<Geometry::Type> geoms;
-   mesh_ho.GetGeometries(mesh_ho.Dimension(), geoms);
+   mesh_ho->GetGeometries(mesh_ho->Dimension(), geoms);
    for (int ig = 0; ig < geoms.Size(); ++ig)
    {
       Geometry::Type geom = geoms[ig];
@@ -1402,8 +1287,7 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space
 
    ParBilinearForm pMho(pfes_ho_scalar.get());
    pMho.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   pMho.AddDomainIntegrator(coeff_ho ? new MassIntegrator(*coeff_ho.coeff)
-                            : new MassIntegrator);
+   pMho.AddDomainIntegrator(new MassIntegrator);
    pMho.Assemble();
 
    // Processor local lumped Mass
@@ -1413,16 +1297,7 @@ void L2ProjectionGridTransfer::L2ProjectionH1Space::EAL2ProjectionH1Space
 
    ParBilinearForm pMlor(pfes_lor_scalar.get());
    pMlor.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   {
-      MassIntegrator *mi = coeff_lor ? new MassIntegrator(*coeff_lor.coeff)
-                           : new MassIntegrator;
-      const int order = 2*fes_lor.GetMaxElementOrder()
-                        + mesh_lor.GetTypicalElementTransformation()->OrderW()
-                        + coeff_lor.order;
-      mi->SetIntegrationRule(
-         IntRules.Get(mesh_lor.GetTypicalElementGeometry(), order));
-      pMlor.AddDomainIntegrator(mi);
-   }
+   pMlor.AddDomainIntegrator(new MassIntegrator);
    pMlor.Assemble();
 
    Vector ones_lor(pMlor.Width()); ones_lor = 1.0;
@@ -1695,7 +1570,7 @@ std::unique_ptr<SparseMatrix>>
          int ilor = lor_els[iref];
          ElementTransformation* el_tr = fes_lor.GetElementTransformation(ilor);
 
-         int order = 2 * fe_lor.GetOrder() + el_tr->OrderW() + coeff_lor.order;
+         int order = 2 * fe_lor.GetOrder() + el_tr->OrderW();
          const IntegrationRule* ir = &IntRules.Get(geom, order);
          ML_el = 0.0;
          for (int i = 0; i < ir->GetNPoints(); ++i)
@@ -1703,13 +1578,7 @@ std::unique_ptr<SparseMatrix>>
             const IntegrationPoint& ip_lor = ir->IntPoint(i);
             fe_lor.CalcShape(ip_lor, shape_lor);
             el_tr->SetIntPoint(&ip_lor);
-            real_t w = ip_lor.weight;
-            if (coeff_lor)
-            {
-               w *= coeff_lor.coeff->Eval(*el_tr, ip_lor);
-            }
-            shape_lor *= el_tr->Weight() * w;
-            ML_el += shape_lor;
+            ML_el += (shape_lor *= (el_tr->Weight() * ip_lor.weight));
          }
          fes_lor.GetElementDofs(ilor, dofs_lor);
          ML_inv.AddElementVector(dofs_lor, ML_el);
@@ -2155,8 +2024,8 @@ void L2ProjectionGridTransfer::BuildF()
    {
       if (!Parallel())
       {
-         F = new L2ProjectionH1Space(
-            dom_fes, ran_fes, coeff_ho, coeff_lor, use_ea, d_mt);
+         F = new L2ProjectionH1Space(dom_fes, ran_fes,
+                                     use_ea, d_mt);
       }
       else
       {
@@ -2165,15 +2034,15 @@ void L2ProjectionGridTransfer::BuildF()
             static_cast<mfem::ParFiniteElementSpace&>(dom_fes);
          const mfem::ParFiniteElementSpace& ran_pfes =
             static_cast<mfem::ParFiniteElementSpace&>(ran_fes);
-         F = new L2ProjectionH1Space(
-            dom_pfes, ran_pfes, coeff_ho, coeff_lor, use_ea, d_mt);
+         F = new L2ProjectionH1Space(dom_pfes, ran_pfes,
+                                     use_ea, d_mt);
 #endif
       }
    }
    else
    {
-      F = new L2ProjectionL2Space(
-         dom_fes, ran_fes, coeff_ho, coeff_lor, use_ea, d_mt);
+      F = new L2ProjectionL2Space(dom_fes, ran_fes,
+                                  use_ea, d_mt);
    }
 }
 

fem/transfer.hpp

@@ -19,8 +19,6 @@
 #include "pfespace.hpp"
 #endif
 
-#include <cstddef>
-
 namespace mfem
 {
 
@@ -164,18 +162,6 @@ public:
 };
 
 
-struct CoefficientWithOrder
-{
-   Coefficient *coeff;
-   int order;
-   CoefficientWithOrder() : coeff(nullptr), order(0) { }
-   CoefficientWithOrder(std::nullptr_t) : coeff(nullptr), order(0) { }
-   CoefficientWithOrder(Coefficient &coeff_) : coeff(&coeff_), order(1) { }
-   CoefficientWithOrder(Coefficient &coeff_, int order_)
-      : coeff(&coeff_), order(order_) { }
-   operator bool() const { return coeff != nullptr; }
-};
-
 /** @brief Transfer data in L2 and H1 finite element spaces between a coarse
     mesh and an embedded refined mesh using L2 projection. */
 /** The forward, coarse-to-fine, transfer uses L2 projection. The backward,
@@ -221,8 +207,6 @@ public:
    protected:
       const FiniteElementSpace& fes_ho;
       const FiniteElementSpace& fes_lor;
-      CoefficientWithOrder coeff_ho;
-      CoefficientWithOrder coeff_lor;
 
       MemoryType d_mt;
       Array<int> offsets;
@@ -230,15 +214,8 @@ public:
 
       L2Projection(const FiniteElementSpace& fes_ho_,
                    const FiniteElementSpace& fes_lor_,
-                   CoefficientWithOrder coeff_ho_,
-                   CoefficientWithOrder coeff_lor_,
                    MemoryType d_mt_ = Device::GetHostMemoryType());
 
-      L2Projection(const FiniteElementSpace& fes_ho_,
-                   const FiniteElementSpace& fes_lor_,
-                   MemoryType d_mt_ = Device::GetHostMemoryType())
-         : L2Projection(fes_ho_, fes_lor_, nullptr, nullptr, d_mt_) { }
-
       void BuildHo2Lor(int nel_ho, int nel_lor,
                        const CoarseFineTransformations& cf_tr);
 
@@ -248,11 +225,11 @@ public:
                          IntegrationPointTransformation& ip_tr,
                          DenseMatrix& M_mixed_el) const;
 
-      void ElemMixedEvaluation(Geometry::Type geom, const FiniteElement& fe_ho,
-                               const FiniteElement& fe_lor,
-                               IntegrationPointTransformation& ip_tr,
-                               const IntegrationRule& ir,
-                               DenseMatrix& B_L, DenseMatrix& B_H) const;
+      void ElemMixedMass(Geometry::Type geom, const FiniteElement& fe_ho,
+                         const FiniteElement& fe_lor,
+                         ElementTransformation* el_tr,
+                         IntegrationPointTransformation& ip_tr,
+                         DenseMatrix& B_L, DenseMatrix& B_H) const;
    public:
       /* Returns the Mixed Mass M_LH via device element assembly by building the
       basis functions and data at the quadrature points. */
@@ -310,17 +287,9 @@ public:
    public:
       L2ProjectionL2Space(const FiniteElementSpace& fes_ho_,
                           const FiniteElementSpace& fes_lor_,
-                          CoefficientWithOrder coeff_ho_,
-                          CoefficientWithOrder coeff_lor_,
                           const bool use_ea_,
                           MemoryType d_mt_ = Device::GetHostMemoryType());
 
-      L2ProjectionL2Space(const FiniteElementSpace& fes_ho_,
-                          const FiniteElementSpace& fes_lor_,
-                          const bool use_ea_,
-                          MemoryType d_mt_ = Device::GetHostMemoryType())
-         : L2ProjectionL2Space(fes_ho_, fes_lor_, nullptr, nullptr, use_ea_, d_mt_) { }
-
       /*Same as above but assembles and stores R_ea, P_ea */
       void EAL2ProjectionL2Space();
 
@@ -387,30 +356,13 @@ public:
    public:
       L2ProjectionH1Space(const FiniteElementSpace &fes_ho_,
                           const FiniteElementSpace &fes_lor_,
-                          CoefficientWithOrder coeff_ho_,
-                          CoefficientWithOrder coeff_lor_,
                           const bool use_ea_,
                           MemoryType d_mt_ = Device::GetHostMemoryType());
-
-      L2ProjectionH1Space(const FiniteElementSpace& fes_ho_,
-                          const FiniteElementSpace& fes_lor_,
-                          const bool use_ea_,
-                          MemoryType d_mt_ = Device::GetHostMemoryType())
-         : L2ProjectionH1Space(fes_ho_, fes_lor_, nullptr, nullptr, use_ea_, d_mt_) { }
-
 #ifdef MFEM_USE_MPI
       L2ProjectionH1Space(const ParFiniteElementSpace &pfes_ho_,
                           const ParFiniteElementSpace &pfes_lor_,
-                          CoefficientWithOrder coeff_ho_,
-                          CoefficientWithOrder coeff_lor_,
                           const bool use_ea_,
                           MemoryType d_mt_ = Device::GetHostMemoryType());
-
-      L2ProjectionH1Space(const ParFiniteElementSpace& fes_ho_,
-                          const ParFiniteElementSpace& fes_lor_,
-                          const bool use_ea_,
-                          MemoryType d_mt_ = Device::GetHostMemoryType())
-         : L2ProjectionH1Space(fes_ho_, fes_lor_, nullptr, nullptr, use_ea_, d_mt_) { }
 #endif
       /// Same as above but assembles action of R through 4 parts:
       ///   ( )  inv( lumped(M_L) ), which is a diagonal matrix (essentially a vector)
@@ -556,38 +508,18 @@ public:
       virtual ~L2Prolongation() { }
    };
 
-   /// Coefficient for the mixed L2 inner product.
-   CoefficientWithOrder coeff_ho;
-   /// Coefficient for the low-order L2 inner product.
-   CoefficientWithOrder coeff_lor;
    L2Projection   *F; ///< Forward, coarse-to-fine, operator
    L2Prolongation *B; ///< Backward, fine-to-coarse, operator
    bool force_l2_space;
 
 public:
-   /// Construct the unweighted L2 projection grid transfer.
    L2ProjectionGridTransfer(FiniteElementSpace &coarse_fes_,
                             FiniteElementSpace &fine_fes_,
                             bool force_l2_space_ = false,
                             MemoryType d_mt_ = Device::GetHostMemoryType()) // move to method
       : GridTransfer(coarse_fes_, fine_fes_),
-        coeff_ho(nullptr), coeff_lor(nullptr), F(nullptr), B(nullptr),
-        force_l2_space(force_l2_space_) { }
-
-   /// @brief Construct the weighted L2 projection grid transfer.
-   ///
-   /// The low-order inner product is weighted by @a coeff_lor, and the mixed
-   /// inner product is weighted by @a coeff_ho.
-   L2ProjectionGridTransfer(FiniteElementSpace &coarse_fes_,
-                            FiniteElementSpace &fine_fes_,
-                            CoefficientWithOrder coeff_ho_,
-                            CoefficientWithOrder coeff_lor_,
-                            bool force_l2_space_ = false,
-                            MemoryType d_mt_ = Device::GetHostMemoryType()) // move to method
-      : GridTransfer(coarse_fes_, fine_fes_),
-        coeff_ho(coeff_ho_), coeff_lor(coeff_lor_), F(nullptr), B(nullptr),
-        force_l2_space(force_l2_space_) { }
-
+        F(NULL), B(NULL), force_l2_space(force_l2_space_)
+   { }
    virtual ~L2ProjectionGridTransfer();
 
    const Operator &ForwardOperator() override;
@@ -595,7 +527,6 @@ public:
    const Operator &BackwardOperator() override;
 
    bool SupportsBackwardsOperator() const override;
-
 private:
    void BuildF();
 };

general/backends.hpp

@@ -14,7 +14,7 @@
 
 #include "../config/config.hpp"
 
-#if defined(MFEM_USE_CUDA)
+#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 #include <cusparse.h>
 #include <library_types.h>
 #include <cuda_runtime.h>
@@ -22,7 +22,7 @@
 #endif
 #include "cuda.hpp"
 
-#if defined(MFEM_USE_HIP)
+#if defined(MFEM_USE_HIP) && defined(__HIP__)
 #include <hip/hip_runtime.h>
 #endif
 #include "hip.hpp"
@@ -45,17 +45,15 @@
 #endif
 
 #if !defined(MFEM_USE_CUDA_OR_HIP)
-// MFEM_DEVICE_SYNC is made available for debugging purposes
-#define MFEM_DEVICE_SYNC
-// MFEM_STREAM_SYNC is used for UVM and MPI GPU-Aware kernels
-#define MFEM_STREAM_SYNC
-#endif
-
-#if !defined(MFEM_USE_CUDA_OR_HIP_LANG)
+constexpr bool mfem_use_gpu = false;
 #define MFEM_DEVICE
 #define MFEM_HOST
 #define MFEM_LAMBDA
 // #define MFEM_HOST_DEVICE // defined in config/config.hpp
+// MFEM_DEVICE_SYNC is made available for debugging purposes
+#define MFEM_DEVICE_SYNC
+// MFEM_STREAM_SYNC is used for UVM and MPI GPU-Aware kernels
+#define MFEM_STREAM_SYNC
 #define MFEM_LAUNCH_BOUNDS(...)
 #endif
 
@@ -68,23 +66,6 @@
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) MFEM_FOREACH_THREAD(i,k,N)
-// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
-// Example (3,2,1) block:
-// 0 (0,0), 1 (1,0), 2 (2,0)
-// 3 (1,0), 4 (1,1), 5 (2,1)
-#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
-   for (int iz = 0; iz < SZ; ++iz)                                             \
-      for (int iy = 0; iy < SY; ++iy)                                          \
-         for (int ix = 0; ix < SX; ++ix)
-// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
-// contiguous in x. This intentionally offsets threads within the block to avoid
-// shared memory bank conflicts.
-// Example (3,2,1) block assigned to work on (2,2,1) items:
-// 0 (0,0), 1 (1,0), 2 (N/A)
-// 3 (1,0), 4 (1,1), 5 (N/A)
-#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
-                                             OY, OZ)                           \
-   MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)
 #endif
 
 // 'double' and 'float' atomicAdd implementation for previous versions of CUDA
@@ -128,23 +109,4 @@ MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
 #endif
 }
 
-namespace mfem::internal
-{
-
-#if defined(MFEM_USE_CUDA_OR_HIP) && !defined(MFEM_USE_CUDA_OR_HIP_LANG)
-static constexpr bool can_compile_kernels = false;
-#else
-static constexpr bool can_compile_kernels = true;
-#endif
-
-template <bool can_compile_kernels = can_compile_kernels>
-void RequireKernelCompilation()
-{
-   static_assert(
-      can_compile_kernels,
-      "The calling function needs to be compiled with CUDA/HIP language!");
-}
-
-}
-
 #endif // MFEM_BACKENDS_HPP

general/communication.cpp

@@ -1108,126 +1108,6 @@ void GroupCommunicator::ReduceEnd(T *ldata, int layout,
    num_requests = 0;
 }
 
-template <class T>
-void GroupCommunicator::ReduceMarked(T *ldata, const Array<int> &marker,
-                                     int layout,
-                                     void (*Op)(OpData<T>)) const
-{
-   if (comm_lock == 0) { return; }
-   // The above also handles the case (group_buf_size == 0).
-   MFEM_VERIFY(comm_lock == 2, "object is NOT locked for Reduce");
-
-   switch (mode)
-   {
-      case byGroup: // ***** Communication by groups *****
-      {
-         OpData<T> opd;
-         opd.ldata = ldata;
-         Array<int> group_num_req(group_ldof.Size());
-         for (int gr = 1; gr < group_ldof.Size(); gr++)
-         {
-            group_num_req[gr] =
-               gtopo.IAmMaster(gr) ? gtopo.GetGroupSize(gr)-1 : 0;
-         }
-         int idx;
-         while (MPI_Waitany(num_requests, requests, &idx, MPI_STATUS_IGNORE),
-                idx != MPI_UNDEFINED)
-         {
-            int gr = request_marker[idx];
-            if (gr == -1) { continue; } // skip send requests
-
-            // Delay the processing of a group until all receive requests, for
-            // that group, are done:
-            if ((--group_num_req[gr]) != 0) { continue; }
-
-            opd.nldofs = group_ldof.RowSize(gr);
-            // groups without dofs are skipped, so here nldofs > 0.
-
-            opd.buf = (T *)group_buf.GetData() + buf_offsets[gr];
-            opd.ldofs = (layout == 0) ?
-                        group_ldof.GetRow(gr) : group_ltdof.GetRow(gr);
-            opd.nb = gtopo.GetGroupSize(gr)-1;
-
-            // Apply operation only to marked DOFs. The receive buffer is
-            // neighbor-major with stride opd.nldofs, i.e. the contributions to
-            // DOF i are buf[j*opd.nldofs + i] for j = 0 ... opd.nb-1. Setting
-            // nldofs = 1 for a single DOF changes that stride to 1, so the
-            // strided values must first be gathered into a contiguous buffer.
-            Array<T> single_buf(opd.nb);
-            for (int i = 0; i < opd.nldofs; i++)
-            {
-               if (marker[opd.ldofs[i]])
-               {
-                  for (int j = 0; j < opd.nb; j++)
-                  {
-                     single_buf[j] = opd.buf[j*opd.nldofs + i];
-                  }
-
-                  // Create a temporary OpData with just this one DOF
-                  OpData<T> single_opd;
-                  single_opd.ldata = ldata;
-                  single_opd.buf = single_buf.GetData();
-                  single_opd.ldofs = opd.ldofs + i;
-                  single_opd.nldofs = 1;
-                  single_opd.nb = opd.nb;
-
-                  // Apply the operation
-                  Op(single_opd);
-               }
-            }
-         }
-         break;
-      }
-
-      case byNeighbor: // ***** Communication by neighbors *****
-      {
-         MPI_Waitall(num_requests, requests, MPI_STATUSES_IGNORE);
-
-         for (int nbr = 1; nbr < nbr_send_groups.Size(); nbr++)
-         {
-            // In Reduce operation: send_groups <--> recv_groups
-            const int num_recv_groups = nbr_send_groups.RowSize(nbr);
-            if (num_recv_groups > 0)
-            {
-               const int *grp_list = nbr_send_groups.GetRow(nbr);
-               const T *buf = (T*)group_buf.GetData() + buf_offsets[nbr];
-               for (int i = 0; i < num_recv_groups; i++)
-               {
-                  // Custom version of ReduceGroupFromBuffer that checks marker
-                  int gr = grp_list[i];
-                  const int *ldofs = (layout == 0) ?
-                                     group_ldof.GetRow(gr) : group_ltdof.GetRow(gr);
-                  const int nldofs = group_ldof.RowSize(gr);
-
-                  for (int j = 0; j < nldofs; j++)
-                  {
-                     if (marker[ldofs[j]])
-                     {
-                        // Create a temporary OpData with just this one DOF
-                        OpData<T> opd;
-                        opd.ldata = ldata;
-                        opd.buf = const_cast<T*>(buf) + j;
-                        opd.ldofs = ldofs + j;
-                        opd.nldofs = 1;
-                        opd.nb = 1;
-
-                        // Apply the operation
-                        Op(opd);
-                     }
-                  }
-
-                  buf += nldofs;
-               }
-            }
-         }
-         break;
-      }
-   }
-
-   comm_lock = 0; // 0 - no lock
-   num_requests = 0;
-}
-
 template <class T>
 void GroupCommunicator::Sum(OpData<T> opd)
 {
@@ -1291,8 +1171,6 @@ void GroupCommunicator::Max(OpData<T> opd)
 template <class T>
 void GroupCommunicator::BitOR(OpData<T> opd)
 {
-   static_assert(std::is_integral<T>::value,
-                 "BitOR reduction requires an integral type.");
    for (int i = 0; i < opd.nldofs; i++)
    {
       T data = opd.ldata[opd.ldofs[i]];
@@ -1304,33 +1182,6 @@ void GroupCommunicator::BitOR(OpData<T> opd)
    }
 }
 
-template <class T>
-void GroupCommunicator::MaxAbs(OpData<T> opd)
-{
-   for (int i = 0; i < opd.nldofs; i++)
-   {
-      T data = opd.ldata[opd.ldofs[i]];
-      T abs_data = std::abs(data);
-
-      for (int j = 0; j < opd.nb; j++)
-      {
-         T b = opd.buf[j*opd.nldofs+i];
-         T abs_b = std::abs(b);
-
-         // On an equal-magnitude tie keep the more positive value, so
-         // opposite-sign ties resolve deterministically to the positive one.
-         if (abs_data < abs_b || (abs_data == abs_b && data < b))
-         {
-            data = b;
-            abs_data = abs_b;
-         }
-      }
-
-      opd.ldata[opd.ldofs[i]] = data;
-   }
-}
-
-
 void GroupCommunicator::PrintInfo(std::ostream &os) const
 {
    char c = '\0';
@@ -1467,24 +1318,18 @@ template void GroupCommunicator::BcastEnd<int>(int *, int) const;
 template void GroupCommunicator::ReduceBegin<int>(const int *) const;
 template void GroupCommunicator::ReduceEnd<int>(
    int *, int, void (*)(OpData<int>)) const;
-template void GroupCommunicator::ReduceMarked<int>(
-   int*, const Array<int>&, int, void (*)(OpData<int>)) const;
 
 template void GroupCommunicator::BcastBegin<double>(double *, int) const;
 template void GroupCommunicator::BcastEnd<double>(double *, int) const;
 template void GroupCommunicator::ReduceBegin<double>(const double *) const;
 template void GroupCommunicator::ReduceEnd<double>(
    double *, int, void (*)(OpData<double>)) const;
-template void GroupCommunicator::ReduceMarked<double>(
-   double*, const Array<int>&, int, void (*)(OpData<double>)) const;
 
 template void GroupCommunicator::BcastBegin<float>(float *, int) const;
 template void GroupCommunicator::BcastEnd<float>(float *, int) const;
 template void GroupCommunicator::ReduceBegin<float>(const float *) const;
 template void GroupCommunicator::ReduceEnd<float>(
    float *, int, void (*)(OpData<float>)) const;
-template void GroupCommunicator::ReduceMarked<float>(
-   float*, const Array<int>&, int, void (*)(OpData<float>)) const;
 
 // @endcond
 
@@ -1493,17 +1338,14 @@ template void GroupCommunicator::Sum<int>(OpData<int>);
 template void GroupCommunicator::Min<int>(OpData<int>);
 template void GroupCommunicator::Max<int>(OpData<int>);
 template void GroupCommunicator::BitOR<int>(OpData<int>);
-template void GroupCommunicator::MaxAbs<int>(OpData<int>);
 
 template void GroupCommunicator::Sum<double>(OpData<double>);
 template void GroupCommunicator::Min<double>(OpData<double>);
 template void GroupCommunicator::Max<double>(OpData<double>);
-template void GroupCommunicator::MaxAbs<double>(OpData<double>);
 
 template void GroupCommunicator::Sum<float>(OpData<float>);
 template void GroupCommunicator::Min<float>(OpData<float>);
 template void GroupCommunicator::Max<float>(OpData<float>);
-template void GroupCommunicator::MaxAbs<float>(OpData<float>);
 
 
 #ifdef __bgq__

general/communication.hpp

@@ -22,7 +22,6 @@
 #include "globals.hpp"
 #include <mpi.h>
 #include <cstdint>
-#include <type_traits>
 
 // can't directly use MPI_CXX_BOOL because Microsoft's MPI implementation
 // doesn't include MPI_CXX_BOOL. Fallback to MPI_C_BOOL if unavailable.
@@ -409,38 +408,14 @@ public:
    template <class T> void Reduce(Array<T> &ldata, void (*Op)(OpData<T>)) const
    { Reduce<T>((T *)ldata, Op); }
 
-   /// Reduce operation Sum, instantiated for int, double and float
+   /// Reduce operation Sum, instantiated for int and double
    template <class T> static void Sum(OpData<T>);
-   /// Reduce operation Min, instantiated for int, double and float
+   /// Reduce operation Min, instantiated for int and double
    template <class T> static void Min(OpData<T>);
-   /// Reduce operation Max, instantiated for int, double and float
+   /// Reduce operation Max, instantiated for int and double
    template <class T> static void Max(OpData<T>);
    /// Reduce operation bitwise OR, instantiated for int only
    template <class T> static void BitOR(OpData<T>);
-   /// Reduce operation selecting the signed value with the largest absolute
-   /// value, instantiated for int, double and float. The result keeps its sign;
-   /// it is not the non-negative absolute value. Equal-magnitude ties are
-   /// broken deterministically toward the more positive value, so opposite-sign
-   /// ties resolve to the positive one regardless of accumulation order.
-   template <class T> static void MaxAbs(OpData<T>);
-
-   /** @brief Finalize reduction operation started with ReduceBegin(), but only apply
-       the reduction to DOFs marked in the marker array.
-
-       @note The reduction is carried out in the signed type @a T, so the result
-       is signed even for bitwise operations.
-   */
-   template <class T>
-   void ReduceMarked(T *ldata, const Array<int> &marker, int layout,
-                     void (*Op)(OpData<T>)) const;
-
-   /** @brief Reduce within each group where the master is the root, but only for marked DOFs. */
-   template <class T>
-   void Reduce(T *ldata, const Array<int> &marker, void (*Op)(OpData<T>)) const
-   {
-      ReduceBegin(ldata);
-      ReduceMarked(ldata, marker, 0, Op);
-   }
 
    /// Print information about the GroupCommunicator from all MPI ranks.
    void PrintInfo(std::ostream &out = mfem::out) const;

general/cuda.hpp

@@ -18,8 +18,14 @@
 // CUDA block size used by MFEM.
 #define MFEM_CUDA_BLOCKS 256
 
-#if defined(MFEM_USE_CUDA)
+#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 #define MFEM_USE_CUDA_OR_HIP
+constexpr bool mfem_use_gpu = true;
+#define MFEM_DEVICE __device__
+#define MFEM_HOST __host__
+#define MFEM_LAMBDA __host__
+#define MFEM_LAUNCH_BOUNDS __launch_bounds__
+// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
 #define MFEM_DEVICE_SYNC MFEM_GPU_CHECK(cudaDeviceSynchronize())
 #define MFEM_STREAM_SYNC MFEM_GPU_CHECK(cudaStreamSynchronize(0))
 // Define a CUDA error check macro, MFEM_GPU_CHECK(x), where x returns/is of
@@ -34,15 +40,6 @@
     }                                                                          \
   } while (0)
 
-// Macros defined only when compiling with CUDA language
-#if defined(__CUDACC__)
-#define MFEM_USE_CUDA_OR_HIP_LANG
-#define MFEM_DEVICE __device__
-#define MFEM_HOST __host__
-#define MFEM_LAMBDA __host__
-#define MFEM_LAUNCH_BOUNDS __launch_bounds__
-// #define MFEM_HOST_DEVICE __host__ __device__ // defined in config/config.hpp
-
 // Define the MFEM inner threading macros
 #if defined(__CUDA_ARCH__)
 #define MFEM_SHARED __shared__
@@ -52,31 +49,13 @@
 #define MFEM_THREAD_SIZE(k) blockDim.k
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=threadIdx.k; i<N; i+=blockDim.k)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) if(const int i=threadIdx.k; i<N)
-// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
-// Example (3,2,1) block:
-// 0 (0,0), 1 (1,0), 2 (2,0)
-// 3 (1,0), 4 (1,1), 5 (2,1)
-#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
-   if (int ix = threadIdx.k % (SX), iy = threadIdx.k / (SX), iz = iy / (SY);   \
-       (iy %= (SY)), (threadIdx.k < (SX) * (SY) * (SZ)))
-// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
-// contiguous in x. This intentionally offsets threads within the block to avoid
-// shared memory bank conflicts.
-// Example (3,2,1) block assigned to work on (2,2,1) items:
-// 0 (0,0), 1 (1,0), 2 (N/A)
-// 3 (1,0), 4 (1,1), 5 (N/A)
-#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
-                                             OY, OZ)                           \
-   if (int ix = threadIdx.k % (OX), iy = threadIdx.k / (OX), iz = iy / (OY);   \
-       (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__CUDA_ARCH__)
-#endif // defined(__CUDACC__)
-#endif // defined(MFEM_USE_CUDA)
+#endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 
 namespace mfem
 {
 
-#if defined(MFEM_USE_CUDA)
+#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 // Function used by the macro MFEM_GPU_CHECK.
 void mfem_cuda_error(cudaError_t err, const char *expr, const char *func,
                      const char *file, int line);

general/error.cpp

@@ -171,7 +171,7 @@ void mfem_error(const char *msg)
 #ifdef MFEM_USE_EXCEPTIONS
    if (mfem_error_action == MFEM_ERROR_THROW)
    {
-      throw ErrorException(msg ? msg : "");
+      throw ErrorException(msg);
    }
 #endif
 

general/error.hpp

@@ -15,7 +15,7 @@
 #include "../config/config.hpp"
 #include <iomanip>
 #include <sstream>
-#if defined(MFEM_USE_HIP)
+#ifdef MFEM_USE_HIP
 #include <hip/hip_runtime.h>
 #endif
 
@@ -153,13 +153,21 @@ void mfem_warning(const char *msg = NULL);
 
 
 // Additional abort functions for HIP
-#if defined(__HIP_DEVICE_COMPILE__)
+#if defined(MFEM_USE_HIP)
+#ifndef __HIP_DEVICE_COMPILE__
+template<typename T>
+__host__ void abort_msg(T & msg)
+{
+   MFEM_ABORT(msg);
+}
+#else
 template<typename T>
 __device__ void abort_msg(T & msg)
 {
    abort();
 }
 #endif
+#endif
 
 // Abort inside a device kernel
 #if defined(__CUDA_ARCH__)