Merge branch 'gpu_em' into gpu-maxwell

Merge branch 'gpu-qinterp-integ' into gpu_em
applied detJ at the wrong place
2026-07-02 18:55:08 -07:00 · 2026-07-02 18:10:10 -07:00 · 2026-07-02 18:09:44 -07:00 · 2026-07-02 08:43:50 -07:00 · 2026-07-02 08:41:58 -07:00 · 2026-07-01 18:31:20 -07:00
228 changed files with 24968 additions and 7846 deletions
@@ -94,6 +94,16 @@ inputs:
    description: If true, do not set any CXXFLAGS or LDFLAGS.
    default: false

+  # Unfortunately, "uses:" fields cannot have references to variables like
+  # ${{env.MFEM_ACTIONS_VERSION}}, so the branch/tag name has to be hard coded.
+  # Therefore, in the future, when updating the version of the
+  # mfem/github-actions to use, we'll have to replace:
+  # - all definitions of MFEM_ACTIONS_VERSION and
+  # - all "uses:" fields that refer to mfem/github-actions.
+  MFEM_ACTIONS_VERSION:
+    description: Version (branch or tag) of the mfem/github-actions to use.
+    default: v2.7
+
 runs:
  using: 'composite'
  steps:
@@ -118,6 +128,7 @@ runs:
        echo UBSAN_LDFLAGS=${{inputs.UBSAN_LDFLAGS}} >> $GITHUB_ENV
        echo MSAN_CXXFLAGS=${{inputs.MSAN_CXXFLAGS}} >> $GITHUB_ENV
        echo MSAN_LDFLAGS=${{inputs.MSAN_LDFLAGS}} >> $GITHUB_ENV
+        echo MFEM_ACTIONS_VERSION=${{inputs.MFEM_ACTIONS_VERSION}} >> $GITHUB_ENV
      shell: bash

    - name: Env (dir)
@@ -53,7 +53,7 @@ runs:
      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
      shell: bash

-    - uses: mfem/github-actions/build-mfem@v2.5
+    - uses: mfem/github-actions/build-mfem@v2.7
      if: ${{steps.debug.outputs.cache-hit != 'true'}}
      env:
        CXXFLAGS: ${{env.CXXFLAGS}}
@@ -82,7 +82,7 @@ runs:
      run: find . -type f -name '*.o' -delete
      shell: bash

-    - uses: actions/upload-artifact@v4
+    - uses: actions/upload-artifact@v7
      with:
        name: build-${{inputs.par}}-${{inputs.sanitizer}}
        path: mfem/build
@@ -12,6 +12,11 @@
 name: 'Install MPI'
 description: 'Installs MPI and set up its environment variables'

+inputs:
+  NO_FLAGS:
+    description: If true, do not set any CXXFLAGS or LDFLAGS.
+    default: false
+
 runs:
  using: 'composite'
  steps:
@@ -27,6 +32,7 @@ runs:
      shell: bash

    - name: Env (bis)
+      if: ${{ inputs.NO_FLAGS != 'true' }}
      run: |
        echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MPI_INC}} >> $GITHUB_ENV
        echo LDFLAGS=${{env.LDFLAGS}} ${{env.MPI_LIB}} >> $GITHUB_ENV
@@ -49,7 +49,7 @@ runs:
        par: ${{inputs.par}}
        sanitizer: ${{inputs.sanitizer}}

-    - uses: actions/download-artifact@v4
+    - uses: actions/download-artifact@v8
      with:
        name: build-${{inputs.par}}-${{inputs.sanitizer}}
        path: mfem/build
@@ -37,14 +37,14 @@ runs:
      with:
        path: ${{env.HYPRE_DIR}}
        fail-on-cache-miss: true
-        key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
+        key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-${{env.MFEM_ACTIONS_VERSION}}

    - uses: actions/cache/restore@v5  # Cache for Metis
      if: ${{inputs.par == 'true'}}
      with:
        path: ${{env.METIS_DIR}}
        fail-on-cache-miss: true
-        key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
+        key: ${{runner.os}}-build-${{env.METIS_DIR}}-${{env.MFEM_ACTIONS_VERSION}}

    - name: Hypre/Metis links
      if: ${{inputs.par == 'true'}}
@@ -0,0 +1,42 @@
+# MFEM Pull Request Review Agent Guide
+
+## Purpose and scope
+Review MFEM PRs for correctness, maintainability, performance, portability, test coverage, and MFEM consistency. Use the diff and PR context; reference source files, tests, and CI results when available. Follow `CONTRIBUTING.md`, especially Developer Guidelines, PR rules, checklist, and testing.
+
+## Critical review pillars
+- Correctness and numerical behavior
+- API and user-facing impact
+- Performance implications
+- Maintainability and portability
+
+## Review workflow
+1. Read the PR description, linked issues, and intended behavior.
+2. Inspect the diff before commenting.
+3. Identify affected MFEM components, examples, tests, build or docs changes, and downstream APIs.
+4. Analyze the code against the critical review pillars.
+5. Compare the change against nearby code and MFEM patterns; flag unmotivated deviations.
+6. Check whether tests and documentation were updated appropriately.
+7. Review CI results and suggest actions.
+8. Produce a structured review with prioritized findings.
+9. Always limit conclusions to available evidence.
+
+## MFEM-specific review checklist
+- Component-aware scope: identify the touched subsystem (FEM, solvers, preconditioners, linear algebra, mesh, examples, miniapps, build, or docs) and assess its impact against the review pillars.
+- Numerical and algorithmic behavior: assess issues in convergence, stability, tolerances, precision, iteration limits, and failure handling. If clear opportunities exist to improve the algorithmic approach, call them out with expected impact.
+- API and user-facing impact: assess backward compatibility, user-visible behavior and default changes, migration impact, deprecations, and whether documentation clearly explains user-facing API changes.
+- Data structure and memory semantics: assess ownership, lifetime, aliasing, container behavior, and device-host synchronization.
+- Parallel and serial behavior: assess whether the change preserves equivalent semantics in serial and parallel modes where applicable; if logic is currently mode-specific, check whether extension to the other mode is straightforward (clear abstractions, no hard-wired assumptions), document constraints, and call out expected behavior differences explicitly.
+- Backend and portability impact: assess likely cross-backend risks in CPU, CUDA, HIP, OCCA, RAJA, partial assembly, fallback paths, compiler compatibility, and platform assumptions.
+- Build, dependency, and configuration impact: assess CMake or make changes, optional dependency behavior, and feature-flag interactions.
+- Tests and docs alignment: check available regression or unit coverage evidence for changed behavior, and ensure docs are updated for new flags, APIs, options, or behavior changes.
+- MFEM developer-guideline fit: keep code lean, simple, general, logically separated, and portable; suggest C++17 improvements when they clearly improve safety, clarity, or maintainability.
+- New source files, examples, or miniapps: if a PR adds source/header files, verify they are properly wired into the relevant `makefile` and `CMakeLists.txt`, referenced in docs where applicable (including `doc/CodeDocumentation.dox`), and added to top-level `.gitignore` only when generated artifacts require it.
+- Changelog: verify `CHANGELOG` is updated if the PR introduces significant new features or user-facing changes.
+- MFEM conventions: use `real_t`; use `mfem::out`/`mfem::err` instead of `std::cout`/`std::cerr` in library code; flag large/binary files; if AI assistance is apparent but undisclosed, suggest following `CONTRIBUTING.md`.
+- Edge cases: if the PR touches complex or error-prone areas, suggest additional tests for edge cases, failure modes, and parallel behavior.
+
+## Commenting guidelines
+- Keep comments concise, actionable, and grounded in the diff.
+- Focus on correctness, behavior changes, and user impact over style nits.
+- Be professional, concise, collaborative, technically precise, and avoid unsupported assumptions.
+
@@ -13,7 +13,7 @@ Note that some of these scripts use the shared MFEM GitHub Actions from the exte

  <https://github.com/mfem/github-actions>

-For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch in the above from which the action is taken.
+For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch (or tag) in the above from which the action is taken.

 The current CI workflows are:

@@ -40,6 +40,7 @@ env:
  METIS_ARCHIVE_MAC: metis-4.0.3-mac.tgz
  METIS_TOP_DIR: metis-4.0.3
  MFEM_TOP_DIR: mfem
+  MFEM_ACTIONS_VERSION: v2.7

 # Note for future improvements:
 #
@@ -170,20 +171,6 @@ jobs:
          env
        shell: bash

-      # For info on Xcode see:
-      #  - https://github.com/actions/runner-images/issues/12541
-      #  - https://github.com/actions/runner-images/blob/releases/macos-15-arm64/20250811/images/macos/macos-15-arm64-Readme.md#xcode
-      - name: Xcode version setup (MacOS)
-        if: matrix.os == 'macos-latest'
-        run: |
-          XCODE_PATH="/Applications/Xcode_16.4.app"
-          echo "> sudo xcode-select -s ${XCODE_PATH}"
-          sudo xcode-select -s ${XCODE_PATH}
-          echo "> g++ -v"
-          g++ -v
-          echo "> clang++ -v"
-          clang++ -v
-
      # Only get MPI if defined for the job.
      # TODO: It would be nice to have only one step, e.g. with a dedicated
      # action, but I (@adrienbernede) don't see how at the moment.
@@ -228,11 +215,11 @@ jobs:
        uses: actions/cache@v5
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5
+          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-${{ env.MFEM_ACTIONS_VERSION }}

      - name: get hypre
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.5
+        uses: mfem/github-actions/build-hypre@v2.7
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
@@ -242,7 +229,7 @@ jobs:

      - name: get hypre (Windows)
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.5
+        uses: mfem/github-actions/build-hypre@v2.7
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
@@ -258,11 +245,11 @@ jobs:
        uses: actions/cache@v5
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-${{ env.MFEM_ACTIONS_VERSION }}

      - name: install metis
        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
+        uses: mfem/github-actions/build-metis@v2.7
        with:
          archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
          dir: ${{ env.METIS_TOP_DIR }}
@@ -304,7 +291,7 @@ jobs:

      # MFEM build and test
      - name: build
-        uses: mfem/github-actions/build-mfem@v2.5
+        uses: mfem/github-actions/build-mfem@v2.7
        env:
          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
        with:
@@ -375,8 +362,10 @@ jobs:
      # Code coverage (process and upload reports)
      - name: codecov
        if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.5
+        uses: mfem/github-actions/upload-coverage@v2.7
        with:
-          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}
          project_dir: ${{ env.MFEM_TOP_DIR }}
          directories: "fem general linalg mesh"
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -32,6 +32,7 @@ env:
  METIS_ARCHIVE: metis-4.0.3.tar.gz
  METIS_TOP_DIR: metis-4.0.3
  COVERAGE_ENV: mfem-coverage
+  MFEM_ACTIONS_VERSION: v2.7

 jobs:
  gitignore:
@@ -53,33 +54,34 @@ jobs:
        uses: actions/cache@v5
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-v2.5
+          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-int32-fp64-${{ env.MFEM_ACTIONS_VERSION }}

      - name: Get Hypre
        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.5
+        uses: mfem/github-actions/build-hypre@v2.7
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
          target: int32
+          precision: fp64

      - name: Cache Metis Install
        id: metis-cache
        uses: actions/cache@v5
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-${{ env.MFEM_ACTIONS_VERSION }}

      - name: Install Metis
        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
+        uses: mfem/github-actions/build-metis@v2.7
        with:
          archive: ${{ env.METIS_ARCHIVE }}
          dir: ${{ env.METIS_TOP_DIR }}

      # MFEM build and test
      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.5
+        uses: mfem/github-actions/build-mfem@v2.7
        with:
          os: ${{ runner.os }}
          target: opt
@@ -19,18 +19,22 @@ jobs:
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/sanitize/config
+        with:
+          NO_FLAGS: true
      - name: Cache
        id: cache
        uses: actions/cache@v5
        with:
          path: ${{env.HYPRE_DIR}}
-          key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
+          key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-${{ env.MFEM_ACTIONS_VERSION }}
      - name: Setup
        if: steps.cache.outputs.cache-hit != 'true'
        uses: ./.github/actions/sanitize/mpi
+        with:
+          NO_FLAGS: true
      - name: Build
        if: steps.cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.5
+        uses: mfem/github-actions/build-hypre@v2.7
        with:
          archive: ${{env.HYPRE_TGZ}}
          dir: ${{env.HYPRE_DIR}}
@@ -19,18 +19,22 @@ jobs:
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/sanitize/config
+        with:
+          NO_FLAGS: true
      - name: Cache
        id: cache
        uses: actions/cache@v5
        with:
          path: ${{env.METIS_DIR}}
-          key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
+          key: ${{runner.os}}-build-${{env.METIS_DIR}}-${{env.MFEM_ACTIONS_VERSION}}
      - name: Setup
        if: steps.cache.outputs.cache-hit != 'true'
        uses: ./.github/actions/sanitize/mpi
+        with:
+          NO_FLAGS: true
      - name: Build
        if: steps.cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
+        uses: mfem/github-actions/build-metis@v2.7
        with:
          archive: ${{env.METIS_TGZ}}
          dir: ${{env.METIS_DIR}}
@@ -146,7 +146,7 @@ jobs:
        if: ${{steps.restore.outputs.cache-hit != 'true'}}
        working-directory: mfem/build/tests/unit
        run: find . -type f -name '*.o' -delete
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
        with:
          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
          path: mfem/build/tests/unit/${{env.unit_tests}}
@@ -172,7 +172,7 @@ jobs:
          par: ${{inputs.par}}
          sanitizer: ${{inputs.sanitizer}}
          cache-path: mfem/build/tests/unit/${{env.unit_tests}}
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v8
        if: ${{steps.restore.outputs.cache-hit != 'true'}}
        with:
          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
@@ -85,3 +85,8 @@ opt_par_gcc_10_pumi:
  extends: .mfem_job_on_dane
  variables:
    SPEC: "%gcc@10.3.1 +pumi"
+
+opt_par_gcc_10_gslib:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 +gslib"
@@ -63,3 +63,8 @@ opt_mpi_cuda_hypre_cuda_gcc:
  extends: .mfem_job_on_matrix
  variables:
    SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90 ^hypre+cuda"
+
+opt_mpi_cuda_gcc_gslib:
+  extends: .mfem_job_on_matrix
+  variables:
+    SPEC: "%gcc@10.3.1 +mpi +cuda +gslib cuda_arch=90 ^hypre+cuda"
@@ -32,9 +32,9 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}

 # run
 if [[ "${MACHINE_NAME}" == "dane" ]]; then
-  salloc --nodes=1 -t 60 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  srun --nodes=1 -t 60 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  srun --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
  exit 1
@@ -8,43 +8,87 @@
                               https://mfem.org


-Version 4.10 (development)
-==========================
-
-Discretization improvements
---------------------------
- Replaced legacy simplex quadrature rules with symmetric positive-weight
-  rules for triangles (orders 0-25) and tetrahedra (orders 0-20). These
-  rules guarantee all-positive weights and interior quadrature points,
-  improving numerical stability. Higher orders fall back to Grundmann-Moller.
-  Triangle rules: Witherden & Vincent, Comput. Math. Appl. 69(10):1232-1241,
-  2015.
-  Tet rules (d=1-13): Witherden & Vincent (ibid).
-  Tet rules (d=14-20): Chuluunbaatar et al., Comput. Math. Appl. 124:89-97,
-  2022.
-
-
 Version 4.9.1 (development)
 ===========================

+- Added policy for AI-assisted contribution to CONTRIBUTING.md.
+
 Discretization improvements
 ---------------------------
- Improved the gridfunction projection routines. Projections work for Scalar,
-  Vector and VectorFE, also NURBS versions. Optionally different types of
-  projections can be selected, default behaviour has not changed.
+- Added GPU-enabled partial assembly for simplicial Bernstein H1 basis based on
+  ragged tensor algorithms (see DOI: 10.1137/11082539X) for mass and diffusion
+  integrators.

- Added methods to estimate function extremum using piecewise linear bounds +
+- Replaced legacy simplex quadrature rules with symmetric positive weight rules
+  for triangles (orders 0-25) and tetrahedra (orders 0-20). These rules
+  guarantee all-positive weights and interior quadrature points, improving
+  numerical stability. Higher orders fall back to Grundmann-Moller.
+  * Triangle rules: Witherden and Vincent, DOI: 10.1016/j.camwa.2015.03.017
+  * Tet rules (d=1-13): Witherden and  Vincent (same as above)
+  * Tet rules (d=14-20): Chuluunbaatar et al., DOI: 10.1016/j.camwa.2022.08.016
+
+- Added support for general 1D Gauss-Jacobi quadrature rules and Stroud conical
+  quadrature rules on triangles and tetrahedra.
+
+- Improved the GridFunction projection routines. Projections work for Scalar,
+  Vector and VectorFE, also NURBS versions. Optionally different types of
+  projections can be selected, default behavior has not changed.
+
+- Added GridFunction projection methods for trace spaces, i.e., project
+  coefficients on the mesh skeleton.
+
+- Added methods to estimate function extremum using piecewise linear bounds plus
  recursive subdivision.

+- Extend FindPointsGSLIB to support surface meshes.
+
 Meshing improvements
 --------------------
+- Added option to guarantee mesh validity during TMOP-based r-adaptivity, using
+  bounds on the determinant of the mesh transformation Jacobian.
+
+- Added PA support for TMOP's adaptive limiting functionality. Multiple
+  GridFunctions and Coefficients can be combined to form a composite term.
+
 - Improved support for 1D NURBS meshes with variable order, including using
  the patches construct for 1D NURBS meshes.

+- Added the option to include material interfaces (faces separating elements
+  with different element attributes) as additional boundary elements, for
+  parallel visualization, e.g. with GLVis. This is supported by both the Print
+  and PrintAsOne methods of ParMesh. See ParMesh::SetPrintInterfaces().
+
+Linear and nonlinear solvers
+----------------------------
+- Added support for trace spaces in PRefinementTransferOperator. This is used in
+  PRefinement multigrid methods for problems posed on trace spaces (see e.g. the
+  DPG miniapps).
+
+GPU computing
+-------------
+- Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
+
+- Added NVIDIA cuDSS library interface. Implementation examples have been
+  added to ex1 and ex1p. See https://developer.nvidia.com/cudss for more
+  details. Supported versions >= 0.6.0.
+
+- Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
+- Added support for FiniteElement::MapType::INTEGRAL spaces to
+  QuadratureInterpolator.
+
 New and updated examples and miniapps
 -------------------------------------
- Electromagnetics/lorentz miniapp has been updated to leverage the ParticleSet
-  capability.
+- The Lorentz miniapp (in miniapps/electromagnetics) has been updated to
+  leverage the ParticleSet capability.
+
+- Added (Complex)PRefinementMultigrid solver option in the DPG miniapps.
+
+Miscellaneous
+-------------
+- Fixed signed DOF handling in ParGridFunction reading (read constructor) and
+  saving via SaveAsOne(). Simplified the process of applying the DOF signs by
+  using the new method ApplyDofSigns() in class ParFiniteElementSpace: the
+  method will return immediately if no sign flips are needed.


 Version 4.9, released on Dec 11, 2025
@@ -433,6 +433,15 @@ if (MFEM_USE_STRUMPACK)
  endif()
 endif()

+# cuDSS can only be enabled in CUDA
+if (MFEM_USE_CUDSS)
+  if (MFEM_USE_CUDA)
+    find_package(CUDSS REQUIRED)
+  else()
+    message(FATAL_ERROR " *** cuDSS requires that CUDA be enabled.")
+  endif()
+endif()
+
 # GnuTLS
 if (MFEM_USE_GNUTLS)
  find_package(_GnuTLS REQUIRED)
@@ -631,7 +640,7 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB HDF5
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
+    ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CUDSS CALIPER CODIPACK
    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
    ALGOIM ENZYME CUDA::cudart)

@@ -3,12 +3,13 @@
 </p>

 <p align="center">
-<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-blue.svg"></a>
+<a href="https://github.com/mfem/mfem/releases/latest"><img alt="GitHub release" src="https://img.shields.io/github/v/release/mfem/mfem"></a>
+<a href="https://github.com/mfem/mfem/actions/workflows/repo-check.yml?query=branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml?query=branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml?query=branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
-<a href="https://docs.mfem.org/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+<a href="https://docs.mfem.org/html/index.html"><img alt="Documentation" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>


@@ -24,6 +25,14 @@ must be made under this license.
 Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
 in the MFEM community, you agree to abide by its rules.

+## AI Policy
+- Use of AI code generation in MFEM is allowed but must be disclosed, e.g. by
+  selecting the `AI-assisted` label on the PR.
+- By submitting a PR, the author acknowledges that they have reviewed and
+  understand the changes they are proposing.
+- PR authors are still responsible for correctness, licensing, and attribution
+  of all changes.
+
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull
@@ -76,7 +85,7 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  follow the [MFEM PR Rules](#mfem-pr-rules).
 - When your contribution is fully working and ready to be reviewed, add
  the `ready-for-review` label.
- PRs are treated similarly to journal submission with an "editor" assigning two
+- PRs are treated similarly to journal submission, with an "editor" assigning two
  reviewers to evaluate the changes.
 - The reviewers have 3 weeks to evaluate the PR and work with the author to
  fix issues and implement improvements.
@@ -117,7 +126,7 @@ The MFEM source code has the following structure:
  │   ├── petsc
  │   ├── pumi
  │   ├── sundials
-  |   └── superlu
+  │   └── superlu
  ├── fem
  │   ├── ceed
  │   ├── dfem
@@ -129,10 +138,6 @@ The MFEM source code has the following structure:
  │   ├── moonolith
  │   ├── qinterp
  │   └── tmop
-  │   |   ├── assemble
-  │   |   ├── metrics
-  │   |   ├── mult
-  │   |   └── tools
  ├── general
  ├── linalg
  │   ├── batched
@@ -145,11 +150,10 @@ The MFEM source code has the following structure:
  │   ├── common
  │   ├── contact
  │   ├── dfem
+  │   ├── diag-smoothers
  │   ├── dpg
  │   ├── electromagnetics
  │   ├── fluids
-  │   │   ├── navier
-  │   │   └── schrodinger-flow
  │   ├── gslib
  │   ├── hdiv-linear-solver
  │   ├── hooke
@@ -159,6 +163,7 @@ The MFEM source code has the following structure:
  │   ├── nurbs
  │   ├── parelag
  │   ├── performance
+  │   ├── plasma
  │   ├── shifted
  │   ├── solvers
  │   ├── spde
@@ -189,15 +194,15 @@ respectively.

 - The main finite element classes are:
  + [`FiniteElement`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElementCollection.html)
  + [`FiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1FiniteElementSpace.html)
  + [`GridFunction`](https://docs.mfem.org/html/classmfem_1_1GridFunction.html)
  + [`BilinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html)
-  + [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
+  + [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)

 - The main linear algebra classes and sources are
  + [`Operator`](https://docs.mfem.org/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html)
-  + [`Vector`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
+  + [`Vector`](https://docs.mfem.org/html/classmfem_1_1Vector.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
  + [`DenseMatrix`](https://docs.mfem.org/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://docs.mfem.org/html/classmfem_1_1SparseMatrix.html)
  + Sparse [smoothers](https://docs.mfem.org/html/sparsesmoothers_8hpp.html) and linear [solvers](https://docs.mfem.org/html/solvers_8hpp.html)

@@ -209,8 +214,8 @@ shared geometric entities between different tasks. The parallel source files
 have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.

 - The main parallel classes are
-  + [`ParMesh`](https://docs.mfem.org/html/solvers_8hpp.html)
-  + [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
+  + [`ParMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
+  + [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParNCMesh.html)
  + [`ParFiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1ParFiniteElementSpace.html)
  + [`ParGridFunction`](https://docs.mfem.org/html/classmfem_1_1ParGridFunction.html)
  + [`ParBilinearForm`](https://docs.mfem.org/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://docs.mfem.org/html/classmfem_1_1ParLinearForm.html)
@@ -220,14 +225,14 @@ have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
 #### GPU and general device support

 GPU and multi-core CPU support is based on device kernels supporting different
-backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
+backends (CUDA, HIP, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
 device/host memory manager.

 - The main device-relevant classes and sources are:
  + [`Device`](https://docs.mfem.org/html/device_8hpp.html)
  + [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
  + the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
-  + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
+  + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html), [`hip.hpp`](https://docs.mfem.org/html/hip_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files

 #### Utilities, building and documentation
 - The `general/` directory contains C++ classes that serve as utilities for
@@ -241,8 +246,8 @@ device/host memory manager.
 - `examples` and `miniapps` respectively gather simple and more fully-featured
  demonstrations of the usage on MFEM. They both rely on `data/` for the
  collection of meshes.
- The `tests/` directory contains a unit test suite and will later contain more
-  tests that run example codes.
+- The `tests/` directory contains a unit test suite, additional tests, and
+  benchmarks.

 See also the [code overview](https://mfem.org/code-overview/) section on the MFEM
 website.
@@ -276,8 +281,8 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  the top of https://github.com/mfem.
 - Consider making your membership public by going to https://github.com/orgs/mfem/people
  and clicking on the organization visibility drop box next to your name.
- Project discussions and announcements will be posted at
-  https://github.com/orgs/mfem/teams/everyone.
+- Project discussions and announcements will be posted at https://github.com/orgs/mfem/discussions,
+  tagging the `@mfem/everyone` team when appropriate.

 #### Structure
 - The MFEM source code is in the [mfem](https://github.com/mfem/mfem)
@@ -337,11 +342,12 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  - Well-designed simple code is frequently more general and powerful.
  - Lean code base is easier to understand by new collaborators.
  - New features should be added only if they are necessary or generally useful.
-  - Introduction of language constructions not currently used in MFEM should be
+  - Introduction of language constructs not currently used in MFEM should be
    justified and generally avoided (to maintain portability to various systems
    and compilers, including early access hardware).
-  - We prefer basic C++ and the C++03 standard, to keep the code readable by
-    a large audience and to make sure it compiles anywhere.
+  - We prefer basic C++. Use C++17 features judiciously, prioritizing readability,
+    consistency with existing MFEM code, and portability to different systems,
+    compilers and device backends.

 - *Keep the code general and reasonably efficient*
  - The main goal is fast prototyping for research and application development.
@@ -384,7 +390,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 - When your branch is ready for other developers to review / comment on
  the code, create a pull request towards `mfem:master`.

- Pull request typically have titles like:
+- Pull requests typically have titles like:

     `Description [new-feature-dev]`

@@ -405,12 +411,12 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 - Add a description, appropriate labels and assign yourself to the PR. The MFEM
  team will add reviewers as appropriate.

- List outstanding TODO items in the description, see PR #222 for an example.
+- List outstanding TODO items in the description.

 - When your contribution is fully working and ready to be reviewed, add
-  the `ready-for-review` label.
+  or request the `ready-for-review` label.

- PRs are treated similarly to journal submission with an "editor" assigning
+- PRs are treated similarly to journal submission, with an "editor" assigning
  two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
  the PR and work with the author to implement improvements and fix issues.

@@ -436,7 +442,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  checks in GitHub Actions enforce MFEM-specific rules which are explained in
  the error messages and the `tests/scripts` directory.

- Also note that the tests `branch-history` and `repos-checks` found in GitHub
+- Also note that the tests `branch-history` and `repo-check` found in GitHub
  Actions can be triggered automatically before each push using git hooks. See
  the [git hooks README](config/githooks/README.md) for a detailed explanation.

@@ -493,15 +499,15 @@ Everyone on the MFEM team can be asked to serve as a reviewer on a PR in their a

 3. To ensure the quality of the PR by making sure that the code adheres to the [Developer Guidelines](#developer-guidelines), e.g. all methods, data members, and functions have documentation, including data ownership and lifetime, new examples/miniapps have a corresponding PR in mfem/web, major features have `CHANGELOG` entries, etc.

-3. To seek help from the editors in case of difficulties.
+4. To seek help from the editors in case of difficulties.

-4. To complete the review in a timely manner: 3 weeks from assignment.
+5. To complete the review in a timely manner: 3 weeks from assignment.

-5. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
+6. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.

-6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
+7. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.

-7. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
+8. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.

 #### Responsibilities of Authors

@@ -527,30 +533,30 @@ Before a PR can be merged, it should satisfy the following:
 - [ ] Code builds.
 - [ ] Code passes `make style`.
 - [ ] Update `CHANGELOG`:
-    - [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
-    - [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
+   - [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
+   - [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
 - [ ] Update `INSTALL`:
-    - [ ] Had a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
-    - [ ] Have the version ranges for any required or optional libraries changed?
-    - [ ] Does `make` or `cmake` have a new target?
-    - [ ] Did the requirements or the installation process change? *(rare)*
+   - [ ] Has a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
+   - [ ] Have the version ranges for any required or optional libraries changed?
+   - [ ] Does `make` or `cmake` have a new target?
+   - [ ] Did the requirements or the installation process change? *(rare)*
 - [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.github`
-    - [ ] `.appveyor.yml`
+   - [ ] `.github`
+   - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
-    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
-    - [ ] Add new patterns (just for the new files above) and re-run the above test.
+   - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
+   - [ ] Add new patterns (just for the new files above) and re-run the above test.
 - [ ] New examples:
-    - [ ] All sample runs at the top of the example source file work.
-    - [ ] Update `examples/makefile`:
+   - [ ] All sample runs at the top of the example source file work.
+   - [ ] Update `examples/makefile`:
      - [ ] Add the example code to the appropriate `SEQ_EXAMPLES` and `PAR_EXAMPLES` variables.
      - [ ] Add any files generated by it to the `clean` target.
      - [ ] Add the example binary and any files generated by it to the top-level `.gitignore` file.
-    - [ ] Update `examples/CMakeLists.txt`:
+   - [ ] Update `examples/CMakeLists.txt`:
      - [ ] Add the example code to the `ALL_EXE_SRCS` variable.
      - [ ] Make sure `THIS_TEST_OPTIONS` is set correctly for the new example.
   - [ ] List the new example in `doc/CodeDocumentation.dox`.
-   - [ ] If new examples directory (e.g.`examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
+   - [ ] If new examples directory (e.g. `examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
   - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
      - [ ] Update or add example-specific documentation, see e.g. the `src/examples.md`.
      - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
@@ -567,13 +573,13 @@ Before a PR can be merged, it should satisfy the following:
      - [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
      - [ ] Consider adding a new test for the new miniapp.
   - [ ] List the new miniapp in `doc/CodeDocumentation.dox`
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
+   - [ ] If new miniapps directory (e.g. `miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
+   - [ ] If new miniapps directory (e.g. `miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
   - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
-     - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
-     - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
-     - [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
-     - [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
+      - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
+      - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
+      - [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
+      - [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
 - [ ] New capability:
   - [ ] All new public, protected, and private classes, methods, data members, and functions have full Doxygen-style documentation in source comments. Documentation should include descriptions of member data, function arguments and return values, template parameters, and prerequisites for calling new functions.
   - [ ] Pointer arguments and return values must specify whether ownership is being transferred or lent with the call.
@@ -675,7 +681,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 - [ ] Update URL shortlinks:
    - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. https://mfem.github.io/releases/mfem-3.1.tgz.
    - [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
-    - [ ] Add the new shortlinks to the MFEM packages in `spack`, `homebrew/science`, `VisIt`, etc.
+    - [ ] Add the new shortlinks to the MFEM package in `spack`.
 - [ ] Update website in `mfem/web` repo:
    - Update version and shortlinks in `src/index.md` and `src/download.md`.
    -  Use [cloc-1.62.pl](http://cloc.sourceforge.net/) and `ls -lh` to estimate the SLOC and the tarball size in `src/download.md`.
@@ -727,22 +733,24 @@ commit or push, see the [README](config/githooks/README.md) in the `config/githo
 directory.


-### Linux and Mac smoke tests
+### GitHub Actions smoke tests
+
 We use GitHub Actions to drive the default tests on the `master` and `next`
 branches. See the `.github/workflows` files and the logs at
 [https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).

-Testing using GitHub Actions should be kept lightweight, as there is a time
-constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
+GitHub Actions testing should be kept lightweight, as there is a time
+constraint on jobs. The current workflows cover Linux, macOS, and Windows
+configurations.

 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.

+### Additional Windows smoke test

-### Windows smoke test
-We use Appveyor to test building with the MS Visual C++ compiler in a Windows
-environment, as well as to test the CMake build. See the `.appveyor` file and the
-build logs at
+We also use Appveyor to test building with the MS Visual C++ compiler in a Windows
+environment, as well as to test the CMake build. See the `.appveyor.yml` file
+and the build logs at
 [https://ci.appveyor.com/project/mfem/mfem](https://ci.appveyor.com/project/mfem/mfem).

 CMake is used to generate the MSVC Project files and drive the build.  A release
@@ -38,14 +38,13 @@ the option MFEM_USE_METIS.
 MFEM also includes support for devices such as GPUs, and programming models such
 as CUDA, HIP, OCCA, OpenMP and RAJA.

- Starting with version 4.0, MFEM requires a C++11 compiler. We recommend using
-  a newer compiler, e.g. GCC version 4.9 or higher.
+- Starting with version 4.9, MFEM requires a C++17 compiler.

 - CUDA support requires an NVIDIA GPU and an installation of the CUDA Toolkit
  https://developer.nvidia.com/cuda-toolkit

 - HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocmdocs.amd.com
+  https://rocm.docs.amd.com

 - OCCA support requires the OCCA library
  https://libocca.org
@@ -83,9 +82,9 @@ Serial build:
 Parallel build:
   (download hypre and METIS 4 from above URLs)
   (build METIS 4 in ../metis-4.0 relative to mfem/)
+   (for METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
   (build hypre in ../hypre relative to mfem/)
   make parallel -j 4
-(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   make cuda -j 4
@@ -115,14 +114,14 @@ Serial build:
 Parallel build:
   (download hypre and METIS 4 from above URLs)
   (build METIS 4 in ../metis-4.0 relative to mfem/)
+   (for METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
   (build hypre in ../hypre relative to mfem/)
   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
   make -j 4
-(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 Parallel build with fetching of hypre and METIS:
-   mkdir <mfem-buil-dir> ; cd <mfem-build-dir>
+   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DMFEM_FETCH_TPLS=YES
   make -j 4

@@ -134,7 +133,8 @@ CUDA build:

 HIP build:
   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
-   cmake <mfem-source-dir> -DMFEM_USE_HIP=YES -DHIP_ARCH=gfx942 -DCMAKE_CXX_COMPILER=amdclang++ -DCMAKE_HIP_COMPILER=amdclang++
+   cmake <mfem-source-dir> -DMFEM_USE_HIP=YES -DHIP_ARCH=gfx942 \
+         -DCMAKE_CXX_COMPILER=amdclang++ -DCMAKE_HIP_COMPILER=amdclang++
   make -j 4

 Example codes (serial/parallel, depending on the build):
@@ -269,6 +269,7 @@ Compilers:
   CXX      - C++ compiler, serial build
   MPICXX   - MPI C++ compiler, parallel build
   CUDA_CXX - The CUDA compiler, 'nvcc' or 'clang++'
+   HIP_CXX  - The HIP compiler, e.g. 'hipcc'

 Compiler options:
   OPTIM_FLAGS - Options for optimized build
@@ -395,6 +396,11 @@ MFEM_USE_STRUMPACK = YES/NO
   classes. When enabled, this option uses the STRUMPACK_* library options, see
   below.

+MFEM_USE_CUDSS = YES/NO
+   Enable MFEM functionality based on the cuDSS library. When using cuDSS, CUDA
+   support must be also enabled in MFEM, i.e. MFEM_USE_CUDA=YES must be set.
+   When enabled, this option uses the CUDSS_* library options, see below.
+
 MFEM_USE_GINKGO = YES/NO
   Enable MFEM functionality based on the Ginkgo library, which provides
   iterative linear solvers and preconditioners with OpenMP, CUDA backends, see
@@ -554,13 +560,13 @@ MFEM_USE_RAJA = YES/NO
 MFEM_USE_OCCA = YES/NO
   Enables support for the OCCA library in MFEM. OCCA is an open-source library
   which aims to make it easy to program different types of devices (e.g. CPU,
-   GPU, FPGA) by providing an unified API for interacting with JIT-compiled
+   GPU, FPGA) by providing a unified API for interacting with JIT-compiled
   backends. In order to use the OCCA CUDA backend, CUDA support must be enabled
   in MFEM as well, i.e. MFEM_USE_CUDA=YES must be set.

 MFEM_USE_GSLIB = YES/NO
   Enables MFEM functionality based on the GSLIB library, and specifically its
-   FindPoints component, which provides a robust algorithms to evaluate finite
+   FindPoints component, which provides robust algorithms to evaluate finite
   element functions in a collection of points in physical space. When enabled,
   the user can use the GSLIB-FindPoints methods as shown in miniapps/gslib.

@@ -719,9 +725,18 @@ The specific libraries and their options are:
  Options: STRUMPACK_OPT, STRUMPACK_LIB.
  Versions: STRUMPACK >= 3.0.0.

- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
-  C++ compiler that supports the C++-17 standard. For additional requirements
-  and dependencies of specific modules, see the Ginkgo webpage below.
+- CUDSS (optional), used when MFEM_USE_CUDSS = YES. Note that CUDSS requires 
+  CUDA 12.x toolkit and the cuDSS libraries. The supported communication backend
+  is OpenMPI 4.x (default), and OpenMPI 4.x or a later version must be pre-built.
+  The source files in the cuDSS tarball provide guidance for developing custom 
+  MPI implementations.
+  URL: https://developer.nvidia.com/cudss
+       https://docs.nvidia.com/cuda/cudss/advanced_features.html#communication-layer-library-in-cudss
+  Options: CUDSS_OPT, CUDSS_LIB.
+  Versions: cuDSS >= 0.6.0.
+
+- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Ginkgo may have additional
+  requirements and module-specific dependencies; see the webpage below.
  URL: https://ginkgo-project.github.io
  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
           Debug).
@@ -793,7 +808,7 @@ The specific libraries and their options are:
  Options: CONDUIT_OPT, CONDUIT_LIB.
  Versions: Conduit >= 0.3.1.

- ADIOS2 (optional) used when MFEM_USE_ADIOS2 = YES.
+- ADIOS2 (optional), used when MFEM_USE_ADIOS2 = YES.
  URL: https://adios2.readthedocs.io/
  Versions: ADIOS >= 2.5.0.

@@ -869,7 +884,7 @@ The specific libraries and their options are:
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
  Versions: RAJA >= 2022.10.3.

- Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
+- Moonolith (optional), used when MFEM_USE_MOONOLITH = YES.
  URL: https://bitbucket.org/zulianp/par_moonolith
  Options: MOONOLITH_DIR
  Versions: MOONOLITH >= 1.1.0.
@@ -957,7 +972,7 @@ CMAKE_BUILD_TYPE which can be set to standard values like "Debug", and "Release"
 To use a specific generator use the "-G <generator>" option of cmake:

   cmake <mfem-source-dir> -G "Xcode"
-   cmake <mfem-source-dir> -G "Visual Studio 12 2013"
+   cmake <mfem-source-dir> -G "Visual Studio 17 2022"
   cmake <mfem-source-dir> -G "MinGW Makefiles"

 With CMake it is possible to build MFEM as a shared library using the standard
@@ -1202,7 +1217,7 @@ larger problems, there are two options:
 Specific options for HIP
 ========================
 MFEM expects the `ROCM_PATH` environment variable to be set to the path of the
-ROCM install, as well as having `$ROCM_PATH/bin` in `PATH`.
+ROCm install, as well as having `$ROCM_PATH/bin` in `PATH`.

 Specific options for RAJA+HIP+MPI
 =================================
@@ -28,6 +28,7 @@ license files. These software products and their licenses are as follows:
 * AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
 * Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
 * Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
+* gslib (fem/gslib.{cpp,hpp}, mesh/bb_grid_map.{cpp,hpp}) -- BSD 3-clause license
 * Picojson (fem/picojson.h) -- Custom 2-clause license
 * TinyXML2 (general/tinyxml2.{cpp,h}) -- zlib license
 * Zstr (general/zstr.hpp) -- MIT license
@@ -35,6 +35,7 @@ set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
+set(MFEM_USE_CUDSS @MFEM_USE_CUDSS@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
 set(MFEM_USE_MAGMA @MFEM_USE_MAGMA@)
@@ -109,6 +110,10 @@ if (MFEM_USE_RAJA)
   find_dependency(RAJA)
 endif()

+if (MFEM_USE_CUDSS)
+  find_dependency(cudss)
+endif (MFEM_USE_CUDSS)
+
 if (MFEM_USE_UMPIRE)
   find_dependency(umpire)
 endif()
@@ -108,6 +108,15 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK

+// Enable MFEM functionality based on the cuDSS library.
+#cmakedefine MFEM_USE_CUDSS
+
+// CUDSS communication layer library path
+#cmakedefine MFEM_CUDSS_COMM_LIB "@MFEM_CUDSS_COMM_LIB@"
+
+// CUDSS threading layer library path
+#cmakedefine MFEM_CUDSS_THREADING_LIB "@MFEM_CUDSS_THREADING_LIB@"
+
 // Enable functionality based on the Ginkgo library.
 #cmakedefine MFEM_USE_GINKGO

@@ -0,0 +1,68 @@
+if (NOT cudss_DIR AND CUDSS_DIR)
+  set(cudss_DIR ${CUDSS_DIR}/lib/cmake/cudss)
+endif()
+message(STATUS "Looking for CUDSS ...")
+message(STATUS "   in CUDSS_DIR = ${CUDSS_DIR}")
+message(STATUS "      cudss_DIR = ${cudss_DIR}")
+find_package(cudss)
+set(CUDSS_FOUND ${cudss_FOUND})
+set(CUDSS_LIBRARIES "cudss")
+if (CUDSS_FOUND)
+  message(STATUS
+    "Found CUDSS target: ${CUDSS_LIBRARIES} (version: ${cudss_VERSION})")
+else()
+  set(msg STATUS)
+  if (CUDSS_FIND_REQUIRED)
+    set(msg FATAL_ERROR)
+  endif()
+  message(${msg}
+    "CUDSS not found. Please set CUDSS_DIR to the install prefix.")
+endif()
+
+if(CUDSS_FOUND AND TARGET cudss)
+  get_target_property(CUDSS_LIBRARY_LOCATION cudss IMPORTED_LOCATION)
+  if(NOT CUDSS_LIBRARY_LOCATION)
+    get_target_property(CUDSS_LIBRARY_LOCATION cudss IMPORTED_LOCATION_RELEASE)
+  endif()
+  if(CUDSS_LIBRARY_LOCATION)
+    get_filename_component(CUDSS_LIBRARY_DIR "${CUDSS_LIBRARY_LOCATION}" DIRECTORY)
+  else()
+    message(WARNING "Could not determine the location of the cuDSS library.")
+  endif()
+else()
+    message(WARNING "cuDSS target not available; cannot determine library directory.")
+endif()
+
+# Set the full name of the cuDSS threading library if OpenMP is enabled.
+# The threading layer library (libcudss_mtlayer_gomp.so) is located under the
+# cuDSS library directory by default.
+if (MFEM_USE_OPENMP)
+  find_file(
+    CUDSS_THREADING_LIB
+    NAMES libcudss_mtlayer_gomp.so
+    PATHS ${CUDSS_LIBRARY_DIR}
+    NO_DEFAULT_PATH
+  )
+  if (NOT DEFINED MFEM_CUDSS_THREADING_LIB AND CUDSS_THREADING_LIB)
+    set(MFEM_CUDSS_THREADING_LIB "${CUDSS_THREADING_LIB}")
+  endif()
+  message(STATUS "CUDSS threading layer library: ${MFEM_CUDSS_THREADING_LIB}")
+endif()
+
+# Set the full name of the cuDSS communication library if MFEM use OpenMPI.
+# The communication layer library (libcudss_commlayer_mpi.so) is located under the
+# cuDSS library directory by default.
+# The communication layer library is used pre-built communication layers for OpenMPI 
+# by default.
+if (MFEM_USE_MPI)
+  find_file(
+    CUDSS_COMM_LIB
+    NAMES libcudss_commlayer_openmpi.so
+    PATHS ${CUDSS_LIBRARY_DIR}
+    NO_DEFAULT_PATH
+  )
+  if (NOT DEFINED MFEM_CUDSS_COMM_LIB AND CUDSS_COMM_LIB)
+    set(MFEM_CUDSS_COMM_LIB "${CUDSS_COMM_LIB}")
+  endif()
+  message(STATUS "CUDSS communication layer library: ${MFEM_CUDSS_COMM_LIB}")
+endif()
@@ -18,19 +18,17 @@

 if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS)
  enable_language(C)
-  string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
  set(GSLIB_FETCH_VERSION 1.0.9)
-  set(GSLIB_C_FLAGS ${CMAKE_C_FLAGS_${BUILD_TYPE}})
-  if (CMAKE_C_FLAGS)
-    set(GSLIB_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
-  endif()
-  if (BUILD_SHARED_LIBS)
-    set(GSLIB_C_FLAGS "${GSLIB_C_FLAGS} -fPIC")
-  endif()
  add_library(GSLIB STATIC IMPORTED)
+  # set options (technically flags because GSLIB does not use cmake)
+  string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
+  set(GSLIB_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
+  if (BUILD_SHARED_LIBS)
+    set(GSLIB_FLAGS "${GSLIB_FLAGS} -fPIC")
+  endif()
  # define external project and create future include directory so it is present
  # to pass CMake checks at end of MFEM configuration step
-  message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_C_FLAGS}")
+  message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_FLAGS}")
  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/gslib)
  include(ExternalProject)
  ExternalProject_Add(gslib
@@ -40,7 +38,7 @@ if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS)
    UPDATE_DISCONNECTED TRUE
    PREFIX ${PREFIX}
    CONFIGURE_COMMAND ""
-    BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$<BOOL:${MFEM_USE_MPI}> "CFLAGS= ${GSLIB_C_FLAGS}"
+    BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$<BOOL:${MFEM_USE_MPI}> "CFLAGS=${GSLIB_FLAGS}"
    INSTALL_COMMAND "")
  file(MAKE_DIRECTORY ${PREFIX}/include)
  # set imported library target properties
@@ -44,6 +44,9 @@ if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS)
  # set options and associated dependencies
  set(HYPRE_CMAKE_OPTIONS "")
  list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
+  if (BUILD_SHARED_LIBS)
+    list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON)
+  endif()
  # collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL.
  get_cmake_property(all_vars VARIABLES)
  foreach(var ${all_vars})
@@ -95,7 +98,6 @@ if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS)
    UPDATE_DISCONNECTED TRUE
    SOURCE_SUBDIR src
    PREFIX ${HYPRE_INSTALL}
-    BUILD_COMMAND ${CMAKE_COMMAND} --build . -- -j${CMAKE_BUILD_PARALLEL_LEVEL}
    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${HYPRE_INSTALL} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${HYPRE_CMAKE_OPTIONS})
  file(MAKE_DIRECTORY ${HYPRE_INSTALL}/include)
  # set imported library target properties
@@ -19,10 +19,18 @@
 #   - METIS_VERSION_5 (cache variable)

 if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS)
+  enable_language(C)
  set(METIS_FETCH_VERSION 4.0.3)
  add_library(METIS STATIC IMPORTED)
+  # set options (technically flags because METIS does not use cmake)
+  set(METIS_FLAGS "-Wno-implicit-int -Wno-incompatible-pointer-types")
+  string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
+  set(METIS_FLAGS "${METIS_FLAGS} ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
+  if (BUILD_SHARED_LIBS)
+    set(METIS_FLAGS "${METIS_FLAGS} -fPIC")
+  endif()
  # define external project
-  message(STATUS "Will fetch METIS ${METIS_FETCH_VERSION} to be built with default options")
+  message(STATUS "Will fetch METIS ${METIS_FETCH_VERSION} to be built with ${METIS_FLAGS}")
  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/metis)
  include(ExternalProject)
  ExternalProject_Add(metis
@@ -32,7 +40,7 @@ if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS)
    UPDATE_DISCONNECTED TRUE
    PREFIX ${PREFIX}
    CONFIGURE_COMMAND tar -xzf ../metis/metis-${METIS_FETCH_VERSION}-mac.tgz --strip=1
-    BUILD_COMMAND $(MAKE) COPTIONS=-Wno-incompatible-pointer-types
+    BUILD_COMMAND $(MAKE) clean && $(MAKE) "OPTFLAGS=${METIS_FLAGS}"
    INSTALL_COMMAND mkdir -p ${PREFIX}/lib && cp libmetis.a ${PREFIX}/lib/)
  # set imported library target properties
  add_dependencies(METIS metis)
@@ -157,4 +157,10 @@ constexpr real_t operator""_r(unsigned long long v)
 #endif
 #endif // MFEM_USE_MPI not defined

+#ifndef MFEM_USE_CUDA
+#ifdef MFEM_USE_CUDSS
+#error Building with cuDSS (MFEM_USE_CUDSS=YES) requires CUDA (MFEM_USE_CUDA=YES)
+#endif
+#endif // MFEM_USE_CUDSS not defined
+
 #endif // MFEM_CONFIG_HPP
@@ -97,9 +97,6 @@
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE

-// Enable MFEM functionality based on the ARPACK library.
-// #define MFEM_USE_ARPACK
-
 // Enable MFEM functionality based on the SuperLU_DIST library.
 // #define MFEM_USE_SUPERLU
 // #define MFEM_USE_SUPERLU5
@@ -111,6 +108,15 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK

+// Enable MFEM functionality based on the cuDSS library.
+// #define MFEM_USE_CUDSS
+
+// CUDSS communication layer library path
+// #define MFEM_CUDSS_COMM_LIB "@MFEM_CUDSS_COMM_LIB@"
+
+// CUDSS threading layer library path
+// #define MFEM_CUDSS_THREADING_LIB "@MFEM_CUDSS_THREADING_LIB@"
+
 // Enable MFEM features based on the Ginkgo library.
 // #define MFEM_USE_GINKGO

@@ -32,11 +32,13 @@ MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
-MFEM_USE_ARPACK        = @MFEM_USE_ARPACK@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
 MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
+MFEM_USE_CUDSS         = @MFEM_USE_CUDSS@
+MFEM_CUDSS_COMM_LIB    = @MFEM_CUDSS_COMM_LIB@
+MFEM_CUDSS_THREADING_LIB = @MFEM_CUDSS_THREADING_LIB@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_MAGMA         = @MFEM_USE_MAGMA@
@@ -38,6 +38,7 @@ option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
 option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
+option(MFEM_USE_CUDSS "Enable cuDSS usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
 option(MFEM_USE_MAGMA "Enable MAGMA usage" OFF)
@@ -153,6 +153,7 @@ MFEM_USE_SUPERLU       = NO
 MFEM_USE_SUPERLU5      = NO
 MFEM_USE_MUMPS         = NO
 MFEM_USE_STRUMPACK     = NO
+MFEM_USE_CUDSS         = NO
 MFEM_USE_GINKGO        = NO
 MFEM_USE_AMGX          = NO
 MFEM_USE_MAGMA         = NO
@@ -178,7 +179,6 @@ MFEM_USE_ALGOIM        = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
-MFEM_USE_ARPACK        = NO
 MFEM_USE_MKL_CPARDISO  = NO
 MFEM_USE_MKL_PARDISO   = NO
 MFEM_USE_MOONOLITH     = NO
@@ -369,6 +369,19 @@ STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 $(SCOTCH_LIB) $(SCALAPACK_LIB)

+# CUDSS library configuration
+CUDSS_DIR         = @MFEM_DIR@/../cudss
+CUDSS_INCLUDE_DIR = $(CUDSS_DIR)/include
+CUDSS_LIBRARY_DIR = $(CUDSS_DIR)/lib
+CUDSS_OPT         = -I$(CUDSS_INCLUDE_DIR)
+CUDSS_LIB         = \
+ $(XLINKER)-rpath,$(CUDSS_LIBRARY_DIR) -L$(CUDSS_LIBRARY_DIR) -lcudss
+# The cuDSS communication and threading libraries. 
+MFEM_CUDSS_COMM_LIB = $(abspath $(wildcard $(or $(CUDSS_COMM_LIB),\
+   $(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so))))
+MFEM_CUDSS_THREADING_LIB = $(abspath $(wildcard $(or $(CUDSS_THREADING_LIB),\
+   $(subst @MFEM_DIR@,$(MFEM_DIR),$(CUDSS_LIBRARY_DIR)/libcudss_mtlayer_gomp.so))))
+
 # Ginkgo library configuration
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
 GINKGO_SEARCH_DIR = $(subst @MFEM_DIR@,$(MFEM_DIR),$(GINKGO_DIR))
@@ -428,14 +441,6 @@ NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
 $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
 -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB)

-# ARPACK library configuration
-ARPACK_DIR = @MFEM_DIR@/../ARPACK
-ifeq ($(MFEM_USE_MPI),YES)
-ARPACK_LIB = -L$(ARPACK_DIR) -lparpack -larpack
-else
-ARPACK_LIB = -L$(ARPACK_DIR) -larpack
-endif
-
 # PETSc library configuration (version greater or equal to 3.8 or the dev branch)
 PETSC_ARCH := arch-linux2-c-debug
 PETSC_DIR  := $(MFEM_DIR)/../petsc/$(PETSC_ARCH)
@@ -630,7 +635,7 @@ PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG
 AXOM_DIR = @MFEM_DIR@/../axom
 TRIBOL_DIR = @MFEM_DIR@/../tribol
 TRIBOL_OPT = -I$(TRIBOL_DIR)/include -I$(AXOM_DIR)/include
-TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
+TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -ltribol_shared -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
   -laxom_slam -laxom_slic -laxom_core

 # Enzyme configuration
@@ -49,13 +49,6 @@ list(APPEND ALL_EXE_SRCS
  ex41.cpp
  )

-if (MFEM_USE_ARPACK)
-list(APPEND ALL_EXE_SRCS
-  ex11.pp
-  ex13.pp
-)
-endif()
-
 if (MFEM_USE_MPI)
  list(APPEND ALL_EXE_SRCS
    ex0p.cpp
@@ -222,7 +215,7 @@ if (MFEM_ENABLE_TESTING)
    add_test(NAME ex1p_ceed_np=${MFEM_MPI_NP}
      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:ex1p> "-no-vis" "-d ceed-cpu" "-pa" "-a"
+      $<TARGET_FILE:ex1p> "-no-vis" "-d" "ceed-cpu" "-pa" "-a"
      ${MPIEXEC_POSTFLAGS})
  endif()
 endif()
@@ -64,7 +64,7 @@ PARALLEL_NAME := Parallel AMGX example
 $(MFEM_LIB_FILE):
 	$(error The MFEM library is not build)

-clean: clean-build
+clean: clean-build clean-exec

 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
@@ -64,12 +64,12 @@ ex1p-test-par: ex1p
 $(MFEM_LIB_FILE):
 	$(error The MFEM library is not built)

-clean: clean-build clean-exec $(SUBDIRS_CLEAN)
+clean: clean-build clean-exec

 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
 	rm -rf *.dSYM *.TVD.*breakpoints

 clean-exec:
-	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
-	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
+	@rm -f refined.mesh mesh.*
+	@rm -f sol.*
@@ -50,6 +50,10 @@
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
 //
+// Device simplices sample runs:
+//               ex1 -pa -d gpu -m ../data/inline-tet.mesh
+//               ex1 -pa -d gpu -m ../data/inline-tri.mesh
+//
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Poisson problem
 //               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
@@ -138,25 +142,25 @@ int main(int argc, char *argv[])
   }

   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
+   //    Lagrange finite elements of the specified order.
+   //    - If order < 1, we instead use an isoparametric/isogeometric space.
+   //    - If the mesh is simplicial and partial assembly is requested,
+   //      we use the positive basis, which supports device execution.
   FiniteElementCollection *fec;
-   bool delete_fec;
+   auto basis_type = (pa && mesh.IsSimplexMesh()) ?
+                     BasisType::Positive : BasisType::GaussLobatto;
   if (order > 0)
   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
+      fec = new H1_FECollection(order, dim, basis_type);
   }
   else if (mesh.GetNodes())
   {
      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
      cout << "Using isoparametric FEs: " << fec->Name() << endl;
   }
   else
   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
+      fec = new H1_FECollection(order = 1, dim, basis_type);
   }
   FiniteElementSpace fespace(&mesh, fec);
   cout << "Number of finite element unknowns: "
@@ -224,17 +228,29 @@ int main(int argc, char *argv[])
   // 11. Solve the linear system A X = B.
   if (!pa)
   {
-#ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
+#ifdef MFEM_USE_CUDSS
+      if (Device::Allows(Backend::CUDA_MASK))
+      {
+         // Use cuDSS to solve the system.
+         CuDSSSolver cudss_solver;
+         cudss_solver.SetOperator(*A);
+         cudss_solver.Mult(B, X);
+      }
+      else
 #endif
+      {
+#ifndef MFEM_USE_SUITESPARSE
+         // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
+         GSSmoother M((SparseMatrix&)(*A));
+         PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
+#else
+         // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+         UMFPackSolver umf_solver;
+         umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+         umf_solver.SetOperator(*A);
+         umf_solver.Mult(B, X);
+#endif
+      }
   }
   else
   {
@@ -273,17 +289,14 @@ int main(int argc, char *argv[])
   if (visualization)
   {
      char vishost[] = "localhost";
-      int  visport   = 19916;
+      int visport = 19916;
      socketstream sol_sock(vishost, visport);
      sol_sock.precision(8);
      sol_sock << "solution\n" << mesh << x << flush;
   }

   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   if (order > 0) { delete fec; }

   return 0;
 }
@@ -1,298 +0,0 @@
-//                       MFEM Example 11 - Serial Version
-//
-// Compile with: make ex11
-//
-// Sample runs:  ex11 -m ../data/square-disc.mesh
-//               ex11 -m ../data/star.mesh
-//               ex11 -m ../data/star-mixed.mesh
-//               ex11 -m ../data/periodic-annulus-sector.msh
-//               ex11 -m ../data/square-disc-p2.vtk -o 2
-//               ex11 -m ../data/square-disc-p3.mesh -o 3
-//               ex11 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex11 -m ../data/disc-nurbs.mesh -o -1 -n 20
-//               ex11 -m ../data/star-surf.mesh
-//               ex11 -m ../data/square-disc-surf.mesh
-//               ex11 -m ../data/inline-segment.mesh
-//               ex11 -m ../data/inline-quad.mesh
-//               ex11 -m ../data/inline-tri.mesh
-//               ex11 -m ../data/amr-quad.mesh
-//               ex11 -m ../data/amr-hex.mesh
-//               ex11 -m ../data/mobius-strip.mesh -n 8
-//
-// Description:  This example code demonstrates the use of MFEM to solve the
-//               eigenvalue problem -Delta u = lambda u with homogeneous
-//               Dirichlet boundary conditions.
-//
-//               We compute a number of the lowest eigenmodes by discretizing
-//               the Laplacian and Mass operators using a FE space of the
-//               specified order, or an isoparametric/isogeometric space if
-//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of the ARPACK eigenvalue solver
-//               (regular inverse mode). Reusing a single GLVis visualization
-//               window for multiple eigenfunctions is also illustrated.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-#ifdef MFEM_USE_ARPACK
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int ser_ref_levels = 3;
-   int order = 1;
-   int nev = 5;
-   double dbc_eig = 1e3;
-   bool visualization = 1;
-   bool arp_solver = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&nev, "-n", "--num-eigs",
-                  "Number of desired eigenmodes.");
-   args.AddOption(&dbc_eig, "-d", "--dbc-eig",
-                  "Eigenvalues associated with Dirichlet BC "
-                  "(should be larger than the maximum desired eigenvalue).");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the (serial) mesh from the given mesh file on all processors. We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh;
-   ifstream imesh(mesh_file);
-   if (!imesh)
-   {
-      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
-      return 2;
-   }
-   mesh = new Mesh(imesh, 1, 1);
-   imesh.close();
-   int dim = mesh->Dimension();
-
-   // 3. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
-   //    specified on the command line with -rs).
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 4. Define a finite element space on the mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (mesh->GetNodes())
-   {
-      fec = mesh->GetNodes()->OwnFEC();
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   int size = fespace->GetVSize();
-
-   cout << "Number of unknowns: " << size << endl;
-
-   // 5. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
-   //    element space. The first corresponds to the Laplacian operator -Delta,
-   //    while the second is a simple mass matrix needed on the right hand side
-   //    of the generalized eigenvalue problem below. The boundary conditions
-   //    are implemented by elimination with special values on the diagonal to
-   //    shift the Dirichlet eigenvalues out of the computational range. After
-   //    serial and parallel assembly we extract the corresponding parallel
-   //    matrices A and M.
-   ConstantCoefficient one(1.0);
-   Array<int> ess_bdr;
-   if (mesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-   }
-
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   if (mesh->bdr_attributes.Size() == 0)
-   {
-      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
-      // closed surface.
-      a->AddDomainIntegrator(new MassIntegrator(one));
-   }
-   a->Assemble();
-   if (mesh->bdr_attributes.Size() != 0)
-   {
-      a->EliminateEssentialBCDiag(ess_bdr, dbc_eig);
-   }
-   a->Finalize();
-
-   BilinearForm *m = new BilinearForm(fespace);
-   m->AddDomainIntegrator(new MassIntegrator(one));
-   m->Assemble();
-   if (mesh->bdr_attributes.Size() != 0)
-   {
-      // shift the eigenvalue corresponding to eliminated dofs to a large value
-      m->EliminateEssentialBCDiag(ess_bdr, 1.0);
-   }
-   m->Finalize();
-
-   Solver       * solver = NULL;
-
-#ifndef MFEM_USE_SUITESPARSE
-   // 6. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-   //    solve the system A X = B with PCG.
-   cout << "Building CGSolver" << endl;
-   GSSmoother M(m->SpMat());
-   CGSolver * cg_solver = new CGSolver;
-   cg_solver->SetPreconditioner(M);
-   cg_solver->SetRelTol(1.0e-12);
-   solver = cg_solver;
-#else
-   // 7. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   cout << "Building UMFPackSolver" << endl;
-   UMFPackSolver * umf_solver = new UMFPackSolver;
-   umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   solver = umf_solver;
-#endif
-   solver->SetOperator(m->SpMat());
-
-   // 7. Define and configure the ARPACK eigensolver
-   SymGenEigensolver * eig_solver = NULL;
-
-   if (arp_solver)
-   {
-      // ArPackSymGen * arpack = new ArPackSymGen();
-      ArPackSAUPD * arpack = new ArPackSAUPD();
-      arpack->SetMode(2);
-      arpack->SetNumModes(nev);
-      arpack->SetMaxIter(400);
-      arpack->SetTol(1e-8);
-      arpack->SetPrintLevel(2);
-      arpack->SetSolver(*solver);
-
-      eig_solver = arpack;
-   }
-
-   eig_solver->SetOperators(*a, *m);
-
-   // 8. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //    parallel grid function to represent each of the eigenmodes returned by
-   //    the solver.
-   Array<double> eigenvalues;
-   eig_solver->Solve();
-   eig_solver->GetEigenvalues(eigenvalues);
-
-   cout << endl;
-   std::ios::fmtflags old_fmt = cout.flags();
-   cout.setf(std::ios::scientific);
-   std::streamsize old_prec = cout.precision(14);
-   for (int i=0; i<nev; i++)
-   {
-      cout << "Eigenvalue lambda   " << eigenvalues[i] << endl;
-   }
-   cout.precision(old_prec);
-   cout.flags(old_fmt);
-   cout << endl;
-
-   GridFunction x(fespace);
-
-   // 9. Save the refined mesh and the modes in parallel. This output can be
-   //    viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
-   {
-      ostringstream mesh_name, mode_name;
-      mesh_name << "ex11.mesh";
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      mesh->Print(mesh_ofs);
-
-      for (int i=0; i<nev; i++)
-      {
-         // convert eigenvector from Vector to GridFunction
-         x = eig_solver->GetEigenvector(i);
-
-         mode_name << "mode_" << setfill('0') << setw(2) << i;
-
-         ofstream mode_ofs(mode_name.str().c_str());
-         mode_ofs.precision(8);
-         x.Save(mode_ofs);
-         mode_name.str("");
-      }
-   }
-
-   // 10. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mode_sock(vishost, visport);
-      mode_sock.precision(8);
-
-      for (int i=0; i<nev; i++)
-      {
-         cout << "Eigenmode " << i+1 << '/' << nev
-              << ", Lambda = " << eigenvalues[i] << endl;
-
-         // convert eigenvector from Vector to GridFunction
-         x = eig_solver->GetEigenvector(i);
-
-         mode_sock << "solution\n" << *mesh << x << flush
-                   << "window_title 'Eigenmode " << i+1 << '/' << nev
-                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
-
-         char c;
-         cout << "press (q)uit or (c)ontinue --> " << flush;
-         cin >> c;
-
-         if (c != 'c')
-         {
-            break;
-         }
-      }
-      mode_sock.close();
-   }
-
-   // 11. Free the used memory.
-   delete eig_solver;
-   delete solver;
-   delete m;
-   delete a;
-
-   delete fespace;
-   if (order > 0)
-   {
-      delete fec;
-   }
-   delete mesh;
-
-   return 0;
-}
-
-#endif // MFEM_USE_ARPACK
@@ -72,8 +72,6 @@ int main(int argc, char *argv[])
   int seed = 75;
   bool slu_solver  = false;
   bool sp_solver = false;
-   bool lob_solver = true;
-   bool arp_solver = false;
   bool cpardiso_solver = false;
   bool visualization = 1;

@@ -99,10 +97,6 @@ int main(int argc, char *argv[])
   args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
                  "--no-strumpack", "Use the STRUMPACK Solver.");
 #endif
-#ifdef MFEM_USE_ARPACK
-   args.AddOption(&arp_solver, "-arp", "--arpack", "-no-arp",
-                  "--no-arpack", "Use the Parallel ARPACK Solver.");
-#endif
 #ifdef MFEM_USE_MKL_CPARDISO
   args.AddOption(&cpardiso_solver, "-cpardiso", "--cpardiso", "-no-cpardiso",
                  "--no-cpardiso", "Use the MKL CPardiso Solver.");
@@ -119,11 +113,6 @@ int main(int argc, char *argv[])
              << "         Defaulting to SuperLU." << endl;
      sp_solver = false;
   }
-   if (arp_solver)
-   {
-      lob_solver = false;
-   }
-
   // The command line options are also passed to the STRUMPACK
   // solver. So do not exit if some options are not recognized.
   if (!sp_solver)
@@ -254,119 +243,70 @@ int main(int argc, char *argv[])
   // 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
   //    preconditioner for A to be used within the solver. Set the matrices
   //    which define the generalized eigenproblem A x = lambda M x.
-   Solver * solver = NULL;
   Solver * precond = NULL;
   if (!slu_solver && !sp_solver && !cpardiso_solver)
   {
      HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
      amg->SetPrintLevel(0);
      precond = amg;
-
-      if (arp_solver)
-      {
-         HyprePCG * pcg = new HyprePCG(*A);
-         pcg->SetTol(1e-12);
-         pcg->SetPreconditioner(*amg);
-         solver = pcg;
-      }
   }
-#ifdef MFEM_USE_SUPERLU
-   else if (slu_solver)
+   else
   {
-      SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
-      superlu->SetPrintStatistics(false);
-      superlu->SetSymmetricPattern(true);
-      superlu->SetColumnPermutation(superlu::PARMETIS);
-      superlu->SetOperator(*Arow);
-
-      if (arp_solver)
-      {
-         solver = superlu;
-      }
-      else
+#ifdef MFEM_USE_SUPERLU
+      if (slu_solver)
      {
+         SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
+         superlu->SetPrintStatistics(false);
+         superlu->SetSymmetricPattern(true);
+         superlu->SetColumnPermutation(superlu::PARMETIS);
+         superlu->SetOperator(*Arow);
         precond = superlu;
      }
-   }
 #endif
 #ifdef MFEM_USE_STRUMPACK
-   else if (sp_solver)
-   {
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv,
-                                                        MPI_COMM_WORLD);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      strumpack->SetMatching(strumpack::MatchingJob::NONE);
-      strumpack->SetCompression(strumpack::CompressionType::NONE);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-      if (arp_solver)
-      {
-         solver = strumpack;
-      }
-      else
+      if (sp_solver)
      {
+         STRUMPACKSolver * strumpack = new STRUMPACKSolver(MPI_COMM_WORLD, argc, argv);
+         strumpack->SetPrintFactorStatistics(true);
+         strumpack->SetPrintSolveStatistics(false);
+         strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+         strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+         strumpack->SetMatching(strumpack::MatchingJob::NONE);
+         strumpack->SetCompression(strumpack::CompressionType::NONE);
+         strumpack->SetOperator(*Arow);
+         strumpack->SetFromCommandLine();
         precond = strumpack;
      }
-   }
 #endif
 #ifdef MFEM_USE_MKL_CPARDISO
-   else if (cpardiso_solver)
-   {
-      auto cpardiso = new CPardisoSolver(A->GetComm());
-      cpardiso->SetMatrixType(CPardisoSolver::MatType::REAL_STRUCTURE_SYMMETRIC);
-      cpardiso->SetPrintLevel(1);
-      cpardiso->SetOperator(*A);
-      if (arp_solver)
-      {
-         solver = cpardiso;
-      }
-      else
+      if (cpardiso_solver)
      {
+         auto cpardiso = new CPardisoSolver(A->GetComm());
+         cpardiso->SetMatrixType(CPardisoSolver::MatType::REAL_STRUCTURE_SYMMETRIC);
+         cpardiso->SetPrintLevel(1);
+         cpardiso->SetOperator(*A);
         precond = cpardiso;
      }
-   }
 #endif
-
-   SymGenEigensolver * eig_solver = NULL;
-
-   if (lob_solver)
-   {
-      HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
-      lobpcg->SetNumModes(nev);
-      lobpcg->SetRandomSeed(seed);
-      lobpcg->SetPreconditioner(*precond);
-      lobpcg->SetMaxIter(200);
-      lobpcg->SetTol(1e-8);
-      lobpcg->SetPrecondUsageMode(1);
-      lobpcg->SetPrintLevel(1);
-
-      eig_solver = lobpcg;
   }
-#ifdef MFEM_USE_ARPACK
-   else if (arp_solver)
-   {
-      ArPackPSAUPD * arpack = new ArPackPSAUPD(MPI_COMM_WORLD);
-      arpack->SetNumModes(nev);
-      arpack->SetMaxIter(400);
-      arpack->SetTol(1e-8);
-      arpack->SetMode(3);
-      arpack->SetPrintLevel(2);
-      arpack->SetSolver(*solver);

-      eig_solver = arpack;
-   }
-#endif
-   eig_solver->SetOperators(*A, *M);
+   HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
+   lobpcg->SetNumModes(nev);
+   lobpcg->SetRandomSeed(seed);
+   lobpcg->SetPreconditioner(*precond);
+   lobpcg->SetMaxIter(200);
+   lobpcg->SetTol(1e-8);
+   lobpcg->SetPrecondUsageMode(1);
+   lobpcg->SetPrintLevel(1);
+   lobpcg->SetMassMatrix(*M);
+   lobpcg->SetOperator(*A);

   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
   //    parallel grid function to represent each of the eigenmodes returned by
   //    the solver.
   Array<real_t> eigenvalues;
-   eig_solver->Solve();
-   eig_solver->GetEigenvalues(eigenvalues);
+   lobpcg->Solve();
+   lobpcg->GetEigenvalues(eigenvalues);
   ParGridFunction x(fespace);

   // 10. Save the refined mesh and the modes in parallel. This output can be
@@ -381,8 +321,8 @@ int main(int argc, char *argv[])

      for (int i=0; i<nev; i++)
      {
-         // convert eigenvector from Vector to ParGridFunction
-         x.Distribute(eig_solver->GetEigenvector(i));
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x = lobpcg->GetEigenvector(i);

         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
                   << setfill('0') << setw(6) << myid;
@@ -410,8 +350,8 @@ int main(int argc, char *argv[])
                 << ", Lambda = " << eigenvalues[i] << endl;
         }

-         // convert eigenvector from Vector to ParGridFunction
-         x.Distribute(eig_solver->GetEigenvector(i));
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x = lobpcg->GetEigenvector(i);

         mode_sock << "parallel " << num_procs << " " << myid << "\n"
                   << "solution\n" << *pmesh << x << flush
@@ -435,8 +375,7 @@ int main(int argc, char *argv[])
   }

   // 12. Free the used memory.
-   delete eig_solver;
-   delete solver;
+   delete lobpcg;
   delete precond;
   delete M;
   delete A;
@@ -1,381 +0,0 @@
-//                       MFEM Example 11 - Parallel Version
-//
-// Compile with: make ex11p
-//
-// Sample runs:  mpirun -np 4 ex11p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex11p -m ../data/star.mesh
-//               mpirun -np 4 ex11p -m ../data/escher.mesh
-//               mpirun -np 4 ex11p -m ../data/fichera.mesh
-//               mpirun -np 4 ex11p -m ../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex11p -m ../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex11p -m ../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex11p -m ../data/disc-nurbs.mesh -o -1 -n 20
-//               mpirun -np 4 ex11p -m ../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex11p -m ../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex11p -m ../data/star-surf.mesh
-//               mpirun -np 4 ex11p -m ../data/square-disc-surf.mesh
-//               mpirun -np 4 ex11p -m ../data/inline-segment.mesh
-//               mpirun -np 4 ex11p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex11p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex11p -m ../data/mobius-strip.mesh -n 8
-//               mpirun -np 4 ex11p -m ../data/klein-bottle.mesh -n 10
-//
-// Description:  This example code demonstrates the use of MFEM to solve the
-//               eigenvalue problem -Delta u = lambda u with homogeneous
-//               Dirichlet boundary conditions.
-//
-//               We compute a number of the lowest eigenmodes by discretizing
-//               the Laplacian and Mass operators using a FE space of the
-//               specified order, or an isoparametric/isogeometric space if
-//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of the LOBPCG and ARPACK
-//               eigenvalue solvers together with the BoomerAMG preconditioner
-//               in HYPRE, as well as optionally the SuperLU parallel direct
-//               solver. Reusing a single GLVis visualization window for
-//               multiple eigenfunctions is also illustrated.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int ser_ref_levels = 2;
-   int par_ref_levels = 1;
-   int order = 1;
-   int nev = 5;
-   bool slu_solver = false;
-   bool use_arpack = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&nev, "-n", "--num-eigs",
-                  "Number of desired eigenmodes.");
-#ifdef MFEM_USE_SUPERLU
-   args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
-                  "--no-superlu", "Use the SuperLU Solver.");
-#endif
-#ifdef MFEM_USE_ARPACK
-   args.AddOption(&use_arpack, "-arpack", "--use-arpack", "-no-arpack",
-                  "--no-arpack",
-                  "Enable or disable the use of ARPACK.");
-#endif
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh;
-   ifstream imesh(mesh_file);
-   if (!imesh)
-   {
-      if (myid == 0)
-      {
-         cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
-      }
-      MPI_Finalize();
-      return 2;
-   }
-   mesh = new Mesh(imesh, 1, 1);
-   imesh.close();
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
-   //    specified on the command line with -rs).
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution (1 time by
-   //    default, or specified on the command line with -rp). Once the parallel
-   //    mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of unknowns: " << size << endl;
-   }
-
-   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
-   //    element space. The first corresponds to the Laplacian operator -Delta,
-   //    while the second is a simple mass matrix needed on the right hand side
-   //    of the generalized eigenvalue problem below. The boundary conditions
-   //    are implemented by elimination with special values on the diagonal to
-   //    shift the Dirichlet eigenvalues out of the computational range. After
-   //    serial and parallel assembly we extract the corresponding parallel
-   //    matrices A and M.
-   ConstantCoefficient one(1.0);
-   Array<int> ess_bdr;
-   if (pmesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-   }
-
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   if (pmesh->bdr_attributes.Size() == 0)
-   {
-      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
-      // closed surface.
-      a->AddDomainIntegrator(new MassIntegrator(one));
-   }
-   a->Assemble();
-   a->EliminateEssentialBCDiag(ess_bdr, 1.0);
-   a->Finalize();
-
-   ParBilinearForm *m = new ParBilinearForm(fespace);
-   m->AddDomainIntegrator(new MassIntegrator(one));
-   m->Assemble();
-   // shift the eigenvalue corresponding to eliminated dofs to a large value
-   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
-   m->Finalize();
-
-   HypreParMatrix *A = a->ParallelAssemble();
-   HypreParMatrix *M = m->ParallelAssemble();
-
-#ifdef MFEM_USE_SUPERLU
-   Operator * Arow = NULL;
-   if (slu_solver)
-   {
-      Arow = new SuperLURowLocMatrix(*A);
-   }
-#endif
-
-   delete a;
-   delete m;
-
-   // 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
-   //    preconditioner for A to be used within the solver. Set the matrices
-   //    which define the generalized eigenproblem A x = lambda M x.
-   Eigensolver * esolver = NULL;
-   Solver      *  solver = NULL;
-   Solver      * precond = NULL;
-
-   if (!slu_solver)
-   {
-      HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
-      amg->SetPrintLevel(0);
-      precond = amg;
-
-#ifdef MFEM_USE_ARPACK
-      if ( use_arpack )
-      {
-         HyprePCG * pcg = new HyprePCG(*A);
-         pcg->SetTol(1e-12);
-         pcg->SetMaxIter(200);
-         pcg->SetPreconditioner(*amg);
-         pcg->SetPrintLevel(0);
-
-         solver = pcg;
-      }
-#endif
-   }
-#ifdef MFEM_USE_SUPERLU
-   else
-   {
-      SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
-      superlu->SetPrintStatistics(false);
-      superlu->SetSymmetricPattern(true);
-      superlu->SetColumnPermutation(superlu::PARMETIS);
-      superlu->SetOperator(*Arow);
-
-      solver  = use_arpack?superlu:NULL;
-      precond = use_arpack?NULL:superlu;
-   }
-#endif
-
-   if ( use_arpack )
-   {
-      ParArPackSym * arpack = new ParArPackSym(MPI_COMM_WORLD);
-      arpack->SetMode(3);
-      arpack->SetPrintLevel(2);
-      arpack->SetSolver(*solver);
-
-      esolver = arpack;
-   }
-   else
-   {
-      HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
-      lobpcg->SetPreconditioner(*precond);
-      lobpcg->SetPrecondUsageMode(1);
-      lobpcg->SetPrintLevel(1);
-
-      esolver = lobpcg;
-   }
-
-   esolver->SetNumModes(nev);
-   esolver->SetMaxIter(100);
-   esolver->SetTol(1e-8);
-
-   esolver->SetMassMatrix(*M);
-   esolver->SetOperator(*A);
-
-   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //    parallel grid function to represent each of the eigenmodes returned by
-   //    the solver.
-   Array<double> eigenvalues;
-   esolver->Solve();
-   esolver->GetEigenvalues(eigenvalues);
-
-   if ( myid == 0 && use_arpack )
-   {
-      cout << endl;
-      for (int i=0; i<eigenvalues.Size(); i++)
-      {
-         cout << "Eigenvalue lambda " << eigenvalues[i] << endl;
-      }
-      cout << endl;
-   }
-
-   ParGridFunction x(fespace);
-
-   // 10. Save the refined mesh and the modes in parallel. This output can be
-   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
-   {
-      ostringstream mesh_name, mode_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      for (int i=0; i<nev; i++)
-      {
-         // convert eigenvector from HypreParVector to ParGridFunction
-         x.Distribute(esolver->GetEigenvector(i));
-
-         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
-                   << setfill('0') << setw(6) << myid;
-
-         ofstream mode_ofs(mode_name.str().c_str());
-         mode_ofs.precision(8);
-         x.Save(mode_ofs);
-         mode_name.str("");
-      }
-   }
-
-   // 11. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mode_sock(vishost, visport);
-      mode_sock.precision(8);
-
-      for (int i=0; i<nev; i++)
-      {
-         if ( myid == 0 )
-         {
-            cout << "Eigenmode " << i+1 << '/' << nev
-                 << ", Lambda = " << eigenvalues[i] << endl;
-         }
-
-         // convert eigenvector from HypreParVector to ParGridFunction
-         x.Distribute(esolver->GetEigenvector(i));
-
-         mode_sock << "parallel " << num_procs << " " << myid << "\n"
-                   << "solution\n" << *pmesh << x << flush
-                   << "window_title 'Eigenmode " << i+1 << '/' << nev
-                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
-
-         char c;
-         if (myid == 0)
-         {
-            cout << "press (q)uit or (c)ontinue --> " << flush;
-            cin >> c;
-         }
-         MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
-
-         if (c != 'c')
-         {
-            break;
-         }
-      }
-      mode_sock.close();
-   }
-
-   // 12. Free the used memory.
-   delete esolver;
-   delete solver;
-   delete precond;
-   delete M;
-   delete A;
-
-   delete fespace;
-   if (order > 0)
-   {
-      delete fec;
-   }
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
@@ -276,8 +276,8 @@ int main(int argc, char *argv[])

      for (int i=0; i<nev; i++)
      {
-         // convert eigenvector from Vector to ParGridFunction
-         x.Distribute(lobpcg->GetEigenvector(i));
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x = lobpcg->GetEigenvector(i);

         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
                   << setfill('0') << setw(6) << myid;
@@ -303,7 +303,7 @@ int main(int argc, char *argv[])
      pmesh->Print(adios2output);
      for (int i=0; i<nev; i++)
      {
-         x.Distribute(lobpcg->GetEigenvector(i));
+         x = lobpcg->GetEigenvector(i);
         // x is a temporary that must be saved immediately
         x.Save(adios2output, "mode_" + std::to_string(i));
      }
@@ -326,8 +326,8 @@ int main(int argc, char *argv[])
                 << ", Lambda = " << eigenvalues[i] << endl;
         }

-         // convert eigenvector from Vector to ParGridFunction
-         x.Distribute(lobpcg->GetEigenvector(i));
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x = lobpcg->GetEigenvector(i);

         mode_sock << "parallel " << num_procs << " " << myid << "\n"
                   << "solution\n" << *pmesh << x << flush
@@ -1,282 +0,0 @@
-//                       MFEM Example 13
-//
-// Compile with: make ex3p
-//
-// Sample runs:  ex13 -m ../data/star.mesh -s 5
-//               ex13 -m ../data/square-disc.mesh -o 2 -n 4 // minres fails to conv.
-//               ex13 -m ../data/beam-hex.mesh
-//               ex13 -m ../data/square-disc.mesh -rs 1 -s 26
-//               ex13 -m ../data/square-disc-nurbs.mesh -rs 3 -s 26
-//               ex13 -m ../data/amr-quad.mesh -o 2 // minres fails to conv.
-//               ex13 -m ../data/mobius-strip.mesh -n 8
-//
-// Description:  This example code solves a simple 3D electromagnetic
-//               eigenmode problem corresponding to the second order
-//               Maxwell equation curl curl E = lambda E with boundary
-//               condition E x n = 0. We discretize with Nedelec finite
-//               elements.
-//
-//               The example demonstrates the use of H(curl) finite element
-//               spaces with the curl-curl and the (vector finite element) mass
-//               bilinear form, as well as the use of the ARPACK eigenmode
-//               solver for symmetric matrices using the shift-invert mode.
-//
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-#ifdef MFEM_USE_ARPACK
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/beam-tet.mesh";
-   int order = 1;
-   int nev = 5;
-   int sr = 2;
-   double sigma = 11.0;
-   bool visualization = 1;
-   bool arp_solver = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&nev, "-n", "--num-eigs",
-                  "Number of desired eigenmodes.");
-   args.AddOption(&sr, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&sigma, "-s", "--shift",
-                  "Average of the desired eigenvalue range.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file.  We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
-   //    with the same code.
-   Mesh *mesh;
-   ifstream imesh(mesh_file);
-   if (!imesh)
-   {
-      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
-      return 2;
-   }
-   mesh = new Mesh(imesh, 1, 1);
-   imesh.close();
-   int dim = mesh->Dimension();
-
-   // 3. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement.
-   {
-      int ref_levels = sr;
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 4. Define a finite element space on the mesh. Here we use the lowest
-   //    order Nedelec finite elements, but we can easily switch
-   //    to higher-order spaces by changing the value of p.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   int size = fespace->GetVSize();
-
-   cout << "Number of unknowns: " << size << endl;
-   cout << "Number of boundary attributes: " << mesh->bdr_attributes.Max()
-        << endl;
-
-   // 5. Set up the parallel bilinear form corresponding to the EM diffusion
-   //    operator curl muinv curl - sigma I, by adding the curl-curl and the
-   //    mass domain integrators and finally imposing homogeneous Dirichlet
-   //    boundary conditions. The boundary conditions are implemented by
-   //    marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet). After serial and parallel assembly we extract the
-   //    parallel matrices A and M.
-   Coefficient *muinv    = new ConstantCoefficient(1.0);
-   Coefficient *negSigma = new ConstantCoefficient(-sigma);
-
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*negSigma));
-   a->Assemble();
-   Array<int> ess_bdr(mesh->bdr_attributes.Max());
-   ess_bdr = 1;
-   a->EliminateEssentialBC(ess_bdr);
-   a->Finalize();
-
-   BilinearForm *m = new BilinearForm(fespace);
-   m->AddDomainIntegrator(new VectorFEMassIntegrator());
-   m->Assemble();
-   m->EliminateEssentialBCDiag(ess_bdr, sqrt(numeric_limits<double>::min()));
-   m->Finalize();
-
-   // 6. Define a parallel grid function to approximate each of the
-   // eigenmodes returned by the solver.  Use this as a template to
-   // create a special multi-vector object needed by the eigensolver
-   // which is then initialized with random values.
-   GridFunction x(fespace);
-   x = 0.0;
-
-   // 7. Define and configure the GMRES
-   //    solver to be used within the eigensolver.
-   Solver    * solver = NULL;
-   if ( false )
-   {
-      GMRESSolver *  gmres = new GMRESSolver();
-
-      gmres->SetOperator(*a);
-      gmres->SetRelTol(1e-8);
-      gmres->SetMaxIter(1000);
-      gmres->SetPrintLevel(0);
-      solver = gmres;
-   }
-   else
-   {
-#ifndef MFEM_USE_SUITESPARSE
-      cout << "Building MINRESSolver" << endl;
-      MINRESSolver * minres = new MINRESSolver();
-
-      minres->SetRelTol(1e-12);
-      minres->SetMaxIter(1000);
-      minres->SetPrintLevel(0);
-      solver = minres;
-#else
-      // 7. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      cout << "Building UMFPackSolver" << endl;
-      UMFPackSolver * umf_solver = new UMFPackSolver;
-      umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      solver = umf_solver;
-#endif
-   }
-   solver->SetOperator(a->SpMat());
-
-   // 7. Define and configure the ARPACK eigensolver
-   SymGenEigensolver * eig_solver = NULL;
-   if (arp_solver)
-   {
-      ArPackSAUPD * arpack = new ArPackSAUPD();
-
-      arpack->SetNumModes(nev);
-      arpack->SetMaxIter(400);
-      arpack->SetTol(1e-8);
-      arpack->SetShift(sigma);
-      arpack->SetMode(3);
-      arpack->SetPrintLevel(2);
-      arpack->SetSolver(*solver);
-
-      eig_solver = arpack;
-   }
-
-   eig_solver->SetOperators(*a, *m);
-
-   // Obtain the eigenvalues and eigenvectors
-   Array<double> eigenvalues(nev);
-   eigenvalues = -1.0;
-
-   // arpack->Solve(eigenvalues, *eigenvectors);
-   eig_solver->Solve();
-
-   eig_solver->GetEigenvalues(eigenvalues);
-
-   cout << endl;
-   std::ios::fmtflags old_fmt = cout.flags();
-   cout.setf(std::ios::scientific);
-   std::streamsize old_prec = cout.precision(14);
-   for (int i=0; i<min(nev,eigenvalues.Size()); i++)
-   {
-      cout << "Eigenvalue lambda   " << eigenvalues[i] << endl;
-   }
-   cout.precision(old_prec);
-   cout.flags(old_fmt);
-   cout << endl;
-
-   VisItDataCollection visit_dc("Example13", mesh);
-   GridFunction ** mode = new GridFunction*[min(nev,eigenvalues.Size())];
-   for (int i=0; i<min(nev,eigenvalues.Size()); i++)
-   {
-      mode[i] = new GridFunction(fespace);
-      *mode[i] = eig_solver->GetEigenvector(i);
-
-      ostringstream modeName;
-      modeName << "mode_" << setfill('0') << setw(2) << i;
-      visit_dc.RegisterField(modeName.str().c_str(),mode[i]);
-   }
-   visit_dc.Save();
-
-   // 8. Save the refined mesh and the modes. This output can
-   //    be viewed later using GLVis: "glvis -m mesh -g mode".
-   {
-      ofstream mesh_ofs("refined.mesh");
-      mesh_ofs.precision(8);
-      mesh->Print(mesh_ofs);
-
-      for (int i=0; i<min(nev,eigenvalues.Size()); i++)
-      {
-         x = eig_solver->GetEigenvector(i);
-
-         ostringstream modeName;
-         modeName << "mode_" << setfill('0') << setw(2) << i;
-
-         ofstream mode_ofs(modeName.str().c_str());
-         mode_ofs.precision(8);
-         x.Save(mode_ofs);
-         modeName.str("");
-      }
-   }
-
-   // 9. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mode_sock(vishost, visport);
-      mode_sock.precision(8);
-
-      for (int i=0; i<min(nev,eigenvalues.Size()); i++)
-      {
-         x = eig_solver->GetEigenvector(i);
-
-         mode_sock << "solution\n" << *mesh << x << flush;
-
-         char c;
-         cout << "press (q)uit or (c)ontinue --> " << flush;
-         cin >> c;
-
-         if (c != 'c')
-         {
-            break;
-         }
-      }
-      mode_sock.close();
-   }
-
-   // 10. Free the used memory.
-   delete a;
-   delete m;
-   delete negSigma;
-   delete muinv;
-   delete eig_solver;
-   delete solver;
-   // delete X;
-   delete fespace;
-   delete fec;
-   delete mesh;
-
-   return 0;
-}
-
-#endif // MFEM_USE_ARPACK
@@ -215,8 +215,8 @@ int main(int argc, char *argv[])

      for (int i=0; i<nev; i++)
      {
-         // convert eigenvector from Vector to ParGridFunction
-         x.Distribute(ame->GetEigenvector(i));
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x = ame->GetEigenvector(i);

         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
                   << setfill('0') << setw(6) << myid;
@@ -244,8 +244,8 @@ int main(int argc, char *argv[])
                 << ", Lambda = " << eigenvalues[i] << endl;
         }

-         // convert eigenvector from Vector to ParGridFunction
-         x.Distribute(ame->GetEigenvector(i));
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x = ame->GetEigenvector(i);

         mode_sock << "parallel " << num_procs << " " << myid << "\n"
                   << "solution\n" << *pmesh << x << flush
@@ -42,7 +42,11 @@
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/beam-tet.mesh
+//
+// Device simplices sample runs:
+//               mpirun -np 4 ex1p -pa -d gpu -m ../data/inline-tet.mesh
+//               mpirun -np 4 ex1p -pa -d gpu -m ../data/inline-tri.mesh
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Poisson problem
@@ -83,6 +87,9 @@ int main(int argc, char *argv[])
   const char *device_config = "cpu";
   bool visualization = true;
   bool algebraic_ceed = false;
+#ifdef MFEM_USE_CUDSS
+   bool cudss_solver = false;
+#endif

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -102,6 +109,10 @@ int main(int argc, char *argv[])
   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
                  "-no-a", "--no-algebraic",
                  "Use algebraic Ceed solver");
+#endif
+#ifdef MFEM_USE_CUDSS
+   args.AddOption(&cudss_solver, "-cudss", "--cudss-solver", "-no-cudss",
+                  "--no-cudss-solver", "Use the cuDSS Solver.");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -158,19 +169,20 @@ int main(int argc, char *argv[])
   }

   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
+   //    use continuous Lagrange finite elements of the specified order.
+   //    - If order < 1, we instead use an isoparametric/isogeometric space.
+   //    - If the mesh is simplicial and partial assembly is requested,
+   //      we use the positive basis, which supports device execution.
   FiniteElementCollection *fec;
-   bool delete_fec;
+   auto basis_type = (pa && pmesh.IsSimplexMesh()) ?
+                     BasisType::Positive : BasisType::GaussLobatto;
   if (order > 0)
   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
+      fec = new H1_FECollection(order, dim, basis_type);
   }
   else if (pmesh.GetNodes())
   {
      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
      if (myid == 0)
      {
         cout << "Using isoparametric FEs: " << fec->Name() << endl;
@@ -178,8 +190,7 @@ int main(int argc, char *argv[])
   }
   else
   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
+      fec = new H1_FECollection(order = 1, dim, basis_type);
   }
   ParFiniteElementSpace fespace(&pmesh, fec);
   HYPRE_BigInt size = fespace.GlobalTrueVSize();
@@ -248,33 +259,51 @@ int main(int argc, char *argv[])
   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
   //     * With partial assembly, use Jacobi smoothing, for now.
-   Solver *prec = NULL;
-   if (pa)
+#ifdef MFEM_USE_CUDSS
+   if (!pa && (Device::Allows(Backend::CUDA_MASK) && cudss_solver))
   {
-      if (UsesTensorBasis(fespace))
-      {
-         if (algebraic_ceed)
-         {
-            prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
-         }
-         else
-         {
-            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-         }
-      }
+      // Solve using a direct solver with cuDSS
+      CuDSSSolver cudss_solver(MPI_COMM_WORLD);
+      cudss_solver.SetMatrixSymType(
+         CuDSSSolver::SYMMETRIC_POSITIVE_DEFINITE);
+      cudss_solver.SetMatrixViewType(CuDSSSolver::UPPER);
+      cudss_solver.SetOperator(*A);
+      cudss_solver.Mult(B, X);
   }
   else
+#endif
   {
-      prec = new HypreBoomerAMG;
+      Solver *prec = NULL;
+      if (pa)
+      {
+         if (UsesTensorBasis(fespace))
+         {
+            if (algebraic_ceed)
+            {
+               prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
+            }
+            else
+            {
+               prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+            }
+         }
+      }
+      else
+      {
+         prec = new HypreBoomerAMG;
+      }
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(1);
+      if (prec)
+      {
+         cg.SetPreconditioner(*prec);
+      }
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete prec;
   }
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   if (prec) { cg.SetPreconditioner(*prec); }
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete prec;

   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
@@ -308,10 +337,7 @@ int main(int argc, char *argv[])
   }

   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   if (order > 0) { delete fec; }

   return 0;
 }
@@ -95,6 +95,15 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

+   if (amg_elast && !static_cond && reorder_space)
+   {
+      if (myid == 0)
+         cerr << "\nThe AMG elasticity solver requires ordering byVDIM! "
+              << "Ignoring the specified option -nodes/--by-nodes.\n"
+              << endl;
+      reorder_space = false;
+   }
+
   // 3. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
   Device device(device_config);
@@ -228,7 +228,7 @@ int main(int argc, char *argv[])
      for (int i=0; i<nev; i++)
      {
         // convert eigenvector from HypreParVector to ParGridFunction
-         x.Distribute(ame->GetEigenvector(i));
+         x = ame->GetEigenvector(i);
         curl.Mult(x, dx);

         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
@@ -295,7 +295,7 @@ int main(int argc, char *argv[])
            }

            // convert eigenvector from HypreParVector to ParGridFunction
-            x.Distribute(ame->GetEigenvector(i));
+            x = ame->GetEigenvector(i);
            curl.Mult(x, dx);

            {
@@ -469,7 +469,7 @@ int main(int argc, char *argv[])
            }

            // convert eigenvector from HypreParVector to ParGridFunction
-            x.Distribute(ame->GetEigenvector(i));
+            x = ame->GetEigenvector(i);
            curl.Mult(x, dx);

            {
@@ -599,7 +599,7 @@ int main(int argc, char *argv[])
            }

            // convert eigenvector from HypreParVector to ParGridFunction
-            x.Distribute(ame->GetEigenvector(i));
+            x = ame->GetEigenvector(i);
            curl.Mult(x, dx);

            mode_sock << "parallel " << num_procs << " " << myid << "\n"
@@ -658,7 +658,7 @@ void ScalarWaveGuide(int mode, ParGridFunction &x)
   lobpcg.SetOperator(*A);
   lobpcg.Solve();

-   x.Distribute(lobpcg.GetEigenvector(mode));
+   x = lobpcg.GetEigenvector(mode);

   delete A;
   delete M;
@@ -714,7 +714,7 @@ void VectorWaveGuide(int mode, ParGridFunction &x)
   ame.SetOperator(*A);
   ame.Solve();

-   x.Distribute(ame.GetEigenvector(mode));
+   x = ame.GetEigenvector(mode);

   delete A;
   delete M;
@@ -780,7 +780,7 @@ void PseudoScalarWaveGuide(int mode, ParGridFunction &x_l2)
   lobpcg.SetOperator(*A);
   lobpcg.Solve();

-   x.Distribute(lobpcg.GetEigenvector(mode));
+   x = lobpcg.GetEigenvector(mode);

   x_l2.ProjectCoefficient(xCoef);

@@ -76,4 +76,4 @@ clean-build:
 	rm -rf *.dSYM *.TVD.*breakpoints

 clean-exec:
-	@rm -f refined.mesh sol.gf
+	@rm -f refined.mesh sol.gf mesh.* sol.*
@@ -31,9 +31,6 @@ SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
   ex22p ex24p ex25p ex26p ex34p ex35p

-ifeq ($(MFEM_USE_ARPACK),YES)
-   SEQ_EXAMPLES += ex11 ex13
-endif
 ifeq ($(MFEM_USE_LAPACK),YES)
   SEQ_EXAMPLES += ex38
 endif
@@ -74,6 +71,7 @@ endif

 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
+SUBDIRS_TEST_NOCLEAN = $(addsuffix /test-noclean,$(SUBDIRS))
 SUBDIRS_CLEAN = $(addsuffix /clean,$(SUBDIRS))
 SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))

@@ -90,8 +88,9 @@ SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))

 all: $(EXAMPLES) $(SUBDIRS_ALL)

-.PHONY: $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_CLEAN) $(SUBDIRS_TPRINT)
-$(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_CLEAN):
+.PHONY: $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) \
+   $(SUBDIRS_CLEAN) $(SUBDIRS_TPRINT)
+$(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) $(SUBDIRS_CLEAN):
 	$(MAKE) -C $(@D) $(@F)
 $(SUBDIRS_TPRINT):
 	@$(MAKE) -C $(@D) $(@F)
@@ -110,6 +109,7 @@ endif
 MFEM_TESTS = EXAMPLES
 include $(MFEM_TEST_MK)
 test: $(SUBDIRS_TEST)
+test-noclean: $(SUBDIRS_TEST_NOCLEAN)
 test-print: $(SUBDIRS_TPRINT)

 # Testing: Parallel vs. serial runs
@@ -171,8 +171,12 @@ set(SRCS
  tmop_tools.cpp
  tmop_amr.cpp
  gslib.cpp
+  gslib/findptsedge_local_2.cpp
+  gslib/findptsedge_local_3.cpp
+  gslib/findptssurf_local_3.cpp
  gslib/findpts_local_2.cpp
  gslib/findpts_local_3.cpp
+  gslib/interpolate_local_1.cpp
  gslib/interpolate_local_2.cpp
  gslib/interpolate_local_3.cpp
  transfer.cpp
@@ -191,12 +195,14 @@ set(HDRS
  integ/bilininteg_dgtrace_kernels.hpp
  integ/bilininteg_vecdiffusion_kernels.hpp
  integ/bilininteg_convection_kernels.hpp
+  integ/bilininteg_diffusion_pa_simplices.hpp
  integ/bilininteg_diffusion_kernels.hpp
  integ/bilininteg_elasticity_kernels.hpp
  integ/bilininteg_hcurl_kernels.hpp
  integ/bilininteg_hdiv_kernels.hpp
  integ/bilininteg_hcurlhdiv_kernels.hpp
  integ/bilininteg_mass_kernels.hpp
+  integ/bilininteg_mass_pa_simplices.hpp
  integ/bilininteg_vecdiffusion_pa.hpp
  integ/bilininteg_vecmass_pa.hpp
  coefficient.hpp
@@ -1255,6 +1255,31 @@ void BilinearForm::Mult(const Vector &x, Vector &y) const
   }
 }

+void BilinearForm::AddMult(const Vector &x, Vector &y, const real_t a) const
+{
+   if (ext)
+   {
+      ext->AddMult(x, y, a);
+   }
+   else
+   {
+      mat->AddMult(x, y, a);
+   }
+}
+
+void BilinearForm::AddMultTranspose(const Vector &x, Vector &y,
+                                    const real_t a) const
+{
+   if (ext)
+   {
+      ext->AddMultTranspose(x, y, a);
+   }
+   else
+   {
+      mat->AddMultTranspose(x, y, a);
+   }
+}
+
 void BilinearForm::MultTranspose(const Vector & x, Vector & y) const
 {
   if (ext)
@@ -307,8 +307,8 @@ public:
   { mat->Mult(x, y); mat_e->AddMult(x, y); }

   /// Add the matrix vector multiple to a vector:  $ y += a M x $
-   void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const override
-   { mat -> AddMult (x, y, a); }
+   void AddMult(const Vector &x, Vector &y,
+                const real_t a = 1.0) const override;

   /** @brief Add the original uneliminated matrix vector multiple to a vector.
       The original matrix is $ M + Me $ so we have:
@@ -318,8 +318,7 @@ public:

   /// Add the matrix transpose vector multiplication:  $ y += a M^T x $
   void AddMultTranspose(const Vector & x, Vector & y,
-                         const real_t a = 1.0) const override
-   { mat->AddMultTranspose(x, y, a); }
+                         const real_t a = 1.0) const override;

   /** @brief Add the original uneliminated matrix transpose vector
       multiple to a vector. The original matrix is $ M + M_e $
@@ -1345,7 +1345,8 @@ real_t DiffusionIntegrator::ComputeFluxEnergy
 }

 const IntegrationRule &DiffusionIntegrator::GetRule(
-   const FiniteElement &trial_fe, const FiniteElement &test_fe)
+   const FiniteElement &trial_fe, const FiniteElement &test_fe,
+   const bool stroud)
 {
   int order;
   if (trial_fe.Space() == FunctionSpace::Pk)
@@ -1362,7 +1363,15 @@ const IntegrationRule &DiffusionIntegrator::GetRule(
   {
      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
   }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
+
+   if (stroud)
+   {
+      return StroudIntRules.Get(trial_fe.GetGeomType(), order);
+   }
+   else
+   {
+      return IntRules.Get(trial_fe.GetGeomType(), order);
+   }
 }

 MassIntegrator::MassIntegrator(const IntegrationRule *ir)
@@ -1449,7 +1458,8 @@ void MassIntegrator::AssembleElementMatrix2(

 const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
                                               const FiniteElement &test_fe,
-                                               const ElementTransformation &Trans)
+                                               const ElementTransformation &Trans,
+                                               const bool stroud)
 {
   // int order = trial_fe.GetOrder() + test_fe.GetOrder();
   const int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
@@ -1458,7 +1468,15 @@ const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
   {
      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
   }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
+
+   if (stroud)
+   {
+      return StroudIntRules.Get(trial_fe.GetGeomType(), order);
+   }
+   else
+   {
+      return IntRules.Get(trial_fe.GetGeomType(), order);
+   }
 }


@@ -1110,20 +1110,36 @@ public:
   MixedDotProductIntegrator(VectorCoefficient &vq)
      : MixedScalarVectorIntegrator(vq, true) {}

-   inline virtual bool VerifyFiniteElementTypes(
+   inline bool VerifyFiniteElementTypes(
      const FiniteElement & trial_fe,
-      const FiniteElement & test_fe) const
+      const FiniteElement & test_fe) const override
   {
      return (trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
              test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
   }

-   inline virtual const char * FiniteElementTypeFailureMessage() const
+   inline const char * FiniteElementTypeFailureMessage() const override
   {
      return "MixedDotProductIntegrator:  "
             "Trial space must be a vector field "
             "and the test space must be a scalar field";
   }
+
+   using BilinearFormIntegrator::AssemblePA;
+   void AssemblePA(const FiniteElementSpace &trial_fes,
+                   const FiniteElementSpace &test_fes) override;
+
+   void AddMultPA(const Vector&, Vector&) const override;
+   void AddMultTransposePA(const Vector&, Vector&) const override;
+
+private:
+   Vector pa_data;
+   const DofToQuad *mapsO = nullptr;      ///< Not owned. Trial open map.
+   const DofToQuad *mapsC = nullptr;      ///< Not owned. Trial closed map.
+   const DofToQuad *mapsTest = nullptr;   ///< Not owned. Scalar test map.
+   const GeometricFactors *geom = nullptr;///< Not owned.
+   int dim = 0, ne = 0, dofs1D = 0, dofs1Dtest = 0, quad1D = 0;
+   bool test_map_integral = false;
 };

 /** Class for integrating the bilinear form $a(u,v) := (-\vec{V} \cdot u, \nabla \cdot v)$ in 2D or
@@ -2184,11 +2200,22 @@ public:
                                   const Vector&, const Vector&,
                                   Vector&, const int, const int);

+   using ApplySimplexKernelType = void(*)(const int, const bool, const Array<int>&,
+                                          const Array<int>&,
+                                          const Array<int>&, const Array<int>&, const Array<int>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Vector&, const Vector&,
+                                          Vector&, const int, const int);
+
   using DiagonalKernelType = void(*)(const int, const bool, const Array<real_t>&,
                                      const Array<real_t>&, const Vector&, Vector&,
                                      const int, const int);

   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
+   MFEM_REGISTER_KERNELS(ApplySimplexPAKernels, ApplySimplexKernelType, (int, int,
+                                                                         int));
   MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
   struct Kernels { Kernels(); };

@@ -2341,7 +2368,8 @@ public:
   void AddMultPatchPA(const int patch, const Vector &x, Vector &y) const;

   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe);
+                                         const FiniteElement &test_fe,
+                                         const bool stroud = false);

   bool SupportsCeed() const override { return DeviceCanUseCeed(); }

@@ -2352,6 +2380,13 @@ public:
   {
      ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
      DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
+      AddSimplexSpecialization<DIM,D1D,Q1D>();
+   }
+
+   template <int DIM, int D1D, int Q1D>
+   static void AddSimplexSpecialization()
+   {
+      ApplySimplexPAKernels::Specialization<DIM,D1D,Q1D>::Add();
   }
 protected:
   const IntegrationRule* GetDefaultIntegrationRule(
@@ -2388,11 +2423,22 @@ public:
                                   const Array<real_t>&, const Vector&,
                                   const Vector&, Vector&, const int, const int);

+   using ApplySimplexKernelType = void(*)(const int, const Array<int>&,
+                                          const Array<int>&,
+                                          const Array<int>&, const Array<int>&, const Array<int>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Vector&, const Vector&, Vector&,
+                                          const int, const int);
+
   using DiagonalKernelType =  void(*)(const int, const Array<real_t>&,
                                       const Vector&, Vector&, const int,
                                       const int);

   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
+   MFEM_REGISTER_KERNELS(ApplySimplexPAKernels, ApplySimplexKernelType, (int, int,
+                                                                         int));
   MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
   struct Kernels { Kernels(); };

@@ -2441,7 +2487,8 @@ public:

   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                         const FiniteElement &test_fe,
-                                         const ElementTransformation &Trans);
+                                         const ElementTransformation &Trans,
+                                         const bool stroud = false);

   bool SupportsCeed() const override { return DeviceCanUseCeed(); }

@@ -2452,6 +2499,13 @@ public:
   {
      ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
      DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
+      AddSimplexSpecialization<DIM,D1D,Q1D>();
+   }
+
+   template <int DIM, int D1D, int Q1D>
+   static void AddSimplexSpecialization()
+   {
+      ApplySimplexPAKernels::Specialization<DIM,D1D,Q1D>::Add();
   }

 protected:
@@ -2957,11 +3011,10 @@ public:
    vector (diagonal matrix), or matrix), trial function $u$ is in $H(curl$ or
    $H(div)$, and test function $v$ is in $H(curl$, $H(div)$, or $v=(v_1,\dots,v_n)$, where
    $v_i$ are in $H^1$. */
-class VectorFEMassIntegrator: public BilinearFormIntegrator
+class VectorFEMassIntegrator : public BilinearFormIntegrator
 {
 private:
-   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq)
-   { Q = q; DQ = dq; MQ = mq; }
+   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq);

 #ifndef MFEM_THREAD_SAFE
   Vector shape;
@@ -2984,7 +3037,8 @@ protected:
   const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
   const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D, trial_fetype, test_fetype;
+   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D;
+   FiniteElement::DerivType trial_fetype, test_fetype;
   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
@@ -3015,6 +3069,29 @@ public:
                   const bool add) override;

   const Coefficient *GetCoefficient() const { return Q; }
+
+   using ApplyKernelType =
+      void (*)(const int NE, bool symmetric, const bool scalar_coeff,
+               const Array<real_t> &trialBO, const Array<real_t> &trialBC,
+               const Array<real_t> &testBOt, const Array<real_t> &testBCt,
+               const Vector &pa_data, const Vector &x, Vector &y,
+               const int triald1d, const int testd1d, const int q1d);
+
+   /// parameters: trial_fetype, test_fetype, ndims, trial_d1d, test_d1d, q1d
+   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType,
+                         (FiniteElement::DerivType, FiniteElement::DerivType,
+                          int, int, int, int));
+
+   struct Kernels { Kernels(); };
+
+   template <FiniteElement::DerivType TrialType,
+             FiniteElement::DerivType TestType, int DIM, int TRIAL_D1D,
+             int TEST_D1D, int Q1D>
+   static void AddSpecialization()
+   {
+      ApplyPAKernels::Specialization<TrialType, TestType, DIM, TRIAL_D1D,
+                     TEST_D1D, Q1D>::Add();
+   }
 };

 /** Integrator for $(Q \nabla \cdot u, v)$ where $u=(u_1,\cdots,u_n)$ and all $u_i$ are in the same
@@ -3868,7 +3945,7 @@ class DiscreteInterpolator : public BilinearFormIntegrator { };


 /** Class for constructing the gradient as a DiscreteLinearOperator from an
-    $H^1$-conforming space to an $H(curl$-conforming space. The range space can be
+    $H^1$-conforming space to an $H(curl)$-conforming space. The range space can be
    vector $L_2$ space as well. */
 class GradientInterpolator : public DiscreteInterpolator
 {
@@ -3977,12 +4054,48 @@ public:
    discrete curl matrix. */
 class CurlInterpolator : public DiscreteInterpolator
 {
+   int dim, ne;
+   // "dof" are the domain fespace dof counts
+   int ndof_o;
+   // "quads" are the range fespace dof counts
+   int nquad_o;
+
+   Vector pa_data;
+
 public:
+   CurlInterpolator();
+
   void AssembleElementMatrix2(const FiniteElement &dom_fe,
                               const FiniteElement &ran_fe,
                               ElementTransformation &Trans,
                               DenseMatrix &elmat) override
   { ran_fe.ProjectCurl(dom_fe, Trans, elmat); }
+
+   void AssemblePA(const FiniteElementSpace &dom_fes,
+                   const FiniteElementSpace &ran_fes) override;
+   void AssemblePA(const FiniteElementSpace &fes) override
+   {
+      AssemblePA(fes, fes);
+   }
+   void AddMultPA(const Vector &x, Vector &y) const override;
+   void AddMultTransposePA(const Vector &x, Vector &y) const override;
+
+   using ApplyKernelType = void (*)(const int ne, const int ndof_o,
+                                    const int nquad_o, const Vector &pa,
+                                    const Vector &x, Vector &y);
+
+   /// arguments: DIM, ndof_o, nquad_o
+   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
+   /// arguments: DIM, ndof_o, nquad_o
+   MFEM_REGISTER_KERNELS(ApplyTPAKernels, ApplyKernelType, (int, int, int));
+
+   template <int DIM, int NDOF_O, int NQUAD_O> static void AddSpecialization()
+   {
+      ApplyPAKernels::Specialization<DIM, NDOF_O, NQUAD_O>::Add();
+      ApplyTPAKernels::Specialization<DIM, NDOF_O, NQUAD_O>::Add();
+   }
+
+   struct Kernels { Kernels(); };
 };


@@ -54,6 +54,8 @@ void Coefficient::Project(QuadratureFunction &qf)
   QuadratureSpaceBase &qspace = *qf.GetSpace();
   const int ne = qspace.GetNE();
   Vector values;
+   // GetValues makes a reference, but we need it to be valid on Host
+   qf.HostWrite();
   for (int iel = 0; iel < ne; ++iel)
   {
      qf.GetValues(iel, values);
@@ -327,6 +329,8 @@ void VectorCoefficient::Project(QuadratureFunction &qf)
   const int ne = qspace.GetNE();
   DenseMatrix values;
   Vector col;
+   // GetValues makes a reference, but we need it to be valid on Host
+   qf.HostWrite();
   for (int iel = 0; iel < ne; ++iel)
   {
      qf.GetValues(iel, values);
@@ -695,6 +699,8 @@ void MatrixCoefficient::Project(QuadratureFunction &qf, bool transpose)
   QuadratureSpaceBase &qspace = *qf.GetSpace();
   const int ne = qspace.GetNE();
   DenseMatrix values, matrix;
+   // GetValues makes a reference, but we need it to be valid on Host
+   qf.HostWrite();
   for (int iel = 0; iel < ne; ++iel)
   {
      qf.GetValues(iel, values);
@@ -830,15 +830,9 @@ ParComplexGridFunction::ParComplexGridFunction(ParMesh *m, std::istream &input)
      int vsize = pfes->GetVSize();
      Vector::Load(input, 2*vsize);

-      real_t *data_  = const_cast<real_t*>(HostRead());
-      for (int i = 0; i < vsize; i++)
-      {
-         if (pfes->GetDofSign(i) < 0)
-         {
-            data_[i] = -data_[i];
-            data_[i+vsize] = -data_[i+vsize];
-         }
-      }
+      real_t *h_data = HostReadWrite();
+      pfes->ApplyDofSigns(h_data);
+      pfes->ApplyDofSigns(h_data + vsize);


      // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering
@@ -1051,15 +1045,14 @@ void ParComplexGridFunction::Save(std::ostream &os) const
   os << '\n';

   int vsize = pfes->GetVSize();
-   real_t *data_  = const_cast<real_t*>(HostRead());
-   for (int i = 0; i < vsize; i++)
-   {
-      if (pfes->GetDofSign(i) < 0)
-      {
-         data_[i] = -data_[i];
-         data_[i+vsize] = -data_[i+vsize];
-      }
-   }
+   // We use const_cast + HostRead (instead of HostReadWrite) because we only
+   // need to change the host data temporarily and this way we do not invalidate
+   // the data if it is on device. If we use HostReadWrite here, later calls to
+   // Read or ReadWrite will need to copy the data from host to device. With the
+   // approach used here, the host-to-device copy is avoided.
+   real_t *h_data = const_cast<real_t*>(HostRead());
+   pfes->ApplyDofSigns(h_data);
+   pfes->ApplyDofSigns(h_data + vsize);

   if (pfes->GetOrdering() == Ordering::byNODES)
   {
@@ -1070,14 +1063,8 @@ void ParComplexGridFunction::Save(std::ostream &os) const
      Vector::Print(os, pfes->GetVDim());
   }

-   for (int i = 0; i < vsize; i++)
-   {
-      if (pfes->GetDofSign(i) < 0)
-      {
-         data_[i] = -data_[i];
-         data_[i+vsize] = -data_[i+vsize];
-      }
-   }
+   pfes->ApplyDofSigns(h_data);
+   pfes->ApplyDofSigns(h_data + vsize);

   os.flush();
 }
@@ -114,6 +114,10 @@ void ConduitDataCollection::Save()
                                         n_mesh["fields"][name]);
   }

+   // TODO: in parallel, we need to call ParFiniteElementSpace::ApplyDofSigns
+   // for all ParGridFunction objects before and after saving, see
+   // ParGridFunction::Save.
+
   // save mesh data
   SaveMeshAndFields(myid,
                     n_mesh,
@@ -57,7 +57,7 @@ void DGMassApply(const int e,
      }
      else if (DIM == 3)
      {
-         SmemPAMassApply3D_Element<TD1D,TQ1D,ACCUM>(e, NE, B, pa_data, x, y);
+         SmemPAMassApply3D_Element<TD1D,TQ1D,NBZ,ACCUM>(e, NE, B, pa_data, x, y);
      }
      else
      {
@@ -167,7 +167,15 @@ public:
      /** @brief Full multidimensional representation which does not use tensor
          product structure. The ordering of the degrees of freedom is the
          same as TENSOR, but the sizes of B and G are the same as FULL.*/
-      LEXICOGRAPHIC_FULL
+      LEXICOGRAPHIC_FULL,
+
+      /** @brief Ragged tensor product representation using 1D matrices/tensors
+          with dimensions using 1D number of quadrature points and ragged tensor degrees of
+          freedom. */
+      /** Used only for partial assembly of the H1 positive basis. The
+          size of B is d1d x qnpt x dim. Since different Gauss-Jacobi quadrature rules
+          are employed in each dimension, we need to store dim arrays. */
+      RAGGED_TENSOR
   };

   /// Describes the contents of the #B, #Bt, #G, and #Gt arrays, see #Mode.
@@ -228,6 +236,39 @@ public:
      const Array<DofToQuad*> &dof2quad_array,
      const IntegrationRule &ir,
      DofToQuad::Mode mode);
+
+   virtual ~DofToQuad() = default;
+};
+
+/** @brief Structure representing the matrices/tensors needed to evaluate (in
+    reference space) the values, gradients, divergences, or curls of a positive
+    FiniteElement on simplices at the quadrature points of Stroud conical quadrature. */
+class RaggedDofToQuad : public DofToQuad
+{
+public:
+   /** @brief Special basis function structures for positive (Bernstein) basis with
+      partial assembly. The storage layout of Ba1 is ndof x nqpt for scalar elements.
+      The storage layout of Ba2 is ndof x ndof x nqpt. In particular, we have
+            Ba2(iqpt, a1, a2) = B^{p-a1}_{a2}(x_{iqpt}). */
+   Array<real_t> Ba1, Ba2, Ba3;
+   Array<real_t> Ba1t, Ba2t, Ba3t;
+
+   /** @brief Special structures for gradients of positive basis with partial assembly.
+      The gradient arrays exploit properties of the Bernstein basis which allow grad(B^p_alpha)
+      to be expressed as the sum of products of B^{p-1}_alpha and the barycentric coordinates.
+      Thus, Ga1 and Ga2 simply contain the ragged tensor product components of B^{p-1}_alpha */
+   Array<real_t> Ga1, Ga2, Ga3;
+   Array<real_t> Ga1t, Ga2t, Ga3t;
+
+   /** @brief Mapping from the Bernstein multi-index (a_1, ..., a_d) to the lexicographic
+       dof index. */
+   Array<int> lex_map;
+
+   Array<int> forward_map2d_diff, forward_map3d_diff;
+   Array<int> inverse_map2d_diff, inverse_map3d_diff;
+
+   Array<int> forward_map2d_mass, forward_map3d_mass;
+   Array<int> inverse_map2d_mass, inverse_map3d_mass;
 };

 /// Describes the function space on each element
@@ -1405,6 +1446,8 @@ public:
                                dof2quad_array_open);
   }

+   const Poly_1D::Basis &GetOpenBasis1D() const { return obasis1d; }
+
   virtual ~VectorTensorFiniteElement();
 };

@@ -557,6 +557,101 @@ H1Pos_TriangleElement::H1Pos_TriangleElement(const int p)
      }
 }

+const DofToQuad &H1Pos_TriangleElement::GetRaggedTensorDofToQuad(
+   const FiniteElement &fe, const IntegrationRule &ir,
+   DofToQuad::Mode mode,
+   Array<DofToQuad*> &dof2quad_array)
+{
+   DofToQuad *d2q = nullptr;
+   MFEM_VERIFY(mode == DofToQuad::RAGGED_TENSOR, "invalid mode requested");
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
+#endif
+   {
+      for (int i = 0; i < dof2quad_array.Size(); i++)
+      {
+         d2q = dof2quad_array[i];
+         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
+      }
+      if (!d2q)
+      {
+         d2q = new RaggedDofToQuad;
+         const int ndof = fe.GetOrder() + 1; // verify
+         const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
+         d2q->FE = &fe;
+         d2q->IntRule = &ir;
+         d2q->mode = mode;
+         d2q->ndof = ndof;
+         d2q->nqpt = nqpt;
+
+         RaggedDofToQuad *rd2q = static_cast<RaggedDofToQuad*>(d2q);
+         rd2q->Ba1.SetSize(nqpt*ndof);
+         // second component of ragged tensor basis, technically dof*(dof-1)/2 entries
+         rd2q->Ba2.SetSize((int)nqpt*ndof*ndof);
+         rd2q->Ba1t.SetSize(nqpt*ndof);
+         rd2q->Ba2t.SetSize((int)nqpt*ndof*ndof);
+         // stores first component of ragged tensor basis with order p-1, for gradients only
+         rd2q->Ga1.SetSize(nqpt*(ndof -1));
+         // stores second component of ragged tensor basis with order p-1
+         rd2q->Ga2.SetSize(nqpt*(ndof-1)*(ndof -1));
+         rd2q->Ga1t.SetSize(nqpt*(ndof -1));
+         rd2q->Ga2t.SetSize(nqpt*(ndof-1)*(ndof -1));
+         rd2q->lex_map.SetSize(ndof * ndof);
+         Vector shape_a1(ndof), shape_a2(ndof * ndof);
+         Vector shape_Ga1(ndof-1), shape_Ga2((ndof-1) * (ndof-1));
+         for (int i = 0; i < nqpt; i++)
+         {
+            // The first 'nqpt' points in the first dimension 'ir' have the same x-coordinates as those
+            // of the 1D rule (ie. (2,0) Gauss-Jacobi rule). The first 'nqpt' points in the second dimension
+            // 'ir' have the same y-coordinates as those of the 1D rule for second dimension (i.e. (1,0)
+            // Gauss-Jacobi rule). Additionally, the Bernstein PA algorithms expect evaluation of the
+            // component 1D bases at the Stroud nodes pulled back to the unit square, so perform the pullback
+            // on the fly.
+            const real_t x = ir.IntPoint(i).x;
+            const real_t y = ir.IntPoint(nqpt*i).y / (1.0 - ir.IntPoint(nqpt*i).x);
+
+            Poly_1D::CalcBernstein(ndof-1, x, shape_a1);
+            Poly_1D::CalcBernstein(ndof-2, x, shape_Ga1);
+            for (int j = 0; j < ndof; j++)
+            {
+               rd2q->Ba1t[i+nqpt*j] = rd2q->Ba1[j+ndof*i] = shape_a1(j);
+               if (j < ndof-1)
+               {
+                  rd2q->Ga1t[i+nqpt*j] = rd2q->Ga1[j+(ndof-1)*i] = shape_Ga1(j);
+                  Poly_1D::CalcBernstein(ndof-2-j, y, shape_Ga2);
+               }
+
+               Poly_1D::CalcBernstein(ndof-1-j, y, shape_a2);
+               for (int k = 0; k < ndof-j; k++)
+               {
+                  rd2q->Ba2t[i + nqpt*(j + ndof*k)] = rd2q->Ba2[k + ndof*(j + ndof*i)] = shape_a2(
+                                                                                            k);
+                  if (j < ndof-1 && k < ndof-j-1)
+                  {
+                     rd2q->Ga2t[i + nqpt*(j + (ndof-1)*k)] = rd2q->Ga2[k + (ndof-1)*(j +
+                                                                                     (ndof-1)*i)] = shape_Ga2(k);
+                  }
+               }
+            }
+         }
+
+         // stores the mapping from 2D Bernstein multi-index (i,j,p-i-j) to the
+         // lexicographic DOF ordering
+         for (int i = 0; i < ndof; i++)
+         {
+            for (int j = 0; j < ndof-i; j++)
+            {
+               int idx = ((2 * (ndof-1) + 3) - j) * j / 2 + i;
+               rd2q->lex_map[j + ndof*i] = idx;
+            }
+         }
+         dof2quad_array.Append(d2q);
+      }
+   }
+   return *d2q;
+}
+
 // static method
 void H1Pos_TriangleElement::CalcShape(
   const int p, const real_t l1, const real_t l2, real_t *shape)
@@ -749,6 +844,213 @@ H1Pos_TetrahedronElement::H1Pos_TetrahedronElement(const int p)
         }
 }

+const DofToQuad &H1Pos_TetrahedronElement::GetRaggedTensorDofToQuad(
+   const FiniteElement &fe, const IntegrationRule &ir,
+   DofToQuad::Mode mode,
+   Array<DofToQuad*> &dof2quad_array)
+{
+   DofToQuad *d2q = nullptr;
+   MFEM_VERIFY(mode == DofToQuad::RAGGED_TENSOR, "invalid mode requested");
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
+#endif
+   {
+      for (int i = 0; i < dof2quad_array.Size(); i++)
+      {
+         d2q = dof2quad_array[i];
+         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
+      }
+      if (!d2q)
+      {
+         d2q = new RaggedDofToQuad;
+         const int ndof = fe.GetOrder() + 1; // verify
+         const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
+         const int basis_dim2d = ndof*(ndof+1) / 2;
+         const int basis_dim3d = ndof*(ndof+1)*(ndof+2) / 6;
+         const int basis_dim2d_diff = (ndof-1)*(ndof) / 2;
+         const int basis_dim3d_diff = (ndof-1)*(ndof)*(ndof+1) / 6;
+         d2q->FE = &fe;
+         d2q->IntRule = &ir;
+         d2q->mode = mode;
+         d2q->ndof = ndof;
+         d2q->nqpt = nqpt;
+
+         RaggedDofToQuad *rd2q = static_cast<RaggedDofToQuad*>(d2q);
+         rd2q->Ba1.SetSize(nqpt * ndof);
+         // second component of ragged tensor basis, technically dof*(dof-1)/2 entries
+         rd2q->Ba2.SetSize(nqpt * basis_dim2d);
+         // third component of ragged tensor basis, technically dof*(dof-1)/2 entries
+         rd2q->Ba3.SetSize(nqpt * basis_dim3d);
+         rd2q->Ba1t.SetSize(nqpt * ndof);
+         rd2q->Ba2t.SetSize(nqpt * basis_dim2d);
+         rd2q->Ba3t.SetSize(nqpt * basis_dim3d);
+         // stores first component of ragged tensor basis with order p-1, for gradients only
+         rd2q->Ga1.SetSize(nqpt * (ndof-1));
+         // stores second component of ragged tensor basis with order p-1
+         rd2q->Ga2.SetSize(nqpt * basis_dim2d_diff);
+         // stores third component of ragged tensor basis with order p-1
+         rd2q->Ga3.SetSize(nqpt * basis_dim3d_diff);
+         rd2q->Ga1t.SetSize(nqpt * (ndof-1));
+         rd2q->Ga2t.SetSize(nqpt * basis_dim2d_diff);
+         rd2q->Ga3t.SetSize(nqpt * basis_dim3d_diff);
+         rd2q->lex_map.SetSize(ndof * ndof * ndof);
+
+         rd2q->forward_map2d_diff.SetSize((ndof-1) * (ndof-1));
+         rd2q->forward_map3d_diff.SetSize((ndof-1) * (ndof-1) * (ndof-1));
+         rd2q->inverse_map2d_diff.SetSize(2 * basis_dim2d_diff);
+         rd2q->inverse_map3d_diff.SetSize(3 * basis_dim3d_diff);
+
+         rd2q->forward_map2d_mass.SetSize(ndof * ndof);
+         rd2q->forward_map3d_mass.SetSize(ndof * ndof * ndof);
+         rd2q->inverse_map2d_mass.SetSize(2 * basis_dim2d);
+         rd2q->inverse_map3d_mass.SetSize(2 * basis_dim3d);
+
+         // forward and inverse maps for multi-index to collpased 1d index for diffusion, can combine
+         // these four loops, but need four idx's and clause for shorter diff loops
+         int idx = 0;
+         for (int i = 0; i < ndof-1; i++)
+         {
+            for (int j = 0; j < ndof-i-1; j++)
+            {
+               rd2q->forward_map2d_diff[j + (ndof-1)*i] = idx;
+               rd2q->inverse_map2d_diff[2*idx] = i;
+               rd2q->inverse_map2d_diff[1 + 2*idx] = j;
+               idx++;
+            }
+         }
+
+         idx = 0;
+         for (int k = 0; k < ndof-1; k++)
+         {
+            for (int j = 0; j < ndof-k-1; j++)
+            {
+               for (int i = 0; i < ndof-k-j-1; i++)
+               {
+                  rd2q->forward_map3d_diff[k + (ndof-1)*(j + (ndof-1)*i)] = idx;
+                  rd2q->inverse_map3d_diff[3*idx] = i;
+                  rd2q->inverse_map3d_diff[1 + 3*idx] = j;
+                  rd2q->inverse_map3d_diff[2 + 3*idx] = k;
+                  idx++;
+               }
+            }
+         }
+
+         // forward and inverse maps for multi-index to collpased 1d index for mass
+         idx = 0;
+         for (int j = 0; j < ndof; j++)
+         {
+            for (int i = 0; i < ndof-j; i++)
+            {
+               rd2q->forward_map2d_mass[j + ndof*i] = idx;
+               rd2q->inverse_map2d_mass[2*idx] = i;
+               rd2q->inverse_map2d_mass[1 + 2*idx] = j;
+               idx++;
+            }
+         }
+
+         idx = 0;
+         for (int k = 0; k < ndof; k++)
+         {
+            for (int j = 0; j < ndof-k; j++)
+            {
+               for (int i = 0; i < ndof-k-j; i++)
+               {
+                  rd2q->forward_map3d_mass[k + ndof*(j + ndof*i)] = idx;
+                  rd2q->inverse_map3d_mass[2*idx] = i;
+                  rd2q->inverse_map3d_mass[1 + 2*idx] = j;
+                  // d2q->inverse_map3d_mass[2 + 3*idx] = k;
+                  idx++;
+               }
+            }
+         }
+
+         Vector shape_a1(ndof), shape_a2(ndof * ndof), shape_a3(ndof * ndof * ndof);
+         Vector shape_Ga1(ndof-1), shape_Ga2(ndof-1), shape_Ga3(ndof-1);
+         for (int i = 0; i < nqpt; i++)
+         {
+            // The first 'nqpt' points in the first dimension 'ir' have the same x-coordinates as those
+            // of the 1D rule (ie. (2,0) Gauss-Jacobi rule). The first 'nqpt' points in the second dimension
+            // 'ir' have the same y-coordinates as those of the 1D rule for second dimension (i.e. (1,0)
+            // Gauss-Jacobi rule). The first 'nqpt' points in the third dimension have the same z-coordinates
+            // as those of the 1D rule for the third dimension (i.e. Gauss-Legendre rule). Additionally,
+            // the Bernstein PA algorithms expect evaluation of the component 1D bases at the Stroud nodes
+            // pulled back to the unit cube, so perform the pullback on the fly.
+            const real_t x = ir.IntPoint(i).x;
+            const real_t y = ir.IntPoint(nqpt*i).y / (1.0 - ir.IntPoint(nqpt*i).x);
+            const real_t z = ir.IntPoint(nqpt*nqpt*i).z / (1.0 - ir.IntPoint(
+                                                              nqpt*nqpt*i).x - ir.IntPoint(nqpt*nqpt*i).y);
+            Poly_1D::CalcBernstein(ndof-1, x, shape_a1);
+            Poly_1D::CalcBernstein(ndof-2, x, shape_Ga1);
+            for (int j = 0; j < ndof; j++)
+            {
+               rd2q->Ba1t[i+nqpt*j] = rd2q->Ba1[j+ndof*i] = shape_a1(j);
+               if (j < ndof-1)
+               {
+                  rd2q->Ga1t[i+nqpt*j] = rd2q->Ga1[j+(ndof-1)*i] = shape_Ga1(j);
+                  Poly_1D::CalcBernstein(ndof-2-j, y, shape_Ga2);
+               }
+
+               Poly_1D::CalcBernstein(ndof-1-j, y, shape_a2);
+               for (int k = 0; k < ndof-j; k++)
+               {
+                  const int a_2d_mass = rd2q->forward_map2d_mass[k + ndof*j];
+                  rd2q->Ba2t[i + nqpt*a_2d_mass] = rd2q->Ba2[a_2d_mass + basis_dim2d*i] =
+                                                      shape_a2(
+                                                         k);
+                  if (j < ndof-1 && k < ndof-j-1)
+                  {
+                     const int a_2d_diff = rd2q->forward_map2d_diff[k + (ndof-1)*j];
+                     rd2q->Ga2t[i + nqpt*a_2d_diff] = rd2q->Ga2[a_2d_diff + basis_dim2d_diff*i] =
+                                                         shape_Ga2(k);
+                     Poly_1D::CalcBernstein(ndof-2-j-k, z, shape_Ga3);
+                  }
+
+                  Poly_1D::CalcBernstein(ndof-1-j-k, z, shape_a3);
+                  for (int m = 0; m < ndof-j-k; m++)
+                  {
+                     const int a_3d_mass = rd2q->forward_map3d_mass[m + ndof*(k + ndof*j)];
+                     rd2q->Ba3t[i + nqpt*a_3d_mass] = rd2q->Ba3[a_3d_mass + basis_dim3d*i] =
+                                                         shape_a3(
+                                                            m);
+                     if (j < ndof-1 && k < ndof-j-1 && m < ndof-j-k-1)
+                     {
+                        // // collapsed 1D access
+                        // d2q->Ga3[i + nqpt*(m + d2q->offset3d[k + (ndof-1)*j])] = shape_Ga3(m);
+                        // collapsed 1D access with forward mapping
+                        const int a_3d_diff = rd2q->forward_map3d_diff[m + (ndof-1)*(k + (ndof-1)*j)];
+                        rd2q->Ga3t[i + nqpt*a_3d_diff] = rd2q->Ga3[a_3d_diff + basis_dim3d_diff*i] =
+                                                            shape_Ga3(m);
+                     }
+                  }
+               }
+            }
+         }
+
+         // stores the mapping from 3D Bernstein multi-index (i,j,k,p-i-j-k) to the
+         // lexicographic DOF ordering
+         int p = ndof - 1;
+         for (int i = 0; i < ndof; i++)
+         {
+            for (int j = 0; j < ndof-i; j++)
+            {
+               for (int k = 0; k < ndof-i-j; k++)
+               {
+                  int dof = (p+1)*(p+2)*(p+3) / 6;
+                  int tet = (p-k)*(p-k+1)*(p-k+2) / 6;
+                  int tri = (p+1-k-j)*(p+2-k-j)/2;
+                  int multi_idx = dof - tet - tri + i;
+                  rd2q->lex_map[k + ndof*(j + ndof*i)] = multi_idx;
+               }
+            }
+         }
+
+         dof2quad_array.Append(d2q);
+      }
+   }
+   return *d2q;
+}
+
 // static method
 void H1Pos_TetrahedronElement::CalcShape(
   const int p, const real_t l1, const real_t l2, const real_t l3,
@@ -191,6 +191,21 @@ public:
   /// Construct the H1Pos_TriangleElement of order @a p
   H1Pos_TriangleElement(const int p);

+   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
+                                 DofToQuad::Mode mode) const override
+   {
+      return (mode == DofToQuad::RAGGED_TENSOR) ?
+             GetRaggedTensorDofToQuad(*this, ir, mode, dof2quad_array) :
+             FiniteElement::GetDofToQuad(ir, mode);
+   }
+
+   static const DofToQuad &GetRaggedTensorDofToQuad(
+      const FiniteElement &fe, const IntegrationRule &ir,
+      DofToQuad::Mode mode,
+      Array<DofToQuad*> &dof2quad_array);
+
+   const Array<int> &GetDofMap() const { return dof_map; }
+
   // The size of shape is (p+1)(p+2)/2 (dof).
   static void CalcShape(const int p, const real_t x, const real_t y,
                         real_t *shape);
@@ -220,6 +235,21 @@ public:
   /// Construct the H1Pos_TetrahedronElement of order @a p
   H1Pos_TetrahedronElement(const int p);

+   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
+                                 DofToQuad::Mode mode) const override
+   {
+      return (mode == DofToQuad::RAGGED_TENSOR) ?
+             GetRaggedTensorDofToQuad(*this, ir, mode, dof2quad_array) :
+             FiniteElement::GetDofToQuad(ir, mode);
+   }
+
+   static const DofToQuad &GetRaggedTensorDofToQuad(
+      const FiniteElement &fe, const IntegrationRule &ir,
+      DofToQuad::Mode mode,
+      Array<DofToQuad*> &dof2quad_array);
+
+   const Array<int> &GetDofMap() const { return dof_map; }
+
   // The size of shape is (p+1)(p+2)(p+3)/6 (dof).
   static void CalcShape(const int p, const real_t x, const real_t y,
                         const real_t z, real_t *shape);
@@ -250,6 +250,14 @@ public:
       its GetOrder() method. */
   virtual FiniteElementCollection *Clone(int p) const;

+   /** @brief Return the order parameter used to construct this collection.
+    *  This differs from GetOrder() depending on the collection type. */
+   virtual int GetConstructorOrder() const
+   {
+      MFEM_ABORT("Collection " << Name() << " does not support GetConstructorOrder");
+      return -1;
+   }
+
 protected:
   const int base_p; ///< Order as returned by GetOrder().

@@ -314,6 +322,9 @@ public:
   FiniteElementCollection *Clone(int p) const override
   { return new H1_FECollection(p, dim, b_type); }

+   int GetConstructorOrder() const override
+   { return base_p; }
+
   virtual ~H1_FECollection();
 };

@@ -343,6 +354,10 @@ class H1_Trace_FECollection : public H1_FECollection
 public:
   H1_Trace_FECollection(const int p, const int dim,
                         const int btype = BasisType::GaussLobatto);
+
+   FiniteElementCollection *Clone(int p) const override
+   { return new H1_Trace_FECollection(p, dim+1, b_type); }
+
 };

 /// Arbitrary order "L2-conforming" discontinuous finite elements.
@@ -396,6 +411,9 @@ public:
   FiniteElementCollection *Clone(int p) const override
   { return new L2_FECollection(p, dim, b_type, m_type); }

+   int GetConstructorOrder() const override
+   { return base_p; }
+
   virtual ~L2_FECollection();
 };

@@ -456,6 +474,9 @@ public:
   FiniteElementCollection *Clone(int p) const override
   { return new RT_FECollection(p, dim, cb_type, ob_type); }

+   int GetConstructorOrder() const override
+   { return base_p-1; }
+
   virtual ~RT_FECollection();
 };

@@ -536,6 +557,9 @@ public:
   FiniteElementCollection *Clone(int p) const override
   { return new ND_FECollection(p, dim, cb_type, ob_type); }

+   int GetConstructorOrder() const override
+   { return dim>1 ? base_p : base_p+1; }
+
   virtual ~ND_FECollection();
 };

@@ -548,6 +572,9 @@ public:
   ND_Trace_FECollection(const int p, const int dim,
                         const int cb_type = BasisType::GaussLobatto,
                         const int ob_type = BasisType::GaussLegendre);
+
+   FiniteElementCollection *Clone(int p) const override
+   { return new ND_Trace_FECollection(p, dim+1, cb_type, ob_type); }
 };

 /// Arbitrary order 3D H(curl)-conforming Nedelec finite elements in 1D.
@@ -4631,9 +4631,8 @@ FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)

 ElementDofOrdering GetEVectorOrdering(const FiniteElementSpace& fes)
 {
-   return UsesTensorBasis(fes)?
+   return (UsesTensorBasis(fes) || fes.UsesRaggedTensorBasis()) ?
          ElementDofOrdering::LEXICOGRAPHIC:
          ElementDofOrdering::NATIVE;
 }
-
 } // namespace mfem
@@ -1514,6 +1514,18 @@ public:
      return dynamic_cast<const L2_FECollection*>(fec) != NULL;
   }

+   /// @brief Return true if the mesh contains only one topology, the elements are
+   /// all triangles or tetrahedrons, and the elements are ragged tensor elements
+   /// i.e. Bernstein/positive basis.
+   bool UsesRaggedTensorBasis() const
+   {
+      bool simplex = this->GetMesh()->IsSimplexMesh();
+      bool positive =
+         dynamic_cast<const mfem::H1Pos_TriangleElement *>(this->GetTypicalFE()) ||
+         dynamic_cast<const mfem::H1Pos_TetrahedronElement *>(this->GetTypicalFE());
+      return simplex && positive;
+   }
+
   /** In variable-order spaces on nonconforming (NC) meshes, this function
       controls whether strict conformity is enforced in cases where coarse
       edges/faces have higher polynomial order than their fine NC neighbors.
@@ -2256,6 +2256,104 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
   }
 }

+void GridFunction::AccumulateAndCountTraceValues(
+   Coefficient *coeff[], VectorCoefficient *vcoeff,
+   Array<int> &values_counter)
+{
+   if (vcoeff)
+   {
+      MFEM_VERIFY(fes->GetVDim() == vcoeff->GetVDim(),
+                  "vcoeff vdim != fes VDim");
+      MFEM_VERIFY(fes->GetTypicalTraceElement()->GetMapType() ==
+                  FiniteElement::VALUE &&
+                  fes->GetTypicalTraceElement()->GetRangeType() ==
+                  FiniteElement::SCALAR,
+                  "Can only call ProjectTraceCoefficient on scalar value-type "
+                  "trace elements. "
+                  "Use ProjectTraceCoefficientNormal for RT and "
+                  "ProjectTraceCoefficientTangent for ND finite elements.");
+   }
+
+   Array<int> vdofs;
+   Vector vc;
+
+   values_counter.SetSize(Size());
+   values_counter = 0;
+
+   const int vdim = fes->GetVDim();
+   HostReadWrite();
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+
+      const FiniteElement *fe = fes->GetFaceElement(i);
+      const int fdof = fe->GetDof();
+      ElementTransformation *transf = fes->GetMesh()->GetFaceTransformation(i);
+      const IntegrationRule &ir = fe->GetNodes();
+      fes->GetFaceVDofs(i, vdofs);
+
+      for (int j = 0; j < fdof; j++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(j);
+         transf->SetIntPoint(&ip);
+         if (vcoeff) { vcoeff->Eval(vc, *transf, ip); }
+         for (int d = 0; d < vdim; d++)
+         {
+            if (!vcoeff && !coeff[d]) { continue; }
+
+            real_t val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
+            int ind = vdofs[fdof*d+j];
+            if ( ind < 0 )
+            {
+               val = -val, ind = -1-ind;
+            }
+            if (++values_counter[ind] == 1)
+            {
+               (*this)(ind) = val;
+            }
+            else
+            {
+               (*this)(ind) += val;
+            }
+         }
+      }
+   }
+}
+
+void GridFunction::AccumulateAndCountTraceTangentValues(
+   VectorCoefficient &vcoeff, Array<int> &values_counter)
+{
+   MFEM_VERIFY(fes->GetVDim() == 1, "fespace VDim != 1");
+   MFEM_VERIFY(fes->GetTypicalTraceElement()
+               ->GetRangeType() == FiniteElement::VECTOR &&
+               fes->GetTypicalTraceElement()
+               ->GetMapType() == FiniteElement::H_CURL,
+               "Not an ND FE space!");
+   MFEM_VERIFY(fes->GetTypicalTraceElement()->GetPhysRangeDim(
+                  fes->GetMesh()->SpaceDimension()) == vcoeff.GetVDim(),
+               "vcoeff vdim != PhysRangeDim");
+
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Array<int> dofs;
+   Vector lvec;
+
+   values_counter.SetSize(Size());
+   values_counter = 0;
+
+   HostReadWrite();
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+      fe = fes->GetFaceElement(i);
+      T = fes->GetMesh()->GetFaceTransformation(i);
+      fes->GetFaceVDofs(i, dofs);
+      lvec.SetSize(fe->GetDof());
+      fe->Project(vcoeff, *T, lvec);
+      accumulate_dofs(dofs, lvec, *this, values_counter);
+   }
+}
+
 void GridFunction::ComputeMeans(AvgType type, Array<int> &zones_per_vdof)
 {
   switch (type)
@@ -2698,6 +2796,74 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff,
   }
 }

+void GridFunction::ProjectTraceCoefficient(Coefficient *coeff[])
+{
+   Array<int> values_counter;
+   AccumulateAndCountTraceValues(coeff, NULL, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
+void GridFunction::ProjectTraceCoefficient(Coefficient &coeff)
+{
+   MFEM_VERIFY(FESpace()->GetVDim() == 1, "ProjectTraceCoefficient(Coefficient&)"
+               "is only valid for scalar GridFunction");
+   Coefficient *coeff_p = &coeff;
+   ProjectTraceCoefficient(&coeff_p);
+}
+
+void GridFunction::ProjectTraceCoefficient(VectorCoefficient &vcoeff)
+{
+   MFEM_VERIFY(FESpace()->GetVDim() == vcoeff.GetVDim(),
+               "Incompatible vcoeff vdim and fes vdim");
+   Array<int> values_counter;
+   AccumulateAndCountTraceValues(NULL, &vcoeff, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
+void GridFunction::ProjectTraceCoefficientNormal(VectorCoefficient &vcoeff)
+{
+   MFEM_VERIFY(fes->GetVDim() == 1, "fespace VDim != 1");
+   MFEM_VERIFY(fes->GetTypicalTraceElement()->GetRangeType() ==
+               FiniteElement::SCALAR &&
+               fes->GetTypicalTraceElement()->GetMapType() ==
+               FiniteElement::INTEGRAL, "Not an RT FE space!");
+   MFEM_VERIFY(vcoeff.GetVDim() == fes->GetMesh()->SpaceDimension(),
+               "vcoeff vdim (" << vcoeff.GetVDim()
+               << ") != SpaceDimension ("
+               << fes->GetMesh()->SpaceDimension() << ")");
+
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Array<int> dofs;
+   int dim = vcoeff.GetVDim();
+   Vector vc(dim), nor(dim), lvec;
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+      fe = fes->GetFaceElement(i);
+      T = fes->GetMesh()->GetFaceTransformation(i);
+      const IntegrationRule &ir = fe->GetNodes();
+      lvec.SetSize(fe->GetDof());
+      for (int j = 0; j < ir.GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(j);
+         T->SetIntPoint(&ip);
+         vcoeff.Eval(vc, *T, ip);
+         CalcOrtho(T->Jacobian(), nor);
+         lvec(j) = (vc * nor);
+      }
+      fes->GetFaceVDofs(i, dofs);
+      SetSubVector(dofs, lvec);
+   }
+}
+
+void GridFunction::ProjectTraceCoefficientTangent(VectorCoefficient &vcoeff)
+{
+   Array<int> values_counter;
+   AccumulateAndCountTraceTangentValues(vcoeff, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
 void GridFunction::ProjectCoefficientGlobalL2(VectorCoefficient &vcoeff,
                                              real_t rtol, int iter)
 {
@@ -5286,6 +5452,7 @@ PLBound GridFunction::GetBounds(Vector &lower, Vector &upper,
 {
   int max_order = fes->GetMaxElementOrder();
   PLBound plb(fes, ref_factor*(max_order+1));
+
   Vector lel, uel;
   GetElementBounds(plb, lel, uel, vdim);

@@ -578,6 +578,13 @@ protected:
                                           const Array<int> &bdr_attr,
                                           Array<int> &values_counter);

+   void AccumulateAndCountTraceValues(Coefficient *coeff[],
+                                      VectorCoefficient *vcoeff,
+                                      Array<int> &values_counter);
+
+   void AccumulateAndCountTraceTangentValues(VectorCoefficient &vcoeff,
+                                             Array<int> &values_counter);
+
   // Complete the computation of averages; called e.g. after
   // AccumulateAndCountZones().
   void ComputeMeans(AvgType type, Array<int> &zones_per_vdof);
@@ -663,6 +670,23 @@ public:
      ProjectBdrCoefficient(&coeff_p, attr);
   }

+   /// Project a Coefficient on a GridFunction defined on H1 trace space
+   void ProjectTraceCoefficient(Coefficient *coeff[]);
+   void ProjectTraceCoefficient(Coefficient &coeff);
+
+   /** @brief Project a VectorCoefficient @a vcoeff on a GridFunction
+       defined on a Vector H1 trace space. Note that this also works
+       for a scalar H1 trace space, where only the first component of
+       @a vcoeff is used. */
+   void ProjectTraceCoefficient(VectorCoefficient &vcoeff);
+   /** @brief Project a VectorCoefficient on a GridFunction
+       defined on an RT trace space */
+   void ProjectTraceCoefficientNormal(VectorCoefficient &vcoeff);
+   /** @brief Project a VectorCoefficient on a GridFunction
+       defined on an ND trace space */
+   void ProjectTraceCoefficientTangent(VectorCoefficient &vcoeff);
+
+
   /** @brief Project a VectorCoefficient on the GridFunction, modifying only
       DOFs on the boundary associated with the boundary attributes marked in
       the @a attr array. */
@@ -1767,8 +1791,8 @@ public:
                             const int ref_factor=1, const int vdim=-1) const;

   /// Computes the \ref PLBound for the gridfunction with number of control
-   /// points based on \p ref_factor, and returns the bounds for each element
-   /// ordered byNodes:
+   /// points based on @a ref_factor, and returns the bounds for each element
+   /// ordered byNODES:
   /// lower_{0,0}, lower_{1,0}, ..., lower_{ne-1,0},
   /// lower_{0,1}, ..., lower_{ne-1,vdim-1}. We also return the
   /// PLBound object used to compute the bounds.
@@ -1802,7 +1826,7 @@ public:
                         const int vdim = -1) const;

   /// Compute bounds on the grid function for all the elements. The bounds
-   /// are returned in @b lower and @b upper, ordered byNodes:
+   /// are returned in @b lower and @b upper, ordered byNODES:
   /// lower_{0,0}, lower_{1,0}, ..., lower_{ne-1,0},
   /// lower_{0,1}, ..., lower_{ne-1,vdim-1}
   void GetElementBounds(const PLBound &plb, Vector &lower, Vector &upper,
@@ -21,6 +21,45 @@

 #ifdef MFEM_USE_GSLIB

+/* gslib license and copyright statement for code adapted from gslib:
+
+Copyright (c) 2008-2024, UCHICAGO ARGONNE, LLC.
+
+The UChicago Argonne, LLC as Operator of Argonne National
+Laboratory holds copyright in the Software. The copyright holder
+reserves all rights except those expressly granted to licensees,
+and U.S. Government license rights.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the disclaimer below.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the disclaimer (as noted below)
+in the documentation and/or other materials provided with the
+distribution.
+
+3. Neither the name of ANL nor the names of its contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+UCHICAGO ARGONNE, LLC, THE U.S. DEPARTMENT OF
+ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
 namespace gslib
 {
 struct comm;
@@ -86,7 +125,7 @@ protected:
   void *fdataD;
   struct gslib::crystal *cr;             // gslib's internal data
   struct gslib::comm *gsl_comm;          // gslib's internal data
-   int dim, points_cnt;                   // mesh dimension and number of points
+   int dim, spacedim, points_cnt;         // mesh dimension and number of points
   Array<unsigned int> gsl_code, gsl_proc, gsl_elem, gsl_mfem_elem;
   Vector gsl_mesh, gsl_ref, gsl_dist, gsl_mfem_ref;
   Array<unsigned int> recv_proc, recv_index; // data for custom interpolation
@@ -104,18 +143,23 @@ protected:
   bool       gpu_to_cpu_fallback = false;

   // Device specific data used for FindPoints
-   struct
+   struct DEV_STRUCT
   {
      bool setup_device = false;
      bool find_device  = false;
-      int local_hash_size, dof1d, dof1d_sol, h_o_size, h_nx;
+      int local_hash_size, dof1d, dof1d_sol, lh_nx, gh_nx;
      double newt_tol; // Tolerance specified during setup for Newton solve
      struct gslib::crystal *cr;
      struct gslib::hash_data_3 *hash3;
      struct gslib::hash_data_2 *hash2;
      mutable Vector bb, wtend, gll1d, lagcoeff, gll1d_sol, lagcoeff_sol;
-      mutable Array<unsigned int> loc_hash_offset;
-      mutable Vector loc_hash_min, loc_hash_fac;
+      mutable Array<unsigned int> lh_offset, gh_offset;
+      mutable Vector lh_min, lh_fac, gh_min, gh_fac;
+      // Tolerance to mark points found on the surface as CODE_INTERNAL
+      // or CODE_BORDER. This is needed because we cannot only use reference
+      // space coordinates to determine if a point is located inside the
+      // element or not.
+      mutable double surf_dist_tol;
   } DEV;

   /// Use GSLIB for communication and interpolation
@@ -127,80 +171,143 @@ protected:
                                   Vector &field_out,
                                   const int field_out_ordering);

-   /// Since GSLIB is designed to work with quads/hexes, we split every
-   /// triangle/tet/prism/pyramid element into quads/hexes.
+   /** @brief Since GSLIB is designed to work with quads/hexes, we split every
+    *  triangle/tet/prism/pyramid element into quads/hexes. */
   virtual void SetupSplitMeshes();

-   /// Setup integration points that will be used to interpolate the nodal
-   /// location at points expected by GSLIB.
+   /** @brief Setup integration points that will be used to interpolate the
+    *  nodal location at points expected by GSLIB. */
   virtual void SetupIntegrationRuleForSplitMesh(Mesh *mesh,
                                                 IntegrationRule *irule,
                                                 int order);

-   /// Helper function that calls \ref SetupSplitMeshes and
-   /// \ref SetupIntegrationRuleForSplitMesh.
+   /** @brief Helper function that calls \ref SetupSplitMeshes and
+    * \ref SetupIntegrationRuleForSplitMesh. */
   virtual void SetupSplitMeshesAndIntegrationRules(const int order);

   /// Get GridFunction value at the points expected by GSLIB.
   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals) const;

-   /// Map {r,s,t} coordinates from [-1,1] to [0,1] for MFEM. For simplices,
-   /// find the original element number (that was split into micro quads/hexes)
-   /// during the setup phase.
+   /** @brief Map {r,s,t} coordinates from [-1,1] to [0,1] for MFEM. For
+    *  simplices, find the original element number (that was split into
+    *  micro quads/hexes) during the setup phase. */
   virtual void MapRefPosAndElemIndices();

-   // Device functions
-   // FindPoints locally on device for 3D.
+   /// FindPoints locally on device for 3D.
   void FindPointsLocal3(const Vector &point_pos, int point_pos_ordering,
                         Array<unsigned int> &gsl_code_dev_l,
                         Array<unsigned int> &gsl_elem_dev_l, Vector &gsl_ref_l,
                         Vector &gsl_dist_l, int npt);

-   // FindPoints locally on device for 2D.
+   /// FindPoints locally on device for 2D.
   void FindPointsLocal2(const Vector &point_pos, int point_pos_ordering,
                         Array<unsigned int> &gsl_code_dev_l,
                         Array<unsigned int> &gsl_elem_dev_l, Vector &gsl_ref_l,
                         Vector &gsl_dist_l, int npt);

-   // Interpolate on device for 3D.
+   /// FindPoints locally on device for 3D surface elements.
+   void FindPointsSurfLocal3(const Vector &point_pos,
+                             int point_pos_ordering,
+                             Array<unsigned int> &gsl_code_dev_l,
+                             Array<unsigned int> &gsl_elem_dev_l,
+                             Vector &gsl_ref_l,
+                             Vector &gsl_dist_l,
+                             int npt);
+
+   /// FindPoints locally on device for 3D edge elements.
+   void FindPointsEdgeLocal3(const Vector &point_pos,
+                             int point_pos_ordering,
+                             Array<unsigned int> &gsl_code_dev_l,
+                             Array<unsigned int> &gsl_elem_dev_l,
+                             Vector &gsl_ref_l,
+                             Vector &gsl_dist_l,
+                             int npt);
+
+   /// FindPoints locally on device for 2D edge elements.
+   void FindPointsEdgeLocal2(const Vector &point_pos,
+                             int point_pos_ordering,
+                             Array<unsigned int> &gsl_code_dev_l,
+                             Array<unsigned int> &gsl_elem_dev_l,
+                             Vector &gsl_ref_l,
+                             Vector &gsl_dist_l,
+                             int npt);
+
+   /// Interpolate on device for 3D.
   void InterpolateLocal3(const Vector &field_in,
                          Array<int> &gsl_elem_dev_l,
                          Vector &gsl_ref_l,
                          Vector &field_out,
                          int npt, int ncomp,
-                          int nel, int dof1dsol);
-   // Interpolate on device for 2D.
+                          int dof1dsol);
+
+   /// Interpolate on device for 2D.
   void InterpolateLocal2(const Vector &field_in,
                          Array<int> &gsl_elem_dev_l,
                          Vector &gsl_ref_l,
                          Vector &field_out,
                          int npt, int ncomp,
-                          int nel, int dof1dsol);
+                          int dof1dsol);

-   // Prepare data for device functions.
+   /// Interpolate on device for 1D.
+   void InterpolateLocal1(const Vector &field_in,
+                          Array<int> &gsl_elem_dev_l,
+                          Vector &gsl_ref_l,
+                          Vector &field_out,
+                          int npt, int ncomp, int dof1dsol);
+
+   /// Prepare data for device execution for volume meshes.
   void SetupDevice();

-   /** Searches positions given in physical space by @a point_pos.
+   /** @brief Searches positions given in physical space by @a point_pos.
       These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
       byVDim: (XYZ,XYZ,....XYZ) specified by @a point_pos_ordering. */
   void FindPointsOnDevice(const Vector &point_pos,
                           const int point_pos_ordering = Ordering::byNODES);

-   /** Interpolation of field values at prescribed reference space positions.
-       @param[in] field_in_evec E-vector of grid function to be interpolated.
-                                Assumed ordering is NDOFSxVDIMxNEL
-       @param[in] nel           Number of elements in the mesh.
-       @param[in] ncomp         Number of components in the field.
-       @param[in] dof1dsol      Number of degrees of freedom in each reference
-                                space direction.
-       @param[in] ordering      Ordering of the out field values: byNodes/byVDIM
-
-       @param[out] field_out  Interpolated values. For points that are not found
-                              the value is set to #default_interp_value. */
+   /** @brief Interpolation of field values at prescribed reference space
+    *         positions.
+    *  @param[in] field_in_evec E-vector of grid function to be interpolated.
+    *                           Assumed ordering is NDOFSxVDIMxNEL
+    *  @param[in] nel           Number of elements in the mesh.
+    *  @param[in] ncomp         Number of components in the field.
+    *  @param[in] dof1dsol      Number of degrees of freedom in each reference
+    *                           space direction.
+    *  @param[in] ordering      Ordering of the out field values: byNodes/byVDIM
+    *
+    *  @param[out] field_out    Interpolated values. For points that are not
+    *                           found the value is set to
+    *                           #default_interp_value. */
   void InterpolateOnDevice(const Vector &field_in_evec, Vector &field_out,
                            const int nel, const int ncomp,
                            const int dof1dsol, const int ordering);

+   /** @brief Interpolation of field values at prescribed reference space
+    *         positions for surface meshes. */
+   void InterpolateSurfBase(const Vector &field_in, Vector &field_out,
+                            const int nel, const int ncomp,
+                            const int dof1dsol, const int field_out_ordering);
+
+   /// Preprocess 2D surface mesh needed for FindPoints.
+   void findptsedge_setup_2(DEV_STRUCT &devs,
+                            const double *const elx[2],
+                            const unsigned n,
+                            const uint nel,
+                            const unsigned m,
+                            const double bbox_tol,
+                            const uint local_hash_size,
+                            const uint global_hash_size);
+
+   /// Preprocess 3D surface mesh needed for FindPoints.
+   void findptssurf_setup_3(DEV_STRUCT &devs,
+                            const double *const elx[3],
+                            const unsigned n,
+                            const uint nel,
+                            const unsigned m,
+                            const double bbox_tol,
+                            const uint local_hash_size,
+                            const uint global_hash_size,
+                            const int rD);
+
 public:
   /// Serial constructor
   FindPointsGSLIB();
@@ -224,8 +331,10 @@ public:
   FindPointsGSLIB(const FindPointsGSLIB&) = delete;
   FindPointsGSLIB& operator=(const FindPointsGSLIB&) = delete;

-   /** Initializes the internal mesh in gslib, by sending the positions of the
-       Gauss-Lobatto nodes of the input Mesh object \p m.
+   /** @brief Preprocess the internal mesh in gslib.
+
+       @details Initializes the internal mesh in gslib, by sending the
+       positions of the Gauss-Lobatto nodes of the input Mesh object \p m.
       Note: not tested with periodic (L2).
       Note: the input mesh \p m must have Nodes set.

@@ -236,13 +345,22 @@ public:
                            search methods.
       @param[in] npt_max   (Optional) Number of points for simultaneous
                            iteration. This alters performance and
-                            memory footprint.*/
-
+                            memory footprint.
+   */
   void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12,
              const int npt_max = 256);
-   /** Searches positions given in physical space by \p point_pos.
-       These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
+
+   /// Preprocess the surface mesh to compute data for FindPoints.
+   void SetupSurf(Mesh &m,
+                  const double bb_t = 0.1,
+                  const double newt_tol = 1.0e-12,
+                  const int npt_max = 256);
+
+   /** @brief Searches positions given in physical space by \p point_pos.
+
+       @details These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
       byVDim: (XYZ,XYZ,....XYZ) specified by \p point_pos_ordering.
+
       This function populates the following member variables:
       #gsl_code        Return codes for each point: inside element (0),
                        element boundary (1), not found (2).
@@ -261,19 +379,34 @@ public:
       #gsl_dist        Distance between the sought and the found point
                        in physical space. */
   void FindPoints(const Vector &point_pos,
-                   const int point_pos_ordering = Ordering::byNODES);
+                   int point_pos_ordering = Ordering::byNODES);
+
   /// Convenience function when point positions are in a ParticleVector
   void FindPoints(const ParticleVector &point_pos)
   {
      FindPoints(point_pos, point_pos.GetOrdering());
   }
+
+   /** @brief Searches positions given in physical space by \p point_pos on
+    *  surface mesh. */
+   void FindPointsSurf(const Vector &point_pos,
+                       int point_pos_ordering = Ordering::byNODES);
+
+   /// Convenience function when point positions are in a ParticleVector
+   void FindPointsSurf(const ParticleVector &point_pos)
+   {
+      FindPointsSurf(point_pos, point_pos.GetOrdering());
+   }
+
   /// Setup FindPoints and search positions
   void FindPoints(Mesh &m, const Vector &point_pos,
                   const int point_pos_ordering = Ordering::byNODES,
                   const double bb_t = 0.1, const double newt_tol = 1.0e-12,
                   const int npt_max = 256);

-   /** Interpolation of field values at prescribed reference space positions.
+   /** @brief Interpolation of field values at prescribed reference space
+    *  positions.
+
       @param[in] field_in    Function values that will be interpolated on the
                              reference positions. Note: it is assumed that
                              \p field_in is in H1 and in the same space as the
@@ -282,19 +415,36 @@ public:
                              the value is set to #default_interp_value.
                              The output ordering is determined from field_in.*/
   virtual void Interpolate(const GridFunction &field_in, Vector &field_out);
+
   /// Interpolation of field values, with output ordering specification.
   virtual void Interpolate(const GridFunction &field_in, Vector &field_out,
                            const int field_out_ordering);
-   /** Search positions and interpolate. The ordering (byNODES or byVDIM) of
-       the output values in \p field_out corresponds to the ordering used
-       in the input GridFunction \p field_in. */
+
+   /** @brief Same as Interpolate but for surface meshes */
+   virtual void InterpolateSurf(const GridFunction &field_in,
+                                Vector &field_out);
+
+   /** @brief Same as Interpolate but for surface meshes with specified output
+       ordering */
+   virtual void InterpolateSurf(const GridFunction &field_in,
+                                Vector &field_out,
+                                const int field_out_ordering);
+
+   /** @brief Search positions and interpolate.
+    *
+    *  @details The ordering (byNODES or byVDIM) of the output values in
+    *  \p field_out corresponds to the ordering used in the input
+    *  GridFunction \p field_in.
+    */
   void Interpolate(const Vector &point_pos, const GridFunction &field_in,
                    Vector &field_out,
-                    const int point_pos_ordering = Ordering::byNODES);
+                    int point_pos_ordering = Ordering::byNODES);
+
   /// Search positions and interpolate with given point and output ordering.
   void Interpolate(const Vector &point_pos, const GridFunction &field_in,
                    Vector &field_out, const int point_pos_ordering,
                    const int field_out_ordering);
+
   /** Setup FindPoints, search positions and interpolate. The ordering (byNODES
       or byVDIM) of the output values in \p field_out corresponds to the
       ordering used in the input GridFunction \p field_in. */
@@ -302,32 +452,36 @@ public:
                    const GridFunction &field_in, Vector &field_out,
                    const int point_pos_ordering = Ordering::byNODES);

-   /// Average type to be used for L2 functions in-case a point is located at
-   /// an element boundary where the function might be multi-valued.
+   /** @brief Average type to be used for L2 functions in-case a point is
+    *  located at an element boundary where the function might be multi-valued.
+    */
   virtual void SetL2AvgType(AvgType avgtype_) { avgtype = avgtype_; }

-   /// Set the default interpolation value for points that are not found in the
-   /// mesh.
+   /** @brief Set the default interpolation value for points that are not found in the mesh. */
   virtual void SetDefaultInterpolationValue(double interp_value_)
   {
      default_interp_value = interp_value_;
   }

-   /// Set the tolerance for detecting points outside the 'curvilinear' boundary
-   /// that gslib may return as found on the boundary. Points found on boundary
-   /// with distance greater than @ bdr_tol are marked as not found.
+   /** @brief Tolerance for detecting points outside the 'curvilinear' boundary.
+    *
+    *  @details When using FindPoints, gslib may return points as found on the
+    *  boundary even when they are slightly outside the domain. This tolerance
+    *  is used to filter such points based on the distance^2 value and mark them
+    *  as not found.*/
   virtual void SetDistanceToleranceForPointsFoundOnBoundary(double bdr_tol_)
   {
      bdr_tol = bdr_tol_;
   }

-   /// Enable/Disable use of CPU functions for GPU data if the gslib version
-   /// is older.
+   /** @brief Enable/Disable use of CPU functions for GPU data if the gslib
+    *  version is older. */
   virtual void SetGPUtoCPUFallback(bool mode) { gpu_to_cpu_fallback = mode; }

-   /** Cleans up memory allocated internally by gslib.
-       Note that in parallel, this must be called before MPI_Finalize(), as it
-       calls MPI_Comm_free() for internal gslib communicators. FreeData is
+   /** @brief Cleans up memory allocated internally by gslib.
+
+       @details Note that in parallel, this must be called before MPI_Finalize,
+       as it calls MPI_Comm_free() for internal gslib communicators. FreeData is
       also called by the class destructor and there are no memory leaks if the
       destructor is called before MPI_Finalize(). If the destructor is called
       after MPI_Finalize(), there will be an error because gslib will try to
@@ -335,8 +489,8 @@ public:
   */
   virtual void FreeData();

-   /// Return code for each point searched by FindPoints: inside element (0), on
-   /// element boundary (1), or not found (2).
+   /** @brief Return code for each point searched by FindPoints:
+    *  inside element (0), element boundary (1), or not found (2). */
   virtual const Array<unsigned int> &GetCode() const { return gsl_code; }
   /// Return element number for each point found by FindPoints.
   virtual const Array<unsigned int> &GetElem() const { return gsl_mfem_elem; }
@@ -344,15 +498,15 @@ public:
   virtual const Array<unsigned int> &GetProc() const { return gsl_proc; }
   /// Return reference coordinates for each point found by FindPoints.
   virtual const Vector &GetReferencePosition() const { return gsl_mfem_ref;  }
-   /// Return distance between the sought and the found point in physical space,
-   /// for each point found by FindPoints.
+   /// Return distance between the sought and the found point in physical space.
   virtual const Vector &GetDist()              const { return gsl_dist; }

-   /// Return element number for each point found by FindPoints corresponding to
-   /// GSLIB mesh. gsl_mfem_elem != gsl_elem for mesh with simplices.
+   /** @brief Return element number for each point found by FindPoints
+    *  corresponding to GSLIB mesh. gsl_mfem_elem != gsl_elem for mesh with
+    *  simplices. */
   virtual const Array<unsigned int> &GetGSLIBElem() const { return gsl_elem; }
-   /// Return reference coordinates in [-1,1] (internal range in GSLIB) for each
-   /// point found by FindPoints.
+   /** @brief Return reference coordinates in [-1,1] (internal range in GSLIB)
+    *  for each point found by FindPoints. */
   virtual const Vector &GetGSLIBReferencePosition() const { return gsl_ref; }

   /// Get array of indices of not-found points.
@@ -395,7 +549,7 @@ public:

   /// Return the axis-aligned bounding boxes (AABB) computed during \ref Setup.
   /// The size of the returned vector is (nel x nverts x dim), where nel is the
-   /// number of elements (after splitting for simplcies), nverts is number of
+   /// number of elements (after splitting for simplicies), nverts is number of
   /// vertices (4 in 2D, 8 in 3D), and dim is the spatial dimension.
   void GetAxisAlignedBoundingBoxes(Vector &aabb) const;

@@ -409,6 +563,18 @@ public:
   /// \p obbV, a vector of size (nel x nverts x dim) .
   void GetOrientedBoundingBoxes(DenseTensor &obbA, Vector &obbC,
                                 Vector &obbV) const;
+
+   /** @brief Return the bounding boxes as a mesh on rank 0.
+    *
+    *  @param[in] type  Bounding-box type: 0 - AABB, 1 - OBB.
+    *
+    *  @return On rank 0, returns a newly allocated mesh containing the
+    *  bounding boxes. The caller owns the returned pointer and is responsible
+    *  for deleting it. On other ranks, returns nullptr.
+    */
+   Mesh *GetBoundingBoxMesh(int type);
+
+   virtual const Vector &GetGLLMesh()           const { return gsl_mesh; }
 };

 /** \brief OversetFindPointsGSLIB enables use of findpts for arbitrary number of
@@ -536,6 +702,116 @@ public:
   void GS(Vector &senddata, GSOp op);
 };

+#if defined(MFEM_USE_MPI)
+/** \brief Class to map a point in physical space to candidate ranks.
+ *
+ *  This class builds a Cartesian-aligned tensor grid that covers the entire
+ *  domain and precomputes which ranks have elements intersecting each
+ *  grid cell. Given a point in physical space, the grid cell containing
+ *  the point is determined, and the list of candidate ranks whose
+ *  elements intersect that cell is returned. This yields a fast, conservative
+ *  point-to-rank candidate query. This is used internally by FindPointsGSLIB
+ *  to speed up point searches in parallel.
+ *
+ *  See Mittal et al., "General Field Evaluation in High-Order Meshes on GPUs".
+ *  (2025). Computers & Fluids. for technical details.
+ *
+ */
+class GlobalBBoxTensorGridMap
+{
+private:
+   struct gslib::crystal *cr = nullptr;               // gslib's internal data
+   struct gslib::comm *gsl_comm = nullptr;            // gslib's internal data
+   int sdim, n_local_cells, num_procs;
+   Array<int> gmap_n;
+   Vector gmap_bnd_min, gmap_bnd_max;
+   Vector gmap_fac;
+   Array<int> ggrid_map;
+
+   void SetupCrystal(const MPI_Comm &comm);
+public:
+   /// Constructor for a given mesh and number of tensor grid divisions
+   GlobalBBoxTensorGridMap(ParMesh &pmesh, int nx);
+
+   /** @brief Constructor for given element bounds and spatial dimension.
+    *
+    *  @details This constructor must be called collectively on \a comm.
+    *  Supports spatial dimensions 1, 2, and 3, and accepts nel == 0 on a rank.
+    *
+    *  Assumes elmin, elmax Ordering::byNodes:
+    *  elmin -> [x_{0,min},x_{1,min},... ,y_{0,min},y_{1,min},..,z_{nel-1,min}]
+    *  elmax -> [x_{0,max},x_{1,max},... ,y_{0,max},y_{1,max},..,z_{nel-1,max}]
+    *  Note elmin, elmax can be obtained using GridFunction::GetElementBounds()
+    *
+    *  When by_max_size=false, n gives the number of tensor-grid divisions in
+    *  each direction. When by_max_size=true, n is a per-rank size hint used to
+    *  derive a uniform global resolution. The communicator-wide sum of n is
+    *  converted to nx = ceil(pow(sum(n), 1./sdim)) in each direction, so n is
+    *  not a hard cap on ggrid_map.Size().
+    */
+   GlobalBBoxTensorGridMap(const MPI_Comm &comm, Vector &elmin,
+                           Vector &elmax, int nel, int sdim, int n,
+                           bool by_max_size);
+
+   /** @brief Constructor for given element bounds, spatial dimension, and
+    *  tensor-grid divisions in each direction.
+    *
+    *  @details This constructor must be called collectively on \a comm.
+    *  Supports spatial dimensions 1, 2, and 3, and accepts nel == 0 on a rank.
+    *  Requires nx.Size() == sdim and positive entries in nx.
+    *
+    *  Assumes elmin, elmax Ordering::byNodes:
+    *  elmin -> [x_{0,min},x_{1,min},... ,y_{0,min},y_{1,min},..,z_{nel-1,min}]
+    *  elmax -> [x_{0,max},x_{1,max},... ,y_{0,max},y_{1,max},..,z_{nel-1,max}]
+    *  Note elmin, elmax can be obtained using GridFunction::GetElementBounds()
+    */
+   GlobalBBoxTensorGridMap(const MPI_Comm &comm, Vector &elmin,
+                           Vector &elmax, int nel, int sdim, Array<int> &nx);
+
+   ~GlobalBBoxTensorGridMap();
+
+   /** @brief Get list of procs corresponding to the list of points.
+    *
+    *  @details This method must be called collectively on the communicator
+    *  used to construct the map. The input points can be ordered byNodes:
+    *  (XXX...,YYY...,ZZZ) or byVDIM: (XYZ,XYZ,...), as specified by
+    *  \a ordering.
+    *
+    *  The output map contains one entry for each input point, keyed by the
+    *  point's local index in \a xyz. Points with no candidate ranks, including
+    *  points outside the global bounding box, have an empty list of candidate
+    *  ranks.
+    */
+   void MapPointsToProcs(Vector &xyz, int ordering,
+                         std::map<int, std::vector<int>> &pt_to_procs) const;
+
+   // Some getters
+   const Array<int> &GetGridMap() const { return ggrid_map; }
+   const Vector &GetGridFac() const { return gmap_fac; }
+   const Vector &GetGridMin() const { return gmap_bnd_min; }
+   const Vector &GetGridMax() const { return gmap_bnd_max; }
+   const Array<int> &GetGridN() const { return gmap_n; }
+
+private:
+   /// Setup the map given element bounds and number of tensor grid divisions.
+   void Setup(const MPI_Comm &comm, Vector &elmin, Vector &elmax,
+              int nel, Array<int> &nx);
+
+   /// Get global hash cell index for a given point.
+   int GetGlobalGridCellFromPoint(Vector &xyz) const;
+
+   /** @brief Get owning proc and local index on that proc for given global
+    *  grid cell index. */
+   void GlobalGridCellToProcAndLocalIndex(int i, int &proc, int &idx) const;
+
+   /// Map a point to proc and local index of the corresponding grid cell
+   void GetProcAndLocalIndexFromPoint(Vector &xyz, int &proc, int &idx) const;
+
+   /// Given local cell index, return list of procs saved in the map
+   Array<int> MapCellToProcs(int l_idx) const;
+};
+#endif // MFEM_USE_MPI
+
 } // namespace mfem

 #endif // MFEM_USE_GSLIB
@@ -562,7 +562,7 @@ newton_area_fin:
      int f = flags >> (2 * dd) & 3u;
      res->r[dd] = f == 0 ? r0[dd] + dr[dd] : (f == 1 ? -1 : 1);
   }
-   res->flags = flags | (p->flags << 5);
+   res->flags = flags | ((p->flags & FLAG_MASK) << 5);
 }

 // Full Newton solve on the face. One of r/s/t is constrained.
@@ -635,7 +635,8 @@ newton_edge_fin:
   res->r[de] = nr;
   res->r[dn]=p->r[dn];
   res->dist2p = -v;
-   res->flags = flags | new_flags | (p->flags << 5);
+   res->flags = flags | new_flags | ((p->flags & FLAG_MASK) << 5);
+#undef EVAL
 }

 // Find closest mesh node to the sought point.
@@ -714,7 +715,6 @@ static void FindPointsLocal2D_Kernel(const int npt,
                                     const double *lagcoeff,
                                     const int pN = 0)
 {
-#define MAX_CONST(a, b) (((a) > (b)) ? (a) : (b))
   const int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
   const int D1D = T_D1D ? T_D1D : pN;
   const int p_NE = D1D*D1D;
@@ -729,7 +729,7 @@ static void FindPointsLocal2D_Kernel(const int npt,
      // 3D1D for seed, 10D1D+6 for area, 3D1D+9 for edge
      constexpr int size1 = 10*MD1 + 6;
      constexpr int size2 = MD1*4;            // edge constraints
-      constexpr int size3 = MD1*MD1*MD1*DIM;  // local element coordinates
+      constexpr int size3 = MD1*MD1*DIM;  // local element coordinates

      MFEM_SHARED double r_workspace[size1];
      MFEM_SHARED findptsElementPoint_t el_pts[2];
@@ -1162,9 +1162,9 @@ void FindPointsGSLIB::FindPointsLocal2(const Vector &point_pos,
   auto pgslm = gsl_mesh.Read();
   auto pwt = DEV.wtend.Read();
   auto pbb = DEV.bb.Read();
-   auto plhm = DEV.loc_hash_min.Read();
-   auto plhf = DEV.loc_hash_fac.Read();
-   auto plho = DEV.loc_hash_offset.ReadWrite();
+   auto plhm = DEV.lh_min.Read();
+   auto plhf = DEV.lh_fac.Read();
+   auto plho = DEV.lh_offset.ReadWrite();
   auto pcode = code.Write();
   auto pelem = elem.Write();
   auto pref = ref.Write();
@@ -1177,30 +1177,32 @@ void FindPointsGSLIB::FindPointsLocal2(const Vector &point_pos,
      case 2:
         return FindPointsLocal2D_Kernel<2>(
                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.h_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
+                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
                   pgll1d, plc);
      case 3:
         return FindPointsLocal2D_Kernel<3>(
                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.h_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
+                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
                   pgll1d, plc);
      case 4:
         return FindPointsLocal2D_Kernel<4>(
                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.h_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
+                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
                   pgll1d, plc);
      case 5:
         return FindPointsLocal2D_Kernel<5>(
                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.h_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
+                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
                   pgll1d, plc);
      default:
         return FindPointsLocal2D_Kernel(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                         pgslm, NE_split_total, pwt, pbb, DEV.h_nx,
+                                         pgslm, NE_split_total, pwt, pbb, DEV.lh_nx,
                                         plhm, plhf, plho, pcode, pelem,
                                         pref, pdist, pgll1d, plc, DEV.dof1d);
   }
 }
+#undef DIM2
+#undef DIM
 #undef CODE_INTERNAL
 #undef CODE_BORDER
 #undef CODE_NOT_FOUND
@@ -706,7 +706,7 @@ newton_vol_fin:
      int f = flags >> (2*dd) & 3u;
      res->r[dd] = f == 0 ? r0[dd]+dr[dd] : (f == 1 ? -1 : 1);
   }
-   res->flags = flags | (p->flags << 7);
+   res->flags = flags | ((p->flags & FLAG_MASK) << 7);
 }

 // Full Newton solve on the face. One of r/s/t is constrained.
@@ -889,7 +889,7 @@ newton_face_fin:
   res->r[dn] = p->r[dn];
   res->r[d1] = r[0];
   res->r[d2] = r[1];
-   res->flags = new_flags | (p->flags << 7);
+   res->flags = new_flags | ((p->flags & FLAG_MASK) << 7);
 }

 // Full Newton solve on the edge. Two of r/s/t are constrained.
@@ -973,7 +973,8 @@ newton_edge_fin:
   res->r[dn1] = p->r[dn1];
   res->r[dn2] = p->r[dn2];
   res->dist2p = -v;
-   res->flags = flags | new_flags | (p->flags << 7);
+   res->flags = flags | new_flags | ((p->flags & FLAG_MASK) << 7);
+#undef EVAL
 }

 // Find closest mesh node to the sought point.
@@ -1252,7 +1253,6 @@ static void FindPointsLocal3DKernel(const int npt,
                     case 0: // findpt_vol
                     {
                        double *wtr = r_workspace_ptr;
-
                        double *resid = wtr+6*D1D;
                        double *jac = resid+3;
                        double *resid_temp = jac+9;
@@ -1503,7 +1503,7 @@ static void FindPointsLocal3DKernel(const int npt,
                              // Hes_T is transposed version (i.e. in col major)
                              // n1*[2, 1, 1, 0, 0]
                              // j==1 => wt_j = wt+n1
-                              double *wt_j = wt+D1D*(2-(row+1) / 2);
+                              double *wt_j = wt+D1D*(2 - (row+1)/2);
                              const double *x = e_x[row+1][d];
                              hes_T[j] = 0.0;
                              for (int k = 0; k < D1D; ++k)
@@ -1522,7 +1522,6 @@ static void FindPointsLocal3DKernel(const int npt,
                              hes[j] += resid[d]*hes_T[j*3+d];
                           }
                        }
-
                        MFEM_SYNC_THREAD;

                        MFEM_FOREACH_THREAD(l,x,1)
@@ -1780,6 +1779,7 @@ static void FindPointsLocal3DKernel(const int npt,
         } //findpts_local
      } //elp
   });
+#undef MAXC
 }

 void FindPointsGSLIB::FindPointsLocal3(const Vector &point_pos,
@@ -1796,9 +1796,9 @@ void FindPointsGSLIB::FindPointsLocal3(const Vector &point_pos,
   auto pgslm = gsl_mesh.Read();
   auto pwt = DEV.wtend.Read();
   auto pbb = DEV.bb.Read();
-   auto plhm = DEV.loc_hash_min.Read();
-   auto plhf = DEV.loc_hash_fac.Read();
-   auto plho = DEV.loc_hash_offset.ReadWrite();
+   auto plhm = DEV.lh_min.Read();
+   auto plhf = DEV.lh_fac.Read();
+   auto plho = DEV.lh_offset.ReadWrite();
   auto pcode = code.Write();
   auto pelem = elem.Write();
   auto pref = ref.Write();
@@ -1809,31 +1809,31 @@ void FindPointsGSLIB::FindPointsLocal3(const Vector &point_pos,
   {
      case 2:
         FindPointsLocal3DKernel<2>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.h_nx, plhm,
+                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
                                    plc);
         break;
      case 3:
         FindPointsLocal3DKernel<3>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.h_nx, plhm,
+                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
                                    plc);
         break;
      case 4:
         FindPointsLocal3DKernel<4>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.h_nx, plhm,
+                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
                                    plc);
         break;
      case 5:
         FindPointsLocal3DKernel<5>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.h_nx, plhm,
+                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
                                    plc);
         break;
      default:
         FindPointsLocal3DKernel(npt, DEV.newt_tol, pp, point_pos_ordering, pgslm,
-                                 NE_split_total, pwt, pbb, DEV.h_nx, plhm, plhf,
+                                 NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
                                 plho, pcode, pelem, pref, pdist, pgll1d, plc,
                                 DEV.dof1d);
   }
@@ -0,0 +1,725 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../gslib.hpp"
+#include "../../general/forall.hpp"
+
+#ifdef MFEM_USE_GSLIB
+
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+#include "gslib.h"
+#ifndef GSLIB_RELEASE_VERSION //gslib v1.0.7
+#define GSLIB_RELEASE_VERSION 10007
+#endif
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic pop
+#endif
+
+namespace mfem
+{
+#if GSLIB_RELEASE_VERSION >= 10009
+#define CODE_INTERNAL 0
+#define CODE_BORDER 1
+#define CODE_NOT_FOUND 2
+#define sDIM 2
+#define sDIM2 4
+#define rDIM 1
+
+struct findptsElementPoint_t
+{
+   double x[sDIM], r, oldr, dist2, dist2p, tr;
+   int flags;
+};
+
+struct findptsElementGEdge_t
+{
+   double *x[sDIM];
+};
+
+struct findptsElementGPT_t
+{
+   double x[sDIM], jac[sDIM*rDIM], hes[sDIM*rDIM];
+};
+
+struct dbl_range_t
+{
+   double min, max;
+};
+
+struct obbox_t
+{
+   double c0[sDIM], A[sDIM*sDIM];
+   dbl_range_t x[sDIM];
+};
+
+struct findptsLocalHashData_t
+{
+   int hash_n;
+   dbl_range_t bnd[sDIM];
+   double fac[sDIM];
+   unsigned int *offset;
+};
+
+static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
+                                                        int i, const double *z,
+                                                        const double *lCoeff,
+                                                        int pN)
+{
+   double u0 = 1, u1 = 0, u2 = 0;
+   for (int j = 0; j < pN; ++j)
+   {
+      if (i != j)
+      {
+         double d_j = 2 * (x-z[j]);
+         u2 = d_j * u2 + u1;
+         u1 = d_j * u1 + u0;
+         u0 = d_j * u0;
+      }
+   }
+   double *p1 = p0 + pN, *p2 = p0 + 2 * pN;
+   p0[i] = lCoeff[i] * u0;
+   p1[i] = 2.0 * lCoeff[i] * u1;
+   p2[i] = 8.0 * lCoeff[i] * u2;
+}
+
+/* positive when possibly inside */
+static MFEM_HOST_DEVICE inline double obbox_axis_test(const obbox_t *const b,
+                                                      const double x[sDIM])
+{
+   double b_d;
+   for (int d=0; d<sDIM; ++d)
+   {
+      b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
+      if (b_d < 0)   // if outside in any dimension
+      {
+         return b_d;
+      }
+   }
+   return b_d;       // only positive if inside
+}
+
+/* positive when given point is possibly inside given obbox b */
+static MFEM_HOST_DEVICE inline double obbox_test(const obbox_t *const b,
+                                                 const double x[sDIM])
+{
+   const double bxyz = obbox_axis_test(b,x);
+   if (bxyz<0)   // test if point is in AABB
+   {
+      return bxyz;
+   }
+   else   // test OBB only if inside AABB
+   {
+      double dxyz[sDIM];
+      for (int d=0; d<sDIM; ++d)
+      {
+         dxyz[d] = x[d] - b->c0[d];
+      }
+      double test = 1;
+      for (int d=0; d<sDIM; ++d)
+      {
+         double rst = 0;
+         for (int e=0; e<sDIM; ++e)
+         {
+            rst += b->A[d*2 + e] * dxyz[e];
+         }
+         double brst = (rst+1)*(1-rst);
+         test = test<0 ? test : brst;
+      }
+      return test;
+   }
+}
+
+/* Hash index in the hash table to the elements that possibly contain the point x */
+static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
+                                              const double x[2])
+{
+   const int n = p->hash_n;
+   int sum = 0;
+   for (int d=sDIM-1; d>=0; --d)
+   {
+      sum *= n;
+      int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
+      sum += i<0 ? 0 : (n-1 < i ? n-1 : i);
+   }
+   return sum;
+}
+
+static MFEM_HOST_DEVICE inline double l2norm2(const double x[2])
+{
+   return x[0] * x[0] + x[1] * x[1];
+}
+
+/* the bit structure of flags is CRR
+   the C bit --- 1<<2 --- is set when the point is converged
+   RR is 0 = 00b if r is unconstrained,
+         1 = 01b if r is constrained at -1, i.e., rmin
+         2 = 10b if r is constrained at +1, i.e., rmax
+*/
+
+#define CONVERGED_FLAG (1u<<2)
+#define FLAG_MASK 0x07u // = 111b
+
+/* returns 1 if r direction (the only free direction in 2D) is constrained.
+   returns 1 if either 1st or 2nd bit of flags is set.
+*/
+static MFEM_HOST_DEVICE inline int num_constrained(const int flags)
+{
+   return ((flags | flags>>1) & 1u);
+}
+
+/* pi=0, r=-1; pi=1, r=+1 */
+static MFEM_HOST_DEVICE inline int point_index(const int x)
+{
+   return ((x>>1) & 1u);
+}
+
+/* check reduction in objective against prediction, and adjust
+   trust region radius (p->tr) accordingly;
+   may reject the prior step, returning 1; otherwise returns 0
+   sets out->dist2, out->index, out->x, out->oldr in any event,
+   leaving out->r, out->dr, out->flags to be set when returning 0 */
+static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
+                                                 const double resid[2],
+                                                 const findptsElementPoint_t *p,
+                                                 const double tol)
+{
+   const double dist2 = l2norm2(resid);
+   const double decr = p->dist2 - dist2;
+   const double pred = p->dist2p;
+   out->x[0] = p->x[0];
+   out->x[1] = p->x[1];
+   out->oldr = p->r;
+   out->dist2 = dist2;
+   if (decr >= 0.01*pred)
+   {
+      if (decr >= 0.9*pred)   // very good iteration
+      {
+         out->tr = p->tr*2;
+      }
+      else                    // somewhat good iteration
+      {
+         out->tr = p->tr;
+      }
+      return false;
+   }
+   else
+   {
+      /* reject step; note: the point will pass through this routine
+         again, and we set things up here so it gets classed as a
+         "very good iteration" --- this doubles the trust radius,
+         which is why we divide by 4 below */
+      double v0 = fabs(p->r - p->oldr);
+      out->tr = v0/4.0;
+      out->dist2 = p->dist2;
+      out->r = p->oldr;
+      out->flags = p->flags>>3;
+      out->dist2p = -HUGE_VAL;
+      if (pred < dist2*tol)
+      {
+         out->flags |= CONVERGED_FLAG;
+      }
+      return true;
+   }
+}
+
+static MFEM_HOST_DEVICE inline void newton_edge( findptsElementPoint_t *const
+                                                 out,
+                                                 const double jac[2],
+                                                 const double rhess,
+                                                 const double resid[2],
+                                                 int flags,
+                                                 const findptsElementPoint_t *const p,
+                                                 const double tol )
+{
+   const double tr = p->tr;
+   const double A = jac[0] * jac[0] + jac[1] * jac[1] -
+                    rhess; // A = J^T J - resid_d H_d
+   const double y = jac[0]*resid[0] + jac[1]*resid[1];        // y = J^T resid
+
+   const double oldr = p->r;
+   double dr, newr, tdr, tnewr, v, tv;
+   int new_flags=0, tnew_flags=0;
+
+#define EVAL(dr) ( (dr*A - 2*y) * dr )
+   if (A>0)
+   {
+      dr = y/A;
+      if (fabs(dr)<tol)
+      {
+         dr=0.0;
+         newr = oldr;
+      }
+      else
+      {
+         newr = oldr+dr;
+      }
+
+      if (fabs(dr)<tr && fabs(newr)<1)
+      {
+         v = EVAL(dr);
+         goto newton_edge_fin;
+      }
+   }
+
+   if ((newr=oldr-tr) > -1)
+   {
+      dr = -tr;
+   }
+   else
+   {
+      newr = -1, dr = -1-oldr, new_flags = flags|1u;
+   }
+   v = EVAL(dr);
+
+   if ((tnewr=oldr+tr) < 1)
+   {
+      tdr = tr;
+   }
+   else
+   {
+      tnewr = 1, tdr = 1-oldr, tnew_flags = flags|2u;
+   }
+   tv = EVAL(tdr);
+
+   if (tv<v)
+   {
+      newr = tnewr, dr = tdr, v = tv, new_flags = tnew_flags;
+   }
+#undef EVAL
+
+newton_edge_fin:
+   // check convergence by testing if change in r is less than tol
+   if (fabs(dr)<tol)
+   {
+      new_flags |= CONVERGED_FLAG;
+   }
+   out->r = newr;
+   out->dist2p = -v;
+   out->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
+}
+
+static MFEM_HOST_DEVICE void seed_j( const double *elx[sDIM],
+                                     const double x[sDIM],
+                                     const double *z,
+                                     double       *dist2,
+                                     double       *r,
+                                     const int    ir,
+                                     const int    pN )
+{
+   double dx[sDIM];
+   for (int d=0; d<sDIM; ++d)
+   {
+      dx[d] = x[d] - elx[d][ir];
+   }
+   dist2[ir] = HUGE_VAL;
+   const double dist2_rs = l2norm2(dx);
+   if (dist2[ir]>dist2_rs)
+   {
+      dist2[ir] = dist2_rs;
+      r[ir] = z[ir];
+   }
+}
+
+template<int T_D1D = 0>
+static void FindPointsEdgeLocal2D_Kernel( const int     npt,
+                                          const double  tol,
+                                          const double  dist2tol,
+                                          const double  *x,
+                                          const int     point_pos_ordering,
+                                          const double  *xElemCoord,
+                                          const int     nel,
+                                          const double  *wtend,
+                                          const double  *boxinfo,
+                                          const int     hash_n,
+                                          const double  *hashMin,
+                                          const double  *hashFac,
+                                          unsigned int  *hashOffset,
+                                          unsigned int *const code_base,
+                                          unsigned int *const el_base,
+                                          double *const r_base,
+                                          double *const dist2_base,
+                                          const double  *gll1D,
+                                          const double  *lagcoeff,
+                                          const int     pN = 0 )
+{
+   const int MD1   = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+   const int D1D   = T_D1D ? T_D1D : pN;
+   const int p_NEL = nel*D1D;
+   MFEM_VERIFY(MD1<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(pN<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(D1D!=0, "Polynomial order not specified.");
+   const int nThreads = D1D*sDIM;
+
+   mfem::forall_2D(npt, nThreads, 1, [=] MFEM_HOST_DEVICE (int i)
+   {
+      // 2D1D for seed, 3D1D + 7 for edge
+      constexpr int size1 = 3*MD1 + 7;
+      // edge coordinates = D1D*2
+      constexpr int size2 = 2*MD1;
+      // local element coordinates in shared memory
+      constexpr int size3 = MD1*sDIM;
+
+      MFEM_SHARED findptsElementPoint_t el_pts[2];
+      MFEM_SHARED double r_workspace[size1];
+
+      MFEM_SHARED double constraint_workspace[size2];
+
+      MFEM_SHARED double elem_coords[MD1 <= 6 ? size3 : 1];
+
+      double *r_workspace_ptr = r_workspace;
+      findptsElementPoint_t *fpt, *tmp;
+      fpt = el_pts + 0;
+      tmp = el_pts + 1;
+
+      // x and y coord index within point_pos for point i
+      int id_x = point_pos_ordering == 0 ? i     : i*sDIM;
+      int id_y = point_pos_ordering == 0 ? i+npt : i*sDIM+1;
+      double x_i[2] = {x[id_x], x[id_y]};
+
+      unsigned int *code_i = code_base  + i;
+      double *dist2_i      = dist2_base + i;
+
+      //---------------- map_points_to_els --------------------
+      findptsLocalHashData_t hash;
+      for (int d=0; d<sDIM; ++d)
+      {
+         hash.bnd[d].min = hashMin[d];
+         hash.fac[d] = hashFac[d];
+      }
+      hash.hash_n = hash_n;
+      hash.offset = hashOffset;
+
+      const int hi                  = hash_index(&hash, x_i);
+      const unsigned int *elp       = hash.offset + hash.offset[hi];
+      const unsigned int *const ele = hash.offset + hash.offset[hi+1];
+      *code_i  = CODE_NOT_FOUND;
+      *dist2_i = HUGE_VAL;
+
+      for (; elp!=ele; ++elp)
+      {
+         const unsigned int el = *elp;
+
+         obbox_t box;
+         int n_box_ents = 3*sDIM + sDIM2;
+
+         for (int idx = 0; idx < sDIM; ++idx)
+         {
+            box.c0[idx] = boxinfo[n_box_ents*el + idx];
+            box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
+            box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
+         }
+
+         for (int idx = 0; idx < sDIM2; ++idx)
+         {
+            box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
+         }
+
+         if (obbox_test(&box,x_i)>=0)
+         {
+            //------------ findpts_local ------------------
+            {
+               if (MD1 <= 6)
+               {
+                  MFEM_FOREACH_THREAD(j,x,D1D*sDIM)
+                  {
+                     const int qp = j % D1D;
+                     const int d = j / D1D;
+                     elem_coords[qp + d*D1D] =
+                        xElemCoord[qp + el*D1D + d*p_NEL];
+                  }
+                  MFEM_SYNC_THREAD;
+               }
+
+               const double *elx[sDIM];
+               for (int d=0; d<sDIM; d++)
+               {
+                  elx[d] = MD1<= 6 ? &elem_coords[d*D1D] :
+                           xElemCoord + d*p_NEL + el*D1D;
+               }
+               MFEM_SYNC_THREAD;
+               //// findpts_el ////
+               {
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     fpt->dist2 = HUGE_VAL;
+                     fpt->dist2p = 0;
+                     fpt->tr = 1;
+                  }
+                  MFEM_FOREACH_THREAD(j,x,sDIM)
+                  {
+                     fpt->x[j] = x_i[j];
+                  }
+                  MFEM_SYNC_THREAD;
+
+                  {
+                     double *dist2_temp = r_workspace_ptr;
+                     double *r_temp = dist2_temp + D1D;
+                     MFEM_FOREACH_THREAD(j,x,D1D)
+                     {
+                        seed_j(elx, x_i, gll1D, dist2_temp, r_temp, j, D1D);
+                     }
+                     MFEM_SYNC_THREAD;
+
+                     MFEM_FOREACH_THREAD(j,x,1)
+                     {
+                        for (int ir=0; ir<D1D; ++ir)
+                        {
+                           if (dist2_temp[ir]<fpt->dist2)
+                           {
+                              fpt->dist2 = dist2_temp[ir];
+                              fpt->r = r_temp[ir];
+                           }
+                        }
+                     }
+                     MFEM_SYNC_THREAD;
+                  } //seed done
+
+                  // Initialize tmp struct with fpt values before starting Newton iterations
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     tmp->dist2 = HUGE_VAL;
+                     tmp->dist2p = 0;
+                     tmp->tr = 1;
+                     tmp->flags = 0;
+                     tmp->r = fpt->r;
+                  }
+                  MFEM_FOREACH_THREAD(j,x,sDIM)
+                  {
+                     tmp->x[j] = fpt->x[j];
+                  }
+                  MFEM_SYNC_THREAD;
+
+
+                  for (int step=0; step<50; step++)
+                  {
+                     int nc = num_constrained(tmp->flags & FLAG_MASK);
+                     switch (nc)
+                     {
+                        case 0:
+                        {
+                           double *wt = r_workspace_ptr;
+                           double *resid = wt + 3*D1D;
+                           double *jac = resid + sDIM;
+                           double *hess = jac + sDIM*rDIM;
+
+                           findptsElementGEdge_t edge;
+                           MFEM_FOREACH_THREAD(j,x,D1D)
+                           {
+                              for (int d=0; d<sDIM; ++d)
+                              {
+                                 edge.x[d] = constraint_workspace + d*D1D;
+                                 edge.x[d][j] = elx[d][j];
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           // compute basis function info upto 2nd derivative
+                           MFEM_FOREACH_THREAD(j,x,D1D)
+                           {
+                              lag_eval_second_der(wt, tmp->r, j, gll1D,
+                                                  lagcoeff, D1D);
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,sDIM)
+                           {
+                              resid[j] = tmp->x[j];
+                              jac[j] = 0.0;
+                              hess[j] = 0.0;
+                              for (int k=0; k<D1D; ++k)
+                              {
+                                 resid[j] -= wt[      k]*edge.x[j][k];
+                                 jac[j]   += wt[D1D+k]*edge.x[j][k];
+                                 hess[j]  += wt[2*D1D+k]*edge.x[j][k];
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,1)
+                           {
+                              hess[2] = resid[0]*hess[0] + resid[1]*hess[1];
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,1)
+                           {
+                              if (!reject_prior_step_q(fpt, resid, tmp, tol))
+                              {
+                                 newton_edge(fpt, jac, hess[2], resid,
+                                             tmp->flags & FLAG_MASK, tmp, tol);
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+                           break;
+                        }
+                        case 1:    // r is constrained to either -1 or 1
+                        {
+                           MFEM_FOREACH_THREAD(j,x,1)
+                           {
+                              const int pi = point_index(tmp->flags &
+                                                         FLAG_MASK);
+                              const double *wt   = wtend + pi*3*D1D;
+                              findptsElementGPT_t gpt;
+                              for (int d=0; d<sDIM; ++d)
+                              {
+                                 gpt.x[d] = elx[d][pi*(D1D-1)];
+                                 gpt.jac[d] = 0.0;
+                                 gpt.hes[d] = 0.0;
+                                 for (int k=0; k<D1D; ++k)
+                                 {
+                                    gpt.jac[d] += wt[D1D  +k]*elx[d][k];
+                                    gpt.hes[d] += wt[2*D1D+k]*elx[d][k];
+                                 }
+                              }
+
+                              const double *const pt_x = gpt.x;
+                              const double *const jac = gpt.jac;
+                              const double *const hes = gpt.hes;
+                              double resid[sDIM], steep, sr;
+                              resid[0] = fpt->x[0] - pt_x[0];
+                              resid[1] = fpt->x[1] - pt_x[1];
+                              steep = jac[0]*resid[0] + jac[1]*resid[1];
+                              sr = steep*tmp->r;
+                              if ( !reject_prior_step_q(fpt, resid, tmp, tol) )
+                              {
+                                 if (sr<0)
+                                 {
+                                    const double rhess = resid[0]*hes[0] +
+                                                         resid[1]*hes[1];
+                                    newton_edge(fpt, jac, rhess,
+                                                resid, 0, tmp, tol);
+                                 }
+                                 else // sr==0
+                                 {
+                                    fpt->r = tmp->r;
+                                    fpt->dist2p = 0;
+                                    fpt->flags = tmp->flags | CONVERGED_FLAG;
+                                 }
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+                           break;
+                        } // case 1
+                     } //switch
+                     if (fpt->flags & CONVERGED_FLAG)
+                     {
+                        break;
+                     }
+                     MFEM_SYNC_THREAD;
+                     MFEM_FOREACH_THREAD(j,x,1)
+                     {
+                        *tmp = *fpt;
+                     }
+                     MFEM_SYNC_THREAD;
+                  } //for int step<50
+               } //findpts_el
+
+               bool converged_internal =
+                  ((fpt->flags&FLAG_MASK) == CONVERGED_FLAG) &&
+                  (fpt->dist2<dist2tol);
+
+               if (*code_i == CODE_NOT_FOUND || converged_internal ||
+                   fpt->dist2 < *dist2_i)
+               {
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     *(el_base+i) = el;
+                     *code_i = converged_internal ? CODE_INTERNAL : CODE_BORDER;
+                     *dist2_i = fpt->dist2;
+                     *(r_base+i) = fpt->r;
+                  }
+                  MFEM_SYNC_THREAD;
+                  if (converged_internal)
+                  {
+                     break;
+                  }
+               }
+            } //findpts_local
+         } //obbox_test
+      } //elp
+   });
+}
+
+void FindPointsGSLIB::FindPointsEdgeLocal2( const Vector &point_pos,
+                                            int point_pos_ordering,
+                                            Array<unsigned int> &code,
+                                            Array<unsigned int> &elem,
+                                            Vector &ref,
+                                            Vector &dist,
+                                            int npt )
+{
+   if (npt==0)
+   {
+      return;
+   }
+   MFEM_VERIFY(dim==1 && spacedim==2,"Function for 2D edges only");
+   bool use_dev = point_pos.UseDevice();
+   auto pp = point_pos.Read(use_dev);
+   auto pgslm = gsl_mesh.Read(use_dev);
+   auto pwt = DEV.wtend.Read(use_dev);
+   auto pbb = DEV.bb.Read(use_dev);
+   auto plhm = DEV.lh_min.Read(use_dev);
+   auto plhf = DEV.lh_fac.Read(use_dev);
+   auto plho = DEV.lh_offset.ReadWrite(use_dev);
+   auto pcode = code.Write(use_dev);
+   auto pelem = elem.Write(use_dev);
+   auto pref = ref.Write(use_dev);
+   auto pdist = dist.Write(use_dev);
+   auto pgll1d = DEV.gll1d.ReadWrite(use_dev);
+   auto plc = DEV.lagcoeff.Read(use_dev);
+   double dist2tol = DEV.surf_dist_tol;
+   switch (DEV.dof1d)
+   {
+      case 2:
+         return FindPointsEdgeLocal2D_Kernel<2>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+      case 3:
+         return FindPointsEdgeLocal2D_Kernel<3>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+      case 4:
+         return FindPointsEdgeLocal2D_Kernel<4>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+      default:
+         return FindPointsEdgeLocal2D_Kernel(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc, DEV.dof1d);
+   }
+}
+#undef sDIM
+#undef rDIM
+#undef sDIM2
+#undef CODE_INTERNAL
+#undef CODE_BORDER
+#undef CODE_NOT_FOUND
+#else
+void FindPointsGSLIB::FindPointsEdgeLocal2( const Vector &point_pos,
+                                            int point_pos_ordering,
+                                            Array<unsigned int> &code,
+                                            Array<unsigned int> &elem,
+                                            Vector &ref,
+                                            Vector &dist,
+                                            int npt ) {} ;
+#endif
+} // namespace mfem
+
+#endif //ifdef MFEM_USE_GSLIB
@@ -0,0 +1,733 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../gslib.hpp"
+#include "../../general/forall.hpp"
+
+#ifdef MFEM_USE_GSLIB
+
+
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+#include "gslib.h"
+#ifndef GSLIB_RELEASE_VERSION //gslib v1.0.7
+#define GSLIB_RELEASE_VERSION 10007
+#endif
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic pop
+#endif
+
+namespace mfem
+{
+#if GSLIB_RELEASE_VERSION >= 10009
+#define CODE_INTERNAL 0
+#define CODE_BORDER 1
+#define CODE_NOT_FOUND 2
+#define sDIM 3
+#define rDIM 1
+#define sDIM2 (sDIM*sDIM)
+#define rDIM2 (rDIM*rDIM)
+
+struct findptsElementPoint_t
+{
+   double x[sDIM], r, oldr, dist2, dist2p, tr;
+   int flags;
+};
+
+struct findptsElementGEdge_t
+{
+   double *x[sDIM], *dxdn[sDIM], *d2xdn[sDIM];
+};
+
+struct findptsElementGPT_t
+{
+   double x[sDIM], jac[sDIM], hes[sDIM*(1+1)];
+};
+
+struct dbl_range_t
+{
+   double min, max;
+};
+
+struct obbox_t
+{
+   double c0[sDIM], A[sDIM*sDIM];
+   dbl_range_t x[sDIM];
+};
+
+struct findptsLocalHashData_t
+{
+   int hash_n;
+   dbl_range_t bnd[sDIM];
+   double fac[sDIM];
+   unsigned int *offset;
+};
+
+static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
+                                                        int i, const double *z,
+                                                        const double *lCoeff,
+                                                        int pN)
+{
+   double u0 = 1, u1 = 0, u2 = 0;
+   for (int j=0; j<pN; ++j)
+   {
+      if (i!=j)
+      {
+         double d_j = 2 * (x-z[j]);
+         u2 = d_j * u2 + u1;
+         u1 = d_j * u1 + u0;
+         u0 = d_j * u0;
+      }
+   }
+   double *p1 = p0 + pN, *p2 = p0 + 2 * pN;
+   p0[i] = lCoeff[i] * u0;
+   p1[i] = 2.0 * lCoeff[i] * u1;
+   p2[i] = 8.0 * lCoeff[i] * u2;
+}
+
+/* positive when possibly inside */
+static MFEM_HOST_DEVICE inline double obbox_axis_test(const obbox_t *const b,
+                                                      const double x[sDIM])
+{
+   double b_d;
+   for (int d=0; d<sDIM; ++d)
+   {
+      b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
+      if (b_d < 0)   // if outside in any dimension
+      {
+         return b_d;
+      }
+   }
+   return b_d;       // only positive if inside in all dimensions
+}
+
+/* positive when possibly inside */
+static MFEM_HOST_DEVICE inline double obbox_test(const obbox_t *const b,
+                                                 const double x[sDIM])
+{
+   const double bxyz = obbox_axis_test(b, x);
+   if (bxyz<0)
+   {
+      return bxyz;
+   }
+   else
+   {
+      double dxyz[3];
+      // dxyz: distance of the point from the center of the OBB
+      for (int d=0; d<sDIM; ++d)
+      {
+         dxyz[d] = x[d] - b->c0[d];
+      }
+      // transform dxyz to the local coordinate system of the OBB,
+      // and check if the point is inside the OBB [-1,1]^sDIM
+      double test = 1;
+      for (int d=0; d<sDIM; ++d)
+      {
+         double rst = 0;
+         for (int e=0; e<sDIM; ++e)
+         {
+            rst += b->A[d*sDIM + e] * dxyz[e];
+         }
+         double brst = (rst+1)*(1-rst);
+         test = test<0 ? test : brst;
+      }
+      return test;
+   }
+}
+
+/* Hash index in the hash table to the elements that possibly contain the point x */
+static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
+                                              const double x[sDIM])
+{
+   const int n = p->hash_n;
+   int sum = 0;
+   for (int d=sDIM-1; d>=0; --d)
+   {
+      sum *= n;
+      int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
+      sum += i<0 ? 0 : (n-1 < i ? n-1 : i);
+   }
+   return sum;
+}
+
+
+static MFEM_HOST_DEVICE inline double norm2(const double x[sDIM])
+{
+   return ( x[0]*x[0] + x[1]*x[1] + x[2]*x[2] );
+}
+
+/* the bit structure of flags is CRR
+   the C bit --- 1<<2 --- is set when the point is converged
+   RR is 0 = 00b if r is unconstrained,
+         1 = 01b if r is constrained at -1, i.e., rmin
+         2 = 10b if r is constrained at +1, i.e., rmax
+*/
+#define CONVERGED_FLAG (1u<<2)
+#define FLAG_MASK 0x07u
+
+/* returns the number of constrained reference coordinates, max 2
+*/
+static MFEM_HOST_DEVICE inline int num_constrained(const int flags)
+{
+   const int y = (flags | flags>>1);
+   return (y & 1u) + (y>>2 & 1u);
+}
+
+static MFEM_HOST_DEVICE inline int point_index(const int x)
+{
+   return ((x>>1)&1u) | ((x>>2)&2u);
+}
+
+/* check reduction in objective against prediction, and adjust
+   trust region radius (p->tr) accordingly;
+   may reject the prior step, returning 1; otherwise returns 0
+   sets out->dist2, out->index, out->x, out->oldr in any event,
+   leaving out->r, out->dr, out->flags to be set when returning 0 */
+static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
+                                                 const double resid[3],
+                                                 const findptsElementPoint_t *p,
+                                                 const double tol)
+{
+   const double dist2 = norm2(resid);
+   const double decr  = p->dist2 - dist2;
+   const double pred  = p->dist2p;
+   for (int d=0; d<sDIM; ++d)
+   {
+      out->x[d] = p->x[d];
+   }
+   out->oldr = p->r;
+   out->dist2 = dist2;
+   if (decr>=0.01*pred)
+   {
+      if (decr>=0.9*pred)   // very good iteration
+      {
+         out->tr = 2*p->tr;
+      }
+      else   // good iteration
+      {
+         out->tr = p->tr;
+      }
+      return false;
+   }
+   else   // if the iteration in not good
+   {
+      /* reject step; note: the point will pass through this routine
+         again, and we set things up here so it gets classed as a
+         "very good iteration" --- this doubles the trust radius,
+         which is why we divide by 4 below */
+      double v0 = fabs(p->r - p->oldr);
+      out->tr = v0/4.0;
+      out->dist2 = p->dist2;
+      out->r = p->oldr;
+      out->flags = p->flags>>3;
+      out->dist2p = -HUGE_VAL;
+      if (pred<dist2*tol)
+      {
+         out->flags |= CONVERGED_FLAG;
+      }
+      return true;
+   }
+}
+
+static MFEM_HOST_DEVICE inline void newton_edge(findptsElementPoint_t *const
+                                                out,
+                                                const double jac[sDIM*rDIM],
+                                                const double rhes,
+                                                const double resid[sDIM],
+                                                int flags,
+                                                const findptsElementPoint_t *const p,
+                                                const double tol)
+{
+   const double tr = p->tr;
+   /* A = J^T J - resid_d H_d */
+   const double A = jac[0]*jac[0]+ jac[1] * jac[1] + jac[2] * jac[2]
+                    - rhes;
+   /* y = J^T r */
+   const double y = jac[0]*resid[0] + jac[1]*resid[1] + jac[0+2]*resid[2];
+
+   const double oldr = p->r;
+   double dr, nr, tdr, tnr;
+   double v, tv;
+   int new_flags = 0, tnew_flags = 0;
+
+#define EVAL(dr) (dr*A - 2*y)*dr
+
+   /* if A is not SPD, quadratic model has no minimum */
+   if (A>0)
+   {
+      dr = y/A;
+
+      if (fabs(dr)<tol)
+      {
+         dr=0.0;
+         nr = oldr;
+      }
+      else
+      {
+         nr = oldr+dr;
+      }
+      if ( fabs(dr)<tr && fabs(nr)<1 )
+      {
+         v = EVAL(dr);
+         goto newton_edge_fin;
+      }
+   }
+
+   if ( (nr=oldr-tr)>-1 )
+   {
+      dr = -tr;
+   }
+   else
+   {
+      nr = -1, dr = -1-oldr, new_flags = flags | 1u;
+   }
+   v = EVAL(dr);
+
+   if ( (tnr = oldr+tr)<1 )
+   {
+      tdr = tr;
+   }
+   else
+   {
+      tnr = 1, tdr = 1-oldr, tnew_flags = flags | 2u;
+   }
+   tv = EVAL(tdr);
+
+   if (tv<v)
+   {
+      nr = tnr, dr = tdr, v = tv, new_flags = tnew_flags;
+   }
+
+newton_edge_fin:
+   /* check convergence */
+   if ( fabs(dr)<tol )
+   {
+      new_flags |= CONVERGED_FLAG;
+   }
+   out->r = nr;
+   out->dist2p = -v;
+   out->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
+#undef EVAL
+}
+
+static MFEM_HOST_DEVICE void seed_j(const double *elx[sDIM],
+                                    const double x[sDIM],
+                                    const double *z,
+                                    double *dist2,
+                                    double *r,
+                                    const int ir,
+                                    const int pN)
+{
+   if (ir>=pN)
+   {
+      return;
+   }
+
+   double dx[sDIM];
+   for (int d=0; d<sDIM; ++d)
+   {
+      dx[d] = x[d] - elx[d][ir];
+   }
+   dist2[ir] = norm2(dx);;
+   r[ir] = z[ir];
+}
+
+template<int T_D1D = 0>
+static void FindPointsEdgeLocal3D_Kernel(const int npt,
+                                         const double tol,
+                                         const double dist2tol,
+                                         const double *x,
+                                         const int point_pos_ordering,
+                                         const double *xElemCoord,
+                                         const int nel,
+                                         const double *wtend,
+                                         const double *boxinfo,
+                                         const int hash_n,
+                                         const double *hashMin,
+                                         const double *hashFac,
+                                         unsigned int *hashOffset,
+                                         unsigned int *const code_base,
+                                         unsigned int *const el_base,
+                                         double *const r_base,
+                                         double *const dist2_base,
+                                         const double *gll1D,
+                                         const double *lagcoeff,
+                                         const int pN = 0)
+{
+   const int MD1   = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+   const int D1D   = T_D1D ? T_D1D : pN;
+   const int p_NEL = nel*D1D;
+   MFEM_VERIFY(MD1<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(pN<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(D1D!=0, "Polynomial order not specified.");
+   const int nThreads = D1D*sDIM;
+
+   mfem::forall_2D(npt, nThreads, 1, [=] MFEM_HOST_DEVICE (int i)
+   {
+      constexpr int size1 = 3*MD1 + 13;
+      constexpr int size2 = 3*MD1;
+      constexpr int size3 = MD1*sDIM;
+
+      MFEM_SHARED findptsElementPoint_t el_pts[2];
+      MFEM_SHARED double r_workspace[size1];
+
+      MFEM_SHARED double constraint_workspace[size2];
+
+      MFEM_SHARED double elem_coords[MD1 <= 6 ? size3 : 1];
+
+      double *r_workspace_ptr = r_workspace;
+      findptsElementPoint_t *fpt, *tmp;
+      fpt = el_pts + 0;
+      tmp = el_pts + 1;
+
+      int id_x = point_pos_ordering==0 ?       i :   i*sDIM;
+      int id_y = point_pos_ordering==0 ?   npt+i : 1+i*sDIM;
+      int id_z = point_pos_ordering==0 ? 2*npt+i : 2+i*sDIM;
+      double x_i[3] = {x[id_x], x[id_y], x[id_z]};
+
+      unsigned int *code_i = code_base  + i;
+      double *dist2_i      = dist2_base + i;
+
+      //// map_points_to_els ////
+      findptsLocalHashData_t hash;
+      for (int d=0; d<sDIM; ++d)
+      {
+         hash.bnd[d].min = hashMin[d];
+         hash.fac[d]     = hashFac[d];
+      }
+      hash.hash_n = hash_n;
+      hash.offset = hashOffset;
+
+      const unsigned int hi   = hash_index(&hash, x_i);
+      const unsigned int *elp       = hash.offset + hash.offset[hi];
+      const unsigned int *const ele = hash.offset + hash.offset[hi+1];
+      *code_i  = CODE_NOT_FOUND;
+      *dist2_i = HUGE_VAL;
+
+      for (; elp!=ele; ++elp)
+      {
+         const unsigned int el = *elp;
+         obbox_t box;
+         int n_box_ents = 3*sDIM + sDIM2;
+
+         for (int idx = 0; idx < sDIM; ++idx)
+         {
+            box.c0[idx] = boxinfo[n_box_ents*el + idx];
+            box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
+            box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
+         }
+         for (int idx = 0; idx < sDIM2; ++idx)
+         {
+            box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
+         }
+
+         if (obbox_test(&box, x_i)>=0)
+         {
+            //// findpts_local ////
+            {
+               if (MD1 <= 6)
+               {
+                  MFEM_FOREACH_THREAD(j,x,D1D*sDIM)
+                  {
+                     const int qp = j % D1D;
+                     const int d = j / D1D;
+                     elem_coords[qp + d*D1D] =
+                        xElemCoord[qp + el*D1D + d*p_NEL];
+                  }
+                  MFEM_SYNC_THREAD;
+               }
+
+               const double *elx[sDIM];
+               for (int d=0; d<sDIM; d++)
+               {
+                  elx[d] = MD1<= 6 ? &elem_coords[d*D1D] :
+                           xElemCoord + d*p_NEL + el*D1D;
+               }
+               MFEM_SYNC_THREAD;
+               //// findpts_el ////
+               {
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     fpt->dist2 = HUGE_VAL;
+                     fpt->dist2p = 0;
+                     fpt->tr = 1.0;
+                  }
+                  MFEM_FOREACH_THREAD(j,x,sDIM)
+                  {
+                     fpt->x[j] = x_i[j];
+                  }
+                  MFEM_SYNC_THREAD;
+
+                  //// seed ////
+                  {
+                     double *dist2_temp = r_workspace_ptr;
+                     double *r_temp = dist2_temp + D1D;
+                     MFEM_FOREACH_THREAD(j,x,nThreads)
+                     {
+                        seed_j(elx, x_i, gll1D, dist2_temp, r_temp, j, D1D);
+                     }
+                     MFEM_SYNC_THREAD;
+
+                     MFEM_FOREACH_THREAD(j,x,1)
+                     {
+                        fpt->dist2 = HUGE_VAL;
+                        for (int ir=0; ir<D1D; ++ir)
+                        {
+                           if (dist2_temp[ir] < fpt->dist2)
+                           {
+                              fpt->dist2 = dist2_temp[ir];
+                              fpt->r = r_temp[ir];
+                           }
+                        }
+                     }
+                     MFEM_SYNC_THREAD;
+                  } //seed done
+
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     tmp->dist2 = HUGE_VAL;
+                     tmp->dist2p = 0;
+                     tmp->tr = 1;
+                     tmp->flags = 0;
+                     tmp->r = fpt->r;
+                  }
+                  MFEM_FOREACH_THREAD(j,x,sDIM)
+                  {
+                     tmp->x[j] = fpt->x[j];
+                  }
+                  MFEM_SYNC_THREAD;
+
+                  for (int step=0; step<50; step++)
+                  {
+                     switch (num_constrained(tmp->flags & FLAG_MASK))
+                     {
+                        case 0:
+                        {
+                           double *wt = r_workspace_ptr;
+                           double *resid = wt + 3*D1D;
+                           double *jac = resid + sDIM;
+                           double *hess = jac + sDIM*rDIM;
+
+                           findptsElementGEdge_t edge;
+                           MFEM_FOREACH_THREAD(j,x,D1D)
+                           {
+                              for (int d=0; d<sDIM; ++d)
+                              {
+                                 edge.x[d] = constraint_workspace + d*D1D;
+                                 edge.x[d][j] = elx[d][j];
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,D1D)
+                           {
+                              lag_eval_second_der(wt, tmp->r, j, gll1D,
+                                                  lagcoeff, D1D);
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,sDIM)
+                           {
+                              resid[j] = tmp->x[j];
+                              jac[j] = 0.0;
+                              hess[j] = 0.0;
+                              for (int k=0; k<D1D; ++k)
+                              {
+                                 resid[j] -= wt[      k]*edge.x[j][k];
+                                 jac[j]   += wt[D1D+k]*edge.x[j][k];
+                                 hess[j]  += wt[2*D1D+k]*edge.x[j][k];
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,1)
+                           {
+                              hess[3] = resid[0]*hess[0] + resid[1]*hess[1] +
+                                        resid[2]*hess[2];
+                           }
+
+                           MFEM_FOREACH_THREAD(l,x,1)
+                           {
+                              if (!reject_prior_step_q(fpt,resid,tmp,tol))
+                              {
+                                 newton_edge(fpt,jac,hess[3],resid,
+                                             tmp->flags&FLAG_MASK,tmp,tol);
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+                           break;
+                        }
+                        case 1:
+                        {
+                           MFEM_FOREACH_THREAD(j,x,1)
+                           {
+                              const int pi = point_index(tmp->flags &
+                                                         FLAG_MASK);
+                              const double *wt = wtend + pi*3*D1D;
+                              findptsElementGPT_t gpt;
+                              for (int d=0; d<sDIM; ++d)
+                              {
+                                 gpt.x[d] = elx[d][pi*(D1D-1)];
+                                 gpt.jac[d] = 0.0;
+                                 gpt.hes[d] = 0.0;
+                                 for (int k=0; k<D1D; ++k)
+                                 {
+                                    gpt.jac[d] += wt[D1D  +k]*elx[d][k];
+                                    gpt.hes[d] += wt[2*D1D+k]*elx[d][k];
+                                 }
+                              }
+
+                              const double *const pt_x = gpt.x;
+                              const double *const jac = gpt.jac;
+                              const double *const hes = gpt.hes;
+                              double resid[sDIM], steep, sr;
+                              resid[0] = fpt->x[0] - pt_x[0];
+                              resid[1] = fpt->x[1] - pt_x[1];
+                              resid[2] = fpt->x[2] - pt_x[2];
+                              steep = jac[0]*resid[0] + jac[1]*resid[1] +
+                                      jac[2]*resid[2];
+                              sr = steep*tmp->r;
+                              if (!reject_prior_step_q(fpt, resid, tmp, tol))
+                              {
+                                 if (sr<0)
+                                 {
+                                    const double rhess = resid[0]*hes[0] +
+                                                         resid[1]*hes[1] +
+                                                         resid[2]*hes[2];
+                                    newton_edge(fpt, jac, rhess,
+                                                resid, 0, tmp, tol);
+                                 }
+                                 else   // sr==0
+                                 {
+                                    fpt->r = tmp->r;
+                                    fpt->dist2p = 0;
+                                    fpt->flags = tmp->flags | CONVERGED_FLAG;
+                                 }
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+                           break;
+                        } // case 1
+                     } //switch
+                     if (fpt->flags & CONVERGED_FLAG)
+                     {
+                        break;
+                     }
+                     MFEM_SYNC_THREAD;
+
+                     MFEM_FOREACH_THREAD(j,x,1)
+                     {
+                        *tmp = *fpt;
+                     }
+                     MFEM_SYNC_THREAD;
+                  } // for step<50
+               } // findpts_el
+
+               bool converged_internal =
+                  ((fpt->flags&FLAG_MASK) == CONVERGED_FLAG) &&
+                  (fpt->dist2<dist2tol);
+               if (*code_i==CODE_NOT_FOUND || converged_internal ||
+                   fpt->dist2<*dist2_i)
+               {
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     *(el_base+i) = el;
+                     *code_i  = converged_internal?CODE_INTERNAL:CODE_BORDER;
+                     *dist2_i = fpt->dist2;
+                     *(r_base+i) = fpt->r;
+                  }
+                  MFEM_SYNC_THREAD;
+                  if (converged_internal)
+                  {
+                     break;
+                  }
+               }
+            } // findpts_local
+         } // obbox_test
+      } // elp
+   });
+}
+
+void FindPointsGSLIB::FindPointsEdgeLocal3(const Vector &point_pos,
+                                           int point_pos_ordering,
+                                           Array<unsigned int> &code,
+                                           Array<unsigned int> &elem,
+                                           Vector &ref,
+                                           Vector &dist,
+                                           int npt)
+{
+   if (npt == 0)
+   {
+      return;
+   }
+   MFEM_VERIFY(spacedim==3 && dim == 1,"Function for 3D edges only");
+   bool use_dev = point_pos.UseDevice();
+   auto pp = point_pos.Read(use_dev);
+   auto pgslm = gsl_mesh.Read(use_dev);
+   auto pwt = DEV.wtend.Read(use_dev);
+   auto pbb = DEV.bb.Read(use_dev);
+   auto plhm = DEV.lh_min.Read(use_dev);
+   auto plhf = DEV.lh_fac.Read(use_dev);
+   auto plho = DEV.lh_offset.ReadWrite(use_dev);
+   auto pcode = code.Write(use_dev);
+   auto pelem = elem.Write(use_dev);
+   auto pref = ref.Write(use_dev);
+   auto pdist = dist.Write(use_dev);
+   auto pgll1d = DEV.gll1d.ReadWrite(use_dev);
+   auto plc = DEV.lagcoeff.Read(use_dev);
+   double dist2tol = DEV.surf_dist_tol;
+   switch (DEV.dof1d)
+   {
+      case 2:
+         return FindPointsEdgeLocal3D_Kernel<2>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+      case 3:
+         return FindPointsEdgeLocal3D_Kernel<3>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+      case 4:
+         return FindPointsEdgeLocal3D_Kernel<4>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+      default:
+         return FindPointsEdgeLocal3D_Kernel(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc, DEV.dof1d);
+   }
+}
+#undef rDIM2
+#undef sDIM2
+#undef rDIM
+#undef sDIM
+#undef CODE_INTERNAL
+#undef CODE_BORDER
+#undef CODE_NOT_FOUND
+#else
+void FindPointsGSLIB::FindPointsEdgeLocal3( const Vector &point_pos,
+                                            int point_pos_ordering,
+                                            Array<unsigned int> &code,
+                                            Array<unsigned int> &elem,
+                                            Vector &ref,
+                                            Vector &dist,
+                                            int npt ) {} ;
+#endif
+} // namespace mfem
+
+#endif //ifdef MFEM_USE_GSLIB
@@ -0,0 +1,157 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../gslib.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/kernels.hpp"
+
+#ifdef MFEM_USE_GSLIB
+
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+#include "gslib.h"
+#ifndef GSLIB_RELEASE_VERSION //gslib v1.0.7
+#define GSLIB_RELEASE_VERSION 10007
+#endif
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic pop
+#endif
+namespace mfem
+{
+#if GSLIB_RELEASE_VERSION >= 10009
+#define CODE_INTERNAL 0
+#define CODE_BORDER 1
+#define CODE_NOT_FOUND 2
+
+static MFEM_HOST_DEVICE void lagrange_eval(double *p0, double x,
+                                           int i, int p_Nq,
+                                           double *z, double *lagrangeCoeff)
+{
+   double p_i = (1 << (p_Nq - 1));
+   for (int j=0; j<p_Nq; ++j)
+   {
+      p_i *= j==i ? 1 : x-z[j];
+   }
+   p0[i] = lagrangeCoeff[i] * p_i;
+}
+
+template<int T_D1D = 0>
+static void InterpolateLocal1DKernel(const double *const gf_in,
+                                     int *const el,
+                                     double *const r,
+                                     double *const int_out,
+                                     const int npt,
+                                     const int nfields,
+                                     double *gll1D,
+                                     double *lagcoeff,
+                                     const int pN = 0)
+{
+   const int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+   const int D1D = T_D1D ? T_D1D : pN;
+   const int p_Nq = D1D;
+   MFEM_VERIFY(MD1 <= DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(pN<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(D1D != 0, "Polynomial order not specified.");
+   // for each point of the npt points, create a thread block of size dof1Dsol
+   mfem::forall_2D(npt, D1D, 1, [=] MFEM_HOST_DEVICE (int i)
+   {
+      MFEM_SHARED double wtr[MD1];
+      MFEM_SHARED double sums[MD1];
+
+      // Evaluate basis functions at the reference space coordinates
+      MFEM_FOREACH_THREAD(j,x,D1D)
+      {
+         lagrange_eval(wtr, r[i], j, p_Nq, gll1D, lagcoeff);
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int fld=0; fld<nfields; ++fld)
+      {
+         // If using GetNodalValues, ordering is NDOFS x NEL x VDIM and the
+         // offset would be `el[i] * p_Nq + fld * gf_offset`.
+         // R->Mult produces element vectors in NDOFS x VDIM x NEL layout.
+         const int elemOffset = el[i]*nfields*p_Nq + fld*p_Nq;
+         MFEM_FOREACH_THREAD(j,x,D1D)
+         {
+            sums[j] = wtr[j] * gf_in[elemOffset + j];
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(j,x,1)
+         {
+            double sumv = 0.0;
+            // sum the contributions of each lagrange polynomial
+            for (int jj=0; jj<D1D; ++jj)
+            {
+               sumv += sums[jj];
+            }
+            int_out[fld*npt + i] = sumv;
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+void FindPointsGSLIB::InterpolateLocal1( const Vector &field_in,
+                                         Array<int> &gsl_elem_dev_l,
+                                         Vector &gsl_ref_l,
+                                         Vector &field_out,
+                                         int npt,
+                                         int ncomp,
+                                         int dof1Dsol )
+{
+   MFEM_VERIFY(dim == 1, "Kernel for edges only.");
+   if (npt == 0) { return; }
+   bool use_dev = field_in.UseDevice();
+   auto pfin = field_in.Read(use_dev);
+   auto pgsl = gsl_elem_dev_l.ReadWrite(use_dev);
+   auto pgslr = gsl_ref_l.ReadWrite(use_dev);
+   auto pfout = field_out.Write(use_dev);
+   auto pgll = DEV.gll1d_sol.ReadWrite(use_dev);
+   auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
+   switch (dof1Dsol)
+   {
+      case 2: return InterpolateLocal1DKernel<2>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 3: return InterpolateLocal1DKernel<3>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 4: return InterpolateLocal1DKernel<4>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 5: return InterpolateLocal1DKernel<5>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      default: return InterpolateLocal1DKernel(pfin, pgsl, pgslr, pfout,
+                                                  npt, ncomp,
+                                                  pgll, plcf, dof1Dsol);
+   }
+}
+#undef CODE_INTERNAL
+#undef CODE_BORDER
+#undef CODE_NOT_FOUND
+#else
+void FindPointsGSLIB::InterpolateLocal1(const Vector &field_in,
+                                        Array<int> &gsl_elem_dev_l,
+                                        Vector &gsl_ref_l,
+                                        Vector &field_out,
+                                        int npt, int ncomp,
+                                        int dof1Dsol) {};
+#endif
+
+} // namespace mfem
+
+#endif //ifdef MFEM_USE_GSLIB
@@ -52,8 +52,6 @@ static void InterpolateLocal2DKernel(const double *const gf_in,
                                     double *const int_out,
                                     const int npt,
                                     const int ncomp,
-                                     const int nel,
-                                     const int gf_offset,
                                     double *gll1D,
                                     double *lagcoeff,
                                     const int pN = 0)
@@ -64,6 +62,8 @@ static void InterpolateLocal2DKernel(const double *const gf_in,
   const int p_Np = D1D*D1D;
   MFEM_VERIFY(MD1 <= DofQuadLimits::MAX_D1D,
               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(pN<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
   MFEM_VERIFY(D1D != 0, "Polynomial order not specified.");
   mfem::forall_2D(npt, D1D, D1D, [=] MFEM_HOST_DEVICE (int i)
   {
@@ -82,9 +82,9 @@ static void InterpolateLocal2DKernel(const double *const gf_in,

      for (int fld = 0; fld < Nfields; ++fld)
      {
-         // If using GetNodalValues, ordering is NDOFSxNELxVDIM
-         // const int elemOffset = el[i] * p_Np + fld * gf_offset;
-         //if using R->Mult for L -> E-Vec use below: NDOFSxVDIMxNEL
+         // If using GetNodalValues, ordering is NDOFS x NEL x VDIM and the
+         // offset would be `el[i] * p_Np + fld * gf_offset`.
+         // R->Mult produces element vectors in NDOFS x VDIM x NEL layout.
         const int elemOffset = el[i] * p_Np * Nfields + fld * p_Np;
         MFEM_FOREACH_THREAD(j,x,D1D)
         {
@@ -120,32 +120,32 @@ void FindPointsGSLIB::InterpolateLocal2(const Vector &field_in,
                                        Vector &gsl_ref_l,
                                        Vector &field_out,
                                        int npt, int ncomp,
-                                        int nel, int dof1Dsol)
+                                        int dof1Dsol)
 {
   if (npt == 0) { return; }
-   const int gf_offset = field_in.Size()/ncomp;
-   auto pfin = field_in.Read();
-   auto pgsl = gsl_elem_dev_l.ReadWrite();
-   auto pgslr = gsl_ref_l.ReadWrite();
-   auto pfout = field_out.Write();
-   auto pgll = DEV.gll1d_sol.ReadWrite();
-   auto plcf = DEV.lagcoeff_sol.ReadWrite();
+   bool use_dev = field_in.UseDevice();
+   auto pfin = field_in.Read(use_dev);
+   auto pgsl = gsl_elem_dev_l.ReadWrite(use_dev);
+   auto pgslr = gsl_ref_l.ReadWrite(use_dev);
+   auto pfout = field_out.Write(use_dev);
+   auto pgll = DEV.gll1d_sol.ReadWrite(use_dev);
+   auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
   switch (dof1Dsol)
   {
      case 2: return InterpolateLocal2DKernel<2>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
+                                                    npt, ncomp,
                                                    pgll, plcf);
      case 3: return InterpolateLocal2DKernel<3>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
+                                                    npt, ncomp,
                                                    pgll, plcf);
      case 4: return InterpolateLocal2DKernel<4>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
+                                                    npt, ncomp,
                                                    pgll, plcf);
      case 5: return InterpolateLocal2DKernel<5>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
+                                                    npt, ncomp,
                                                    pgll, plcf);
      default: return InterpolateLocal2DKernel(pfin, pgsl, pgslr, pfout,
-                                                  npt, ncomp, nel, gf_offset,
+                                                  npt, ncomp,
                                                  pgll, plcf, dof1Dsol);
   }
 }
@@ -160,7 +160,7 @@ void FindPointsGSLIB::InterpolateLocal2(const Vector &field_in,
                                        Vector &gsl_ref_l,
                                        Vector &field_out,
                                        int npt, int ncomp,
-                                        int nel, int dof1Dsol) {};
+                                        int dof1Dsol) {};
 #endif
 } // namespace mfem

@@ -52,8 +52,6 @@ static void InterpolateLocal3DKernel(const double *const gf_in,
                                     double *const int_out,
                                     const int npt,
                                     const int ncomp,
-                                     const int nel,
-                                     const int gf_offset,
                                     double *gll1D,
                                     double *lagcoeff,
                                     const int pN = 0)
@@ -84,9 +82,9 @@ static void InterpolateLocal3DKernel(const double *const gf_in,

      for (int fld = 0; fld < Nfields; ++fld)
      {
-         // If using GetNodalValues, ordering is NDOFSxNELxVDIM
-         // const int elemOffset = el[i] * p_Np + fld * gf_offset;
-         //if using R->Mult for L -> E-Vec use below.
+         // If using GetNodalValues, ordering is NDOFS x NEL x VDIM and the
+         // offset would be `el[i] * p_Np + fld * gf_offset`.
+         // R->Mult produces element vectors in NDOFS x VDIM x NEL layout.
         const int elemOffset = el[i] * p_Np * Nfields + fld * p_Np;
         MFEM_FOREACH_THREAD(j,x,D1D)
         {
@@ -125,37 +123,38 @@ void FindPointsGSLIB::InterpolateLocal3(const Vector &field_in,
                                        Vector &gsl_ref_l,
                                        Vector &field_out,
                                        int npt, int ncomp,
-                                        int nel, int dof1Dsol)
+                                        int dof1Dsol)
 {
   if (npt == 0) { return; }
-   const int gf_offset = field_in.Size()/ncomp;
-   auto pfin = field_in.Read();
-   auto pgsle = gsl_elem_dev_l.ReadWrite();
-   auto pgslr = gsl_ref_l.ReadWrite();
-   auto pfout = field_out.Write();
-   auto pgll = DEV.gll1d_sol.ReadWrite();
-   auto plcf = DEV.lagcoeff_sol.ReadWrite();
+   bool use_dev = field_in.UseDevice();
+   auto pfin = field_in.Read(use_dev);
+   auto pgsle = gsl_elem_dev_l.ReadWrite(use_dev);
+   auto pgslr = gsl_ref_l.ReadWrite(use_dev);
+   auto pfout = field_out.Write(use_dev);
+   auto pgll = DEV.gll1d_sol.ReadWrite(use_dev);
+   auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
   switch (dof1Dsol)
   {
      case 2: return InterpolateLocal3DKernel<2>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
+                                                    npt, ncomp,
                                                    pgll, plcf);
      case 3: return InterpolateLocal3DKernel<3>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
+                                                    npt, ncomp,
                                                    pgll, plcf);
      case 4: return InterpolateLocal3DKernel<4>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
+                                                    npt, ncomp,
                                                    pgll, plcf);
      case 5: return InterpolateLocal3DKernel<5>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
+                                                    npt, ncomp,
                                                    pgll, plcf);
      default: return InterpolateLocal3DKernel(pfin, pgsle, pgslr, pfout,
-                                                  npt, ncomp, nel, gf_offset,
+                                                  npt, ncomp,
                                                  pgll, plcf, dof1Dsol);
   }
 }


+#undef MAXC
 #undef CODE_INTERNAL
 #undef CODE_BORDER
 #undef CODE_NOT_FOUND
@@ -165,7 +164,7 @@ void FindPointsGSLIB::InterpolateLocal3(const Vector &field_in,
                                        Vector &gsl_ref_l,
                                        Vector &field_out,
                                        int npt, int ncomp,
-                                        int nel, int dof1Dsol) {};
+                                        int dof1Dsol) {};
 #endif
 } // namespace mfem

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.

 #include "bilininteg_diffusion_kernels.hpp"
+#include "bilininteg_diffusion_pa_simplices.hpp" // IWYU pragma: keep

 namespace mfem
 {
@@ -19,6 +20,13 @@ namespace mfem
 DiffusionIntegrator::Kernels::Kernels()
 {
   // 2D
+   // Q = P, only for simplex
+   DiffusionIntegrator::AddSimplexSpecialization<2,2,1>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,3,2>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,4,3>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,5,4>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,6,5>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,7,6>();
   // Q = P+1
   DiffusionIntegrator::AddSpecialization<2,1,1>();
   DiffusionIntegrator::AddSpecialization<2,2,2>();
@@ -40,7 +48,18 @@ DiffusionIntegrator::Kernels::Kernels()
   DiffusionIntegrator::AddSpecialization<2,8,9>();
   DiffusionIntegrator::AddSpecialization<2,9,10>();
   // others
+   DiffusionIntegrator::AddSimplexSpecialization<2,2,5>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,3,6>();
+
   // 3D
+   // Q = P, only for simplex
+   DiffusionIntegrator::AddSimplexSpecialization<3,2,1>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,3,2>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,4,3>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,5,4>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,6,5>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,7,6>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,8,7>();
   // Q = P+1
   DiffusionIntegrator::AddSpecialization<3,1,1>();
   DiffusionIntegrator::AddSpecialization<3,2,2>();
@@ -12,7 +12,6 @@
 #ifndef MFEM_BILININTEG_DIFFUSION_KERNELS_HPP
 #define MFEM_BILININTEG_DIFFUSION_KERNELS_HPP

-#include "../kernel_dispatch.hpp"
 #include "../../config/config.hpp"
 #include "../../general/array.hpp"
 #include "../../general/forall.hpp"
@@ -20,6 +19,8 @@
 #include "../../linalg/vector.hpp"
 #include "../bilininteg.hpp"

+#include "bilininteg_diffusion_pa_simplices.hpp"
+
 namespace mfem
 {

@@ -637,8 +638,8 @@ inline void SmemPADiffusionApply2D(const int NE,
                                   const bool symmetric,
                                   const Array<real_t> &b_,
                                   const Array<real_t> &g_,
-                                   const Array<real_t> &bt_,
-                                   const Array<real_t> &gt_,
+                                   const Array<real_t> &,
+                                   const Array<real_t> &,
                                   const Vector &d_,
                                   const Vector &x_,
                                   Vector &y_,
@@ -1218,43 +1219,47 @@ inline void SmemPADiffusionApply3D(const int NE,
 namespace
 {
 using ApplyKernelType = DiffusionIntegrator::ApplyKernelType;
+using ApplySimplexKernelType = DiffusionIntegrator::ApplySimplexKernelType;
 using DiagonalKernelType = DiffusionIntegrator::DiagonalKernelType;
 }

-template<int DIM, int T_D1D, int T_Q1D>
+template<int DIM, int D1D, int Q1D>
 ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Kernel()
 {
-   if constexpr (DIM == 2) { return internal::SmemPADiffusionApply2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPADiffusionApply3D<T_D1D, T_Q1D>; }
-   MFEM_ABORT("");
+   if constexpr (DIM == 2) { return internal::SmemPADiffusionApply2D<D1D, Q1D>; }
+   else if constexpr (DIM == 3) { return internal::SmemPADiffusionApply3D<D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
+   return nullptr;
 }

 inline
-ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
+ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int dim, int, int)
 {
-   if (DIM == 2) { return internal::PADiffusionApply2D; }
-   else if (DIM == 3) { return internal::PADiffusionApply3D; }
+   if (dim == 2) { return internal::PADiffusionApply2D; }
+   else if (dim == 3) { return internal::PADiffusionApply3D; }
   else { MFEM_ABORT(""); }
 }

 template<int DIM, int D1D, int Q1D>
 DiagonalKernelType DiffusionIntegrator::DiagonalPAKernels::Kernel()
 {
-   if constexpr (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D,Q1D>; }
+   if constexpr (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D, Q1D>; }
   else if constexpr (DIM == 3) { return internal::SmemPADiffusionDiagonal3D<D1D, Q1D>; }
-   MFEM_ABORT("");
+   else { MFEM_ABORT(""); }
+   return nullptr;
 }

 inline DiagonalKernelType
-DiffusionIntegrator::DiagonalPAKernels::Fallback(int DIM, int, int)
+DiffusionIntegrator::DiagonalPAKernels::Fallback(int dim, int, int)
 {
-   if (DIM == 2) { return internal::PADiffusionDiagonal2D; }
-   else if (DIM == 3) { return internal::PADiffusionDiagonal3D; }
+   if (dim == 2) { return internal::PADiffusionDiagonal2D; }
+   else if (dim == 3) { return internal::PADiffusionDiagonal3D; }
   else { MFEM_ABORT(""); }
+   return nullptr;
 }
+
 /// \endcond DO_NOT_DOCUMENT

 } // namespace mfem

-
 #endif
@@ -15,6 +15,7 @@
 #include "../../mesh/nurbs.hpp"
 #include "../ceed/integrators/diffusion/diffusion.hpp"
 #include "bilininteg_diffusion_kernels.hpp"
+#include "bilininteg_diffusion_pa_simplices.hpp"

 namespace mfem
 {
@@ -68,6 +69,24 @@ void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
      }
 #endif // MFEM_USE_OCCA

+      if (fespace->UsesRaggedTensorBasis())
+      {
+         const auto *rmaps = static_cast<const RaggedDofToQuad*>(maps);
+         return ApplySimplexPAKernels::Run(dim, dofs1D, quad1D, ne, symmetric,
+                                           rmaps->lex_map,
+                                           rmaps->forward_map2d_diff,
+                                           rmaps->inverse_map2d_diff,
+                                           rmaps->forward_map3d_diff,
+                                           rmaps->inverse_map3d_diff,
+                                           rmaps->Ga1,
+                                           rmaps->Ga2,
+                                           rmaps->Ga3,
+                                           rmaps->Ga1t,
+                                           rmaps->Ga2t,
+                                           rmaps->Ga3t,
+                                           Dv, x, y, dofs1D, quad1D);
+      }
+
      ApplyPAKernels::Run(dim, dofs1D, quad1D, ne, symmetric, B, G, Bt,
                          Gt, Dv, x, y, dofs1D, quad1D);
   }
@@ -94,7 +113,8 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   fespace = &fes;
   Mesh *mesh = fes.GetMesh();
   const FiniteElement &el = *fes.GetTypicalFE();
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+   const bool stroud = fes.UsesRaggedTensorBasis();
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, stroud);
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
@@ -119,13 +139,22 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dim = mesh->Dimension();
   ne = fes.GetNE();
   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
+   if (stroud)
+   {
+      maps = &el.GetDofToQuad(*ir, DofToQuad::RAGGED_TENSOR);
+   }
+   else
+   {
+      maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   }
   const int sdim = mesh->SpaceDimension();
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;

   QuadratureSpace qs(*mesh, *ir);
   CoefficientVector coeff(qs, CoefficientStorage::COMPRESSED);
+   // QuadratureSpace expects ir defined in reference simplex for Bernstein
+   // elements with partial assembly

   if (MQ) { coeff.ProjectTranspose(*MQ); }
   else if (VQ) { coeff.Project(*VQ); }
@@ -147,18 +147,15 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
   }); // end of element loop
 }

-void PAHcurlMassApply2D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y)
+void PAHcurlMassApply2D(const int NE, const bool symmetric,
+                        const bool scalar_coeff, const Array<real_t> &bo,
+                        const Array<real_t> &bc, const Array<real_t> &bot,
+                        const Array<real_t> &bct, const Vector &pa_data,
+                        const Vector &x, Vector &y, const int D1D,
+                        const int TestD1D, const int Q1D)
 {
+   MFEM_ASSERT(D1D == TestD1D,
+               "Trial and Test space must have the same number of dofs");
   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
   auto Bc = Reshape(bc.Read(), Q1D, D1D);
   auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
@@ -277,18 +274,15 @@ void PAHcurlMassApply2D(const int D1D,
   }); // end of element loop
 }

-void PAHcurlMassApply3D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y)
+void PAHcurlMassApply3D(const int NE, const bool symmetric,
+                        const bool scalar_coeff, const Array<real_t> &bo,
+                        const Array<real_t> &bc, const Array<real_t> &bot,
+                        const Array<real_t> &bct, const Vector &pa_data,
+                        const Vector &x, Vector &y, const int D1D,
+                        const int TestD1D, const int Q1D)
 {
+   MFEM_VERIFY(D1D == TestD1D,
+               "Trial and test spaces must have same number of dofs");
   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
               "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
@@ -181,228 +181,309 @@ inline void SmemPAHcurlMassAssembleDiagonal3D(const int d1d,
 }

 // PA H(curl) Mass Apply 2D kernel
-void PAHcurlMassApply2D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y);
+void PAHcurlMassApply2D(const int NE, const bool symmetric,
+                        const bool scalar_coeff, const Array<real_t> &bo,
+                        const Array<real_t> &bc, const Array<real_t> &bot,
+                        const Array<real_t> &bct, const Vector &pa_data,
+                        const Vector &x, Vector &y, const int TrialD1D,
+                        const int TestD1D, const int Q1D);

 // PA H(curl) Mass Apply 3D kernel
-void PAHcurlMassApply3D(const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const bool symmetric,
-                        const Array<real_t> &bo,
-                        const Array<real_t> &bc,
-                        const Array<real_t> &bot,
-                        const Array<real_t> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y);
+void PAHcurlMassApply3D(const int NE, const bool symmetric,
+                        const bool scalar_coeff, const Array<real_t> &bo,
+                        const Array<real_t> &bc, const Array<real_t> &bot,
+                        const Array<real_t> &bct, const Vector &pa_data,
+                        const Vector &x, Vector &y, const int TrialD1D,
+                        const int TestD1D, const int Q1D);

 // Shared memory PA H(curl) Mass Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHcurlMassApply3D(const int d1d,
-                                   const int q1d,
-                                   const int NE,
-                                   const bool symmetric,
-                                   const Array<real_t> &bo,
-                                   const Array<real_t> &bc,
-                                   const Array<real_t> &bot,
-                                   const Array<real_t> &bct,
-                                   const Vector &pa_data,
-                                   const Vector &x,
-                                   Vector &y)
+template <int T_D1D = 0, int T_Q1D = 0, int TBATCH = 0, bool ACCUMULATE = true>
+inline void SmemPAHcurlMassApply3D(
+   const int NE, const bool symmetric, const bool scalar_coeff,
+   const Array<real_t> &bo, const Array<real_t> &bc, const Array<real_t> &bot,
+   const Array<real_t> &bct, const Vector &pa_data, const Vector &x, Vector &y,
+   const int d1d = 0, const int = 0, const int q1d = 0)
 {
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
               "Error: d1d > HCURL_MAX_D1D");
   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
               "Error: q1d > HCURL_MAX_Q1D");
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
+   MFEM_ASSERT(Q1D >= D1D, "Expected Q1D >= D1D");
   const int dataSize = symmetric ? 6 : 9;

-   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
-   auto Bc = Reshape(bc.Read(), Q1D, D1D);
-   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
-   auto X = Reshape(x.Read(), 3*(D1D-1)*D1D*D1D, NE);
-   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
+   // assume trial space == test space
+   auto Bo = bo.Read();
+   auto Bc = bc.Read();
+   auto op =
+      Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
+   auto X_ = Reshape(x.Read(), 3 * (D1D - 1) * D1D * D1D, NE);
+   auto y_ = y.ReadWrite();

-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   constexpr int MD_ = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
+   constexpr int MQ_ = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+   constexpr int MDQ_ = std::max(MD_, MQ_);
+   constexpr int MB_ = TBATCH ? TBATCH : 1;
+
+   mfem::forall_2D_batch<MDQ_ * MDQ_ * MDQ_ * MB_>(
+      NE, MDQ_ * MDQ_ * MDQ_, 1, MB_, [=] MFEM_HOST_DEVICE(int e)
   {
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = TBATCH ? TBATCH : 1;
+      int tidz = MFEM_THREAD_ID(z);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+
      constexpr int VDIM = 3;
      constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MDQ = std::max(MD1D, MQ1D);

-      MFEM_SHARED real_t sBo[MQ1D][MD1D];
-      MFEM_SHARED real_t sBc[MQ1D][MD1D];
+      // nvcc limit work-around: can't have Y_ be captured first in
+      // if constexpr, so capture y_ and construct Y_ locally
+      // only works on GPU
+      auto Y = Reshape(y_, VDIM * (D1D - 1) * D1D * D1D, NE);

-      real_t op9[9];
-      MFEM_SHARED real_t sop[9*MQ1D*MQ1D];
-      MFEM_SHARED real_t mass[MQ1D][MQ1D][3];
+      MFEM_SHARED real_t sBo[MDQ * (MD1D - 1)];
+      MFEM_SHARED real_t sBc[MDQ * MD1D];
+      auto BO = Reshape(sBo, Q1D, D1D - 1);
+      auto BC = Reshape(sBc, Q1D, D1D);

-      MFEM_SHARED real_t sX[MD1D][MD1D][MD1D];
+      MFEM_SHARED real_t sX[nbz * VDIM * (MD1D - 1) * MD1D * MD1D];
+      MFEM_SHARED real_t sm0[nbz * VDIM * MDQ * MDQ * MDQ];
+      MFEM_SHARED real_t sm1[nbz * VDIM * MDQ * MDQ * MDQ];

-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      real_t(*X)[nbz][(MD1D - 1) * MD1D * MD1D] =
+         (real_t(*)[nbz][(MD1D - 1) * MD1D * MD1D])(sX);
+      // shapes of buffers always use MQ1D to mitigate shared memory bank
+      // conflicts
+      real_t(*DDQ)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
+      real_t(*DQQ)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
+      real_t(*QQQ)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
+      real_t(*QQD)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
+      real_t(*QDD)[nbz][MQ1D][MQ1D][MQ1D] =
+         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
+
+      // load dofs into smem
+      const int offset = (D1D - 1) * D1D * D1D;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         for (int dim = 0; dim < VDIM; ++dim)
         {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               for (int i=0; i<dataSize; ++i)
-               {
-                  op9[i] = op(qx,qy,qz,i,e);
-               }
-            }
+            X[dim][tidz][ix] = X_(ix + dim * offset, e);
         }
      }
-
-      const int tidx = MFEM_THREAD_ID(x);
-      const int tidy = MFEM_THREAD_ID(y);
-      const int tidz = MFEM_THREAD_ID(z);
-
+      // load basis functions data
      if (tidz == 0)
      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(ix, x, D1D * Q1D) { sBc[ix] = Bc[ix]; }
+         MFEM_FOREACH_THREAD_DIRECT(ix, x, (D1D - 1) * Q1D)
         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
+            sBo[ix] = Bo[ix];
+         }
+      }
+
+      for (int dim0 = 0; dim0 < VDIM; ++dim0)
+      {
+         MFEM_SYNC_THREAD;
+         // sum factor to QQQ = Q_{dim0,dim1} B X_{dim1}
+         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         {
+            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, Q1D, D1Dy, D1Dz,
+                                                 Q1D, Q1D, Q1D)
            {
-               sBc[q][d] = Bc(q,d);
-               if (d < D1D-1)
+               real_t u = 0;
+               for (int dx = 0; dx < D1Dx; ++dx)
               {
-                  sBo[q][d] = Bo(q,d);
+                  real_t b;
+                  if (dim1 == 0)
+                  {
+                     b = BO(qx, dx);
+                  }
+                  else
+                  {
+                     b = BC(qx, dx);
+                  }
+                  u += X[dim1][tidz][dx + (dy + dz * D1Dy) * D1Dx] * b;
+               }
+               DDQ[dim1][tidz][dz][dy][qx] = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         {
+            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
+            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, Q1D, Q1D, D1Dz,
+                                                 Q1D, Q1D, Q1D)
+            {
+               real_t u = 0;
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  real_t b;
+                  if (dim1 == 1)
+                  {
+                     b = BO(qy, dy);
+                  }
+                  else
+                  {
+                     b = BC(qy, dy);
+                  }
+                  u += DDQ[dim1][tidz][dz][dy][qx] * b;
+               }
+               DQQ[dim1][tidz][dz][qy][qx] = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         {
+            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
+            // const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
+            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+            MFEM_FOREACH_THREAD_DIRECT_3D(qx, qy, qz, x, Q1D, Q1D, Q1D)
+            {
+               real_t u = 0;
+               for (int dz = 0; dz < D1Dz; ++dz)
+               {
+                  real_t b;
+                  if (dim1 == 2)
+                  {
+                     b = BO(qz, dz);
+                  }
+                  else
+                  {
+                     b = BC(qz, dz);
+                  }
+                  u += DQQ[dim1][tidz][dz][qy][qx] * b;
+               }
+               // pa_data is row major
+               int idx;
+               if (symmetric)
+               {
+                  int row;
+                  int col;
+                  if (dim0 > dim1)
+                  {
+                     row = dim1;
+                     col = dim0;
+                  }
+                  else
+                  {
+                     row = dim0;
+                     col = dim1;
+                  }
+                  idx = col + VDIM * row - row * (row + 1) / 2;
+               }
+               else
+               {
+                  idx = dim0 * VDIM + dim1;
+               }
+               QQQ[dim1][tidz][qz][qy][qx] = op(qx, qy, qz, idx, e) * u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         // sum factor back to Y
+         // Assume bot and bct == bo^t and bc^t respectively (i.e. test ==
+         // trial functions), skip loading them again.
+         {
+            const int D1Dz = (dim0 == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (dim0 == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (dim0 == 0) ? D1D - 1 : D1D;
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, D1Dz, Q1D, Q1D,
+                                                 Q1D, Q1D, Q1D)
+            {
+               for (int dim1 = 0; dim1 < VDIM; ++dim1)
+               {
+                  real_t u = 0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     real_t b = 0;
+                     if (dim0 == 2)
+                     {
+                        b = BO(qz, dz);
+                     }
+                     else
+                     {
+                        b = BC(qz, dz);
+                     }
+                     u += QQQ[dim1][tidz][qz][qy][qx] * b;
+                  }
+                  QQD[dim1][tidz][qy][qx][dz] = u;
+               }
+            }
+            MFEM_SYNC_THREAD;
+            // threads assigned to mitigate bank conflicts
+            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, D1Dy, D1Dz, Q1D,
+                                                 Q1D, Q1D, Q1D)
+            {
+               for (int dim1 = 0; dim1 < VDIM; ++dim1)
+               {
+                  real_t u = 0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     real_t b;
+                     if (dim0 == 1)
+                     {
+                        b = BO(qy, dy);
+                     }
+                     else
+                     {
+                        b = BC(qy, dy);
+                     }
+                     u += QQD[dim1][tidz][qy][qx][dz] * b;
+                  }
+                  QDD[dim1][tidz][qx][dz][dy] = u;
+               }
+            }
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD_DIRECT_3D(dx, dy, dz, x, D1Dx, D1Dy, D1Dz)
+            {
+               int ix = dx + D1Dx * (dy + D1Dy * dz);
+               real_t u = 0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  real_t b;
+                  if (dim0 == 0)
+                  {
+                     b = BO(qx, dx);
+                  }
+                  else
+                  {
+                     b = BC(qx, dx);
+                  }
+                  for (int dim1 = 0; dim1 < VDIM; ++dim1)
+                  {
+                     u += QDD[dim1][tidz][qx][dz][dy] * b;
+                  }
+               }
+               if constexpr (ACCUMULATE)
+               {
+                  Y(ix + dim0 * offset, e) += u;
+               }
+               else
+               {
+                  Y(ix + dim0 * offset, e) = u;
               }
            }
         }
      }
-      MFEM_SYNC_THREAD;
-
-      for (int qz=0; qz < Q1D; ++qz)
-      {
-         int osc = 0;
-         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
-         {
-            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
-
-            MFEM_FOREACH_THREAD(dz,z,D1Dz)
-            {
-               MFEM_FOREACH_THREAD(dy,y,D1Dy)
-               {
-                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
-                  {
-                     sX[dz][dy][dx] = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            if (tidz == qz)
-            {
-               for (int i=0; i<dataSize; ++i)
-               {
-                  sop[i + (dataSize*tidx) + (dataSize*Q1D*tidy)] = op9[i];
-               }
-
-               MFEM_FOREACH_THREAD(qy,y,Q1D)
-               {
-                  MFEM_FOREACH_THREAD(qx,x,Q1D)
-                  {
-                     real_t u = 0.0;
-
-                     for (int dz = 0; dz < D1Dz; ++dz)
-                     {
-                        const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
-                        for (int dy = 0; dy < D1Dy; ++dy)
-                        {
-                           const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
-                           for (int dx = 0; dx < D1Dx; ++dx)
-                           {
-                              const real_t t = sX[dz][dy][dx];
-                              const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
-                              u += t * wx * wy * wz;
-                           }
-                        }
-                     }
-
-                     mass[qy][qx][c] = u;
-                  } // qx
-               } // qy
-            } // tidz == qz
-
-            osc += D1Dx * D1Dy * D1Dz;
-            MFEM_SYNC_THREAD;
-         } // c
-
-         MFEM_SYNC_THREAD;  // Sync mass[qy][qx][d] and sop
-
-         osc = 0;
-         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
-         {
-            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
-
-            real_t dxyz = 0.0;
-
-            MFEM_FOREACH_THREAD(dz,z,D1Dz)
-            {
-               const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
-
-               MFEM_FOREACH_THREAD(dy,y,D1Dy)
-               {
-                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
-                  {
-                     for (int qy = 0; qy < Q1D; ++qy)
-                     {
-                        const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
-                        for (int qx = 0; qx < Q1D; ++qx)
-                        {
-                           const int os = (dataSize*qx) + (dataSize*Q1D*qy);
-                           const int id1 = os + ((c == 0) ? 0 : ((c == 1) ? (symmetric ? 1 : 3) :
-                                                                 (symmetric ? 2 : 6))); // O11, O21, O31
-                           const int id2 = os + ((c == 0) ? 1 : ((c == 1) ? (symmetric ? 3 : 4) :
-                                                                 (symmetric ? 4 : 7))); // O12, O22, O32
-                           const int id3 = os + ((c == 0) ? 2 : ((c == 1) ? (symmetric ? 4 : 5) :
-                                                                 (symmetric ? 5 : 8))); // O13, O23, O33
-
-                           const real_t m_c = (sop[id1] * mass[qy][qx][0]) + (sop[id2] * mass[qy][qx][1]) +
-                                              (sop[id3] * mass[qy][qx][2]);
-
-                           const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
-                           dxyz += m_c * wx * wy * wz;
-                        }
-                     }
-                  }
-               }
-            }
-
-            MFEM_SYNC_THREAD;
-
-            MFEM_FOREACH_THREAD(dz,z,D1Dz)
-            {
-               MFEM_FOREACH_THREAD(dy,y,D1Dy)
-               {
-                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
-                  {
-                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) += dxyz;
-                  }
-               }
-            }
-
-            osc += D1Dx * D1Dy * D1Dz;
-         } // c loop
-      } // qz
   }); // end of element loop
 }

@@ -62,6 +62,30 @@ void PAHcurlHdivMassApply2D(const int D1D,
                            const Vector &x_,
                            Vector &y_);

+/// H(curl) test, H(div) trial
+inline void
+PAHcurlHdivMassApply2D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false,
+                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
+/// H(div) test, H(curl) trial
+inline void
+PAHdivHcurlMassApply2D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true,
+                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
 // PA H(curl)-H(div) Mass Apply 3D kernel
 void PAHcurlHdivMassApply3D(const int D1D,
                            const int D1Dtest,
@@ -78,6 +102,30 @@ void PAHcurlHdivMassApply3D(const int D1D,
                            const Vector &x_,
                            Vector &y_);

+/// H(curl) test, H(div) trial
+inline void
+PAHcurlHdivMassApply3D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false, false, Bo_,
+                          Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
+/// H(div) test, H(curl) trial
+inline void
+PAHdivHcurlMassApply3D(const int NE, const bool, const bool scalarCoeff,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int D1Dtest, const int Q1D)
+{
+   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true, false, Bo_,
+                          Bc_, Bot_, Bct_, op_, x_, y_);
+}
+
 // PA H(curl)-H(div) Curl Apply 3D kernel
 template<int T_D1D = 0, int T_D1D_TEST = 0, int T_Q1D = 0>
 inline void PAHcurlHdivApply3D(const int d1d,
@@ -816,8 +864,656 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
   }); // end of element loop
 }

+namespace curlinterp
+{
+constexpr int NBZ3D(int ndof_o, int nquad_o, int mdq)
+{
+   if (ndof_o <= 0 || nquad_o <= 0)
+   {
+      return 1;
+   }
+   int ndof_c = ndof_o + 1;
+   int nquad_c = nquad_o + 1;
+   // z dimension is capped at 64 on nvidia and amd gpus
+   int tmp =
+      std::min((128 + mdq * mdq * (mdq - 1) - 1) / (mdq * mdq * (mdq - 1)), 64);
+   int smem_req =
+      sizeof(mfem::real_t) *
+      ((3 * ndof_c * ndof_c * ndof_o + 2 * 2 * mdq * mdq * mdq) * tmp +
+       ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
+   // assume GPU has at least 48k shared memory
+   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
+}
+}
+
+template <int T_NDOF_O, int T_NQUAD_O>
+void CurlInterpolatorApply3DSmem(const int ne, const int ndof_o,
+                                 const int nquad_o, const Vector &pa,
+                                 const Vector &x_, Vector &y_)
+{
+   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+   constexpr int mnq_o =
+      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
+   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
+   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
+   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
+   int mnq = std::max(ndof_o + 1, nquad_o + 1);
+   auto pa_data = pa.Read();
+   auto x_d = x_.Read();
+   auto y_d = y_.ReadWrite();
+   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
+      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MND_O =
+         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+      constexpr int MNQ_O =
+         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
+      int tidz = MFEM_THREAD_ID(z);
+      // Make mnq a local variable since capturing would result in different
+      // captures between host/device versions, and spuriously fails
+      int mnq = std::max(ndof_o + 1, nquad_o + 1);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
+      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
+      const int NDOF_C = NDOF_O + 1;
+      const int NQUAD_C = NQUAD_O + 1;
+      MFEM_SHARED real_t
+      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
+      auto X_ = Reshape(x_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
+      auto Y = Reshape(y_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
+      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
+      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
+      auto Boo =
+         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
+      MFEM_SHARED real_t X[3][nbz][MND_O * (MND_O + 1) * (MND_O + 1)];
+      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
+      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
+
+      // shapes of buffers always use MNDQ to mitigate shared memory bank
+      // conflicts
+      real_t(*DDQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      real_t(*DQQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
+      real_t(*QQQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      const int offset = NDOF_O * NDOF_C * NDOF_C;
+      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
+      {
+         for (int dim = 0; dim < 3; ++dim)
+         {
+            X[dim][tidz][ix] = X_(ix + dim * offset, e);
+         }
+      }
+      // load basis functions data
+      if (tidz == 0)
+      {
+         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
+         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
+      }
+      MFEM_SYNC_THREAD;
+
+      // x: Vz Bcc Gco Boo - Vy Bcc Boo Gco
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Bcc(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Bcc(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Gco(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_O; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Boo(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_O; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Boo(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Gco(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_O) * NQUAD_C, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // y: Vx Boo Bcc Gco - Vz Gco Bcc Boo
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_O; ++dx)
+         {
+            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Gco(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Bcc(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
+                                           NDOF_O, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Bcc(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Gco(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_O; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Boo(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_C) * NQUAD_O + offsetq, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // z: Vy Gco Boo Bcc - Vx Boo Gco Bcc
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Gco(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_O; ++dx)
+         {
+            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_O; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Boo(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Gco(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Bcc(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Bcc(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_O) * NQUAD_O + 2 * offsetq, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+template <int T_NDOF_O, int T_NQUAD_O>
+void CurlInterpolatorTApply3DSmem(const int ne, const int ndof_o,
+                                  const int nquad_o, const Vector &pa,
+                                  const Vector &x_, Vector &y_)
+{
+   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+   constexpr int mnq_o =
+      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
+   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
+   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
+   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
+   int mnq = std::max(ndof_o + 1, nquad_o + 1);
+   auto pa_data = pa.Read();
+   auto x_d = x_.Read();
+   auto y_d = y_.ReadWrite();
+   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
+      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MND_O =
+         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+      constexpr int MNQ_O =
+         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
+      int tidz = MFEM_THREAD_ID(z);
+      // Make mnq a local variable since capturing would result in different
+      // captures between host/device versions, and spuriously fails
+      int mnq = std::max(ndof_o + 1, nquad_o + 1);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
+      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
+      const int NDOF_C = NDOF_O + 1;
+      const int NQUAD_C = NQUAD_O + 1;
+      MFEM_SHARED real_t
+      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
+      auto X_ = Reshape(x_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
+      auto Y = Reshape(y_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
+      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
+      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
+      auto Boo =
+         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
+      MFEM_SHARED real_t X[3][nbz][MNQ_O * MNQ_O * (MNQ_O + 1)];
+      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
+      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
+
+      // shapes of buffers always use MNDQ to mitigate shared memory bank
+      // conflicts
+      real_t(*QQD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      real_t(*QDD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
+      real_t(*DDD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      const int offset = NDOF_O * NDOF_C * NDOF_C;
+      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offsetq)
+      {
+         for (int dim = 0; dim < 3; ++dim)
+         {
+            X[dim][tidz][ix] = X_(ix + dim * offsetq, e);
+         }
+      }
+      // load basis functions data
+      if (tidz == 0)
+      {
+         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
+         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
+      }
+      MFEM_SYNC_THREAD;
+
+      // x: Vy Boo Bcc Gco - Vz Boo Gco Bcc
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Gco(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_C; ++qz)
+         {
+            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_C; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Bcc(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Gco(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Boo(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Boo(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_C) * NDOF_O, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // y: Vz Gco Boo Bcc - Vx Bcc Boo Gco
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_C; ++qz)
+         {
+            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Gco(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Boo(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
+                                           NQUAD_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Boo(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Gco(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_C; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Bcc(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_O) * NDOF_C + offset, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // z: Vx Bcc Gco Boo - Vy Gco Bcc Boo
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Boo(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Boo(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Gco(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_C; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Bcc(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_C; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Bcc(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Gco(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_C) * NDOF_C + 2 * offset, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
 } // namespace internal

+template <int DIM, int NDOF_O, int NQUAD_O>
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyPAKernels::Kernel()
+{
+   if constexpr (DIM == 3)
+   {
+      return internal::CurlInterpolatorApply3DSmem<NDOF_O, NQUAD_O>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+template <int DIM, int NDOF_O, int NQUAD_O>
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyTPAKernels::Kernel()
+{
+   if constexpr (DIM == 3)
+   {
+      return internal::CurlInterpolatorTApply3DSmem<NDOF_O, NQUAD_O>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
 } // namespace mfem

 /// \endcond DO_NOT_DOCUMENT
@@ -294,61 +294,14 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
   }); // end of element loop
 }

-void PAHdivMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const bool symmetric,
-                     const Array<real_t> &Bo,
-                     const Array<real_t> &Bc,
-                     const Array<real_t> &Bot,
-                     const Array<real_t> &Bct,
-                     const Vector &op,
-                     const Vector &x,
-                     Vector &y)
-{
-   const int id = (D1D << 4) | Q1D;
-
-   if (dim == 2)
-   {
-      switch (id)
-      {
-         case 0x22: return SmemPAHdivMassApply2D<2,2>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x33: return SmemPAHdivMassApply2D<3,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x44: return SmemPAHdivMassApply2D<4,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x55: return SmemPAHdivMassApply2D<5,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         default: // fallback
-            return PAHdivMassApply2D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch (id)
-      {
-         case 0x23: return SmemPAHdivMassApply3D<2,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x34: return SmemPAHdivMassApply3D<3,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x45: return SmemPAHdivMassApply3D<4,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x56: return SmemPAHdivMassApply3D<5,6>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x67: return SmemPAHdivMassApply3D<6,7>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x78: return SmemPAHdivMassApply3D<7,8>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         default: // fallback
-            return PAHdivMassApply3D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-      }
-   }
-}
-
-void PAHdivMassApply2D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_)
+void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int TestD1D, const int Q1D)
 {
+   MFEM_VERIFY(D1D == TestD1D,
+               "Trial and test spaces must have same number of dofs");
   auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
   auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
   auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
@@ -468,18 +421,14 @@ void PAHdivMassApply2D(const int D1D,
   }); // end of element loop
 }

-void PAHdivMassApply3D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_)
+void PAHdivMassApply3D(const int NE, const bool symmetric, const bool,
+                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                       const Vector &op_, const Vector &x_, Vector &y_,
+                       const int D1D, const int TestD1D, const int Q1D)
 {
+   MFEM_VERIFY(D1D == TestD1D,
+               "Trial and test spaces must have same number of dofs");
   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
               "Error: D1D > HDIV_MAX_D1D");
   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
@@ -66,58 +66,29 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
                                  const Vector &op_,
                                  Vector &diag_);

-void PAHdivMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const bool symmetric,
-                     const Array<real_t> &Bo,
-                     const Array<real_t> &Bc,
-                     const Array<real_t> &Bot,
-                     const Array<real_t> &Bct,
-                     const Vector &op,
-                     const Vector &x,
-                     Vector &y);
-
 // PA H(div) Mass Apply 2D kernel
-void PAHdivMassApply2D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_);
+void PAHdivMassApply2D(const int NE, const bool symmetric,
+                       const bool scalar_coeff, const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_, const Vector &op_,
+                       const Vector &x_, Vector &y_, const int D1D,
+                       const int TestD1D, const int Q1D);

 // PA H(div) Mass Apply 3D kernel
-void PAHdivMassApply3D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const bool symmetric,
-                       const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_);
+void PAHdivMassApply3D(const int NE, const bool symmetric,
+                       const bool scalar_coeff, const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_, const Vector &op_,
+                       const Vector &x_, Vector &y_, const int D1D,
+                       const int TestD1D, const int Q1D);

 // Shared memory PA H(div) Mass Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHdivMassApply2D(const int NE,
-                                  const bool symmetric,
-                                  const Array<real_t> &Bo_,
-                                  const Array<real_t> &Bc_,
-                                  const Array<real_t> &Bot_,
-                                  const Array<real_t> &Bct_,
-                                  const Vector &op_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply2D(
+   const int NE, const bool symmetric, const bool, const Array<real_t> &Bo_,
+   const Array<real_t> &Bc_, const Array<real_t> &Bot_,
+   const Array<real_t> &Bct_, const Vector &op_, const Vector &x_, Vector &y_,
+   const int d1d = 0, const int = 0, const int q1d = 0)
 {
   MFEM_CONTRACT_VAR(Bot_);
   MFEM_CONTRACT_VAR(Bct_);
@@ -280,18 +251,13 @@ inline void SmemPAHdivMassApply2D(const int NE,
 }

 // Shared memory PA H(div) Mass Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHdivMassApply3D(const int NE,
-                                  const bool symmetric,
-                                  const Array<real_t> &Bo_,
-                                  const Array<real_t> &Bc_,
-                                  const Array<real_t> &Bot_,
-                                  const Array<real_t> &Bct_,
-                                  const Vector &op_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+template <int T_D1D = 0, int T_Q1D = 0>
+inline void
+SmemPAHdivMassApply3D(const int NE, const bool symmetric, const bool,
+                      const Array<real_t> &Bo_, const Array<real_t> &Bc_,
+                      const Array<real_t> &Bot_, const Array<real_t> &Bct_,
+                      const Vector &op_, const Vector &x_, Vector &y_,
+                      const int d1d = 0, const int = 0, const int q1d = 0)
 {
   MFEM_CONTRACT_VAR(Bot_);
   MFEM_CONTRACT_VAR(Bct_);
@@ -14,6 +14,8 @@
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"

+#include "bilininteg_hcurlhdiv_kernels.hpp"
+
 namespace mfem
 {

@@ -1950,4 +1952,138 @@ void IdentityInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
   }
 }

+void CurlInterpolator::AssemblePA(const FiniteElementSpace &dom_fes,
+                                  const FiniteElementSpace &ran_fes)
+{
+   // TODO: 1D and 2D meshes
+   Mesh *mesh = dom_fes.GetMesh();
+   const VectorTensorFiniteElement *dom_el =
+      dynamic_cast<const VectorTensorFiniteElement *>(dom_fes.GetTypicalFE());
+   const VectorTensorFiniteElement *ran_el =
+      dynamic_cast<const VectorTensorFiniteElement *>(ran_fes.GetTypicalFE());
+   MFEM_VERIFY(dom_el != NULL, "Only VectorTensorFiniteElement is supported!");
+   MFEM_VERIFY(ran_el != NULL, "Only VectorTensorFiniteElement is supported!");
+   // only supports H(curl) -> H(div) because of discontinuity requirements
+   MFEM_VERIFY(dom_el->GetDerivType() == FiniteElement::CURL,
+               "Domain space must be H(curl)");
+   MFEM_VERIFY(ran_el->GetDerivType() == FiniteElement::DIV,
+               "Range space must be H(div)");
+
+   const int dims = dom_el->GetDim();
+   MFEM_VERIFY(dims == 3, "");
+   dim = mesh->Dimension();
+
+   ne = dom_fes.GetNE();
+   ndof_o = dom_el->GetOrder();
+   int ndof_c = ndof_o + 1;
+   nquad_o = ran_el->GetOrder();
+   int nquad_c = nquad_o + 1;
+
+   // extract the tensor product range dof locations
+   std::vector<real_t> qc(nquad_c);
+   std::vector<real_t> qo(nquad_o);
+   {
+      const IntegrationRule &ran_nodes = ran_el->GetNodes();
+      const Array<int> &quad_map = ran_el->GetDofMap();
+      for (int i = 0; i < nquad_c; ++i)
+      {
+         int idx = UnsignIndex(quad_map[i]);
+         qc[i] = ran_nodes.IntPoint(idx).x;
+      }
+      int offset = ndof_c * ndof_o * ndof_o;
+      for (int i = 0; i < nquad_o; ++i)
+      {
+         int idx = UnsignIndex(quad_map[i + offset]);
+         qo[i] = ran_nodes.IntPoint(idx).x;
+      }
+   }
+
+   // evaluate closed/open 1D basis (and their derivatives) at closed and
+   // open quads
+   // storage order: GCO, BCC, BOO
+   pa_data.SetSize(ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
+   auto ptr = pa_data.HostWrite();
+   auto &cbasis1d = dom_el->GetBasis1D();
+   auto &obasis1d = dom_el->GetOpenBasis1D();
+   Vector b, g;
+   b.SetSize(ndof_c);
+   g.SetSize(ndof_c);
+   for (int j = 0; j < nquad_o; ++j)
+   {
+      cbasis1d.Eval(qo[j], b, g);
+      for (int i = 0; i < ndof_c; ++i)
+      {
+         ptr[j + i * nquad_o] = g[i];
+      }
+   }
+   ptr += nquad_o * ndof_c;
+
+   for (int j = 0; j < nquad_c; ++j)
+   {
+      cbasis1d.Eval(qc[j], b);
+      for (int i = 0; i < ndof_c; ++i)
+      {
+         ptr[j + i * nquad_c] = b[i];
+      }
+   }
+   ptr += ndof_c * nquad_c;
+
+   b.SetSize(ndof_o);
+   for (int j = 0; j < nquad_o; ++j)
+   {
+      obasis1d.Eval(qo[j], b);
+      for (int i = 0; i < ndof_o; ++i)
+      {
+         ptr[j + i * nquad_o] = b[i];
+      }
+   }
+}
+
+CurlInterpolator::Kernels::Kernels()
+{
+   CurlInterpolator::AddSpecialization<3, 1, 1>();
+   CurlInterpolator::AddSpecialization<3, 2, 2>();
+   CurlInterpolator::AddSpecialization<3, 3, 3>();
+   CurlInterpolator::AddSpecialization<3, 4, 4>();
+   CurlInterpolator::AddSpecialization<3, 5, 5>();
+}
+
+CurlInterpolator::CurlInterpolator() { static Kernels kernels{}; }
+
+void CurlInterpolator::AddMultPA(const Vector &x, Vector &y) const
+{
+   ApplyPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x,
+                       y);
+}
+
+void CurlInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
+{
+   ApplyTPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x,
+                        y);
+}
+
+/// \cond DO_NOT_DOCUMENT
+
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 3)
+   {
+      return internal::CurlInterpolatorApply3DSmem<0, 0>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyTPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 3)
+   {
+      return internal::CurlInterpolatorTApply3DSmem<0, 0>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
 } // namespace mfem
@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.

 #include "bilininteg_mass_kernels.hpp"
+#include "bilininteg_mass_pa_simplices.hpp" // IWYU pragma: keep

 namespace mfem
 {
@@ -39,8 +40,10 @@ MassIntegrator::Kernels::Kernels()
   MassIntegrator::AddSpecialization<2,9,10>();
   // others
   MassIntegrator::AddSpecialization<2,2,4>();
+   MassIntegrator::AddSpecialization<2,2,5>();
   MassIntegrator::AddSpecialization<2,3,6>();
   MassIntegrator::AddSpecialization<2,4,6>();
+
   // 3D
   // Q=P+1
   MassIntegrator::AddSpecialization<3,1,1>();
@@ -19,6 +19,8 @@
 #include "../../linalg/vector.hpp"
 #include "../bilininteg.hpp"

+#include "bilininteg_mass_pa_simplices.hpp"
+
 namespace mfem
 {

@@ -181,6 +183,12 @@ constexpr int NBZ(int D1D)
 {
   return ipow(2, D(D1D) >= 0 ? D(D1D) : 0);
 }
+constexpr int NBZ3D(int MDQ)
+{
+   return MDQ > 0 ? std::min<int>(
+             (128 + MDQ * MDQ * MDQ - 1) / (MDQ * MDQ * MDQ), 64)
+          : 1;
+}
 }

 // Shared memory PA Mass Diagonal 2D kernel
@@ -804,19 +812,23 @@ void PAMassApply3D_Element(const int e,
   }
 }

-template<int T_D1D, int T_Q1D, bool ACCUMULATE = true>
-MFEM_HOST_DEVICE inline
-void SmemPAMassApply3D_Element(const int e,
-                               const int NE,
-                               const real_t *b_,
-                               const real_t *d_,
-                               const real_t *x_,
-                               real_t *y_,
-                               const int d1d = 0,
-                               const int q1d = 0)
+template <int T_D1D, int T_Q1D, int TBATCH, bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline void
+SmemPAMassApply3D_Element(const int e, const int NE, const real_t *b_,
+                          const real_t *d_, const real_t *x_, real_t *y_,
+                          int d1d = 0, int q1d = 0)
 {
-   constexpr int D1D = T_D1D ? T_D1D : d1d;
-   constexpr int Q1D = T_Q1D ? T_Q1D : q1d;
+   static_assert(TBATCH > 0, "TBATCH must be positive");
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+   constexpr int tbatch = TBATCH;
+   const int tidz = MFEM_THREAD_ID(z);
+#else
+   // host always batch size 1
+   constexpr int tbatch = 1;
+   constexpr int tidz = 0;
+#endif
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
   constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
   constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
   constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
@@ -829,33 +841,37 @@ void SmemPAMassApply3D_Element(const int e,
   MFEM_SHARED real_t sDQ[MQ1*MD1];
   real_t (*B)[MD1] = (real_t (*)[MD1]) sDQ;
   real_t (*Bt)[MQ1] = (real_t (*)[MQ1]) sDQ;
-   MFEM_SHARED real_t sm0[MDQ*MDQ*MDQ];
-   MFEM_SHARED real_t sm1[MDQ*MDQ*MDQ];
-   real_t (*X)[MD1][MD1]   = (real_t (*)[MD1][MD1]) sm0;
-   real_t (*DDQ)[MD1][MQ1] = (real_t (*)[MD1][MQ1]) sm1;
-   real_t (*DQQ)[MQ1][MQ1] = (real_t (*)[MQ1][MQ1]) sm0;
-   real_t (*QQQ)[MQ1][MQ1] = (real_t (*)[MQ1][MQ1]) sm1;
-   real_t (*QQD)[MQ1][MD1] = (real_t (*)[MQ1][MD1]) sm0;
-   real_t (*QDD)[MD1][MD1] = (real_t (*)[MD1][MD1]) sm1;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
+   MFEM_SHARED real_t sm0[tbatch][MDQ*MDQ*MDQ];
+   MFEM_SHARED real_t sm1[tbatch][MDQ*MDQ*MDQ];
+   real_t (*X)[MD1][MD1]   = (real_t (*)[MD1][MD1]) (sm0+tidz);
+   real_t (*DDQ)[MD1][MQ1] = (real_t (*)[MD1][MQ1]) (sm1+tidz);
+   real_t (*DQQ)[MQ1][MQ1] = (real_t (*)[MQ1][MQ1]) (sm0+tidz);
+   real_t (*QQQ)[MQ1][MQ1] = (real_t (*)[MQ1][MQ1]) (sm1+tidz);
+   real_t (*QQD)[MQ1][MD1] = (real_t (*)[MQ1][MD1]) (sm0+tidz);
+   real_t (*QDD)[MD1][MD1] = (real_t (*)[MD1][MD1]) (sm1+tidz);
+   MFEM_FOREACH_THREAD(dy, y, D1D)
   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
+      MFEM_FOREACH_THREAD(dx, x, D1D)
      {
         MFEM_UNROLL(MD1)
         for (int dz = 0; dz < D1D; ++dz)
         {
-            X[dz][dy][dx] = x(dx,dy,dz,e);
+            X[dz][dy][dx] = x(dx, dy, dz, e);
         }
      }
-      MFEM_FOREACH_THREAD(dx,x,Q1D)
+      MFEM_FOREACH_THREAD(dx, x, Q1D) { B[dx][dy] = b(dx, dy); }
+   }
+   if (tidz == 0)
+   {
+      MFEM_FOREACH_THREAD(dy, y, D1D)
      {
-         B[dx][dy] = b(dx,dy);
+         MFEM_FOREACH_THREAD(dx, x, Q1D) { B[dx][dy] = b(dx, dy); }
      }
   }
   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
+   MFEM_FOREACH_THREAD(dy, y, D1D)
   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      MFEM_FOREACH_THREAD(qx, x, Q1D)
      {
         real_t u[D1D];
         MFEM_UNROLL(MD1)
@@ -880,9 +896,9 @@ void SmemPAMassApply3D_Element(const int e,
      }
   }
   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   MFEM_FOREACH_THREAD(qy, y, Q1D)
   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      MFEM_FOREACH_THREAD(qx, x, Q1D)
      {
         real_t u[D1D];
         MFEM_UNROLL(MD1)
@@ -907,9 +923,9 @@ void SmemPAMassApply3D_Element(const int e,
      }
   }
   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   MFEM_FOREACH_THREAD(qy, y, Q1D)
   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      MFEM_FOREACH_THREAD(qx, x, Q1D)
      {
         real_t u[Q1D];
         MFEM_UNROLL(MQ1)
@@ -929,22 +945,22 @@ void SmemPAMassApply3D_Element(const int e,
         MFEM_UNROLL(MQ1)
         for (int qz = 0; qz < Q1D; qz++)
         {
-            QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
+            QQQ[qz][qy][qx] = u[qz] * d(qx, qy, qz, e);
         }
      }
   }
   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(di,y,D1D)
+   if (tidz == 0)
   {
-      MFEM_FOREACH_THREAD(q,x,Q1D)
+      MFEM_FOREACH_THREAD(di, y, D1D)
      {
-         Bt[di][q] = b(q,di);
+         MFEM_FOREACH_THREAD(q, x, Q1D) { Bt[di][q] = b(q, di); }
      }
   }
   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   MFEM_FOREACH_THREAD(qy, y, Q1D)
   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
+      MFEM_FOREACH_THREAD(dx, x, D1D)
      {
         real_t u[Q1D];
         MFEM_UNROLL(MQ1)
@@ -969,9 +985,9 @@ void SmemPAMassApply3D_Element(const int e,
      }
   }
   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
+   MFEM_FOREACH_THREAD(dy, y, D1D)
   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
+      MFEM_FOREACH_THREAD(dx, x, D1D)
      {
         real_t u[Q1D];
         MFEM_UNROLL(MQ1)
@@ -996,9 +1012,9 @@ void SmemPAMassApply3D_Element(const int e,
      }
   }
   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
+   MFEM_FOREACH_THREAD(dy, y, D1D)
   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
+      MFEM_FOREACH_THREAD(dx, x, D1D)
      {
         real_t u[D1D];
         MFEM_UNROLL(MD1)
@@ -1020,11 +1036,11 @@ void SmemPAMassApply3D_Element(const int e,
         {
            if (ACCUMULATE)
            {
-               y(dx,dy,dz,e) += u[dz];
+               y(dx, dy, dz, e) += u[dz];
            }
            else
            {
-               y(dx,dy,dz,e) = u[dz];
+               y(dx, dy, dz, e) = u[dz];
            }
         }
      }
@@ -1115,8 +1131,8 @@ inline void PAMassApply3D(const int NE,
   });
 }

-// Shared memory PA Mass Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
+// Shared memory PA Mass Apply 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0, int TBATCH=1>
 inline void SmemPAMassApply3D(const int NE,
                              const Array<real_t> &b_,
                              const Array<real_t> &bt_,
@@ -1126,6 +1142,9 @@ inline void SmemPAMassApply3D(const int NE,
                              const int d1d = 0,
                              const int q1d = 0)
 {
+   static_assert(T_D1D > 0, "T_D1D must be positive");
+   static_assert(T_Q1D > 0, "T_Q1D must be positive");
+   static_assert(TBATCH > 0, "TBATCH must be positive");
   MFEM_CONTRACT_VAR(bt_);
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1137,9 +1156,11 @@ inline void SmemPAMassApply3D(const int NE,
   const auto d = d_.Read();
   const auto x = x_.Read();
   auto y = y_.ReadWrite();
-   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   mfem::forall_2D_batch<T_Q1D * T_Q1D * TBATCH>(NE, Q1D, Q1D, TBATCH,
+                                                 [=] MFEM_HOST_DEVICE(int e)
   {
-      internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
+      internal::SmemPAMassApply3D_Element<T_D1D, T_Q1D, TBATCH>(e, NE, b, d, x,
+                                                                y, d1d, q1d);
   });
 }

@@ -1389,42 +1410,57 @@ using ApplyKernelType = MassIntegrator::ApplyKernelType;
 using DiagonalKernelType = MassIntegrator::DiagonalKernelType;
 }

-template<int DIM, int T_D1D, int T_Q1D>
+template<int DIM, int D1D, int Q1D>
 ApplyKernelType MassIntegrator::ApplyPAKernels::Kernel()
 {
   if constexpr (DIM == 1) { return internal::PAMassApply1D; }
-   else if constexpr (DIM == 2) { return internal::SmemPAMassApply2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPAMassApply3D<T_D1D, T_Q1D>; }
-   MFEM_ABORT("");
+   else if constexpr (DIM == 2) { return internal::SmemPAMassApply2D<D1D, Q1D>; }
+   else if constexpr (DIM == 3)
+   {
+      constexpr int MDQ = D1D >= Q1D ? D1D : Q1D;
+      // max 64 threads in z limit in cuda and hip
+      if constexpr (MDQ > 0)
+      {
+         return internal::SmemPAMassApply3D<D1D, Q1D,
+                internal::mass::NBZ3D(MDQ)>;
+      }
+   }
+   else { MFEM_ABORT(""); }
+   return nullptr;
 }

 inline ApplyKernelType MassIntegrator::ApplyPAKernels::Fallback(
-   int DIM, int, int)
+   int dim, int, int)
 {
-   if (DIM == 1) { return internal::PAMassApply1D; }
-   else if (DIM == 2) { return internal::PAMassApply2D; }
-   else if (DIM == 3) { return internal::PAMassApply3D; }
+   if (dim == 1) { return internal::PAMassApply1D; }
+   else if (dim == 2) { return internal::PAMassApply2D; }
+   else if (dim == 3) { return internal::PAMassApply3D; }
   else { MFEM_ABORT(""); }
+   return nullptr;
 }

-template<int DIM, int T_D1D, int T_Q1D>
+template<int DIM, int D1D, int Q1D>
 DiagonalKernelType MassIntegrator::DiagonalPAKernels::Kernel()
 {
   if constexpr (DIM == 1) { return internal::PAMassAssembleDiagonal1D; }
-   else if constexpr (DIM == 2) { return internal::SmemPAMassAssembleDiagonal2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPAMassAssembleDiagonal3D<T_D1D, T_Q1D>; }
-   MFEM_ABORT("");
+   else if constexpr (DIM == 2) { return internal::SmemPAMassAssembleDiagonal2D<D1D, Q1D>; }
+   else if constexpr (DIM == 3) { return internal::SmemPAMassAssembleDiagonal3D<D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
+   return nullptr;
 }

 inline DiagonalKernelType MassIntegrator::DiagonalPAKernels::Fallback(
-   int DIM, int, int)
+   int dim, int, int)
 {
-   if (DIM == 1) { return internal::PAMassAssembleDiagonal1D; }
-   else if (DIM == 2) { return internal::PAMassAssembleDiagonal2D; }
-   else if (DIM == 3) { return internal::PAMassAssembleDiagonal3D; }
+   if (dim == 1) { return internal::PAMassAssembleDiagonal1D; }
+   else if (dim == 2) { return internal::PAMassAssembleDiagonal2D; }
+   else if (dim == 3) { return internal::PAMassAssembleDiagonal3D; }
   else { MFEM_ABORT(""); }
+   return nullptr;
 }
+
 /// \endcond DO_NOT_DOCUMENT
+
 } // namespace mfem

 #endif
@@ -15,6 +15,7 @@
 #include "../qfunction.hpp"
 #include "../ceed/integrators/mass/mass.hpp"
 #include "bilininteg_mass_kernels.hpp"
+#include "bilininteg_mass_pa_simplices.hpp"

 namespace mfem
 {
@@ -29,9 +30,11 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   // Assuming the same element type
   fespace = &fes;
   Mesh *mesh = fes.GetMesh();
+   dim = mesh->Dimension();
   const FiniteElement &el = *fes.GetTypicalFE();
   ElementTransformation *T0 = mesh->GetTypicalElementTransformation();
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T0);
+   const bool stroud = fes.UsesRaggedTensorBasis();
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T0, stroud);
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
@@ -48,17 +51,25 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
      return;
   }
   int map_type = el.GetMapType();
-   dim = mesh->Dimension();
   ne = fes.GetMesh()->GetNE();
   nq = ir->GetNPoints();
   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::DETERMINANTS, mt);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   if (stroud)
+   {
+      maps = &el.GetDofToQuad(*ir, DofToQuad::RAGGED_TENSOR);
+   }
+   else
+   {
+      maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   }
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(ne*nq, mt);

   QuadratureSpace qs(*mesh, *ir);
   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+   // QuadratureSpace expects ir defined in reference simplex for Bernstein
+   // elements with partial assembly
   {
      const int NE = ne;
      const int NQ = nq;
@@ -147,9 +158,10 @@ void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
   {
      const int D1D = dofs1D;
      const int Q1D = quad1D;
+      const Vector &D = pa_data;
      const Array<real_t> &B = maps->B;
      const Array<real_t> &Bt = maps->Bt;
-      const Vector &D = pa_data;
+
 #ifdef MFEM_USE_OCCA
      if (DeviceCanUseOcca())
      {
@@ -164,7 +176,31 @@ void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
         MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
      }
 #endif // MFEM_USE_OCCA
-      ApplyPAKernels::Run(dim, D1D, Q1D, ne, B, Bt, D, x, y, D1D, Q1D);
+
+      if (fespace->UsesRaggedTensorBasis())
+      {
+         const auto *rmaps = static_cast<const RaggedDofToQuad*>(maps);
+
+         const Array<real_t> &Ba1 = rmaps->Ba1;
+         const Array<real_t> &Ba2 = rmaps->Ba2;
+         const Array<real_t> &Ba3 = rmaps->Ba3;
+         const Array<real_t> &Ba1t = rmaps->Ba1t;
+         const Array<real_t> &Ba2t = rmaps->Ba2t;
+         const Array<real_t> &Ba3t = rmaps->Ba3t;
+         const Array<int> &lex_map = rmaps->lex_map;
+         const Array<int> &forward_map2d = rmaps->forward_map2d_mass;
+         const Array<int> &inverse_map2d = rmaps->inverse_map2d_mass;
+         const Array<int> &forward_map3d = rmaps->forward_map3d_mass;
+         const Array<int> &inverse_map3d = rmaps->inverse_map3d_mass;
+         ApplySimplexPAKernels::Run(dim, D1D, Q1D, ne, lex_map, forward_map2d,
+                                    inverse_map2d,
+                                    forward_map3d, inverse_map3d, Ba1, Ba2, Ba3, Ba1t, Ba2t, Ba3t,
+                                    D, x, y, D1D, Q1D);
+      }
+      else
+      {
+         ApplyPAKernels::Run(dim, D1D, Q1D, ne, B, Bt, D, x, y, D1D, Q1D);
+      }
   }
 }

@@ -177,6 +213,8 @@ void MassIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
   }
   else
   {
+      MFEM_VERIFY(!fespace->UsesRaggedTensorBasis(),
+                  "AbsMultPA not implemented for ragged tensor basis");
      Vector abs_pa_data(pa_data);
      abs_pa_data.Abs();
      Array<real_t> absB(maps->B);
@@ -18,6 +18,779 @@
 namespace mfem
 {

+namespace
+{
+
+void PAHcurlDotSetup2D(const int q1d,
+                       const int ne,
+                       const bool test_map_integral,
+                       const Array<real_t> &w,
+                       const Vector &jacobians,
+                       const Vector &coeff,
+                       Vector &op)
+{
+   auto W = Reshape(w.Read(), q1d, q1d);
+   auto J = Reshape(jacobians.Read(), q1d, q1d, 2, 2, ne);
+   auto C = Reshape(coeff.Read(), 2, q1d, q1d, ne);
+   auto O = Reshape(op.Write(), 2, q1d, q1d, ne);
+
+   mfem::forall_2D(ne, q1d, q1d, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(qy, y, q1d)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            const real_t J11 = J(qx, qy, 0, 0, e);
+            const real_t J12 = J(qx, qy, 1, 0, e);
+            const real_t J21 = J(qx, qy, 0, 1, e);
+            const real_t J22 = J(qx, qy, 1, 1, e);
+            const real_t detJ = (J11 * J22) - (J21 * J12);
+            const real_t scale = W(qx, qy) * (test_map_integral ? 1.0 / detJ : 1.0);
+            const real_t Vx = C(0, qx, qy, e);
+            const real_t Vy = C(1, qx, qy, e);
+
+            O(0, qx, qy, e) = scale * ( J22 * Vx - J12 * Vy);
+            O(1, qx, qy, e) = scale * (-J21 * Vx + J11 * Vy);
+         }
+      }
+   });
+}
+
+void PAHcurlDotSetup3D(const int q1d,
+                       const int ne,
+                       const bool test_map_integral,
+                       const Array<real_t> &w,
+                       const Vector &jacobians,
+                       const Vector &coeff,
+                       Vector &op)
+{
+   auto W = Reshape(w.Read(), q1d, q1d, q1d);
+   auto J = Reshape(jacobians.Read(), q1d, q1d, q1d, 3, 3, ne);
+   auto C = Reshape(coeff.Read(), 3, q1d, q1d, q1d, ne);
+   auto O = Reshape(op.Write(), 3, q1d, q1d, q1d, ne);
+
+   mfem::forall_3D(ne, q1d, q1d, q1d, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(qz, z, q1d)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               const real_t J11 = J(qx, qy, qz, 0, 0, e);
+               const real_t J12 = J(qx, qy, qz, 0, 1, e);
+               const real_t J13 = J(qx, qy, qz, 0, 2, e);
+               const real_t J21 = J(qx, qy, qz, 1, 0, e);
+               const real_t J22 = J(qx, qy, qz, 1, 1, e);
+               const real_t J23 = J(qx, qy, qz, 1, 2, e);
+               const real_t J31 = J(qx, qy, qz, 2, 0, e);
+               const real_t J32 = J(qx, qy, qz, 2, 1, e);
+               const real_t J33 = J(qx, qy, qz, 2, 2, e);
+               const real_t detJ = J11 * (J22 * J33 - J32 * J23)
+                                   - J21 * (J12 * J33 - J32 * J13)
+                                   + J31 * (J12 * J23 - J22 * J13);
+               const real_t scale = W(qx, qy, qz) *
+                                    (test_map_integral ? 1.0 / detJ : 1.0);
+               const real_t Vx = C(0, qx, qy, qz, e);
+               const real_t Vy = C(1, qx, qy, qz, e);
+               const real_t Vz = C(2, qx, qy, qz, e);
+
+               O(0, qx, qy, qz, e) = scale *
+                                     ((J22 * J33 - J23 * J32) * Vx +
+                                      (J13 * J32 - J12 * J33) * Vy +
+                                      (J12 * J23 - J13 * J22) * Vz);
+               O(1, qx, qy, qz, e) = scale *
+                                     ((J23 * J31 - J21 * J33) * Vx +
+                                      (J11 * J33 - J13 * J31) * Vy +
+                                      (J13 * J21 - J11 * J23) * Vz);
+               O(2, qx, qy, qz, e) = scale *
+                                     ((J21 * J32 - J22 * J31) * Vx +
+                                      (J12 * J31 - J11 * J32) * Vy +
+                                      (J11 * J22 - J12 * J21) * Vz);
+            }
+         }
+      }
+   });
+}
+
+void PAHcurlDotApply2D(const int d1d,
+                       const int d1d_test,
+                       const int q1d,
+                       const int ne,
+                       const Array<real_t> &bo,
+                       const Array<real_t> &bc,
+                       const Array<real_t> &bt,
+                       const Vector &pa_data,
+                       const Vector &x,
+                       Vector &y)
+{
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
+   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
+
+   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
+   auto Bc = Reshape(bc.Read(), q1d, d1d);
+   auto Bt = Reshape(bt.Read(), d1d_test, q1d);
+   auto O = Reshape(pa_data.Read(), 2, q1d, q1d, ne);
+   auto X = Reshape(x.Read(), 2 * (d1d - 1) * d1d, ne);
+   auto Y = Reshape(y.ReadWrite(), d1d_test, d1d_test, ne);
+
+   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
+      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
+
+      real_t u0[MAX_Q1D][MAX_Q1D];
+      real_t u1[MAX_Q1D][MAX_Q1D];
+
+      for (int qy = 0; qy < q1d; ++qy)
+      {
+         for (int qx = 0; qx < q1d; ++qx)
+         {
+            u0[qy][qx] = 0.0;
+            u1[qy][qx] = 0.0;
+         }
+      }
+
+      int osc = 0;
+      for (int dy = 0; dy < d1d; ++dy)
+      {
+         real_t mass_x[MAX_Q1D];
+         for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+         for (int dx = 0; dx < d1d - 1; ++dx)
+         {
+            const real_t t = X(dx + (dy * (d1d - 1)) + osc, e);
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bo(qx, dx); }
+         }
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            const real_t wy = Bc(qy, dy);
+            for (int qx = 0; qx < q1d; ++qx) { u0[qy][qx] += mass_x[qx] * wy; }
+         }
+      }
+
+      osc += (d1d - 1) * d1d;
+      for (int dy = 0; dy < d1d - 1; ++dy)
+      {
+         real_t mass_x[MAX_Q1D];
+         for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+         for (int dx = 0; dx < d1d; ++dx)
+         {
+            const real_t t = X(dx + (dy * d1d) + osc, e);
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bc(qx, dx); }
+         }
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            const real_t wy = Bo(qy, dy);
+            for (int qx = 0; qx < q1d; ++qx) { u1[qy][qx] += mass_x[qx] * wy; }
+         }
+      }
+
+      for (int qy = 0; qy < q1d; ++qy)
+      {
+         real_t sol_x[MAX_D1D];
+         for (int dx = 0; dx < d1d_test; ++dx) { sol_x[dx] = 0.0; }
+         for (int qx = 0; qx < q1d; ++qx)
+         {
+            const real_t s = O(0, qx, qy, e) * u0[qy][qx]
+                             + O(1, qx, qy, e) * u1[qy][qx];
+            for (int dx = 0; dx < d1d_test; ++dx)
+            {
+               sol_x[dx] += s * Bt(dx, qx);
+            }
+         }
+         for (int dy = 0; dy < d1d_test; ++dy)
+         {
+            const real_t wy = Bt(dy, qy);
+            for (int dx = 0; dx < d1d_test; ++dx)
+            {
+               Y(dx, dy, e) += sol_x[dx] * wy;
+            }
+         }
+      }
+   });
+}
+
+void PAHcurlDotApplyTranspose2D(const int d1d,
+                                const int d1d_test,
+                                const int q1d,
+                                const int ne,
+                                const Array<real_t> &bo,
+                                const Array<real_t> &bc,
+                                const Array<real_t> &b,
+                                const Vector &pa_data,
+                                const Vector &x,
+                                Vector &y)
+{
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
+   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
+
+   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
+   auto Bc = Reshape(bc.Read(), q1d, d1d);
+   auto B = Reshape(b.Read(), q1d, d1d_test);
+   auto O = Reshape(pa_data.Read(), 2, q1d, q1d, ne);
+   auto X = Reshape(x.Read(), d1d_test, d1d_test, ne);
+   auto Y = Reshape(y.ReadWrite(), 2 * (d1d - 1) * d1d, ne);
+
+   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
+      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
+
+      real_t mass[MAX_Q1D][MAX_Q1D];
+      for (int qy = 0; qy < q1d; ++qy)
+      {
+         for (int qx = 0; qx < q1d; ++qx)
+         {
+            mass[qy][qx] = 0.0;
+         }
+      }
+
+      for (int dy = 0; dy < d1d_test; ++dy)
+      {
+         real_t sol_x[MAX_Q1D];
+         for (int qx = 0; qx < q1d; ++qx) { sol_x[qx] = 0.0; }
+         for (int dx = 0; dx < d1d_test; ++dx)
+         {
+            const real_t t = X(dx, dy, e);
+            for (int qx = 0; qx < q1d; ++qx) { sol_x[qx] += t * B(qx, dx); }
+         }
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            const real_t wy = B(qy, dy);
+            for (int qx = 0; qx < q1d; ++qx) { mass[qy][qx] += sol_x[qx] * wy; }
+         }
+      }
+
+      int osc = 0;
+      for (int qy = 0; qy < q1d; ++qy)
+      {
+         real_t mass_x[MAX_D1D];
+         for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] = 0.0; }
+         for (int qx = 0; qx < q1d; ++qx)
+         {
+            const real_t s = O(0, qx, qy, e) * mass[qy][qx];
+            for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] += s * Bo(qx, dx); }
+         }
+         for (int dy = 0; dy < d1d; ++dy)
+         {
+            const real_t wy = Bc(qy, dy);
+            for (int dx = 0; dx < d1d - 1; ++dx)
+            {
+               Y(dx + (dy * (d1d - 1)) + osc, e) += mass_x[dx] * wy;
+            }
+         }
+      }
+
+      osc += (d1d - 1) * d1d;
+      for (int qy = 0; qy < q1d; ++qy)
+      {
+         real_t mass_x[MAX_D1D];
+         for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] = 0.0; }
+         for (int qx = 0; qx < q1d; ++qx)
+         {
+            const real_t s = O(1, qx, qy, e) * mass[qy][qx];
+            for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] += s * Bc(qx, dx); }
+         }
+         for (int dy = 0; dy < d1d - 1; ++dy)
+         {
+            const real_t wy = Bo(qy, dy);
+            for (int dx = 0; dx < d1d; ++dx)
+            {
+               Y(dx + (dy * d1d) + osc, e) += mass_x[dx] * wy;
+            }
+         }
+      }
+   });
+}
+
+void PAHcurlDotApply3D(const int d1d,
+                       const int d1d_test,
+                       const int q1d,
+                       const int ne,
+                       const Array<real_t> &bo,
+                       const Array<real_t> &bc,
+                       const Array<real_t> &bt,
+                       const Vector &pa_data,
+                       const Vector &x,
+                       Vector &y)
+{
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
+   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
+
+   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
+   auto Bc = Reshape(bc.Read(), q1d, d1d);
+   auto Bt = Reshape(bt.Read(), d1d_test, q1d);
+   auto O = Reshape(pa_data.Read(), 3, q1d, q1d, q1d, ne);
+   auto X = Reshape(x.Read(), 3 * (d1d - 1) * d1d * d1d, ne);
+   auto Y = Reshape(y.ReadWrite(), d1d_test, d1d_test, d1d_test, ne);
+
+   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
+      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
+
+      real_t u[MAX_Q1D][MAX_Q1D][MAX_Q1D][3];
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx)
+            {
+               for (int c = 0; c < 3; ++c) { u[qz][qy][qx][c] = 0.0; }
+            }
+         }
+      }
+
+      int osc = 0;
+      for (int dz = 0; dz < d1d; ++dz)
+      {
+         real_t mass_xy[MAX_Q1D][MAX_Q1D];
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
+         }
+
+         for (int dy = 0; dy < d1d; ++dy)
+         {
+            real_t mass_x[MAX_Q1D];
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+            for (int dx = 0; dx < d1d - 1; ++dx)
+            {
+               const real_t t = X(dx + ((dy + (dz * d1d)) * (d1d - 1)) + osc, e);
+               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bo(qx, dx); }
+            }
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               const real_t wy = Bc(qy, dy);
+               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
+            }
+         }
+
+         for (int qz = 0; qz < q1d; ++qz)
+         {
+            const real_t wz = Bc(qz, dz);
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               for (int qx = 0; qx < q1d; ++qx) { u[qz][qy][qx][0] += mass_xy[qy][qx] * wz; }
+            }
+         }
+      }
+
+      osc += (d1d - 1) * d1d * d1d;
+      for (int dz = 0; dz < d1d; ++dz)
+      {
+         real_t mass_xy[MAX_Q1D][MAX_Q1D];
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
+         }
+
+         for (int dy = 0; dy < d1d - 1; ++dy)
+         {
+            real_t mass_x[MAX_Q1D];
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+            for (int dx = 0; dx < d1d; ++dx)
+            {
+               const real_t t = X(dx + ((dy + (dz * (d1d - 1))) * d1d) + osc, e);
+               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bc(qx, dx); }
+            }
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               const real_t wy = Bo(qy, dy);
+               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
+            }
+         }
+
+         for (int qz = 0; qz < q1d; ++qz)
+         {
+            const real_t wz = Bc(qz, dz);
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               for (int qx = 0; qx < q1d; ++qx) { u[qz][qy][qx][1] += mass_xy[qy][qx] * wz; }
+            }
+         }
+      }
+
+      osc += (d1d - 1) * d1d * d1d;
+      for (int dz = 0; dz < d1d - 1; ++dz)
+      {
+         real_t mass_xy[MAX_Q1D][MAX_Q1D];
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
+         }
+
+         for (int dy = 0; dy < d1d; ++dy)
+         {
+            real_t mass_x[MAX_Q1D];
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+            for (int dx = 0; dx < d1d; ++dx)
+            {
+               const real_t t = X(dx + ((dy + (dz * d1d)) * d1d) + osc, e);
+               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bc(qx, dx); }
+            }
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               const real_t wy = Bc(qy, dy);
+               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
+            }
+         }
+
+         for (int qz = 0; qz < q1d; ++qz)
+         {
+            const real_t wz = Bo(qz, dz);
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               for (int qx = 0; qx < q1d; ++qx) { u[qz][qy][qx][2] += mass_xy[qy][qx] * wz; }
+            }
+         }
+      }
+
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         real_t mass_xy[MAX_D1D][MAX_D1D];
+         for (int dy = 0; dy < d1d_test; ++dy)
+         {
+            for (int dx = 0; dx < d1d_test; ++dx) { mass_xy[dy][dx] = 0.0; }
+         }
+
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            real_t mass_x[MAX_D1D];
+            for (int dx = 0; dx < d1d_test; ++dx) { mass_x[dx] = 0.0; }
+            for (int qx = 0; qx < q1d; ++qx)
+            {
+               const real_t s = O(0, qx, qy, qz, e) * u[qz][qy][qx][0]
+                                + O(1, qx, qy, qz, e) * u[qz][qy][qx][1]
+                                + O(2, qx, qy, qz, e) * u[qz][qy][qx][2];
+               for (int dx = 0; dx < d1d_test; ++dx) { mass_x[dx] += s * Bt(dx, qx); }
+            }
+            for (int dy = 0; dy < d1d_test; ++dy)
+            {
+               const real_t wy = Bt(dy, qy);
+               for (int dx = 0; dx < d1d_test; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
+            }
+         }
+
+         for (int dz = 0; dz < d1d_test; ++dz)
+         {
+            const real_t wz = Bt(dz, qz);
+            for (int dy = 0; dy < d1d_test; ++dy)
+            {
+               for (int dx = 0; dx < d1d_test; ++dx)
+               {
+                  Y(dx, dy, dz, e) += mass_xy[dy][dx] * wz;
+               }
+            }
+         }
+      }
+   });
+}
+
+void PAHcurlDotApplyTranspose3D(const int d1d,
+                                const int d1d_test,
+                                const int q1d,
+                                const int ne,
+                                const Array<real_t> &bo,
+                                const Array<real_t> &bc,
+                                const Array<real_t> &b,
+                                const Vector &pa_data,
+                                const Vector &x,
+                                Vector &y)
+{
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
+   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
+
+   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
+   auto Bc = Reshape(bc.Read(), q1d, d1d);
+   auto B = Reshape(b.Read(), q1d, d1d_test);
+   auto O = Reshape(pa_data.Read(), 3, q1d, q1d, q1d, ne);
+   auto X = Reshape(x.Read(), d1d_test, d1d_test, d1d_test, ne);
+   auto Y = Reshape(y.ReadWrite(), 3 * (d1d - 1) * d1d * d1d, ne);
+
+   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
+      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
+
+      real_t mass[MAX_Q1D][MAX_Q1D][MAX_Q1D];
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx) { mass[qz][qy][qx] = 0.0; }
+         }
+      }
+
+      for (int dz = 0; dz < d1d_test; ++dz)
+      {
+         real_t mass_xy[MAX_Q1D][MAX_Q1D];
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
+         }
+
+         for (int dy = 0; dy < d1d_test; ++dy)
+         {
+            real_t mass_x[MAX_Q1D];
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+            for (int dx = 0; dx < d1d_test; ++dx)
+            {
+               const real_t t = X(dx, dy, dz, e);
+               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * B(qx, dx); }
+            }
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               const real_t wy = B(qy, dy);
+               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
+            }
+         }
+
+         for (int qz = 0; qz < q1d; ++qz)
+         {
+            const real_t wz = B(qz, dz);
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               for (int qx = 0; qx < q1d; ++qx) { mass[qz][qy][qx] += mass_xy[qy][qx] * wz; }
+            }
+         }
+      }
+
+      int osc = 0;
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         real_t mass_xy[MAX_D1D][MAX_D1D];
+         for (int dy = 0; dy < d1d; ++dy)
+         {
+            for (int dx = 0; dx < d1d - 1; ++dx) { mass_xy[dy][dx] = 0.0; }
+         }
+
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            real_t mass_x[MAX_D1D];
+            for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] = 0.0; }
+            for (int qx = 0; qx < q1d; ++qx)
+            {
+               const real_t s = O(0, qx, qy, qz, e) * mass[qz][qy][qx];
+               for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] += s * Bo(qx, dx); }
+            }
+            for (int dy = 0; dy < d1d; ++dy)
+            {
+               const real_t wy = Bc(qy, dy);
+               for (int dx = 0; dx < d1d - 1; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
+            }
+         }
+
+         for (int dz = 0; dz < d1d; ++dz)
+         {
+            const real_t wz = Bc(qz, dz);
+            for (int dy = 0; dy < d1d; ++dy)
+            {
+               for (int dx = 0; dx < d1d - 1; ++dx)
+               {
+                  Y(dx + ((dy + (dz * d1d)) * (d1d - 1)) + osc, e) += mass_xy[dy][dx] * wz;
+               }
+            }
+         }
+      }
+
+      osc += (d1d - 1) * d1d * d1d;
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         real_t mass_xy[MAX_D1D][MAX_D1D];
+         for (int dy = 0; dy < d1d - 1; ++dy)
+         {
+            for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] = 0.0; }
+         }
+
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            real_t mass_x[MAX_D1D];
+            for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] = 0.0; }
+            for (int qx = 0; qx < q1d; ++qx)
+            {
+               const real_t s = O(1, qx, qy, qz, e) * mass[qz][qy][qx];
+               for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] += s * Bc(qx, dx); }
+            }
+            for (int dy = 0; dy < d1d - 1; ++dy)
+            {
+               const real_t wy = Bo(qy, dy);
+               for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
+            }
+         }
+
+         for (int dz = 0; dz < d1d; ++dz)
+         {
+            const real_t wz = Bc(qz, dz);
+            for (int dy = 0; dy < d1d - 1; ++dy)
+            {
+               for (int dx = 0; dx < d1d; ++dx)
+               {
+                  Y(dx + ((dy + (dz * (d1d - 1))) * d1d) + osc, e) += mass_xy[dy][dx] * wz;
+               }
+            }
+         }
+      }
+
+      osc += (d1d - 1) * d1d * d1d;
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         real_t mass_xy[MAX_D1D][MAX_D1D];
+         for (int dy = 0; dy < d1d; ++dy)
+         {
+            for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] = 0.0; }
+         }
+
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            real_t mass_x[MAX_D1D];
+            for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] = 0.0; }
+            for (int qx = 0; qx < q1d; ++qx)
+            {
+               const real_t s = O(2, qx, qy, qz, e) * mass[qz][qy][qx];
+               for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] += s * Bc(qx, dx); }
+            }
+            for (int dy = 0; dy < d1d; ++dy)
+            {
+               const real_t wy = Bc(qy, dy);
+               for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
+            }
+         }
+
+         for (int dz = 0; dz < d1d - 1; ++dz)
+         {
+            const real_t wz = Bo(qz, dz);
+            for (int dy = 0; dy < d1d; ++dy)
+            {
+               for (int dx = 0; dx < d1d; ++dx)
+               {
+                  Y(dx + ((dy + (dz * d1d)) * d1d) + osc, e) += mass_xy[dy][dx] * wz;
+               }
+            }
+         }
+      }
+   });
+}
+
+} // namespace
+
+void MixedDotProductIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
+                                           const FiniteElementSpace &test_fes)
+{
+   Mesh *mesh = trial_fes.GetMesh();
+   const FiniteElement *trial_fel = trial_fes.GetTypicalFE();
+   const FiniteElement *test_fel = test_fes.GetTypicalFE();
+
+   const VectorTensorFiniteElement *trial_el =
+      dynamic_cast<const VectorTensorFiniteElement*>(trial_fel);
+   MFEM_VERIFY(trial_el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   const TensorBasisElement *test_tensor_el =
+      dynamic_cast<const TensorBasisElement*>(test_fel);
+   MFEM_VERIFY(test_tensor_el != NULL,
+               "Only tensor-product scalar test elements are supported!");
+
+   MFEM_VERIFY(trial_el->GetDerivType() == mfem::FiniteElement::CURL,
+               "Only H(curl) trial spaces are supported!");
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == nullptr)
+   {
+      const int order = trial_fel->GetOrder() + test_fel->GetOrder()
+                        + mesh->GetTypicalElementTransformation()->OrderW();
+      ir = &IntRules.Get(trial_fel->GetGeomType(), order);
+   }
+
+   dim = mesh->Dimension();
+   MFEM_VERIFY(dim == 2 || dim == 3, "Unsupported dimension!");
+   MFEM_VERIFY(trial_el->GetDim() == dim && test_fel->GetDim() == dim,
+               "Trial/test dimension mismatch.");
+
+   ne = trial_fes.GetNE();
+   MFEM_VERIFY(ne == test_fes.GetNE(),
+               "Different meshes for test and trial spaces");
+
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   mapsC = &trial_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsO = &trial_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   mapsTest = &test_fel->GetDofToQuad(*ir, DofToQuad::TENSOR);
+
+   dofs1D = mapsC->ndof;
+   dofs1Dtest = mapsTest->ndof;
+   quad1D = mapsC->nqpt;
+   test_map_integral = (test_fel->GetMapType() == FiniteElement::INTEGRAL);
+
+   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
+   MFEM_VERIFY(quad1D == mapsTest->nqpt, "Trial/test quadrature mismatch");
+   MFEM_VERIFY(dofs1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
+   MFEM_VERIFY(dofs1Dtest <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(quad1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
+
+   const int nq = ir->GetNPoints();
+   if (dim == 2) { MFEM_VERIFY(nq == quad1D * quad1D, ""); }
+   else { MFEM_VERIFY(nq == quad1D * quad1D * quad1D, ""); }
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(*VQ, qs, CoefficientStorage::FULL);
+   MFEM_VERIFY(coeff.GetVDim() == dim, "Vector coefficient dimension mismatch.");
+
+   pa_data.SetSize(dim * nq * ne, Device::GetMemoryType());
+
+   if (dim == 2)
+   {
+      PAHcurlDotSetup2D(quad1D, ne, test_map_integral, ir->GetWeights(),
+                        geom->J, coeff, pa_data);
+   }
+   else
+   {
+      PAHcurlDotSetup3D(quad1D, ne, test_map_integral, ir->GetWeights(),
+                        geom->J, coeff, pa_data);
+   }
+}
+
+void MixedDotProductIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (dim == 2)
+   {
+      PAHcurlDotApply2D(dofs1D, dofs1Dtest, quad1D, ne,
+                        mapsO->B, mapsC->B, mapsTest->Bt, pa_data, x, y);
+   }
+   else if (dim == 3)
+   {
+      PAHcurlDotApply3D(dofs1D, dofs1Dtest, quad1D, ne,
+                        mapsO->B, mapsC->B, mapsTest->Bt, pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
+void MixedDotProductIntegrator::AddMultTransposePA(const Vector &x,
+                                                   Vector &y) const
+{
+   if (dim == 2)
+   {
+      PAHcurlDotApplyTranspose2D(dofs1D, dofs1Dtest, quad1D, ne,
+                                 mapsO->B, mapsC->B, mapsTest->B,
+                                 pa_data, x, y);
+   }
+   else if (dim == 3)
+   {
+      PAHcurlDotApplyTranspose3D(dofs1D, dofs1Dtest, quad1D, ne,
+                                 mapsO->B, mapsC->B, mapsTest->B,
+                                 pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
 void MixedScalarCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
                                           const FiniteElementSpace &test_fes)
 {
@@ -320,6 +1093,7 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
   CoefficientVector coeff(qs, CoefficientStorage::FULL);
   if (Q) { coeff.Project(*Q); }
   else if (DQ) { coeff.Project(*DQ); }
+   else if (MQ) { MFEM_ABORT("Not implemented."); }
   else { coeff.SetConstant(1.0); }

   if (trialType == mfem::FiniteElement::CURL && dim == 3)
@@ -0,0 +1,131 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
+#define MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
+
+#include "../../config/config.hpp"
+#include "../../general/array.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/vector.hpp"
+#include "../bilininteg.hpp"
+
+#include "bilininteg_diffusion_kernels.hpp"
+#include "bilininteg_hcurl_kernels.hpp"
+#include "bilininteg_hdiv_kernels.hpp"
+#include "bilininteg_hcurlhdiv_kernels.hpp"
+
+namespace mfem
+{
+/// \cond DO_NOT_DOCUMENT
+namespace internal
+{
+namespace hcurlmass
+{
+constexpr int NBZ3D(int d1d, int q1d)
+{
+   if (d1d <= 1 || q1d <= 0)
+   {
+      return 1;
+   }
+   // assume q1d >= d1d
+   // z dimension is capped at 64 on nvidia and amd gpus
+   int tmp = std::min((128 + q1d * q1d * q1d - 1) / (q1d * q1d * q1d), 64);
+   int smem_req =
+      sizeof(mfem::real_t) *
+      (3 * ((d1d - 1) * d1d * d1d + 2 * q1d * q1d * q1d) * tmp +
+       q1d * (d1d - 1) + q1d * d1d);
+   // assume GPU has at least 48k shared memory
+   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
+}
+} // namespace hcurlmass
+} // namespace internal
+
+template <FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
+          int DIM, int TrialD1D, int TestD1D, int Q1D>
+VectorFEMassIntegrator::ApplyKernelType
+VectorFEMassIntegrator::ApplyPAKernels::Kernel()
+{
+   constexpr bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
+   constexpr bool trial_div = (TrialType == mfem::FiniteElement::DIV);
+   constexpr bool test_curl = (TestType == mfem::FiniteElement::CURL);
+   constexpr bool test_div = (TestType == mfem::FiniteElement::DIV);
+
+   if constexpr (DIM == 3)
+   {
+      if constexpr (trial_curl && test_curl)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            // assume TrialD1D == TestD1D
+            return internal::SmemPAHcurlMassApply3D<
+                   TrialD1D, Q1D, internal::hcurlmass::NBZ3D(TrialD1D, Q1D)>;
+         }
+         else
+         {
+            return internal::PAHcurlMassApply3D;
+         }
+      }
+      else if constexpr (trial_div && test_div)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            // assumes TrialD1D == TestD1D
+            return internal::SmemPAHdivMassApply3D<TrialD1D, Q1D>;
+         }
+         else
+         {
+            return internal::PAHdivMassApply3D;
+         }
+      }
+      else if constexpr (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply3D;
+      }
+      else if constexpr (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply3D;
+      }
+   }
+   else if constexpr (DIM == 2) // 2D
+   {
+      if constexpr (trial_curl && test_curl)
+      {
+         return internal::PAHcurlMassApply2D;
+      }
+      else if constexpr (trial_div && test_div)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            // assumes TrialD1D == TestD1D
+            return internal::SmemPAHdivMassApply2D<TrialD1D, Q1D>;
+         }
+         else
+         {
+            return internal::PAHdivMassApply2D;
+         }
+      }
+      else if constexpr (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply2D;
+      }
+      else if constexpr (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply2D;
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+/// \endcond DO_NOT_DOCUMENT
+}
+
+#endif
@@ -12,13 +12,115 @@
 #include "../bilininteg.hpp"
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
-#include "bilininteg_diffusion_kernels.hpp"
-#include "bilininteg_hcurl_kernels.hpp"
-#include "bilininteg_hdiv_kernels.hpp"
-#include "bilininteg_hcurlhdiv_kernels.hpp"
+#include "bilininteg_vectorfemass_kernels.hpp"

 namespace mfem
 {
+/// \cond DO_NOT_DOCUMENT
+VectorFEMassIntegrator::ApplyKernelType
+VectorFEMassIntegrator::ApplyPAKernels::Fallback(
+   FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
+   int dim, int, int, int)
+{
+   const bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
+   const bool trial_div = (TrialType == mfem::FiniteElement::DIV);
+   const bool test_curl = (TestType == mfem::FiniteElement::CURL);
+   const bool test_div = (TestType == mfem::FiniteElement::DIV);
+
+   if (dim == 3)
+   {
+      if (trial_curl && test_curl)
+      {
+         return internal::PAHcurlMassApply3D;
+      }
+      else if (trial_div && test_div)
+      {
+         return internal::PAHdivMassApply3D;
+      }
+      else if (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply3D;
+      }
+      else if (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply3D;
+      }
+   }
+   else if (dim == 2) // 2D
+   {
+      if (trial_curl && test_curl)
+      {
+         return internal::PAHcurlMassApply2D;
+      }
+      else if (trial_div && test_div)
+      {
+         return internal::PAHdivMassApply2D;
+      }
+      else if (trial_curl && test_div)
+      {
+         return internal::PAHdivHcurlMassApply2D;
+      }
+      else if (trial_div && test_curl)
+      {
+         return internal::PAHcurlHdivMassApply2D;
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+/// \endcond DO_NOT_DOCUMENT
+
+VectorFEMassIntegrator::Kernels::Kernels()
+{
+   // h(curl), h(curl)
+   // P = Q (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 2, 2, 2>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 3, 3, 3>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 4, 4, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 5, 5, 5>();
+
+   // P = Q + 1 (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 2, 2, 3>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 3, 3, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 4, 4, 5>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 5, 5, 6>();
+   // h(div), h(div)
+   // P = Q (2D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 2, 2, 2>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 3, 3, 3>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 4, 4, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 2, 5, 5, 5>();
+
+   // P = Q + 1 (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 2, 2, 3>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 3, 3, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 4, 4, 5>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
+                          FiniteElement::DIV, 3, 5, 5, 6>();
+}
+
+void VectorFEMassIntegrator::Init(Coefficient *q, DiagonalMatrixCoefficient *dq,
+                                  MatrixCoefficient *mq)
+{
+   static Kernels kernels{};
+   Q = q;
+   DQ = dq;
+   MQ = mq;
+}

 void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
@@ -67,8 +169,8 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");

-   trial_fetype = trial_el->GetDerivType();
-   test_fetype = test_el->GetDerivType();
+   trial_fetype = static_cast<FiniteElement::DerivType>(trial_el->GetDerivType());
+   test_fetype = static_cast<FiniteElement::DerivType>(test_el->GetDerivType());

   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
@@ -215,225 +317,36 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)

 void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
-   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
-   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
-   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
-
-   if (dim == 3)
-   {
-      if (trial_curl && test_curl)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            const int ID = (dofs1D << 4) | quad1D;
-            switch (ID)
-            {
-               case 0x23:
-                  return internal::SmemPAHcurlMassApply3D<2,3>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               case 0x34:
-                  return internal::SmemPAHcurlMassApply3D<3,4>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               case 0x45:
-                  return internal::SmemPAHcurlMassApply3D<4,5>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               case 0x56:
-                  return internal::SmemPAHcurlMassApply3D<5,6>(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-               default:
-                  return internal::SmemPAHcurlMassApply3D(
-                            dofs1D, quad1D, ne, symmetric,
-                            mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
-            }
-         }
-         else
-         {
-            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                         mapsO->Bt, mapsC->Bt, pa_data, x, y);
-         }
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                   mapsO->Bt, mapsC->Bt, pa_data, x, y);
-      }
-      else if (trial_curl && test_div)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                          true, false, mapsO->B, mapsC->B, mapsOtest->Bt,
-                                          mapsCtest->Bt, pa_data, x, y);
-      }
-      else if (trial_div && test_curl)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                          false, false, mapsO->B, mapsC->B, mapsOtest->Bt,
-                                          mapsCtest->Bt, pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
-   else // 2D
-   {
-      if (trial_curl && test_curl)
-      {
-         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                      mapsO->Bt, mapsC->Bt, pa_data, x, y);
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                                   mapsO->Bt,
-                                   mapsC->Bt, pa_data, x, y);
-      }
-      else if ((trial_curl && test_div) || (trial_div && test_curl))
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                          trial_curl, false, mapsO->B, mapsC->B,
-                                          mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
+   const bool scalar_coeff = !(DQ || MQ);
+   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
+                       quad1D, ne, symmetric, scalar_coeff, mapsO->B, mapsC->B,
+                       mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y, dofs1D,
+                       dofs1Dtest, quad1D);
 }

 void VectorFEMassIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
 {
-   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
-   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
-   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
-   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
+   const bool scalar_coeff = !(DQ || MQ);

   Vector abs_pa_data(pa_data);
   abs_pa_data.Abs();

   Array<real_t> absBo(mapsO->B);
   Array<real_t> absBc(mapsC->B);
-   Array<real_t> absBto(mapsO->Bt);
-   Array<real_t> absBtc(mapsC->Bt);
+   // Array<real_t> absBto(mapsO->Bt);
+   // Array<real_t> absBtc(mapsC->Bt);
   Array<real_t> absBto_t(mapsOtest->Bt);
   Array<real_t> absBtc_t(mapsCtest->Bt);

   absBo.Abs();
   absBc.Abs();
-   absBto.Abs();
-   absBtc.Abs();
   absBto_t.Abs();
   absBtc_t.Abs();

-   if (dim == 3)
-   {
-      if (trial_curl && test_curl)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            const int ID = (dofs1D << 4) | quad1D;
-            switch (ID)
-            {
-               case 0x23:
-                  return internal::SmemPAHcurlMassApply3D<2,3>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               case 0x34:
-                  return internal::SmemPAHcurlMassApply3D<3,4>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               case 0x45:
-                  return internal::SmemPAHcurlMassApply3D<4,5>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               case 0x56:
-                  return internal::SmemPAHcurlMassApply3D<5,6>(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-               default:
-                  return internal::SmemPAHcurlMassApply3D(
-                            dofs1D, quad1D, ne, symmetric,
-                            absBo, absBc, absBto, absBtc,
-                            abs_pa_data, x, y);
-            }
-         }
-         else
-         {
-            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric,
-                                         absBo, absBc, absBto, absBtc,
-                                         abs_pa_data, x, y);
-         }
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric,
-                                   absBo, absBc, absBto, absBtc,
-                                   abs_pa_data, x, y);
-      }
-      else if (trial_curl && test_div)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
-                                          scalarCoeff, true, false,
-                                          absBo, absBc, absBto_t, absBtc_t,
-                                          abs_pa_data, x, y);
-      }
-      else if (trial_div && test_curl)
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
-                                          scalarCoeff, false, false,
-                                          absBo, absBc, absBto_t, absBtc_t,
-                                          abs_pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
-   else // 2D
-   {
-      if (trial_curl && test_curl)
-      {
-         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric,
-                                      absBo, absBc, absBto, absBtc,
-                                      abs_pa_data, x, y);
-      }
-      else if (trial_div && test_div)
-      {
-         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric,
-                                   absBo, absBc, absBto, absBtc,
-                                   abs_pa_data, x, y);
-      }
-      else if ((trial_curl && test_div) || (trial_div && test_curl))
-      {
-         const bool scalarCoeff = !(DQ || MQ);
-         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne,
-                                          scalarCoeff, trial_curl, false,
-                                          absBo, absBc, absBto_t, absBtc_t,
-                                          abs_pa_data, x, y);
-      }
-      else
-      {
-         MFEM_ABORT("Unknown kernel.");
-      }
-   }
+   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
+                       quad1D, ne, symmetric, scalar_coeff, absBo, absBc,
+                       absBto_t, absBtc_t, abs_pa_data, x, y, dofs1D,
+                       dofs1Dtest, quad1D);
 }

 void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,
@@ -307,6 +307,506 @@ DomainLFIntegrator::AssembleKernels::Kernel()
   MFEM_ABORT("");
 }

+template <int T_D1D = 0, int T_Q1D = 0>
+static void HdivDLFAssemble2D(const int ne, const Array<int> &markers,
+                              const Vector &jac, const Array<real_t> &weights,
+                              const Array<real_t> &testBO,
+                              const Array<real_t> &testBC, const Vector &coeff,
+                              Vector &y, const int d, const int q)
+{
+   MFEM_VERIFY(T_D1D || d <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
+               "Problem size too large.");
+   MFEM_VERIFY(T_Q1D || q <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
+               "Problem size too large.");
+   MFEM_VERIFY(y.Size() == 2 * (d - 1) * d * ne, "");
+
+   constexpr int vdim = 2;
+   const auto F = coeff.Read();
+   const auto M = markers.Read();
+   const auto BO = Reshape(testBO.Read(), q, d-1);
+   const auto BC = Reshape(testBC.Read(), q, d);
+   const auto J = Reshape(jac.Read(), q, q, vdim, vdim, ne);
+   const auto W = Reshape(weights.Read(), q, q);
+   const bool cst = coeff.Size() == vdim;
+   const auto C = cst ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,ne);
+   auto Y = y.ReadWrite();
+
+   mfem::forall_3D(ne, q, q, vdim, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int vdim = 2;
+      if (M[e] == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
+
+      MFEM_SHARED real_t sBot[Q*D];
+      MFEM_SHARED real_t sBct[Q*D];
+      MFEM_SHARED real_t sQQ[vdim*Q*Q];
+      MFEM_SHARED real_t sQD[vdim*Q*D];
+
+      // Bo and Bc into shared memory
+      const DeviceMatrix Bot(sBot, d-1, q);
+      kernels::internal::LoadB<D,Q>(d-1, q, BO, sBot);
+      const DeviceMatrix Bct(sBct, d, q);
+      kernels::internal::LoadB<D,Q>(d, q, BC, sBct);
+
+      const DeviceCube QQ(sQQ, q, q, vdim);
+      const DeviceCube QD(sQD, q, d, vdim);
+
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const real_t cst_val_0 = C(0,0,0,0);
+         const real_t cst_val_1 = C(1,0,0,0);
+         MFEM_FOREACH_THREAD(y,y,q)
+         {
+            MFEM_FOREACH_THREAD(x,x,q)
+            {
+               const real_t J0 = J(x,y,0,vd,e);
+               const real_t J1 = J(x,y,1,vd,e);
+               const real_t C0 = cst ? cst_val_0 : C(0,x,y,e);
+               const real_t C1 = cst ? cst_val_1 : C(1,x,y,e);
+               QQ(x,y,vd) = W(x,y)*(J0*C0 + J1*C1);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         DeviceMatrix Btx = (vd == 0) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(qy,y,q)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               real_t qd = 0.0;
+               for (int qx = 0; qx < q; ++qx)
+               {
+                  qd += QQ(qx,qy,vd) * Btx(dx,qx);
+               }
+               QD(dx,qy,vd) = qd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         const int ny = (vd == 1) ? d : d-1;
+         DeviceMatrix Bty = (vd == 1) ? Bct : Bot;
+         DeviceTensor<4> Yxy(Y, nx, ny, vdim, ne);
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               real_t dd = 0.0;
+               for (int qy = 0; qy < q; ++qy)
+               {
+                  dd += QD(dx,qy,vd) * Bty(dy,qy);
+               }
+               Yxy(dx,dy,vd,e) += dd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+template <int T_D1D = 0, int T_Q1D = 0>
+static void HdivDLFAssemble3D(const int ne, const Array<int> &markers,
+                              const Vector &jac, const Array<real_t> &weights,
+                              const Array<real_t> &testBO,
+                              const Array<real_t> &testBC, const Vector &coeff,
+                              Vector &y, const int d, const int q)
+{
+   MFEM_VERIFY(T_D1D || d <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
+               "Problem size too large.");
+   MFEM_VERIFY(T_Q1D || q <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
+               "Problem size too large.");
+   MFEM_VERIFY(y.Size() == 3 * (d - 1) * (d - 1) * d * ne, "y wrong length");
+
+   constexpr int vdim = 3;
+   const auto F = coeff.Read();
+   const auto M = markers.Read();
+   const auto BO = Reshape(testBO.Read(), q, d-1);
+   const auto BC = Reshape(testBC.Read(), q, d);
+   const auto J = Reshape(jac.Read(), q, q, q, vdim, vdim, ne);
+   const auto W = Reshape(weights.Read(), q, q, q);
+   const bool cst = coeff.Size() == vdim;
+   const auto C = cst ? Reshape(F,vdim,1,1,1,1) : Reshape(F,vdim,q,q,q,ne);
+   auto Y = y.ReadWrite();
+
+   mfem::forall_3D(ne, q, q, vdim, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int vdim = 3;
+      if (M[e] == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
+
+      MFEM_SHARED real_t sBot[Q*D];
+      MFEM_SHARED real_t sBct[Q*D];
+
+      // Bo and Bc into shared memory
+      const DeviceMatrix Bot(sBot, d-1, q);
+      kernels::internal::LoadB<D,Q>(d-1, q, BO, sBot);
+      const DeviceMatrix Bct(sBct, d, q);
+      kernels::internal::LoadB<D,Q>(d, q, BC, sBct);
+
+      MFEM_SHARED real_t sm0[vdim*Q*Q*Q];
+      MFEM_SHARED real_t sm1[vdim*Q*Q*Q];
+      DeviceTensor<4> QQQ(sm1, q, q, q, vdim);
+      DeviceTensor<4> DQQ(sm0, d, q, q, vdim);
+      DeviceTensor<4> DDQ(sm1, d, d, q, vdim);
+
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const real_t cst_val_0 = C(0,0,0,0,0);
+         const real_t cst_val_1 = C(1,0,0,0,0);
+         const real_t cst_val_2 = C(2,0,0,0,0);
+         MFEM_FOREACH_THREAD(y,y,q)
+         {
+            MFEM_FOREACH_THREAD(x,x,q)
+            {
+               for (int z = 0; z < q; ++z)
+               {
+                  const real_t J0 = J(x,y,z,0,vd,e);
+                  const real_t J1 = J(x,y,z,1,vd,e);
+                  const real_t J2 = J(x,y,z,2,vd,e);
+                  const real_t C0 = cst ? cst_val_0 : C(0,x,y,z,e);
+                  const real_t C1 = cst ? cst_val_1 : C(1,x,y,z,e);
+                  const real_t C2 = cst ? cst_val_2 : C(2,x,y,z,e);
+                  QQQ(x,y,z,vd) = W(x,y,z)*(J0*C0 + J1*C1 + J2*C2);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply Bt operator
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         DeviceMatrix Btx = (vd == 0) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(qy,y,q)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               real_t u[Q];
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(Q)
+               for (int qx = 0; qx < q; ++qx)
+               {
+                  MFEM_UNROLL(Q)
+                  for (int qz = 0; qz < q; ++qz)
+                  {
+                     u[qz] += QQQ(qx,qy,qz,vd) * Btx(dx,qx);
+                  }
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { DQQ(dx,qy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         const int ny = (vd == 1) ? d : d-1;
+         DeviceMatrix Bty = (vd == 1) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               real_t u[Q];
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(Q)
+               for (int qy = 0; qy < q; ++qy)
+               {
+                  MFEM_UNROLL(Q)
+                  for (int qz = 0; qz < q; ++qz)
+                  {
+                     u[qz] += DQQ(dx,qy,qz,vd) * Bty(dy,qy);
+                  }
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { DDQ(dx,dy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         const int ny = (vd == 1) ? d : d-1;
+         const int nz = (vd == 2) ? d : d-1;
+         DeviceTensor<5> Yxyz(Y, nx, ny, nz, vdim, ne);
+         DeviceMatrix Btz = (vd == 2) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               real_t u[D];
+               MFEM_UNROLL(D)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  MFEM_UNROLL(D)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += DDQ(dx,dy,qz,vd) * Btz(dz,qz);
+                  }
+               }
+               MFEM_UNROLL(D)
+               for (int dz = 0; dz < nz; ++dz) { Yxyz(dx,dy,dz,vd,e) += u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+/// @param ne number of elements
+/// @param markers array where entry markers[e] == 0 to skip assembly over
+/// element e element
+/// @param jac Spatial Jacobians evaluated at all quadrature points
+/// @param weights 1D quadrature weights
+/// @param testBO 1D open basis test functions
+/// @param testBC 1D closed basis test functions
+/// @param coeff coefficient values evaluated at quadrature points, possibly
+/// compressed.
+/// @param d number of 1D closed dofs
+/// @param q number of 1D quadrature points
+/// @tparam T_D1D maximum number of dofs along any direction, or 0
+/// @tparam T_Q1D maximum number of quadrature points along any direction, or 0
+template <int T_D1D = 0, int T_Q1D = 0>
+static void HcurlDLFAssemble3D(const int ne, const Array<int> &markers,
+                               const Vector &jac, const Array<real_t> &weights,
+                               const Array<real_t> &testBO,
+                               const Array<real_t> &testBC, const Vector &coeff,
+                               Vector &y, const int d, const int q)
+{
+   MFEM_VERIFY(T_D1D || d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
+               "Problem size too large.");
+   MFEM_VERIFY(T_Q1D || q <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
+               "Problem size too large.");
+   MFEM_VERIFY(y.Size() == 3 * (d - 1) * d * d * ne, "y wrong length");
+
+   constexpr int vdim = 3;
+   const auto F = coeff.Read();
+   const auto M = markers.Read();
+   const auto BO = Reshape(testBO.Read(), q, d-1);
+   const auto BC = Reshape(testBC.Read(), q, d);
+   const auto J = Reshape(jac.Read(), q, q, q, vdim, vdim, ne);
+   const auto W = Reshape(weights.Read(), q, q, q);
+   const bool cst = coeff.Size() == vdim;
+   const auto C = cst ? Reshape(F,vdim,1,1,1,1) : Reshape(F,vdim,q,q,q,ne);
+   auto Y = y.ReadWrite();
+
+   mfem::forall_3D(ne, q, q, vdim, [=] MFEM_HOST_DEVICE(int e)
+   {
+      if (M[e] == 0)
+      {
+         // ignore
+         return;
+      }
+
+      constexpr int vdim = 3;
+      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
+
+      MFEM_SHARED real_t sBot[Q * D];
+      MFEM_SHARED real_t sBct[Q * D];
+
+      // Bo and Bc into shared memory
+      const DeviceMatrix Bot(sBot, d - 1, q);
+      kernels::internal::LoadB<D, Q>(d - 1, q, BO, sBot);
+      const DeviceMatrix Bct(sBct, d, q);
+      kernels::internal::LoadB<D, Q>(d, q, BC, sBct);
+
+      MFEM_SHARED real_t sm0[vdim * Q * Q * Q];
+      MFEM_SHARED real_t sm1[vdim * Q * Q * Q];
+      DeviceTensor<4> QQQ(sm1, q, q, q, vdim);
+      DeviceTensor<4> DQQ(sm0, d, q, q, vdim);
+      DeviceTensor<4> DDQ(sm1, d, d, q, vdim);
+
+      const real_t cst_val_0 = C(0, 0, 0, 0, 0);
+      const real_t cst_val_1 = C(1, 0, 0, 0, 0);
+      const real_t cst_val_2 = C(2, 0, 0, 0, 0);
+
+      MFEM_FOREACH_THREAD(vd, z, vdim)
+      {
+         MFEM_FOREACH_THREAD(y, y, q)
+         {
+            MFEM_FOREACH_THREAD(x, x, q)
+            {
+               for (int z = 0; z < q; ++z)
+               {
+                  real_t curr[3];
+                  curr[0] = cst ? cst_val_0 : C(0, x, y, z, e);
+                  curr[1] = cst ? cst_val_1 : C(1, x, y, z, e);
+                  curr[2] = cst ? cst_val_2 : C(2, x, y, z, e);
+
+                  const real_t J11 = J(x, y, z, 0, 0, e);
+                  const real_t J21 = J(x, y, z, 1, 0, e);
+                  const real_t J31 = J(x, y, z, 2, 0, e);
+                  const real_t J12 = J(x, y, z, 0, 1, e);
+                  const real_t J22 = J(x, y, z, 1, 1, e);
+                  const real_t J32 = J(x, y, z, 2, 1, e);
+                  const real_t J13 = J(x, y, z, 0, 2, e);
+                  const real_t J23 = J(x, y, z, 1, 2, e);
+                  const real_t J33 = J(x, y, z, 2, 2, e);
+                  // adj(J)
+                  const real_t A11 = (J22 * J33) - (J23 * J32);
+                  const real_t A12 = (J32 * J13) - (J12 * J33);
+                  const real_t A13 = (J12 * J23) - (J22 * J13);
+                  const real_t A21 = (J31 * J23) - (J21 * J33);
+                  const real_t A22 = (J11 * J33) - (J13 * J31);
+                  const real_t A23 = (J21 * J13) - (J11 * J23);
+                  const real_t A31 = (J21 * J32) - (J31 * J22);
+                  const real_t A32 = (J31 * J12) - (J11 * J32);
+                  const real_t A33 = (J11 * J22) - (J12 * J21);
+                  const real_t A[9] = {A11, A12, A13, A21, A22,
+                                       A23, A31, A32, A33
+                                      };
+                  QQQ(x, y, z, vd) = W(x, y, z) * (A[vd * vdim] * curr[0] +
+                                                   A[vd * vdim + 1] * curr[1] +
+                                                   A[vd * vdim + 2] * curr[2]);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply Bt operator
+      MFEM_FOREACH_THREAD(vd, z, vdim)
+      {
+         const int nx = (vd == 0) ? d - 1 : d;
+         DeviceMatrix Btx = (vd == 0) ? Bot : Bct;
+         MFEM_FOREACH_THREAD(qy, y, q)
+         {
+            MFEM_FOREACH_THREAD(dx, x, nx)
+            {
+               real_t u[Q];
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  u[qz] = 0.0;
+               }
+               MFEM_UNROLL(Q)
+               for (int qx = 0; qx < q; ++qx)
+               {
+                  MFEM_UNROLL(Q)
+                  for (int qz = 0; qz < q; ++qz)
+                  {
+                     u[qz] += QQQ(qx, qy, qz, vd) * Btx(dx, qx);
+                  }
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  DQQ(dx, qy, qz, vd) = u[qz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd, z, vdim)
+      {
+         const int nx = (vd == 0) ? d - 1 : d;
+         const int ny = (vd == 1) ? d - 1 : d;
+         DeviceMatrix Bty = (vd == 1) ? Bot : Bct;
+         MFEM_FOREACH_THREAD(dy, y, ny)
+         {
+            MFEM_FOREACH_THREAD(dx, x, nx)
+            {
+               real_t u[Q];
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  u[qz] = 0.0;
+               }
+               MFEM_UNROLL(Q)
+               for (int qy = 0; qy < q; ++qy)
+               {
+                  MFEM_UNROLL(Q)
+                  for (int qz = 0; qz < q; ++qz)
+                  {
+                     u[qz] += DQQ(dx, qy, qz, vd) * Bty(dy, qy);
+                  }
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  DDQ(dx, dy, qz, vd) = u[qz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd, z, vdim)
+      {
+         const int nx = (vd == 0) ? d - 1 : d;
+         const int ny = (vd == 1) ? d - 1 : d;
+         const int nz = (vd == 2) ? d - 1 : d;
+         DeviceTensor<5> Yxyz(Y, nx, ny, nz, vdim, ne);
+         DeviceMatrix Btz = (vd == 2) ? Bot : Bct;
+         MFEM_FOREACH_THREAD(dy, y, ny)
+         {
+            MFEM_FOREACH_THREAD(dx, x, nx)
+            {
+               real_t u[D];
+               MFEM_UNROLL(D)
+               for (int dz = 0; dz < nz; ++dz)
+               {
+                  u[dz] = 0.0;
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  MFEM_UNROLL(D)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += DDQ(dx, dy, qz, vd) * Btz(dz, qz);
+                  }
+               }
+               MFEM_UNROLL(D)
+               for (int dz = 0; dz < nz; ++dz)
+               {
+                  Yxyz(dx, dy, dz, vd, e) += u[dz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+template <FiniteElement::DerivType TestType, int DIM, int TEST_D1D, int Q1D>
+VectorFEDomainLFIntegrator::AssembleKernelType
+VectorFEDomainLFIntegrator::AssembleKernels::Kernel()
+{
+   if constexpr (TestType == FiniteElement::DIV)
+   {
+      if constexpr (DIM == 2)
+      {
+         return HdivDLFAssemble2D<TEST_D1D, Q1D>;
+      }
+      if constexpr (DIM == 3)
+      {
+         return HdivDLFAssemble3D<TEST_D1D, Q1D>;
+      }
+   }
+   if constexpr (TestType == FiniteElement::CURL)
+   {
+      if constexpr (DIM == 3)
+      {
+         return HcurlDLFAssemble3D<TEST_D1D, Q1D>;
+      }
+   }
+   MFEM_ABORT("");
+}
+
 /// \endcond DO_NOT_DOCUMENT

 } // namespace mfem
@@ -13,317 +13,76 @@
 #include "../../fem/kernels.hpp"
 #include "../fem.hpp"

+#include "lininteg_domain_kernels.hpp"
+
 namespace mfem
 {

-template<int T_D1D = 0, int T_Q1D = 0>
-static void HdivDLFAssemble2D(
-   const int ne, const int d, const int q, const int *markers, const real_t *bo,
-   const real_t *bc, const real_t *j, const real_t *weights,
-   const Vector &coeff, real_t *y)
+VectorFEDomainLFIntegrator::Kernels::Kernels()
 {
-   MFEM_VERIFY(T_D1D || d <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
-               "Problem size too large.");
-   MFEM_VERIFY(T_Q1D || q <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
-               "Problem size too large.");
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 1, 1>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 2, 2>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 3, 3>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 4, 4>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 5, 5>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 6, 6>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 7, 7>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 8, 8>();

-   static constexpr int vdim = 2;
-   const auto F = coeff.Read();
-   const auto M = Reshape(markers, ne);
-   const auto BO = Reshape(bo, q, d-1);
-   const auto BC = Reshape(bc, q, d);
-   const auto J = Reshape(j, q, q, vdim, vdim, ne);
-   const auto W = Reshape(weights, q, q);
-   const bool cst = coeff.Size() == vdim;
-   const auto C = cst ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,ne);
-   auto Y = Reshape(y, 2*(d-1)*d, ne);
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 1, 1>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 2, 2>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 3, 3>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 4, 4>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 5, 5>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 6, 6>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 7, 7>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 8, 8>();

-   mfem::forall_3D(ne, q, q, vdim, [=] MFEM_HOST_DEVICE (int e)
-   {
-      if (M(e) == 0) { return; } // ignore
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 1, 1>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 2, 2>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 3, 3>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 4, 4>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 5, 5>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 6, 6>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 7, 7>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 8, 8>();

-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
-
-      MFEM_SHARED real_t sBot[Q*D];
-      MFEM_SHARED real_t sBct[Q*D];
-      MFEM_SHARED real_t sQQ[vdim*Q*Q];
-      MFEM_SHARED real_t sQD[vdim*Q*D];
-
-      // Bo and Bc into shared memory
-      const DeviceMatrix Bot(sBot, d-1, q);
-      kernels::internal::LoadB<D,Q>(d-1, q, BO, sBot);
-      const DeviceMatrix Bct(sBct, d, q);
-      kernels::internal::LoadB<D,Q>(d, q, BC, sBct);
-
-      const DeviceCube QQ(sQQ, q, q, vdim);
-      const DeviceCube QD(sQD, q, d, vdim);
-
-      MFEM_FOREACH_THREAD(vd,z,vdim)
-      {
-         const real_t cst_val_0 = C(0,0,0,0);
-         const real_t cst_val_1 = C(1,0,0,0);
-         MFEM_FOREACH_THREAD(y,y,q)
-         {
-            MFEM_FOREACH_THREAD(x,x,q)
-            {
-               const real_t J0 = J(x,y,0,vd,e);
-               const real_t J1 = J(x,y,1,vd,e);
-               const real_t C0 = cst ? cst_val_0 : C(0,x,y,e);
-               const real_t C1 = cst ? cst_val_1 : C(1,x,y,e);
-               QQ(x,y,vd) = W(x,y)*(J0*C0 + J1*C1);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,vdim)
-      {
-         const int nx = (vd == 0) ? d : d-1;
-         DeviceMatrix Btx = (vd == 0) ? Bct : Bot;
-         MFEM_FOREACH_THREAD(qy,y,q)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               real_t qd = 0.0;
-               for (int qx = 0; qx < q; ++qx)
-               {
-                  qd += QQ(qx,qy,vd) * Btx(dx,qx);
-               }
-               QD(dx,qy,vd) = qd;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,vdim)
-      {
-         const int nx = (vd == 0) ? d : d-1;
-         const int ny = (vd == 1) ? d : d-1;
-         DeviceMatrix Bty = (vd == 1) ? Bct : Bot;
-         DeviceTensor<4> Yxy(Y, nx, ny, vdim, ne);
-         MFEM_FOREACH_THREAD(dy,y,ny)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               real_t dd = 0.0;
-               for (int qy = 0; qy < q; ++qy)
-               {
-                  dd += QD(dx,qy,vd) * Bty(dy,qy);
-               }
-               Yxy(dx,dy,vd,e) += dd;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 1, 2>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 2, 3>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 3, 4>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 4, 5>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 5, 6>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 6, 7>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 7, 8>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 8, 9>();
 }

-template<int T_D1D = 0, int T_Q1D = 0>
-static void HdivDLFAssemble3D(
-   const int ne, const int d, const int q, const int *markers, const real_t *bo,
-   const real_t *bc, const real_t *j, const real_t *weights,
-   const Vector &coeff, real_t *y)
+/// \cond DO_NOT_DOCUMENT
+VectorFEDomainLFIntegrator::AssembleKernelType
+VectorFEDomainLFIntegrator::AssembleKernels::Fallback(
+   FiniteElement::DerivType TestType, int DIM, int, int)
 {
-   MFEM_VERIFY(T_D1D || d <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
-               "Problem size too large.");
-   MFEM_VERIFY(T_Q1D || q <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
-               "Problem size too large.");
-
-   static constexpr int vdim = 3;
-   const auto F = coeff.Read();
-   const auto M = Reshape(markers, ne);
-   const auto BO = Reshape(bo, q, d-1);
-   const auto BC = Reshape(bc, q, d);
-   const auto J = Reshape(j, q, q, q, vdim, vdim, ne);
-   const auto W = Reshape(weights, q, q, q);
-   const bool cst = coeff.Size() == vdim;
-   const auto C = cst ? Reshape(F,vdim,1,1,1,1) : Reshape(F,vdim,q,q,q,ne);
-   auto Y = Reshape(y, 2*(d-1)*(d-1)*d, ne);
-
-   mfem::forall_3D(ne, q, q, vdim, [=] MFEM_HOST_DEVICE (int e)
+   if (TestType == FiniteElement::DIV)
   {
-      if (M(e) == 0) { return; } // ignore
-
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
-
-      MFEM_SHARED real_t sBot[Q*D];
-      MFEM_SHARED real_t sBct[Q*D];
-
-      // Bo and Bc into shared memory
-      const DeviceMatrix Bot(sBot, d-1, q);
-      kernels::internal::LoadB<D,Q>(d-1, q, BO, sBot);
-      const DeviceMatrix Bct(sBct, d, q);
-      kernels::internal::LoadB<D,Q>(d, q, BC, sBct);
-
-      MFEM_SHARED real_t sm0[vdim*Q*Q*Q];
-      MFEM_SHARED real_t sm1[vdim*Q*Q*Q];
-      DeviceTensor<4> QQQ(sm1, q, q, q, vdim);
-      DeviceTensor<4> DQQ(sm0, d, q, q, vdim);
-      DeviceTensor<4> DDQ(sm1, d, d, q, vdim);
-
-      MFEM_FOREACH_THREAD(vd,z,vdim)
+      if (DIM == 2)
      {
-         const real_t cst_val_0 = C(0,0,0,0,0);
-         const real_t cst_val_1 = C(1,0,0,0,0);
-         const real_t cst_val_2 = C(2,0,0,0,0);
-         MFEM_FOREACH_THREAD(y,y,q)
-         {
-            MFEM_FOREACH_THREAD(x,x,q)
-            {
-               for (int z = 0; z < q; ++z)
-               {
-                  const real_t J0 = J(x,y,z,0,vd,e);
-                  const real_t J1 = J(x,y,z,1,vd,e);
-                  const real_t J2 = J(x,y,z,2,vd,e);
-                  const real_t C0 = cst ? cst_val_0 : C(0,x,y,z,e);
-                  const real_t C1 = cst ? cst_val_1 : C(1,x,y,z,e);
-                  const real_t C2 = cst ? cst_val_2 : C(2,x,y,z,e);
-                  QQQ(x,y,z,vd) = W(x,y,z)*(J0*C0 + J1*C1 + J2*C2);
-               }
-            }
-         }
+         return HdivDLFAssemble2D<0, 0>;
      }
-      MFEM_SYNC_THREAD;
-      // Apply Bt operator
-      MFEM_FOREACH_THREAD(vd,z,vdim)
+      if (DIM == 3)
      {
-         const int nx = (vd == 0) ? d : d-1;
-         DeviceMatrix Btx = (vd == 0) ? Bct : Bot;
-         MFEM_FOREACH_THREAD(qy,y,q)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               real_t u[Q];
-               MFEM_UNROLL(Q)
-               for (int qz = 0; qz < q; ++qz) { u[qz] = 0.0; }
-               MFEM_UNROLL(Q)
-               for (int qx = 0; qx < q; ++qx)
-               {
-                  MFEM_UNROLL(Q)
-                  for (int qz = 0; qz < q; ++qz)
-                  {
-                     u[qz] += QQQ(qx,qy,qz,vd) * Btx(dx,qx);
-                  }
-               }
-               MFEM_UNROLL(Q)
-               for (int qz = 0; qz < q; ++qz) { DQQ(dx,qy,qz,vd) = u[qz]; }
-            }
-         }
+         return HdivDLFAssemble3D<0, 0>;
      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,vdim)
-      {
-         const int nx = (vd == 0) ? d : d-1;
-         const int ny = (vd == 1) ? d : d-1;
-         DeviceMatrix Bty = (vd == 1) ? Bct : Bot;
-         MFEM_FOREACH_THREAD(dy,y,ny)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               real_t u[Q];
-               MFEM_UNROLL(Q)
-               for (int qz = 0; qz < q; ++qz) { u[qz] = 0.0; }
-               MFEM_UNROLL(Q)
-               for (int qy = 0; qy < q; ++qy)
-               {
-                  MFEM_UNROLL(Q)
-                  for (int qz = 0; qz < q; ++qz)
-                  {
-                     u[qz] += DQQ(dx,qy,qz,vd) * Bty(dy,qy);
-                  }
-               }
-               MFEM_UNROLL(Q)
-               for (int qz = 0; qz < q; ++qz) { DDQ(dx,dy,qz,vd) = u[qz]; }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,vdim)
-      {
-         const int nx = (vd == 0) ? d : d-1;
-         const int ny = (vd == 1) ? d : d-1;
-         const int nz = (vd == 2) ? d : d-1;
-         DeviceTensor<5> Yxyz(Y, nx, ny, nz, vdim, ne);
-         DeviceMatrix Btz = (vd == 2) ? Bct : Bot;
-         MFEM_FOREACH_THREAD(dy,y,ny)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               real_t u[D];
-               MFEM_UNROLL(D)
-               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
-               MFEM_UNROLL(Q)
-               for (int qz = 0; qz < q; ++qz)
-               {
-                  MFEM_UNROLL(D)
-                  for (int dz = 0; dz < nz; ++dz)
-                  {
-                     u[dz] += DDQ(dx,dy,qz,vd) * Btz(dz,qz);
-                  }
-               }
-               MFEM_UNROLL(D)
-               for (int dz = 0; dz < nz; ++dz) { Yxyz(dx,dy,dz,vd,e) += u[dz]; }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-static void HdivDLFAssemble(const FiniteElementSpace &fes,
-                            const IntegrationRule *ir,
-                            const Array<int> &markers,
-                            const Vector &coeff,
-                            Vector &y)
-{
-   Mesh &mesh = *fes.GetMesh();
-   const int dim = mesh.Dimension();
-   const FiniteElement *el = fes.GetTypicalFE();
-   const auto *vel = dynamic_cast<const VectorTensorFiniteElement *>(el);
-   MFEM_VERIFY(vel != nullptr, "Must be VectorTensorFiniteElement");
-   const MemoryType mt = Device::GetDeviceMemoryType();
-   const DofToQuad &maps_o = vel->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
-   const DofToQuad &maps_c = vel->GetDofToQuad(*ir, DofToQuad::TENSOR);
-   const int d = maps_c.ndof, q = maps_c.nqpt;
-   constexpr int flags = GeometricFactors::JACOBIANS;
-   const GeometricFactors *geom = mesh.GetGeometricFactors(*ir, flags, mt);
-   decltype(&HdivDLFAssemble2D<>) ker =
-      dim == 2 ? HdivDLFAssemble2D<> : HdivDLFAssemble3D<>;
-
-   if (dim==2)
-   {
-      if (d==1 && q==1) { ker=HdivDLFAssemble2D<1,1>; }
-      if (d==2 && q==2) { ker=HdivDLFAssemble2D<2,2>; }
-      if (d==3 && q==3) { ker=HdivDLFAssemble2D<3,3>; }
-      if (d==4 && q==4) { ker=HdivDLFAssemble2D<4,4>; }
-      if (d==5 && q==5) { ker=HdivDLFAssemble2D<5,5>; }
-      if (d==6 && q==6) { ker=HdivDLFAssemble2D<6,6>; }
-      if (d==7 && q==7) { ker=HdivDLFAssemble2D<7,7>; }
-      if (d==8 && q==8) { ker=HdivDLFAssemble2D<8,8>; }
   }
-
-   if (dim==3)
+   else if (TestType == FiniteElement::CURL)
   {
-      if (d==2 && q==2) { ker=HdivDLFAssemble3D<2,2>; }
-      if (d==3 && q==3) { ker=HdivDLFAssemble3D<3,3>; }
-      if (d==4 && q==4) { ker=HdivDLFAssemble3D<4,4>; }
-      if (d==5 && q==5) { ker=HdivDLFAssemble3D<5,5>; }
-      if (d==6 && q==6) { ker=HdivDLFAssemble3D<6,6>; }
-      if (d==7 && q==7) { ker=HdivDLFAssemble3D<7,7>; }
-      if (d==8 && q==8) { ker=HdivDLFAssemble3D<8,8>; }
+      if (DIM == 3)
+      {
+         return HcurlDLFAssemble3D<0, 0>;
+      }
   }
-
-   MFEM_VERIFY(ker, "No kernel ndof " << d << " nqpt " << q);
-
-   const int ne = mesh.GetNE();
-   const int *M = markers.Read();
-   const real_t *Bo = maps_o.B.Read();
-   const real_t *Bc = maps_c.B.Read();
-   const real_t *J = geom->J.Read();
-   const real_t *W = ir->GetWeights().Read();
-   real_t *Y = y.ReadWrite();
-   ker(ne, d, q, M, Bo, Bc, J, W, coeff, Y);
+   MFEM_ABORT("");
 }
+/// \endcond DO_NOT_DOCUMENT

 void VectorFEDomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
                                                const Array<int> &markers,
@@ -337,15 +96,23 @@ void VectorFEDomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
   QuadratureSpace qs(*fes.GetMesh(), *ir);
   CoefficientVector coeff(QF, qs, CoefficientStorage::COMPRESSED);

-   const int fe_type = fe.GetDerivType();
-   if (fe_type == FiniteElement::DIV)
-   {
-      HdivDLFAssemble(fes, ir, markers, coeff, b);
-   }
-   else
-   {
-      MFEM_ABORT("Not implemented.");
-   }
+   const FiniteElement::DerivType fe_type =
+      static_cast<FiniteElement::DerivType>(fe.GetDerivType());
+
+   Mesh &mesh = *fes.GetMesh();
+   const int dim = mesh.Dimension();
+   const FiniteElement *el = fes.GetTypicalFE();
+   const auto *vel = dynamic_cast<const VectorTensorFiniteElement *>(el);
+   MFEM_VERIFY(vel != nullptr, "Must be VectorTensorFiniteElement");
+   const MemoryType mt = Device::GetDeviceMemoryType();
+   const DofToQuad &maps_o = vel->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   const DofToQuad &maps_c = vel->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   const int d = maps_c.ndof, q = maps_c.nqpt;
+   constexpr int flags = GeometricFactors::JACOBIANS;
+   const GeometricFactors *geom = mesh.GetGeometricFactors(*ir, flags, mt);
+
+   AssembleKernels::Run(fe_type, dim, d, q, mesh.GetNE(), markers, geom->J,
+                        ir->GetWeights(), maps_o.B, maps_c.B, coeff, b, d, q);
 }

 } // namespace mfem
@@ -236,6 +236,58 @@ IntegrationRule::ApplyToKnotIntervals(KnotVector const& kv) const
   return kvir;
 }

+IntegrationRule IntegrationRule::Reorder(const Array<int> &ordering) const
+{
+   const int np = GetNPoints();
+   MFEM_VERIFY(np == ordering.Size(), "Invalid permutation size");
+   IntegrationRule ir(np);
+   ir.SetOrder(GetOrder());
+
+   for (int i = 0; i < np; i++)
+   {
+      IntegrationPoint &ip_new = ir.IntPoint(i);
+      const IntegrationPoint &ip_old = IntPoint(ordering[i]);
+      ip_new.Set(ip_old.x, ip_old.y, ip_old.z, ip_old.weight);
+   }
+
+   return ir;
+}
+
+IntegrationRule DuffyTrans(const IntegrationRule &ir, int dim)
+{
+   IntegrationRule ir_mapped(ir.GetNPoints());
+   ir_mapped.SetOrder(ir.GetOrder());
+
+   if (dim == 2)
+   {
+      for (int i = 0; i < ir.GetNPoints(); i++)
+      {
+         IntegrationPoint &ip_mapped = ir_mapped.IntPoint(i);
+         ip_mapped.y = ir.IntPoint(i).y * (1 - ir.IntPoint(i).x);
+         ip_mapped.x = ir.IntPoint(i).x;
+         ip_mapped.weight = ir.IntPoint(i).weight;
+      }
+      return ir_mapped;
+   }
+   else if (dim == 3)
+   {
+      for (int i = 0; i < ir.GetNPoints(); i++)
+      {
+         IntegrationPoint &ip_mapped = ir_mapped.IntPoint(i);
+         ip_mapped.z = ir.IntPoint(i).z * (1 - ir.IntPoint(i).x) * (1 - ir.IntPoint(
+                                                                       i).y);
+         ip_mapped.y = ir.IntPoint(i).y * (1 - ir.IntPoint(i).x);
+         ip_mapped.x = ir.IntPoint(i).x;
+         ip_mapped.weight = ir.IntPoint(i).weight;
+      }
+      return ir_mapped;
+   }
+   else
+   {
+      MFEM_ABORT("Duffy transformation not implemented for this dimension!");
+   }
+}
+
 #ifdef MFEM_USE_MPFR

 // Class for computing hi-precision (HP) quadrature in 1D
@@ -433,6 +485,142 @@ public:
 #endif // MFEM_USE_MPFR


+void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
+                                        const real_t beta, IntegrationRule* ir)
+{
+   /* The np-point Gauss-Jacobi quadrature rule is exact for polynomials of
+      degree 2np - 1 with weight function w(x) = (1-x)^alpha * x^beta. The
+      nodes are the zeros of the Jacobi polynomial P_{np}^{alpha,beta} and
+      the weights are
+
+      w_i = C / [(1 - x_i^2) * P'_{np}^{alpha,beta}(x_i)^2]
+      C = 2^{alpha + beta + 1} * Gamma(np + alpha + 1) * Gamma(np + beta + 1)
+            / [Gamma(np + alpha + beta + 1) * Gamma(np + 1)].
+
+      The nodes are computed via nonlinear solve (Newton's method) with an
+      initial guess corresponding to Gatteschi's asymptotic expansions of the
+      Jacobi polynomial roots [1].
+
+      The current initial guess has been tested and performs well for
+      np <= 200 and -1 <= alpha, beta <= 4. For larger np, it may be necessary
+      utilize different initial guesses in the vicinity of x = -1,+1 [2].
+
+      [1] Gautschi, W., & Giordano, C. (2008). Luigi Gatteschi’s work on
+          asymptotics of special functions and their zeros. Numerical Algorithms,
+          49, 11-31.
+      [2] Hale, N., & Townsend, A. (2013). Fast and accurate computation of
+          Gauss--Legendre and Gauss--Jacobi quadrature nodes and weights.
+          SIAM Journal on Scientific Computing, 35(2), A652-A674.
+   */
+   ir->SetSize(np);
+   ir->SetPointIndices();
+   ir->SetOrder(2*np - 1);
+
+   if (alpha <= -1.0 || beta <= -1.0)
+   {
+      MFEM_ABORT("Gauss-Jacobi quadrature only defined for alpha > -1 and beta > -1");
+   }
+   // Jacobi weight function is undefined whenever alpha <= -1 or beta <= -1
+
+   if (alpha > 4.0 || beta > 4.0)
+   {
+      MFEM_ABORT("Current Gauss-Jacobi quadrature implementation only tested for alpha <= 4 and beta <= 4");
+   }
+   // current asymptotic expansions for initial guess may perform poorly for large alpha, beta
+
+   switch (np)
+   {
+      case 1:
+         real_t x = (beta - alpha) / (alpha + beta + 2);
+         real_t w = pow(2, alpha + beta + 1) * tgamma(alpha + 2) * tgamma(
+                       beta + 2) / (tgamma(alpha + beta + 2));
+         w = 0.5 * w / pow(2, alpha + beta);
+         // map weight to  to [0,1], with additional 1/(2^(alpha + beta)) factor coming from mapping
+         // the weight (1-x)^alpha * (1+x)^beta to [0,1] as well.
+         ir->IntPoint(0).Set1w(0.5 * x + 0.5,
+                               4.0 * w / ((1.0 - x*x) * (alpha + beta + 2) * (alpha + beta + 2)));
+         return;
+   }
+
+#ifndef MFEM_USE_MPFR
+
+   const int n = np;
+   // common constants for Jacobi polynomials
+   real_t ab = alpha + beta;
+   real_t a2_minus_b2 = (alpha - beta) * (alpha + beta);
+
+   // roots of P^(alpha,beta)_n in the interval [-1,1]
+   for (int i = 1; i <= n; i++)
+   {
+      // rather than using Chebyshev points for initial guess, use Gatteschi's asymptotic expansion for roots of Jacobi
+      // polynomials
+      real_t n_ab_plus_1 = 2 * n + alpha + beta + 1;
+      real_t v = (2 * i + alpha - 0.5) * M_PI / n_ab_plus_1;
+      real_t theta = v + 1.0 / (n_ab_plus_1*n_ab_plus_1) * ((0.25 - alpha*alpha) *
+                                                            1.0/tan(0.5*v) - (0.25 - beta*beta) * tan(0.5*v));
+      real_t z = cos(theta);
+
+      real_t pp, p1, dz, xi = 0.;
+      bool done = false;
+      while (1)
+      {
+         real_t p2 = 1;
+         p1 = ((alpha-beta) + (alpha + beta + 2) * z) / 2;
+         for (int j = 1; j <= n-1; j++)
+         {
+            real_t p3 = p2;
+            p2 = p1;
+
+            real_t jx2_ab = 2 * j + ab;
+            real_t an = (jx2_ab) * (jx2_ab + 2);
+            real_t bn = a2_minus_b2;
+            real_t cn = 2 * (j + alpha) * (j + beta) * (jx2_ab + 2) / (jx2_ab + 1);
+
+            real_t D = (jx2_ab + 1) / (2 * (j + 1) * (j + ab + 1) * (jx2_ab));
+            p1 = ((an * z + bn) * p2 - cn * p3) * D;
+         }
+         // p1 is Jacobi polynomial
+         pp = n * (alpha - beta - (2 * n + ab) * z) * p1 + 2 * (n + alpha) *
+              (n + beta) * p2;
+         pp = pp / ((2 * n + ab) * (1 - z*z));
+         // derivative of the Jacobi polynomial
+         if (done) { break; }
+
+         dz = p1/pp;
+#ifdef MFEM_USE_SINGLE
+         if (std::abs(dz) < 1e-7)
+#elif defined MFEM_USE_DOUBLE
+         if (std::abs(dz) < std::numeric_limits<real_t>::epsilon())
+            // this seems to cause trouble if we try std::abs(dz) < 1e-16
+#else
+         MFEM_ABORT("Floating point type undefined");
+         // if (std::abs(dz) < 1e-16)
+#endif
+         {
+            done = true;
+            xi = z - dz;
+         }
+         z -= dz;
+      }
+      real_t c0 = exp(lgamma(n + alpha + 1) - lgamma(n + ab + 1)) * exp(lgamma(
+                                                                           n + beta + 1) - lgamma(n + 1));
+      // ratio of gamma functions prone to overflow for large n, so compute logarithms
+      // of Gamma function instead, i.e. Gamma(a)/Gamma(b) = exp(lgamma(a) - lgamma(b))
+      ir->IntPoint(n-i).x = 0.5 * xi + 0.5;
+      ir->IntPoint(n-i).weight = 0.5 * c0 * pow(2.0,
+                                                ab + 1) / ((1.0 - xi*xi)*pp*pp) / pow(2, ab);
+      // map nodes and weights to the interval [0,1]
+   }
+
+#else // MFEM_USE_MPFR is defined
+
+   MFEM_ABORT("MPFR implementation of Gauss-Jacobi quadrature not defined yet");
+
+#endif // MFEM_USE_MPFR
+
+}
+
+
 void QuadratureFunctions1D::GaussLegendre(const int np, IntegrationRule* ir)
 {
   ir->SetSize(np);
@@ -2362,6 +2550,194 @@ IntegrationRule *IntegrationRules::CubeIntegrationRule(int Order)
   return CubeIntRules[Order];
 }

+StroudIntegrationRules StroudIntRules;
+
+StroudIntegrationRules::StroudIntegrationRules()
+{
+   const MemoryType h_mt = MemoryType::HOST;
+   SquareStroudIntRules.SetSize(32, h_mt);
+   SquareStroudIntRules = NULL;
+
+   TriangleStroudIntRules.SetSize(32, h_mt);
+   TriangleStroudIntRules = NULL;
+
+   CubeStroudIntRules.SetSize(32, h_mt);
+   CubeStroudIntRules = NULL;
+
+   TetrahedronStroudIntRules.SetSize(32, h_mt);
+   TetrahedronStroudIntRules = NULL;
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   IntRuleLocks.SetSize(Geometry::NUM_GEOMETRIES, h_mt);
+   for (int i = 0; i < Geometry::NUM_GEOMETRIES; i++)
+   {
+      omp_init_lock(&IntRuleLocks[i]);
+   }
+#endif
+}
+
+const IntegrationRule &StroudIntegrationRules::Get(int GeomType, int Order)
+{
+   Array<IntegrationRule *> *ir_array = NULL;
+
+   switch (GeomType)
+   {
+      case Geometry::TRIANGLE:    ir_array = &TriangleStroudIntRules; break;
+      case Geometry::TETRAHEDRON: ir_array = &TetrahedronStroudIntRules; break;
+      case Geometry::INVALID:
+      case Geometry::NUM_GEOMETRIES:
+         MFEM_ABORT("Unknown type of reference element!");
+      default:
+         MFEM_ABORT("Stroud rules only valid for triangular and tetrahedral elements!");
+   }
+
+   if (Order < 0)
+   {
+      Order = 0;
+   }
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_set_lock(&IntRuleLocks[GeomType]);
+#endif
+
+   if (!HaveIntRule(*ir_array, Order))
+   {
+      IntegrationRule *ir = GenerateIntegrationRule(GeomType, Order);
+#ifdef MFEM_DEBUG
+      int RealOrder = Order;
+      while (RealOrder+1 < ir_array->Size() && (*ir_array)[RealOrder+1] == ir)
+      {
+         RealOrder++;
+      }
+      MFEM_VERIFY(RealOrder == ir->GetOrder(), "internal error");
+#else
+      MFEM_CONTRACT_VAR(ir);
+#endif
+   }
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_unset_lock(&IntRuleLocks[GeomType]);
+#endif
+
+   return *(*ir_array)[Order];
+}
+
+void StroudIntegrationRules::DeleteIntRuleArray(
+   Array<IntegrationRule *> &ir_array) const
+{
+   // Many of the intrules have multiple contiguous copies in the ir_array
+   // so we have to be careful to not delete them twice.
+   IntegrationRule *ir = NULL;
+   for (int i = 0; i < ir_array.Size(); i++)
+   {
+      if (ir_array[i] != NULL && ir_array[i] != ir)
+      {
+         ir = ir_array[i];
+         delete ir;
+      }
+   }
+}
+
+StroudIntegrationRules::~StroudIntegrationRules()
+{
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   for (int i = 0; i < Geometry::NUM_GEOMETRIES; i++)
+   {
+      omp_destroy_lock(&IntRuleLocks[i]);
+   }
+#endif
+   DeleteIntRuleArray(SquareStroudIntRules);
+   DeleteIntRuleArray(TriangleStroudIntRules);
+   DeleteIntRuleArray(CubeStroudIntRules);
+   DeleteIntRuleArray(TetrahedronStroudIntRules);
+}
+
+
+IntegrationRule *StroudIntegrationRules::GenerateIntegrationRule(int GeomType,
+                                                                 int Order)
+{
+   switch (GeomType)
+   {
+      case Geometry::TRIANGLE:
+         return TriangleStroudIntegrationRule(Order);
+      case Geometry::TETRAHEDRON:
+         return TetrahedronStroudIntegrationRule(Order);
+      case Geometry::INVALID:
+      case Geometry::NUM_GEOMETRIES:
+         MFEM_ABORT("Unknown type of reference element!");
+      default:
+         MFEM_ABORT("Stroud rules only valid for triangular and tetrahedral elements!");
+   }
+   return NULL;
+}
+
+/* Integration rule in reference triangle according to tensor product Gauss-Jacobi rule.
+  The nodes and weights are used in the original form defined on the reference
+  square to evaluate the component 1D basis functions. Mapping to the reference
+  triangle via IntegrationRule::DuffyTrans() occurs only in evaluation of coefficient
+  vectors, see e.g. MassIntegrator::AssemblePASimplex. */
+IntegrationRule *StroudIntegrationRules::TriangleStroudIntegrationRule(
+   int Order)
+{
+   int RealOrder = GetSegmentRealOrder(Order);
+   // Order is one of {RealOrder-1,RealOrder}
+   // if (!HaveIntRule(SegmentIntRules, RealOrder))
+   // {
+   //    SegmentIntegrationRule(RealOrder);
+   // }
+   IntegrationRule ir_0_0;
+   // Gauss-Jacobi is exact for 2*n-1
+   int n = RealOrder/2 + 1;
+   QuadratureFunctions1D::GaussJacobi(n, 0.0, 0.0, &ir_0_0);
+
+   IntegrationRule ir_1_0;
+   QuadratureFunctions1D::GaussJacobi(n, 1.0, 0.0, &ir_1_0);
+
+   AllocIntRule(TriangleStroudIntRules, RealOrder); // RealOrder >= Order
+   // create rule in unit square
+   TriangleStroudIntRules[RealOrder-1] =
+      TriangleStroudIntRules[RealOrder] =
+         new IntegrationRule(ir_1_0, ir_0_0);
+   // map rule to reference triangle
+   // TriangleStroudIntRules[RealOrder-1]->DuffyTrans(2);
+   *TriangleStroudIntRules[RealOrder-1] =
+      DuffyTrans(*TriangleStroudIntRules[RealOrder-1], 2);
+   return TriangleStroudIntRules[Order];
+}
+
+/* Integration rule in reference tetrahedron according to tensor product Gauss-Jacobi rule.
+  The nodes and weights are used in the original form defined on the reference
+  square to evaluate the component 1D basis functions. Mapping to the reference
+  triangle via IntegrationRule::DuffyTrans() occurs only in evaluation of coefficient
+  vectors, see e.g. MassIntegrator::AssemblePASimplex. */
+IntegrationRule *StroudIntegrationRules::TetrahedronStroudIntegrationRule(
+   int Order)
+{
+   int RealOrder = GetSegmentRealOrder(Order);
+   // Order is one of {RealOrder-1,RealOrder}
+
+   IntegrationRule ir_0_0;
+   int n = RealOrder/2 + 1;
+   QuadratureFunctions1D::GaussJacobi(n, 0.0, 0.0, &ir_0_0);
+
+   IntegrationRule ir_1_0;
+   QuadratureFunctions1D::GaussJacobi(n, 1.0, 0.0, &ir_1_0);
+
+   IntegrationRule ir_2_0;
+   QuadratureFunctions1D::GaussJacobi(n, 2.0, 0.0, &ir_2_0);
+
+   AllocIntRule(TetrahedronStroudIntRules, RealOrder); // RealOrder >= Order
+   // create rule in unit cube
+   TetrahedronStroudIntRules[RealOrder-1] =
+      TetrahedronStroudIntRules[RealOrder] =
+         new IntegrationRule(ir_2_0, ir_1_0, ir_0_0);
+   // map rule to reference tetrahedron
+   // TetrahedronStroudIntRules[RealOrder-1]->DuffyTrans(3);
+   *TetrahedronStroudIntRules[RealOrder-1] =
+      DuffyTrans(*TetrahedronStroudIntRules[RealOrder-1], 3);
+   return TetrahedronStroudIntRules[Order];
+}
+
 IntegrationRule& NURBSMeshRules::GetElementRule(const int elem,
                                                const int patch, const int *ijk,
                                                Array<const KnotVector*> const& kv) const
@@ -269,6 +269,13 @@ public:
   /// applying this rule on each knot interval.
   IntegrationRule* ApplyToKnotIntervals(KnotVector const& kv) const;

+   /** @brief  Returns an integration rule such that the new IntegrationPoints
+    *  are re-ordered based on @a ordering.
+    *
+    *  @details In the new integration rule, ip_new[i] = ip_old[ordering[i]]
+    */
+   IntegrationRule Reorder(const Array<int> &ordering) const;
+
   /// Destroys an IntegrationRule object
   ~IntegrationRule() { }
 };
@@ -378,6 +385,8 @@ public:
       These methods calculate the actual points and weights for the different
       types of quadrature rules. */
   ///@{
+   static void GaussJacobi(const int np, const real_t alpha, const real_t beta,
+                           IntegrationRule* ir);
   static void GaussLegendre(const int np, IntegrationRule* ir);
   static void GaussLobatto(const int np, IntegrationRule *ir);
   static void OpenUniform(const int np, IntegrationRule *ir);
@@ -487,12 +496,71 @@ public:
   ~IntegrationRules();
 };

+/// Container class for integration rules
+class StroudIntegrationRules
+{
+private:
+   Array<IntegrationRule *> SquareStroudIntRules;
+   Array<IntegrationRule *> TriangleStroudIntRules;
+   Array<IntegrationRule *> CubeStroudIntRules;
+   Array<IntegrationRule *> TetrahedronStroudIntRules;
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   Array<omp_lock_t> IntRuleLocks;
+#endif
+
+   void AllocIntRule(Array<IntegrationRule *> &ir_array, int Order) const
+   {
+      if (ir_array.Size() <= Order)
+      {
+         ir_array.SetSize(Order + 1, NULL);
+      }
+   }
+   bool HaveIntRule(Array<IntegrationRule *> &ir_array, int Order) const
+   {
+      return (ir_array.Size() > Order && ir_array[Order] != NULL);
+   }
+   int GetSegmentRealOrder(int Order) const
+   {
+      return Order | 1; // valid for all quad_type's
+   }
+   void DeleteIntRuleArray(Array<IntegrationRule *> &ir_array) const;
+
+   /// The following methods allocate new IntegrationRule objects without
+   /// checking if they already exist.  To avoid memory leaks use
+   /// IntegrationRules::Get(int GeomType, int Order) instead.
+   IntegrationRule *GenerateIntegrationRule(int GeomType, int Order);
+   IntegrationRule *TriangleStroudIntegrationRule(int Order);
+   IntegrationRule *TetrahedronStroudIntegrationRule(int Order);
+
+public:
+   /// Sets initial sizes for the integration rule arrays, but rules
+   /// are defined the first time they are requested with the Get method.
+   explicit StroudIntegrationRules();
+
+   /// Returns a Stroud integration rule for given GeomType and Order.
+   const IntegrationRule &Get(int GeomType, int Order);
+
+   /// Destroys an StroudIntegrationRules object
+   ~StroudIntegrationRules();
+};
+
 /// A global object with all integration rules (defined in intrules.cpp)
 extern MFEM_EXPORT IntegrationRules IntRules;

 /// A global object with all refined integration rules
 extern MFEM_EXPORT IntegrationRules RefinedIntRules;

+/// A global object with all Stroud integration rules (defined in intrules.cpp)
+extern MFEM_EXPORT StroudIntegrationRules StroudIntRules;
+
+/// Duffy Transformation of 2D and 3D tensor product rules of the form
+/// $X(t) = \sum_{i=1}^{d+1} \lambda_i(t) * x_i$, where $x_i$ are the vertices
+/// of the simplex and $\lambda_i = t_i * (1-\lambda_1-...-\lambda_{i-1})$, with
+/// $t$ being the coordinates in the unit square/cube. This function is used only
+/// in the partial assembly of Bernstein elements on simplices and does NOT
+/// modify the quadrature weights.
+IntegrationRule DuffyTrans(const IntegrationRule &ir, int dim);
 }

 #endif
@@ -61,7 +61,7 @@ namespace mfem
 #define MFEM_REGISTER_KERNELS_1(KernelName, KernelType, Params)                \
   MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, (), Params)

-// Version of MFEM_REGISTER_KERNELS without any optional (non-dispatch)
+// Version of MFEM_REGISTER_KERNELS with optional (non-dispatch)
 // parameters (e.g. NBZ).
 #define MFEM_REGISTER_KERNELS_2(KernelName, KernelType, Params, OptParams)     \
   MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, OptParams,           \
@@ -471,6 +471,13 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
   }
 }

+VectorFEDomainLFIntegrator::VectorFEDomainLFIntegrator(
+   VectorCoefficient &F, const IntegrationRule *ir)
+   : DeltaLFIntegrator(F, ir), QF(F)
+{
+   static Kernels kernels{};
+}
+
 void VectorFEDomainLFIntegrator::AssembleRHSElementVect(
   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -369,8 +369,8 @@ private:
   Vector vec;

 public:
-   VectorFEDomainLFIntegrator(VectorCoefficient &F)
-      : DeltaLFIntegrator(F), QF(F) { }
+   VectorFEDomainLFIntegrator(VectorCoefficient &F,
+                              const IntegrationRule *ir = nullptr);

   void AssembleRHSElementVect(const FiniteElement &el,
                               ElementTransformation &Tr,
@@ -387,6 +387,40 @@ public:
                       Vector &b) override;

   using LinearFormIntegrator::AssembleRHSElementVect;
+
+   /// @param ne number of elements
+   /// @param markers array where entry markers[e] == 0 to skip assembly over
+   /// element e element
+   /// @param jac Spatial Jacobians evaluated at all quadrature points
+   /// @param weights 1D quadrature weights
+   /// @param testBO 1D open basis test functions
+   /// @param testBC 1D closed basis test functions
+   /// @param coeff coefficient values evaluated at quadrature points, possibly
+   /// compressed.
+   /// @param d number of 1D closed dofs
+   /// @param q number of 1D quadrature points
+   using AssembleKernelType = void (*)(const int NE, const Array<int> &markers,
+                                       const Vector &jac,
+                                       const Array<real_t> &weights,
+                                       const Array<real_t> &testBO,
+                                       const Array<real_t> &testBC,
+                                       const Vector &coeff, Vector &y,
+                                       const int testd1d, const int q1d);
+
+   /// parameters: test_fetype, ndims, test_d1d, q1d
+   MFEM_REGISTER_KERNELS(AssembleKernels, AssembleKernelType,
+                         (FiniteElement::DerivType, int, int, int));
+
+   struct Kernels
+   {
+      Kernels();
+   };
+
+   template <FiniteElement::DerivType TestType, int DIM, int TEST_D1D, int Q1D>
+   static void AddSpecialization()
+   {
+      AssembleKernels::Specialization<TestType, DIM, TEST_D1D, Q1D>::Add();
+   }
 };

 /// $ (Q, \mathrm{curl}(v))_{\Omega} $ for Nedelec Elements
@@ -258,8 +258,8 @@ void BatchedLOR_DG::Assemble2D()
                  const int w_idx = (n_idx == 0) ? iy : ix;
                  const int x_idx = (n_idx == 0) ? i_0 : j_0;

-                  const real_t J1 = J(ix, iy, n_idx, !n_idx, iel_ho);
-                  const real_t J2 = J(ix, iy, !n_idx, !n_idx, iel_ho);
+                  const real_t J1 = J(ix, iy, n_idx, (int)!n_idx, iel_ho);
+                  const real_t J2 = J(ix, iy, (int)!n_idx, (int)!n_idx, iel_ho);
                  const real_t Jh = (J1*J1 + J2*J2) / detJ(ix, iy, iel_ho);

                  V(v_idx, ix, iy, iel_ho) =
@@ -284,12 +284,12 @@ GeometricMultigrid::GeometricMultigrid(
   ownedProlongations.SetSize(nlevels - 1);
   ownedProlongations = have_ess_bdr;

-   if (have_ess_bdr)
+   essentialTrueDofs.SetSize(nlevels);
+   for (int level = 0; level < nlevels; ++level)
   {
-      essentialTrueDofs.SetSize(nlevels);
-      for (int level = 0; level < nlevels; ++level)
+      essentialTrueDofs[level] = new Array<int>;
+      if (have_ess_bdr)
      {
-         essentialTrueDofs[level] = new Array<int>;
         fespaces.GetFESpaceAtLevel(level).GetEssentialTrueDofs(
            ess_bdr, *essentialTrueDofs[level]);
      }
--- a/Show More
+++ b/Show More