resolving conflicts with master

Add option to skip branch-history check

.github/actions/sanitize/config/action.yml

@@ -94,6 +94,16 @@ inputs:
     description: If true, do not set any CXXFLAGS or LDFLAGS.
     default: false
 
+  # Unfortunately, "uses:" fields cannot have references to variables like
+  # ${{env.MFEM_ACTIONS_VERSION}}, so the branch/tag name has to be hard coded.
+  # Therefore, in the future, when updating the version of the
+  # mfem/github-actions to use, we'll have to replace:
+  # - all definitions of MFEM_ACTIONS_VERSION and
+  # - all "uses:" fields that refer to mfem/github-actions.
+  MFEM_ACTIONS_VERSION:
+    description: Version (branch or tag) of the mfem/github-actions to use.
+    default: v2.7
+
 runs:
   using: 'composite'
   steps:
@@ -118,6 +128,7 @@ runs:
         echo UBSAN_LDFLAGS=${{inputs.UBSAN_LDFLAGS}} >> $GITHUB_ENV
         echo MSAN_CXXFLAGS=${{inputs.MSAN_CXXFLAGS}} >> $GITHUB_ENV
         echo MSAN_LDFLAGS=${{inputs.MSAN_LDFLAGS}} >> $GITHUB_ENV
+        echo MFEM_ACTIONS_VERSION=${{inputs.MFEM_ACTIONS_VERSION}} >> $GITHUB_ENV
       shell: bash
 
     - name: Env (dir)

.github/actions/sanitize/mfem/action.yml

@@ -53,7 +53,7 @@ runs:
       run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
       shell: bash
 
-    - uses: mfem/github-actions/build-mfem@v2.5
+    - uses: mfem/github-actions/build-mfem@v2.7
       if: ${{steps.debug.outputs.cache-hit != 'true'}}
       env:
         CXXFLAGS: ${{env.CXXFLAGS}}
@@ -82,7 +82,7 @@ runs:
       run: find . -type f -name '*.o' -delete
       shell: bash
 
-    - uses: actions/upload-artifact@v4
+    - uses: actions/upload-artifact@v7
       with:
         name: build-${{inputs.par}}-${{inputs.sanitizer}}
         path: mfem/build

.github/actions/sanitize/mpi/action.yml

@@ -12,6 +12,11 @@
 name: 'Install MPI'
 description: 'Installs MPI and set up its environment variables'
 
+inputs:
+  NO_FLAGS:
+    description: If true, do not set any CXXFLAGS or LDFLAGS.
+    default: false
+
 runs:
   using: 'composite'
   steps:
@@ -27,6 +32,7 @@ runs:
       shell: bash
 
     - name: Env (bis)
+      if: ${{ inputs.NO_FLAGS != 'true' }}
       run: |
         echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MPI_INC}} >> $GITHUB_ENV
         echo LDFLAGS=${{env.LDFLAGS}} ${{env.MPI_LIB}} >> $GITHUB_ENV

.github/actions/sanitize/restore/action.yml

@@ -49,7 +49,7 @@ runs:
         par: ${{inputs.par}}
         sanitizer: ${{inputs.sanitizer}}
 
-    - uses: actions/download-artifact@v4
+    - uses: actions/download-artifact@v8
       with:
         name: build-${{inputs.par}}-${{inputs.sanitizer}}
         path: mfem/build

.github/actions/sanitize/setup/action.yml

@@ -37,14 +37,14 @@ runs:
       with:
         path: ${{env.HYPRE_DIR}}
         fail-on-cache-miss: true
-        key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
+        key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-${{env.MFEM_ACTIONS_VERSION}}
 
     - uses: actions/cache/restore@v5  # Cache for Metis
       if: ${{inputs.par == 'true'}}
       with:
         path: ${{env.METIS_DIR}}
         fail-on-cache-miss: true
-        key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
+        key: ${{runner.os}}-build-${{env.METIS_DIR}}-${{env.MFEM_ACTIONS_VERSION}}
 
     - name: Hypre/Metis links
       if: ${{inputs.par == 'true'}}

.github/copilot-instructions.md

@@ -0,0 +1,42 @@
+# MFEM Pull Request Review Agent Guide
+
+## Purpose and scope
+Review MFEM PRs for correctness, maintainability, performance, portability, test coverage, and MFEM consistency. Use the diff and PR context; reference source files, tests, and CI results when available. Follow `CONTRIBUTING.md`, especially Developer Guidelines, PR rules, checklist, and testing.
+
+## Critical review pillars
+- Correctness and numerical behavior
+- API and user-facing impact
+- Performance implications
+- Maintainability and portability
+
+## Review workflow
+1. Read the PR description, linked issues, and intended behavior.
+2. Inspect the diff before commenting.
+3. Identify affected MFEM components, examples, tests, build or docs changes, and downstream APIs.
+4. Analyze the code against the critical review pillars.
+5. Compare the change against nearby code and MFEM patterns; flag unmotivated deviations.
+6. Check whether tests and documentation were updated appropriately.
+7. Review CI results and suggest actions.
+8. Produce a structured review with prioritized findings.
+9. Always limit conclusions to available evidence.
+
+## MFEM-specific review checklist
+- Component-aware scope: identify the touched subsystem (FEM, solvers, preconditioners, linear algebra, mesh, examples, miniapps, build, or docs) and assess its impact against the review pillars.
+- Numerical and algorithmic behavior: assess issues in convergence, stability, tolerances, precision, iteration limits, and failure handling. If clear opportunities exist to improve the algorithmic approach, call them out with expected impact.
+- API and user-facing impact: assess backward compatibility, user-visible behavior and default changes, migration impact, deprecations, and whether documentation clearly explains user-facing API changes.
+- Data structure and memory semantics: assess ownership, lifetime, aliasing, container behavior, and device-host synchronization.
+- Parallel and serial behavior: assess whether the change preserves equivalent semantics in serial and parallel modes where applicable; if logic is currently mode-specific, check whether extension to the other mode is straightforward (clear abstractions, no hard-wired assumptions), document constraints, and call out expected behavior differences explicitly.
+- Backend and portability impact: assess likely cross-backend risks in CPU, CUDA, HIP, OCCA, RAJA, partial assembly, fallback paths, compiler compatibility, and platform assumptions.
+- Build, dependency, and configuration impact: assess CMake or make changes, optional dependency behavior, and feature-flag interactions.
+- Tests and docs alignment: check available regression or unit coverage evidence for changed behavior, and ensure docs are updated for new flags, APIs, options, or behavior changes.
+- MFEM developer-guideline fit: keep code lean, simple, general, logically separated, and portable; suggest C++17 improvements when they clearly improve safety, clarity, or maintainability.
+- New source files, examples, or miniapps: if a PR adds source/header files, verify they are properly wired into the relevant `makefile` and `CMakeLists.txt`, referenced in docs where applicable (including `doc/CodeDocumentation.dox`), and added to top-level `.gitignore` only when generated artifacts require it.
+- Changelog: verify `CHANGELOG` is updated if the PR introduces significant new features or user-facing changes.
+- MFEM conventions: use `real_t`; use `mfem::out`/`mfem::err` instead of `std::cout`/`std::cerr` in library code; flag large/binary files; if AI assistance is apparent but undisclosed, suggest following `CONTRIBUTING.md`.
+- Edge cases: if the PR touches complex or error-prone areas, suggest additional tests for edge cases, failure modes, and parallel behavior.
+
+## Commenting guidelines
+- Keep comments concise, actionable, and grounded in the diff.
+- Focus on correctness, behavior changes, and user impact over style nits.
+- Be professional, concise, collaborative, technically precise, and avoid unsupported assumptions.
+

.github/workflows/README.md

@@ -13,7 +13,7 @@ Note that some of these scripts use the shared MFEM GitHub Actions from the exte
 
   <https://github.com/mfem/github-actions>
 
-For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch in the above from which the action is taken.
+For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch (or tag) in the above from which the action is taken.
 
 The current CI workflows are:
 
@@ -29,16 +29,12 @@ Runs a number of static repository-level sanity checks.
 
 - `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.
 
-## `mfem-analysis.yml` (`build-analysis`)
-
-Checks if the code builds and satisfies minimal requirements.
-
-- `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.
-
 ## `builds-and-tests.yml`
 
 Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.
 
+One matrix job runs `tests/scripts/gitignore` after `make test-noclean` to check generated artifacts against `.gitignore`.
+
 Uses the following GitHub Actions from <https://github.com/mfem/github-actions>:
 
 - `mfem/github-actions/build-hypre`

.github/workflows/builds-and-tests.yml

@@ -40,6 +40,7 @@ env:
   METIS_ARCHIVE_MAC: metis-4.0.3-mac.tgz
   METIS_TOP_DIR: metis-4.0.3
   MFEM_TOP_DIR: mfem
+  MFEM_ACTIONS_VERSION: v2.7
 
 # Note for future improvements:
 #
@@ -110,6 +111,7 @@ jobs:
             build-system: make
             hypre-target: int64
             precision: fp64
+            gitignore-check: YES
           - os: ubuntu-latest
             target: opt
             codecov: NO
@@ -140,6 +142,10 @@ jobs:
 
     continue-on-error: ${{ matrix.enzyme && true || false }}
 
+    # Enable ccache for all jobs except Windows (would need sccache).
+    env:
+      USE_CCACHE: ${{ matrix.os != 'windows-latest' }}
+
     steps:
       # Fix 'No space left on device' errors for Ubuntu builds.
       - name: Run Actions Cleaner
@@ -170,20 +176,6 @@ jobs:
           env
         shell: bash
 
-      # For info on Xcode see:
-      #  - https://github.com/actions/runner-images/issues/12541
-      #  - https://github.com/actions/runner-images/blob/releases/macos-15-arm64/20250811/images/macos/macos-15-arm64-Readme.md#xcode
-      - name: Xcode version setup (MacOS)
-        if: matrix.os == 'macos-latest'
-        run: |
-          XCODE_PATH="/Applications/Xcode_16.4.app"
-          echo "> sudo xcode-select -s ${XCODE_PATH}"
-          sudo xcode-select -s ${XCODE_PATH}
-          echo "> g++ -v"
-          g++ -v
-          echo "> clang++ -v"
-          clang++ -v
-
       # Only get MPI if defined for the job.
       # TODO: It would be nice to have only one step, e.g. with a dedicated
       # action, but I (@adrienbernede) don't see how at the moment.
@@ -228,11 +220,11 @@ jobs:
         uses: actions/cache@v5
         with:
           path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5
+          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-${{ env.MFEM_ACTIONS_VERSION }}
 
       - name: get hypre
         if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.5
+        uses: mfem/github-actions/build-hypre@v2.7
         with:
           archive: ${{ env.HYPRE_ARCHIVE }}
           dir: ${{ env.HYPRE_TOP_DIR }}
@@ -242,7 +234,7 @@ jobs:
 
       - name: get hypre (Windows)
         if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.5
+        uses: mfem/github-actions/build-hypre@v2.7
         with:
           archive: ${{ env.HYPRE_ARCHIVE }}
           dir: ${{ env.HYPRE_TOP_DIR }}
@@ -258,11 +250,11 @@ jobs:
         uses: actions/cache@v5
         with:
           path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-${{ env.MFEM_ACTIONS_VERSION }}
 
       - name: install metis
         if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
+        uses: mfem/github-actions/build-metis@v2.7
         with:
           archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
           dir: ${{ env.METIS_TOP_DIR }}
@@ -302,9 +294,55 @@ jobs:
           echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
           echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
 
+      # Restore the compiler cache (ccache). The key embeds the run id, so new
+      # runs save a fresh snapshot; the restore-keys prefix warm-starts from the
+      # most recent prior run (incl. the base branch for PRs).
+      - name: cache ccache
+        if: ${{ env.USE_CCACHE == 'true' }}
+        uses: actions/cache@v5
+        with:
+          path: .ccache
+          key: ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-${{ github.run_id }}
+          restore-keys: |
+            ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-
+
+      # Configure ccache and select how it is injected into the MFEM build:
+      # - make:  set CXX="ccache g++"; for MPI, OMPI_CXX="ccache g++" so mpicxx
+      #          runs ccache around g++ (not ccache around the mpicxx wrapper).
+      # - cmake: set CMAKE_<LANG>_COMPILER_LAUNCHER=ccache.
+      # - enzyme: wrap the brew clang++ via OMPI_CXX.
+      # The chosen options are passed through build-mfem's 'config-options'
+      # input (see the build step below).
+      - name: configure ccache
+        if: ${{ env.USE_CCACHE == 'true' }}
+        run: |
+          command -v ccache >/dev/null 2>&1 || {
+            if [[ "${{ runner.os }}" == "Linux" ]]; then
+              sudo apt-get update && sudo apt-get install -y ccache
+            else
+              brew install ccache
+            fi
+          }
+          echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
+          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
+          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
+          # Ignore header timestamps (restamped by each checkout) so direct mode hits.
+          echo "CCACHE_SLOPPINESS=include_file_mtime,include_file_ctime,time_macros" >> $GITHUB_ENV
+          # Hash absolute paths relative to the workspace.
+          echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
+          if [[ "${{ matrix.enzyme }}" == "true" ]]; then
+            echo "OMPI_CXX=ccache $LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
+          elif [[ "${{ matrix.build-system }}" == "cmake" ]]; then
+            echo 'CCACHE_CONFIG_OPTS=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache' >> $GITHUB_ENV
+          else
+            echo "OMPI_CXX=ccache g++" >> $GITHUB_ENV
+            echo 'CCACHE_CONFIG_OPTS=CXX="ccache g++" MPICXX="mpicxx"' >> $GITHUB_ENV
+          fi
+        shell: bash
+
       # MFEM build and test
       - name: build
-        uses: mfem/github-actions/build-mfem@v2.5
+        uses: mfem/github-actions/build-mfem@v2.7
         env:
           VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
         with:
@@ -317,9 +355,14 @@ jobs:
           metis-dir: ${{ env.METIS_TOP_DIR }}
           mfem-dir: ${{ env.MFEM_TOP_DIR }}
           precision: ${{ matrix.precision }}
-          config-options: ${{ matrix.config-opts }}
+          config-options: ${{ matrix.config-opts }} ${{ env.CCACHE_CONFIG_OPTS }}
           library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
 
+      - name: ccache stats
+        if: ${{ env.USE_CCACHE == 'true' }}
+        run: ccache -s
+        shell: bash
+
       # Run checks (and only checks) on debug targets
       - name: checks
         if: matrix.build-system == 'make' && matrix.target == 'dbg'
@@ -330,7 +373,13 @@ jobs:
       - name: tests
         if: matrix.build-system == 'make' && (matrix.target == 'opt' || matrix.os == 'ubuntu-latest')
         run: |
-          cd ${{ env.MFEM_TOP_DIR }} && make test
+          cd ${{ env.MFEM_TOP_DIR }}
+          if [[ "${{ matrix.gitignore-check }}" == "YES" ]]; then
+            make test-noclean
+          else
+            make test
+          fi
+        shell: bash
 
       - name: cmake checks
         if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
@@ -375,8 +424,16 @@ jobs:
       # Code coverage (process and upload reports)
       - name: codecov
         if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.5
+        uses: mfem/github-actions/upload-coverage@v2.7
         with:
-          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}
           project_dir: ${{ env.MFEM_TOP_DIR }}
           directories: "fem general linalg mesh"
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: gitignore
+        if: matrix.gitignore-check == 'YES'
+        run: |
+          cd ${{ env.MFEM_TOP_DIR }}/tests/scripts
+          ./runtest gitignore

.github/workflows/cleanup-pr-caches.yml

@@ -0,0 +1,42 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+# A closed PR's caches can never be restored again, so delete them to free
+# space against the 10 GB per-repo cache limit.
+name: Cleanup PR caches
+
+on:
+  pull_request:
+    types: [closed]
+
+permissions:
+  actions: write
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Delete caches for the closed PR
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          PR_REF: refs/pull/${{ github.event.pull_request.number }}/merge
+        run: |
+          echo "Deleting caches for $PR_REF"
+          while :; do
+            ids=$(gh cache list --ref "$PR_REF" --limit 100 --json id --jq '.[].id')
+            [ -n "$ids" ] || break
+            echo "$ids" | while read -r id; do
+              [ -n "$id" ] || continue
+              echo "Deleting cache $id"
+              gh cache delete "$id" || echo "  (already gone)"
+            done
+          done

.github/workflows/codeql-analysis.yml

@@ -14,9 +14,19 @@ name: "Static Analysis"
 on:
   push:
     branches: ["master", "next"]
+    paths-ignore: &docs-only-paths
+      - "**/*.md"
+      - "doc/**"
+      - ".binder/**"
+      - "CITATION.cff"
+      - "LICENSE"
+      - "NOTICE"
+      - "CHANGELOG"
+      - "INSTALL"
   pull_request:
     # The branches below must be a subset of the branches above
     branches: ["master"]
+    paths-ignore: *docs-only-paths
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/mfem-analysis.yml

@@ -1,100 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Build Analysis"
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_TOP_DIR: metis-4.0.3
-  COVERAGE_ENV: mfem-coverage
-
-jobs:
-  gitignore:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: checkout MFEM
-        uses: actions/checkout@v6
-        with:
-          path: mfem
-
-      - name: Get MPI (Linux)
-        run: |
-          sudo apt-get install openmpi-bin libopenmpi-dev
-          export OMPI_MCA_rmaps_base_oversubscribe=1
-
-      - name: Cache Hypre Install
-        id: hypre-cache
-        uses: actions/cache@v5
-        with:
-          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-v2.5
-
-      - name: Get Hypre
-        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.5
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: int32
-
-      - name: Cache Metis Install
-        id: metis-cache
-        uses: actions/cache@v5
-        with:
-          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
-
-      - name: Install Metis
-        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
-        with:
-          archive: ${{ env.METIS_ARCHIVE }}
-          dir: ${{ env.METIS_TOP_DIR }}
-
-      # MFEM build and test
-      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.5
-        with:
-          os: ${{ runner.os }}
-          target: opt
-          codecov: NO
-          mpi: par
-          build-system: make
-          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-          metis-dir: ${{ env.METIS_TOP_DIR }}
-          mfem-dir: mfem
-
-      - name: test (no clean)
-        run: |
-          cd mfem && make test-noclean
-
-      - name: gitignore
-        run: |
-          cd mfem/tests/scripts
-          ./runtest gitignore

.github/workflows/repo-check.yml

@@ -13,6 +13,7 @@ name: "Checks"
 
 permissions:
   actions: write
+  pull-requests: read
 
 on:
   push:
@@ -29,6 +30,11 @@ concurrency:
 # by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
 # and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
 # github.repository").
+#
+# The logic for the branch-history check is slightly different, since that check
+# also inspects the PR's labels to allow for overriding failures. In this case,
+# we run on all 'pull_request' triggers, but only run for 'push' triggers that
+# do not correspond to any open PRs.
 
 jobs:
   file-headers-check:
@@ -128,10 +134,7 @@ jobs:
 
   branch-history:
     if: |
-      github.ref != 'refs/heads/next' &&
-      github.ref != 'refs/heads/master' &&
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
+      github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
     runs-on: ubuntu-latest
     steps:
       - name: checkout mfem
@@ -139,7 +142,27 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: check for pull request
+        id: check_pr
+        if: github.event_name == 'push'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          pr_exists=$(gh pr list --repo "$GITHUB_REPOSITORY" \
+            --head "$GITHUB_REF_NAME" \
+            --state open \
+            --json number \
+            --jq 'length > 0')
+          echo "pr_exists=$pr_exists" >> "$GITHUB_OUTPUT"
+
       - name: branch-history
+        id: branch_history
+        if: |
+          (github.event_name == 'pull_request' ||
+           github.event_name == 'workflow_dispatch' ||
+           steps.check_pr.outputs.pr_exists == 'false')
+        continue-on-error: ${{ contains(github.event.pull_request.labels.*.name,
+                                        'branch-history-override') }}
         run: |
           # We override origin to make sure we point to the main repo.
           # This is to have consistent test results on PRs from forks.
@@ -147,3 +170,9 @@ jobs:
           git remote add origin https://github.com/mfem/mfem.git
           git checkout -b gh-actions-branch-history
           ./config/githooks/pre-push --history
+
+      - name: report branch-history override
+        if: steps.branch_history.outcome == 'failure'
+        run: |
+          echo "::warning::branch-history check failed, but the" \
+            "'branch-history-override' label is set."

.github/workflows/sanitize-build-hypre.yml

@@ -19,18 +19,22 @@ jobs:
     steps:
       - uses: actions/checkout@v6
       - uses: ./.github/actions/sanitize/config
+        with:
+          NO_FLAGS: true
       - name: Cache
         id: cache
         uses: actions/cache@v5
         with:
           path: ${{env.HYPRE_DIR}}
-          key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
+          key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-${{ env.MFEM_ACTIONS_VERSION }}
       - name: Setup
         if: steps.cache.outputs.cache-hit != 'true'
         uses: ./.github/actions/sanitize/mpi
+        with:
+          NO_FLAGS: true
       - name: Build
         if: steps.cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.5
+        uses: mfem/github-actions/build-hypre@v2.7
         with:
           archive: ${{env.HYPRE_TGZ}}
           dir: ${{env.HYPRE_DIR}}

.github/workflows/sanitize-build-metis.yml

@@ -19,18 +19,22 @@ jobs:
     steps:
       - uses: actions/checkout@v6
       - uses: ./.github/actions/sanitize/config
+        with:
+          NO_FLAGS: true
       - name: Cache
         id: cache
         uses: actions/cache@v5
         with:
           path: ${{env.METIS_DIR}}
-          key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
+          key: ${{runner.os}}-build-${{env.METIS_DIR}}-${{env.MFEM_ACTIONS_VERSION}}
       - name: Setup
         if: steps.cache.outputs.cache-hit != 'true'
         uses: ./.github/actions/sanitize/mpi
+        with:
+          NO_FLAGS: true
       - name: Build
         if: steps.cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
+        uses: mfem/github-actions/build-metis@v2.7
         with:
           archive: ${{env.METIS_TGZ}}
           dir: ${{env.METIS_DIR}}

.github/workflows/sanitize-tests.yml

@@ -146,7 +146,7 @@ jobs:
         if: ${{steps.restore.outputs.cache-hit != 'true'}}
         working-directory: mfem/build/tests/unit
         run: find . -type f -name '*.o' -delete
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         with:
           name: tests-${{inputs.par}}-${{inputs.sanitizer}}
           path: mfem/build/tests/unit/${{env.unit_tests}}
@@ -172,7 +172,7 @@ jobs:
           par: ${{inputs.par}}
           sanitizer: ${{inputs.sanitizer}}
           cache-path: mfem/build/tests/unit/${{env.unit_tests}}
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v8
         if: ${{steps.restore.outputs.cache-hit != 'true'}}
         with:
           name: tests-${{inputs.par}}-${{inputs.sanitizer}}

.github/workflows/sanitizers.yml

@@ -17,7 +17,17 @@ permissions:
 on:
   push:
     branches: ["master", "next"]
+    paths-ignore: &docs-only-paths
+      - "**/*.md"
+      - "doc/**"
+      - ".binder/**"
+      - "CITATION.cff"
+      - "LICENSE"
+      - "NOTICE"
+      - "CHANGELOG"
+      - "INSTALL"
   pull_request:
+    paths-ignore: *docs-only-paths
   workflow_dispatch:
 
 concurrency:

CHANGELOG

@@ -11,36 +11,90 @@
 Version 4.9.1 (development)
 ===========================
 
-- Policy for AI-assisted contribution added to CONTRIBUTING.md
+- Added policy for AI-assisted contribution to CONTRIBUTING.md.
 
 Discretization improvements
 ---------------------------
-- Replaced legacy simplex quadrature rules with symmetric positive-weight
-  rules for triangles (orders 0-25) and tetrahedra (orders 0-20). These
-  rules guarantee all-positive weights and interior quadrature points,
-  improving numerical stability. Higher orders fall back to Grundmann-Moller.
-  Triangle rules: Witherden & Vincent, Comput. Math. Appl. 69(10):1232-1241,
-  2015.
-  Tet rules (d=1-13): Witherden & Vincent (ibid).
-  Tet rules (d=14-20): Chuluunbaatar et al., Comput. Math. Appl. 124:89-97,
-  2022.
+- Improved FindPointsGSLIB surface mesh capability with support for simplices
+  and an option to specify axis-aligned bounding box padding for near-surface
+  point queries.
 
-- Improved the gridfunction projection routines. Projections work for Scalar,
+- Added GPU-enabled partial assembly for simplicial Bernstein H1 basis based on
+  ragged tensor algorithms (see DOI: 10.1137/11082539X) for mass and diffusion
+  integrators.
+
+- Replaced legacy simplex quadrature rules with symmetric positive weight rules
+  for triangles (orders 0-25) and tetrahedra (orders 0-20). These rules
+  guarantee all-positive weights and interior quadrature points, improving
+  numerical stability. Higher orders fall back to Grundmann-Moller.
+  * Triangle rules: Witherden and Vincent, DOI: 10.1016/j.camwa.2015.03.017
+  * Tet rules (d=1-13): Witherden and  Vincent (same as above)
+  * Tet rules (d=14-20): Chuluunbaatar et al., DOI: 10.1016/j.camwa.2022.08.016
+
+- Added support for general 1D Gauss-Jacobi quadrature rules and Stroud conical
+  quadrature rules on triangles and tetrahedra.
+
+- Improved the GridFunction projection routines. Projections work for Scalar,
   Vector and VectorFE, also NURBS versions. Optionally different types of
-  projections can be selected, default behaviour has not changed.
+  projections can be selected, default behavior has not changed.
 
-- Added methods to estimate function extremum using piecewise linear bounds +
+- Added GridFunction projection methods for trace spaces, i.e., project
+  coefficients on the mesh skeleton.
+
+- Added methods to estimate function extremum using piecewise linear bounds plus
   recursive subdivision.
 
+- Extend FindPointsGSLIB to support surface meshes.
+
 Meshing improvements
 --------------------
+- Added option to guarantee mesh validity during TMOP-based r-adaptivity, using
+  bounds on the determinant of the mesh transformation Jacobian.
+
+- Added PA support for TMOP's adaptive limiting functionality. Multiple
+  GridFunctions and Coefficients can be combined to form a composite term.
+
 - Improved support for 1D NURBS meshes with variable order, including using
   the patches construct for 1D NURBS meshes.
 
+- Added the option to include material interfaces (faces separating elements
+  with different element attributes) as additional boundary elements, for
+  parallel visualization, e.g. with GLVis. This is supported by both the Print
+  and PrintAsOne methods of ParMesh. See ParMesh::SetPrintInterfaces().
+
+Linear and nonlinear solvers
+----------------------------
+- Added support for trace spaces in PRefinementTransferOperator. This is used in
+  PRefinement multigrid methods for problems posed on trace spaces (see e.g. the
+  DPG miniapps).
+
+- Added interface to MUMPS direct solver for complex-valued problems.
+  Its usage is demonstrated in ex25p. See http://mumps.enseeiht.fr/ for more details.
+  Supported versions >= 5.1.1.
+
+GPU computing
+-------------
+- Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
+
+- Added NVIDIA cuDSS library interface. Implementation examples have been
+  added to ex1 and ex1p. See https://developer.nvidia.com/cudss for more
+  details. Supported versions >= 0.6.0.
+
+- Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
+
 New and updated examples and miniapps
 -------------------------------------
-- Electromagnetics/lorentz miniapp has been updated to leverage the ParticleSet
-  capability.
+- The Lorentz miniapp (in miniapps/electromagnetics) has been updated to
+  leverage the ParticleSet capability.
+
+- Added (Complex)PRefinementMultigrid solver option in the DPG miniapps.
+
+Miscellaneous
+-------------
+- Fixed signed DOF handling in ParGridFunction reading (read constructor) and
+  saving via SaveAsOne(). Simplified the process of applying the DOF signs by
+  using the new method ApplyDofSigns() in class ParFiniteElementSpace: the
+  method will return immediately if no sign flips are needed.
 
 
 Version 4.9, released on Dec 11, 2025

CMakeLists.txt

@@ -76,7 +76,9 @@ set(XSDK_ENABLE_Fortran OFF)
 # Check if we need to enable C or Fortran.
 if (MFEM_USE_CONDUIT OR
     MFEM_USE_SIDRE OR
-    MFEM_USE_PETSC)
+    MFEM_USE_PETSC OR
+    MFEM_USE_MUMPS OR
+    MFEM_USE_COMPLEX_MUMPS)
    # This seems to be needed by:
    #  * find_package(BLAS REQUIRED) and
    #  * find_package(HDF5 REQUIRED) needed, in turn, by:
@@ -84,7 +86,7 @@ if (MFEM_USE_CONDUIT OR
    #  * find_package(PETSc REQUIRED)
    set(XSDK_ENABLE_C ON)
 endif()
-if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
+if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS OR MFEM_USE_COMPLEX_MUMPS)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
@@ -342,7 +344,7 @@ if (MFEM_USE_MPI)
     endif()
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU MUMPS PETSC SLEPC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU MUMPS COMPLEX_MUMPS PETSC SLEPC STRUMPACK PUMI)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -415,7 +417,7 @@ if (MFEM_USE_SUPERLU)
 endif()
 
 # MUMPS can only be enabled in parallel
-if (MFEM_USE_MUMPS)
+if (MFEM_USE_MUMPS OR MFEM_USE_COMPLEX_MUMPS)
   if (MFEM_USE_MPI)
     find_package(MUMPS REQUIRED mumps_common pord)
     set(MFEM_MUMPS_VERSION ${MUMPS_VERSION})
@@ -433,6 +435,15 @@ if (MFEM_USE_STRUMPACK)
   endif()
 endif()
 
+# cuDSS can only be enabled in CUDA
+if (MFEM_USE_CUDSS)
+  if (MFEM_USE_CUDA)
+    find_package(CUDSS REQUIRED)
+  else()
+    message(FATAL_ERROR " *** cuDSS requires that CUDA be enabled.")
+  endif()
+endif()
+
 # GnuTLS
 if (MFEM_USE_GNUTLS)
   find_package(_GnuTLS REQUIRED)
@@ -631,7 +642,7 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
     SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB HDF5
     NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
+    ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CUDSS CALIPER CODIPACK
     BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
     ALGOIM ENZYME CUDA::cudart)
 

CONTRIBUTING.md

@@ -3,12 +3,12 @@
 </p>
 
 <p align="center">
-<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-blue.svg"></a>
+<a href="https://github.com/mfem/mfem/releases/latest"><img alt="GitHub release" src="https://img.shields.io/github/v/release/mfem/mfem"></a>
+<a href="https://github.com/mfem/mfem/actions/workflows/repo-check.yml?query=branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml?query=branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
-<a href="https://docs.mfem.org/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+<a href="https://docs.mfem.org/html/index.html"><img alt="Documentation" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
 
 
@@ -84,7 +84,7 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
   follow the [MFEM PR Rules](#mfem-pr-rules).
 - When your contribution is fully working and ready to be reviewed, add
   the `ready-for-review` label.
-- PRs are treated similarly to journal submission with an "editor" assigning two
+- PRs are treated similarly to journal submission, with an "editor" assigning two
   reviewers to evaluate the changes.
 - The reviewers have 3 weeks to evaluate the PR and work with the author to
   fix issues and implement improvements.
@@ -125,7 +125,7 @@ The MFEM source code has the following structure:
   │   ├── petsc
   │   ├── pumi
   │   ├── sundials
-  |   └── superlu
+  │   └── superlu
   ├── fem
   │   ├── ceed
   │   ├── dfem
@@ -137,10 +137,6 @@ The MFEM source code has the following structure:
   │   ├── moonolith
   │   ├── qinterp
   │   └── tmop
-  │   |   ├── assemble
-  │   |   ├── metrics
-  │   |   ├── mult
-  │   |   └── tools
   ├── general
   ├── linalg
   │   ├── batched
@@ -153,11 +149,10 @@ The MFEM source code has the following structure:
   │   ├── common
   │   ├── contact
   │   ├── dfem
+  │   ├── diag-smoothers
   │   ├── dpg
   │   ├── electromagnetics
   │   ├── fluids
-  │   │   ├── navier
-  │   │   └── schrodinger-flow
   │   ├── gslib
   │   ├── hdiv-linear-solver
   │   ├── hooke
@@ -167,6 +162,7 @@ The MFEM source code has the following structure:
   │   ├── nurbs
   │   ├── parelag
   │   ├── performance
+  │   ├── plasma
   │   ├── shifted
   │   ├── solvers
   │   ├── spde
@@ -197,15 +193,15 @@ respectively.
 
 - The main finite element classes are:
   + [`FiniteElement`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElementCollection.html)
   + [`FiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1FiniteElementSpace.html)
   + [`GridFunction`](https://docs.mfem.org/html/classmfem_1_1GridFunction.html)
   + [`BilinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html)
-  + [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
+  + [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
 
 - The main linear algebra classes and sources are
   + [`Operator`](https://docs.mfem.org/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html)
-  + [`Vector`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
+  + [`Vector`](https://docs.mfem.org/html/classmfem_1_1Vector.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
   + [`DenseMatrix`](https://docs.mfem.org/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://docs.mfem.org/html/classmfem_1_1SparseMatrix.html)
   + Sparse [smoothers](https://docs.mfem.org/html/sparsesmoothers_8hpp.html) and linear [solvers](https://docs.mfem.org/html/solvers_8hpp.html)
 
@@ -217,8 +213,8 @@ shared geometric entities between different tasks. The parallel source files
 have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
 
 - The main parallel classes are
-  + [`ParMesh`](https://docs.mfem.org/html/solvers_8hpp.html)
-  + [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
+  + [`ParMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
+  + [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParNCMesh.html)
   + [`ParFiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1ParFiniteElementSpace.html)
   + [`ParGridFunction`](https://docs.mfem.org/html/classmfem_1_1ParGridFunction.html)
   + [`ParBilinearForm`](https://docs.mfem.org/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://docs.mfem.org/html/classmfem_1_1ParLinearForm.html)
@@ -228,14 +224,14 @@ have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
 #### GPU and general device support
 
 GPU and multi-core CPU support is based on device kernels supporting different
-backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
+backends (CUDA, HIP, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
 device/host memory manager.
 
 - The main device-relevant classes and sources are:
   + [`Device`](https://docs.mfem.org/html/device_8hpp.html)
   + [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
   + the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
-  + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
+  + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html), [`hip.hpp`](https://docs.mfem.org/html/hip_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
 
 #### Utilities, building and documentation
 - The `general/` directory contains C++ classes that serve as utilities for
@@ -249,8 +245,8 @@ device/host memory manager.
 - `examples` and `miniapps` respectively gather simple and more fully-featured
   demonstrations of the usage on MFEM. They both rely on `data/` for the
   collection of meshes.
-- The `tests/` directory contains a unit test suite and will later contain more
-  tests that run example codes.
+- The `tests/` directory contains a unit test suite, additional tests, and
+  benchmarks.
 
 See also the [code overview](https://mfem.org/code-overview/) section on the MFEM
 website.
@@ -284,8 +280,8 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   the top of https://github.com/mfem.
 - Consider making your membership public by going to https://github.com/orgs/mfem/people
   and clicking on the organization visibility drop box next to your name.
-- Project discussions and announcements will be posted at
-  https://github.com/orgs/mfem/teams/everyone.
+- Project discussions and announcements will be posted at https://github.com/orgs/mfem/discussions,
+  tagging the `@mfem/everyone` team when appropriate.
 
 #### Structure
 - The MFEM source code is in the [mfem](https://github.com/mfem/mfem)
@@ -345,11 +341,12 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   - Well-designed simple code is frequently more general and powerful.
   - Lean code base is easier to understand by new collaborators.
   - New features should be added only if they are necessary or generally useful.
-  - Introduction of language constructions not currently used in MFEM should be
+  - Introduction of language constructs not currently used in MFEM should be
     justified and generally avoided (to maintain portability to various systems
     and compilers, including early access hardware).
-  - We prefer basic C++ and the C++03 standard, to keep the code readable by
-    a large audience and to make sure it compiles anywhere.
+  - We prefer basic C++. Use C++17 features judiciously, prioritizing readability,
+    consistency with existing MFEM code, and portability to different systems,
+    compilers and device backends.
 
 - *Keep the code general and reasonably efficient*
   - The main goal is fast prototyping for research and application development.
@@ -392,7 +389,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 - When your branch is ready for other developers to review / comment on
   the code, create a pull request towards `mfem:master`.
 
-- Pull request typically have titles like:
+- Pull requests typically have titles like:
 
      `Description [new-feature-dev]`
 
@@ -413,12 +410,12 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 - Add a description, appropriate labels and assign yourself to the PR. The MFEM
   team will add reviewers as appropriate.
 
-- List outstanding TODO items in the description, see PR #222 for an example.
+- List outstanding TODO items in the description.
 
 - When your contribution is fully working and ready to be reviewed, add
-  the `ready-for-review` label.
+  or request the `ready-for-review` label.
 
-- PRs are treated similarly to journal submission with an "editor" assigning
+- PRs are treated similarly to journal submission, with an "editor" assigning
   two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
   the PR and work with the author to implement improvements and fix issues.
 
@@ -444,7 +441,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   checks in GitHub Actions enforce MFEM-specific rules which are explained in
   the error messages and the `tests/scripts` directory.
 
-- Also note that the tests `branch-history` and `repos-checks` found in GitHub
+- Also note that the tests `branch-history` and `repo-check` found in GitHub
   Actions can be triggered automatically before each push using git hooks. See
   the [git hooks README](config/githooks/README.md) for a detailed explanation.
 
@@ -501,15 +498,15 @@ Everyone on the MFEM team can be asked to serve as a reviewer on a PR in their a
 
 3. To ensure the quality of the PR by making sure that the code adheres to the [Developer Guidelines](#developer-guidelines), e.g. all methods, data members, and functions have documentation, including data ownership and lifetime, new examples/miniapps have a corresponding PR in mfem/web, major features have `CHANGELOG` entries, etc.
 
-3. To seek help from the editors in case of difficulties.
+4. To seek help from the editors in case of difficulties.
 
-4. To complete the review in a timely manner: 3 weeks from assignment.
+5. To complete the review in a timely manner: 3 weeks from assignment.
 
-5. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
+6. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
 
-6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
+7. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
 
-7. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
+8. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
 
 #### Responsibilities of Authors
 
@@ -535,30 +532,30 @@ Before a PR can be merged, it should satisfy the following:
 - [ ] Code builds.
 - [ ] Code passes `make style`.
 - [ ] Update `CHANGELOG`:
-    - [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
-    - [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
+   - [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
+   - [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
 - [ ] Update `INSTALL`:
-    - [ ] Had a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
-    - [ ] Have the version ranges for any required or optional libraries changed?
-    - [ ] Does `make` or `cmake` have a new target?
-    - [ ] Did the requirements or the installation process change? *(rare)*
+   - [ ] Has a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
+   - [ ] Have the version ranges for any required or optional libraries changed?
+   - [ ] Does `make` or `cmake` have a new target?
+   - [ ] Did the requirements or the installation process change? *(rare)*
 - [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.github`
-    - [ ] `.appveyor.yml`
+   - [ ] `.github`
+   - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
-    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
-    - [ ] Add new patterns (just for the new files above) and re-run the above test.
+   - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
+   - [ ] Add new patterns (just for the new files above) and re-run the above test.
 - [ ] New examples:
-    - [ ] All sample runs at the top of the example source file work.
-    - [ ] Update `examples/makefile`:
+   - [ ] All sample runs at the top of the example source file work.
+   - [ ] Update `examples/makefile`:
       - [ ] Add the example code to the appropriate `SEQ_EXAMPLES` and `PAR_EXAMPLES` variables.
       - [ ] Add any files generated by it to the `clean` target.
       - [ ] Add the example binary and any files generated by it to the top-level `.gitignore` file.
-    - [ ] Update `examples/CMakeLists.txt`:
+   - [ ] Update `examples/CMakeLists.txt`:
       - [ ] Add the example code to the `ALL_EXE_SRCS` variable.
       - [ ] Make sure `THIS_TEST_OPTIONS` is set correctly for the new example.
    - [ ] List the new example in `doc/CodeDocumentation.dox`.
-   - [ ] If new examples directory (e.g.`examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
+   - [ ] If new examples directory (e.g. `examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
    - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
       - [ ] Update or add example-specific documentation, see e.g. the `src/examples.md`.
       - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
@@ -575,13 +572,13 @@ Before a PR can be merged, it should satisfy the following:
       - [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
       - [ ] Consider adding a new test for the new miniapp.
    - [ ] List the new miniapp in `doc/CodeDocumentation.dox`
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
+   - [ ] If new miniapps directory (e.g. `miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
+   - [ ] If new miniapps directory (e.g. `miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
    - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
-     - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
-     - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
-     - [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
-     - [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
+      - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
+      - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
+      - [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
+      - [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
 - [ ] New capability:
    - [ ] All new public, protected, and private classes, methods, data members, and functions have full Doxygen-style documentation in source comments. Documentation should include descriptions of member data, function arguments and return values, template parameters, and prerequisites for calling new functions.
    - [ ] Pointer arguments and return values must specify whether ownership is being transferred or lent with the call.
@@ -683,7 +680,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 - [ ] Update URL shortlinks:
     - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. https://mfem.github.io/releases/mfem-3.1.tgz.
     - [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
-    - [ ] Add the new shortlinks to the MFEM packages in `spack`, `homebrew/science`, `VisIt`, etc.
+    - [ ] Add the new shortlinks to the MFEM package in `spack`.
 - [ ] Update website in `mfem/web` repo:
     - Update version and shortlinks in `src/index.md` and `src/download.md`.
     -  Use [cloc-1.62.pl](http://cloc.sourceforge.net/) and `ls -lh` to estimate the SLOC and the tarball size in `src/download.md`.
@@ -735,22 +732,24 @@ commit or push, see the [README](config/githooks/README.md) in the `config/githo
 directory.
 
 
-### Linux and Mac smoke tests
+### GitHub Actions smoke tests
+
 We use GitHub Actions to drive the default tests on the `master` and `next`
 branches. See the `.github/workflows` files and the logs at
 [https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
 
-Testing using GitHub Actions should be kept lightweight, as there is a time
-constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
+GitHub Actions testing should be kept lightweight, as there is a time
+constraint on jobs. The current workflows cover Linux, macOS, and Windows
+configurations.
 
 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.
 
+### Additional Windows smoke test
 
-### Windows smoke test
-We use Appveyor to test building with the MS Visual C++ compiler in a Windows
-environment, as well as to test the CMake build. See the `.appveyor` file and the
-build logs at
+We also use Appveyor to test building with the MS Visual C++ compiler in a Windows
+environment, as well as to test the CMake build. See the `.appveyor.yml` file
+and the build logs at
 [https://ci.appveyor.com/project/mfem/mfem](https://ci.appveyor.com/project/mfem/mfem).
 
 CMake is used to generate the MSVC Project files and drive the build.  A release

INSTALL

@@ -38,14 +38,13 @@ the option MFEM_USE_METIS.
 MFEM also includes support for devices such as GPUs, and programming models such
 as CUDA, HIP, OCCA, OpenMP and RAJA.
 
-- Starting with version 4.0, MFEM requires a C++11 compiler. We recommend using
-  a newer compiler, e.g. GCC version 4.9 or higher.
+- Starting with version 4.9, MFEM requires a C++17 compiler.
 
 - CUDA support requires an NVIDIA GPU and an installation of the CUDA Toolkit
   https://developer.nvidia.com/cuda-toolkit
 
 - HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocmdocs.amd.com
+  https://rocm.docs.amd.com
 
 - OCCA support requires the OCCA library
   https://libocca.org
@@ -83,9 +82,9 @@ Serial build:
 Parallel build:
    (download hypre and METIS 4 from above URLs)
    (build METIS 4 in ../metis-4.0 relative to mfem/)
+   (for METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
    (build hypre in ../hypre relative to mfem/)
    make parallel -j 4
-(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
 
 CUDA build:
    make cuda -j 4
@@ -115,14 +114,14 @@ Serial build:
 Parallel build:
    (download hypre and METIS 4 from above URLs)
    (build METIS 4 in ../metis-4.0 relative to mfem/)
+   (for METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
    (build hypre in ../hypre relative to mfem/)
    mkdir <mfem-build-dir> ; cd <mfem-build-dir>
    cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
    make -j 4
-(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
 
 Parallel build with fetching of hypre and METIS:
-   mkdir <mfem-buil-dir> ; cd <mfem-build-dir>
+   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
    cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DMFEM_FETCH_TPLS=YES
    make -j 4
 
@@ -134,7 +133,8 @@ CUDA build:
 
 HIP build:
    mkdir <mfem-build-dir> ; cd <mfem-build-dir>
-   cmake <mfem-source-dir> -DMFEM_USE_HIP=YES -DHIP_ARCH=gfx942 -DCMAKE_CXX_COMPILER=amdclang++ -DCMAKE_HIP_COMPILER=amdclang++
+   cmake <mfem-source-dir> -DMFEM_USE_HIP=YES -DHIP_ARCH=gfx942 \
+         -DCMAKE_CXX_COMPILER=amdclang++ -DCMAKE_HIP_COMPILER=amdclang++
    make -j 4
 
 Example codes (serial/parallel, depending on the build):
@@ -269,6 +269,7 @@ Compilers:
    CXX      - C++ compiler, serial build
    MPICXX   - MPI C++ compiler, parallel build
    CUDA_CXX - The CUDA compiler, 'nvcc' or 'clang++'
+   HIP_CXX  - The HIP compiler, e.g. 'hipcc'
 
 Compiler options:
    OPTIM_FLAGS - Options for optimized build
@@ -395,6 +396,11 @@ MFEM_USE_STRUMPACK = YES/NO
    classes. When enabled, this option uses the STRUMPACK_* library options, see
    below.
 
+MFEM_USE_CUDSS = YES/NO
+   Enable MFEM functionality based on the cuDSS library. When using cuDSS, CUDA
+   support must be also enabled in MFEM, i.e. MFEM_USE_CUDA=YES must be set.
+   When enabled, this option uses the CUDSS_* library options, see below.
+
 MFEM_USE_GINKGO = YES/NO
    Enable MFEM functionality based on the Ginkgo library, which provides
    iterative linear solvers and preconditioners with OpenMP, CUDA backends, see
@@ -554,13 +560,13 @@ MFEM_USE_RAJA = YES/NO
 MFEM_USE_OCCA = YES/NO
    Enables support for the OCCA library in MFEM. OCCA is an open-source library
    which aims to make it easy to program different types of devices (e.g. CPU,
-   GPU, FPGA) by providing an unified API for interacting with JIT-compiled
+   GPU, FPGA) by providing a unified API for interacting with JIT-compiled
    backends. In order to use the OCCA CUDA backend, CUDA support must be enabled
    in MFEM as well, i.e. MFEM_USE_CUDA=YES must be set.
 
 MFEM_USE_GSLIB = YES/NO
    Enables MFEM functionality based on the GSLIB library, and specifically its
-   FindPoints component, which provides a robust algorithms to evaluate finite
+   FindPoints component, which provides robust algorithms to evaluate finite
    element functions in a collection of points in physical space. When enabled,
    the user can use the GSLIB-FindPoints methods as shown in miniapps/gslib.
 
@@ -719,9 +725,18 @@ The specific libraries and their options are:
   Options: STRUMPACK_OPT, STRUMPACK_LIB.
   Versions: STRUMPACK >= 3.0.0.
 
-- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
-  C++ compiler that supports the C++-17 standard. For additional requirements
-  and dependencies of specific modules, see the Ginkgo webpage below.
+- CUDSS (optional), used when MFEM_USE_CUDSS = YES. Note that CUDSS requires 
+  CUDA 12.x toolkit and the cuDSS libraries. The supported communication backend
+  is OpenMPI 4.x (default), and OpenMPI 4.x or a later version must be pre-built.
+  The source files in the cuDSS tarball provide guidance for developing custom 
+  MPI implementations.
+  URL: https://developer.nvidia.com/cudss
+       https://docs.nvidia.com/cuda/cudss/advanced_features.html#communication-layer-library-in-cudss
+  Options: CUDSS_OPT, CUDSS_LIB.
+  Versions: cuDSS >= 0.6.0.
+
+- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Ginkgo may have additional
+  requirements and module-specific dependencies; see the webpage below.
   URL: https://ginkgo-project.github.io
   Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
            Debug).
@@ -793,7 +808,7 @@ The specific libraries and their options are:
   Options: CONDUIT_OPT, CONDUIT_LIB.
   Versions: Conduit >= 0.3.1.
 
-- ADIOS2 (optional) used when MFEM_USE_ADIOS2 = YES.
+- ADIOS2 (optional), used when MFEM_USE_ADIOS2 = YES.
   URL: https://adios2.readthedocs.io/
   Versions: ADIOS >= 2.5.0.
 
@@ -869,7 +884,7 @@ The specific libraries and their options are:
   Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
   Versions: RAJA >= 2022.10.3.
 
-- Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
+- Moonolith (optional), used when MFEM_USE_MOONOLITH = YES.
   URL: https://bitbucket.org/zulianp/par_moonolith
   Options: MOONOLITH_DIR
   Versions: MOONOLITH >= 1.1.0.
@@ -957,7 +972,7 @@ CMAKE_BUILD_TYPE which can be set to standard values like "Debug", and "Release"
 To use a specific generator use the "-G <generator>" option of cmake:
 
    cmake <mfem-source-dir> -G "Xcode"
-   cmake <mfem-source-dir> -G "Visual Studio 12 2013"
+   cmake <mfem-source-dir> -G "Visual Studio 17 2022"
    cmake <mfem-source-dir> -G "MinGW Makefiles"
 
 With CMake it is possible to build MFEM as a shared library using the standard
@@ -1202,7 +1217,7 @@ larger problems, there are two options:
 Specific options for HIP
 ========================
 MFEM expects the `ROCM_PATH` environment variable to be set to the path of the
-ROCM install, as well as having `$ROCM_PATH/bin` in `PATH`.
+ROCm install, as well as having `$ROCM_PATH/bin` in `PATH`.
 
 Specific options for RAJA+HIP+MPI
 =================================

NOTICE

@@ -28,6 +28,7 @@ license files. These software products and their licenses are as follows:
 * AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
 * Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
 * Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
+* gslib (fem/gslib.{cpp,hpp}, mesh/bb_grid_map.{cpp,hpp}) -- BSD 3-clause license
 * Picojson (fem/picojson.h) -- Custom 2-clause license
 * TinyXML2 (general/tinyxml2.{cpp,h}) -- zlib license
 * Zstr (general/zstr.hpp) -- MIT license

config/cmake/MFEMConfig.cmake.in

@@ -34,7 +34,9 @@ set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
+set(MFEM_USE_COMPLEX_MUMPS @MFEM_USE_COMPLEX_MUMPS@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
+set(MFEM_USE_CUDSS @MFEM_USE_CUDSS@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
 set(MFEM_USE_MAGMA @MFEM_USE_MAGMA@)
@@ -109,6 +111,10 @@ if (MFEM_USE_RAJA)
    find_dependency(RAJA)
 endif()
 
+if (MFEM_USE_CUDSS)
+  find_dependency(cudss)
+endif (MFEM_USE_CUDSS)
+
 if (MFEM_USE_UMPIRE)
    find_dependency(umpire)
 endif()

config/cmake/config.hpp.in

@@ -103,11 +103,21 @@
 
 // Enable MFEM functionality based on the MUMPS library.
 #cmakedefine MFEM_USE_MUMPS
+#cmakedefine MFEM_USE_COMPLEX_MUMPS
 #cmakedefine MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@
 
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK
 
+// Enable MFEM functionality based on the cuDSS library.
+#cmakedefine MFEM_USE_CUDSS
+
+// CUDSS communication layer library path
+#cmakedefine MFEM_CUDSS_COMM_LIB "@MFEM_CUDSS_COMM_LIB@"
+
+// CUDSS threading layer library path
+#cmakedefine MFEM_CUDSS_THREADING_LIB "@MFEM_CUDSS_THREADING_LIB@"
+
 // Enable functionality based on the Ginkgo library.
 #cmakedefine MFEM_USE_GINKGO
 

config/cmake/modules/FindCUDSS.cmake

@@ -0,0 +1,68 @@
+if (NOT cudss_DIR AND CUDSS_DIR)
+  set(cudss_DIR ${CUDSS_DIR}/lib/cmake/cudss)
+endif()
+message(STATUS "Looking for CUDSS ...")
+message(STATUS "   in CUDSS_DIR = ${CUDSS_DIR}")
+message(STATUS "      cudss_DIR = ${cudss_DIR}")
+find_package(cudss)
+set(CUDSS_FOUND ${cudss_FOUND})
+set(CUDSS_LIBRARIES "cudss")
+if (CUDSS_FOUND)
+  message(STATUS
+    "Found CUDSS target: ${CUDSS_LIBRARIES} (version: ${cudss_VERSION})")
+else()
+  set(msg STATUS)
+  if (CUDSS_FIND_REQUIRED)
+    set(msg FATAL_ERROR)
+  endif()
+  message(${msg}
+    "CUDSS not found. Please set CUDSS_DIR to the install prefix.")
+endif()
+
+if(CUDSS_FOUND AND TARGET cudss)
+  get_target_property(CUDSS_LIBRARY_LOCATION cudss IMPORTED_LOCATION)
+  if(NOT CUDSS_LIBRARY_LOCATION)
+    get_target_property(CUDSS_LIBRARY_LOCATION cudss IMPORTED_LOCATION_RELEASE)
+  endif()
+  if(CUDSS_LIBRARY_LOCATION)
+    get_filename_component(CUDSS_LIBRARY_DIR "${CUDSS_LIBRARY_LOCATION}" DIRECTORY)
+  else()
+    message(WARNING "Could not determine the location of the cuDSS library.")
+  endif()
+else()
+    message(WARNING "cuDSS target not available; cannot determine library directory.")
+endif()
+
+# Set the full name of the cuDSS threading library if OpenMP is enabled.
+# The threading layer library (libcudss_mtlayer_gomp.so) is located under the
+# cuDSS library directory by default.
+if (MFEM_USE_OPENMP)
+  find_file(
+    CUDSS_THREADING_LIB
+    NAMES libcudss_mtlayer_gomp.so
+    PATHS ${CUDSS_LIBRARY_DIR}
+    NO_DEFAULT_PATH
+  )
+  if (NOT DEFINED MFEM_CUDSS_THREADING_LIB AND CUDSS_THREADING_LIB)
+    set(MFEM_CUDSS_THREADING_LIB "${CUDSS_THREADING_LIB}")
+  endif()
+  message(STATUS "CUDSS threading layer library: ${MFEM_CUDSS_THREADING_LIB}")
+endif()
+
+# Set the full name of the cuDSS communication library if MFEM use OpenMPI.
+# The communication layer library (libcudss_commlayer_mpi.so) is located under the
+# cuDSS library directory by default.
+# The communication layer library is used pre-built communication layers for OpenMPI 
+# by default.
+if (MFEM_USE_MPI)
+  find_file(
+    CUDSS_COMM_LIB
+    NAMES libcudss_commlayer_openmpi.so
+    PATHS ${CUDSS_LIBRARY_DIR}
+    NO_DEFAULT_PATH
+  )
+  if (NOT DEFINED MFEM_CUDSS_COMM_LIB AND CUDSS_COMM_LIB)
+    set(MFEM_CUDSS_COMM_LIB "${CUDSS_COMM_LIB}")
+  endif()
+  message(STATUS "CUDSS communication layer library: ${MFEM_CUDSS_COMM_LIB}")
+endif()

config/cmake/modules/FindGSLIB.cmake

@@ -18,19 +18,17 @@
 
 if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS)
   enable_language(C)
-  string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
   set(GSLIB_FETCH_VERSION 1.0.9)
-  set(GSLIB_C_FLAGS ${CMAKE_C_FLAGS_${BUILD_TYPE}})
-  if (CMAKE_C_FLAGS)
-    set(GSLIB_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
-  endif()
-  if (BUILD_SHARED_LIBS)
-    set(GSLIB_C_FLAGS "${GSLIB_C_FLAGS} -fPIC")
-  endif()
   add_library(GSLIB STATIC IMPORTED)
+  # set options (technically flags because GSLIB does not use cmake)
+  string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
+  set(GSLIB_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
+  if (BUILD_SHARED_LIBS)
+    set(GSLIB_FLAGS "${GSLIB_FLAGS} -fPIC")
+  endif()
   # define external project and create future include directory so it is present
   # to pass CMake checks at end of MFEM configuration step
-  message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_C_FLAGS}")
+  message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_FLAGS}")
   set(PREFIX ${CMAKE_BINARY_DIR}/fetch/gslib)
   include(ExternalProject)
   ExternalProject_Add(gslib
@@ -40,7 +38,7 @@ if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS)
     UPDATE_DISCONNECTED TRUE
     PREFIX ${PREFIX}
     CONFIGURE_COMMAND ""
-    BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$<BOOL:${MFEM_USE_MPI}> "CFLAGS= ${GSLIB_C_FLAGS}"
+    BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$<BOOL:${MFEM_USE_MPI}> "CFLAGS=${GSLIB_FLAGS}"
     INSTALL_COMMAND "")
   file(MAKE_DIRECTORY ${PREFIX}/include)
   # set imported library target properties

config/cmake/modules/FindHYPRE.cmake

@@ -44,6 +44,9 @@ if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS)
   # set options and associated dependencies
   set(HYPRE_CMAKE_OPTIONS "")
   list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
+  if (BUILD_SHARED_LIBS)
+    list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON)
+  endif()
   # collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL.
   get_cmake_property(all_vars VARIABLES)
   foreach(var ${all_vars})
@@ -95,7 +98,6 @@ if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS)
     UPDATE_DISCONNECTED TRUE
     SOURCE_SUBDIR src
     PREFIX ${HYPRE_INSTALL}
-    BUILD_COMMAND ${CMAKE_COMMAND} --build . -- -j${CMAKE_BUILD_PARALLEL_LEVEL}
     CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${HYPRE_INSTALL} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${HYPRE_CMAKE_OPTIONS})
   file(MAKE_DIRECTORY ${HYPRE_INSTALL}/include)
   # set imported library target properties

config/cmake/modules/FindMETIS.cmake

@@ -19,10 +19,18 @@
 #   - METIS_VERSION_5 (cache variable)
 
 if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS)
+  enable_language(C)
   set(METIS_FETCH_VERSION 4.0.3)
   add_library(METIS STATIC IMPORTED)
+  # set options (technically flags because METIS does not use cmake)
+  set(METIS_FLAGS "-Wno-implicit-int -Wno-incompatible-pointer-types")
+  string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
+  set(METIS_FLAGS "${METIS_FLAGS} ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
+  if (BUILD_SHARED_LIBS)
+    set(METIS_FLAGS "${METIS_FLAGS} -fPIC")
+  endif()
   # define external project
-  message(STATUS "Will fetch METIS ${METIS_FETCH_VERSION} to be built with default options")
+  message(STATUS "Will fetch METIS ${METIS_FETCH_VERSION} to be built with ${METIS_FLAGS}")
   set(PREFIX ${CMAKE_BINARY_DIR}/fetch/metis)
   include(ExternalProject)
   ExternalProject_Add(metis
@@ -32,7 +40,7 @@ if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS)
     UPDATE_DISCONNECTED TRUE
     PREFIX ${PREFIX}
     CONFIGURE_COMMAND tar -xzf ../metis/metis-${METIS_FETCH_VERSION}-mac.tgz --strip=1
-    BUILD_COMMAND $(MAKE) COPTIONS=-Wno-incompatible-pointer-types
+    BUILD_COMMAND $(MAKE) clean && $(MAKE) "OPTFLAGS=${METIS_FLAGS}"
     INSTALL_COMMAND mkdir -p ${PREFIX}/lib && cp libmetis.a ${PREFIX}/lib/)
   # set imported library target properties
   add_dependencies(METIS metis)

config/cmake/modules/FindMUMPS.cmake

@@ -17,13 +17,31 @@
 
 include(MfemCmakeUtilities)
 
-# Toggle which precision of MUMPS to use depending on the precision of MFEM.
+# Decide headers/libs by MFEM precision
 if (MFEM_USE_DOUBLE)
-  set(_mumps_header dmumps_c.h)
-  set(_mumps_lib dmumps)
-elseif(MFEM_USE_SINGLE)
-  set(_mumps_header smumps_c.h)
-  set(_mumps_lib smumps)
+  set(_rmumps_header dmumps_c.h)
+  set(_rmumps_lib    dmumps)
+  set(_cmumps_header zmumps_c.h)
+  set(_cmumps_lib    zmumps)
+elseif (MFEM_USE_SINGLE)
+  set(_rmumps_header smumps_c.h)
+  set(_rmumps_lib    smumps)
+  set(_cmumps_header cmumps_c.h)
+  set(_cmumps_lib    cmumps)
+endif()
+
+# Choose which header/lib mfem_find_package should use as the "primary" one.
+# If both enabled, prefer the real one as primary (either is fine).
+if (MFEM_USE_MUMPS)
+  set(_mumps_header ${_rmumps_header})
+  set(_mumps_lib    ${_rmumps_lib})
+elseif (MFEM_USE_COMPLEX_MUMPS)
+  set(_mumps_header ${_cmumps_header})
+  set(_mumps_lib    ${_cmumps_lib})
+else()
+  # Should not happen in practice because FindMUMPS is only called when enabled,
+  set(_mumps_header ${_rmumps_header})
+  set(_mumps_lib    ${_rmumps_lib})
 endif()
 
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
@@ -31,8 +49,35 @@ mfem_find_package(MUMPS MUMPS MUMPS_DIR
   "Paths to headers required by MUMPS."
   "Libraries required by MUMPS."
   ADD_COMPONENT mumps_common "include" ${_mumps_header} "lib" mumps_common
-  ADD_COMPONENT pord "include" ${_mumps_header} "lib" pord)
+  ADD_COMPONENT pord        "include" ${_mumps_header} "lib" pord)
 
+# If BOTH real and complex are enabled, ensure BOTH solver libs are linked.
+if (MUMPS_FOUND AND MFEM_USE_MUMPS AND MFEM_USE_COMPLEX_MUMPS)
+  # Find the "other" solver library and append it.
+  find_library(_mfem_other_mumps_solver
+    NAMES ${_cmumps_lib}   
+    HINTS ${MUMPS_DIR}
+    PATH_SUFFIXES lib lib64
+    NO_DEFAULT_PATH)
+
+  if (NOT _mfem_other_mumps_solver)
+    # Fall back to system search
+    find_library(_mfem_other_mumps_solver NAMES ${_cmumps_lib})
+  endif()
+
+  if (NOT _mfem_other_mumps_solver)
+    message(FATAL_ERROR
+      "MFEM_USE_MUMPS=ON and MFEM_USE_COMPLEX_MUMPS=ON, but could not find "
+      "the complex solver library '${_cmumps_lib}' in MUMPS_DIR='${MUMPS_DIR}'.")
+  endif()
+
+  # Put solver libs first (important for static link order)
+  # MUMPS_LIBRARIES contains the primary solver already + common + pord.
+  # We prepend the other solver.
+  list(INSERT MUMPS_LIBRARIES 0 ${_mfem_other_mumps_solver})
+endif()
+
+# Version detection 
 if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
   try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
           ${CMAKE_CURRENT_BINARY_DIR}/config

config/cmake/modules/FindSuiteSparse.cmake

@@ -22,15 +22,15 @@ include(MfemCmakeUtilities)
 mfem_find_package(SuiteSparse SuiteSparse SuiteSparse_DIR "" "" "" ""
   "Paths to headers required by SuiteSparse."
   "Libraries required by SuiteSparse."
-  ADD_COMPONENT "UMFPACK" "include;suitesparse" umfpack.h "lib" umfpack
-  ADD_COMPONENT "KLU" "include;suitesparse" klu.h "lib" klu
-  ADD_COMPONENT "AMD" "include;suitesparse" amd.h "lib" amd
-  ADD_COMPONENT "BTF" "include;suitesparse" btf.h "lib" btf
-  ADD_COMPONENT "CHOLMOD" "include;suitesparse" cholmod.h "lib" cholmod
-  ADD_COMPONENT "COLAMD" "include;suitesparse" colamd.h "lib" colamd
-  ADD_COMPONENT "CAMD" "include;suitesparse" camd.h "lib" camd
-  ADD_COMPONENT "CCOLAMD" "include;suitesparse" ccolamd.h "lib" ccolamd
-  ADD_COMPONENT "config" "include;suitesparse" SuiteSparse_config.h "lib"
+  ADD_COMPONENT "UMFPACK" "include;include/suitesparse;suitesparse" umfpack.h "lib" umfpack
+  ADD_COMPONENT "KLU" "include;include/suitesparse;suitesparse" klu.h "lib" klu
+  ADD_COMPONENT "AMD" "include;include/suitesparse;suitesparse" amd.h "lib" amd
+  ADD_COMPONENT "BTF" "include;include/suitesparse;suitesparse" btf.h "lib" btf
+  ADD_COMPONENT "CHOLMOD" "include;include/suitesparse;suitesparse" cholmod.h "lib" cholmod
+  ADD_COMPONENT "COLAMD" "include;include/suitesparse;suitesparse" colamd.h "lib" colamd
+  ADD_COMPONENT "CAMD" "include;include/suitesparse;suitesparse" camd.h "lib" camd
+  ADD_COMPONENT "CCOLAMD" "include;include/suitesparse;suitesparse" ccolamd.h "lib" ccolamd
+  ADD_COMPONENT "config" "include;include/suitesparse;suitesparse" SuiteSparse_config.h "lib"
     suitesparseconfig)
 
 if (SuiteSparse_FOUND AND METIS_VERSION_5)

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -876,12 +876,12 @@ function(mfem_export_mk_files)
       MFEM_USE_ZLIB MFEM_USE_LIBUNWIND MFEM_USE_LAPACK MFEM_THREAD_SAFE
       MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS
       MFEM_USE_SUITESPARSE MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS
-      MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_MAGMA
-      MFEM_USE_GNUTLS MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC
-      MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA
-      MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO
+      MFEM_USE_COMPLEX_MUMPS MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
+      MFEM_USE_MAGMA MFEM_USE_GNUTLS MFEM_USE_NETCDF MFEM_USE_PETSC
+      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS MFEM_USE_CONDUIT
+      MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP
+      MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER MFEM_USE_UMPIRE
+      MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO
       MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
       MFEM_USE_TRIBOL MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME
       MFEM_USE_HDF5)

config/config.hpp

@@ -140,6 +140,9 @@ constexpr real_t operator""_r(unsigned long long v)
 #ifdef MFEM_USE_MUMPS
 #error Building with MUMPS (MFEM_USE_MUMPS=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#ifdef MFEM_USE_COMPLEX_MUMPS
+#error Building with COMPLEX_MUMPS (MFEM_USE_COMPLEX_MUMPS=YES) requires MPI (MFEM_USE_MPI=YES)
+#endif
 #ifdef MFEM_USE_STRUMPACK
 #error Building with STRUMPACK (MFEM_USE_STRUMPACK=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
@@ -157,4 +160,10 @@ constexpr real_t operator""_r(unsigned long long v)
 #endif
 #endif // MFEM_USE_MPI not defined
 
+#ifndef MFEM_USE_CUDA
+#ifdef MFEM_USE_CUDSS
+#error Building with cuDSS (MFEM_USE_CUDSS=YES) requires CUDA (MFEM_USE_CUDA=YES)
+#endif
+#endif // MFEM_USE_CUDSS not defined
+
 #endif // MFEM_CONFIG_HPP

config/config.hpp.in

@@ -103,11 +103,21 @@
 
 // Enable MFEM functionality based on the MUMPS library.
 // #define MFEM_USE_MUMPS
+// #define MFEM_USE_COMPLEX_MUMPS
 // #define MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@
 
 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK
 
+// Enable MFEM functionality based on the cuDSS library.
+// #define MFEM_USE_CUDSS
+
+// CUDSS communication layer library path
+// #define MFEM_CUDSS_COMM_LIB "@MFEM_CUDSS_COMM_LIB@"
+
+// CUDSS threading layer library path
+// #define MFEM_CUDSS_THREADING_LIB "@MFEM_CUDSS_THREADING_LIB@"
+
 // Enable MFEM features based on the Ginkgo library.
 // #define MFEM_USE_GINKGO
 

config/config.mk.in

@@ -36,6 +36,9 @@ MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
 MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
+MFEM_USE_CUDSS         = @MFEM_USE_CUDSS@
+MFEM_CUDSS_COMM_LIB    = @MFEM_CUDSS_COMM_LIB@
+MFEM_CUDSS_THREADING_LIB = @MFEM_CUDSS_THREADING_LIB@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_MAGMA         = @MFEM_USE_MAGMA@

config/defaults.cmake

@@ -37,7 +37,9 @@ option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
 option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
+option(MFEM_USE_COMPLEX_MUMPS "Enable COMPLEX_MUMPS usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
+option(MFEM_USE_CUDSS "Enable cuDSS usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
 option(MFEM_USE_MAGMA "Enable MAGMA usage" OFF)
@@ -152,7 +154,7 @@ set(SuperLUDist_REQUIRED_PACKAGES "MPI" "ParMETIS" "METIS"
 set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.5.0" CACHE PATH
     "Path to the MUMPS library.")
 # MUMPS may also depend on "OpenMP", depending on how it was compiled.
-set(MUMPS_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
+set(MUMPS_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "METIS"
     "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
     "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:

config/defaults.mk

@@ -152,7 +152,9 @@ MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_SUPERLU5      = NO
 MFEM_USE_MUMPS         = NO
+MFEM_USE_COMPLEX_MUMPS = NO
 MFEM_USE_STRUMPACK     = NO
+MFEM_USE_CUDSS         = NO
 MFEM_USE_GINKGO        = NO
 MFEM_USE_AMGX          = NO
 MFEM_USE_MAGMA         = NO
@@ -248,7 +250,7 @@ ifeq (YES,$(MFEM_USE_HIP))
 endif
 
 # METIS library configuration
-ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
+ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS)$(MFEM_USE_COMPLEX_MUMPS),NONONONO)
    ifeq ($(MFEM_USE_METIS_5),NO)
      METIS_DIR = @MFEM_DIR@/../metis-4.0
      METIS_OPT =
@@ -352,13 +354,19 @@ MPI_FORTRAN_LIB = -lmpifort
 # MUMPS library configuration
 MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib
+COMPLEX_MUMPS_OPT = $(MUMPS_OPT)
+
+MUMPS_COMMON_PATH = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib
+MUMPS_COMMON_LIB = -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
 ifeq ($(MFEM_USE_SINGLE),YES)
-   MUMPS_LIB += -lsmumps
+  MUMPS_SOLVER_LIB         = -lsmumps
+  COMPLEX_MUMPS_SOLVER_LIB = -lcmumps
 else
-   MUMPS_LIB += -ldmumps
+  MUMPS_SOLVER_LIB         = -ldmumps
+  COMPLEX_MUMPS_SOLVER_LIB = -lzmumps
 endif
-MUMPS_LIB += -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
+MUMPS_LIB         = $(MUMPS_COMMON_PATH) $(MUMPS_SOLVER_LIB) $(MUMPS_COMMON_LIB)
+COMPLEX_MUMPS_LIB = $(MUMPS_COMMON_PATH) $(COMPLEX_MUMPS_SOLVER_LIB) $(MUMPS_COMMON_LIB)
 
 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
@@ -368,6 +376,19 @@ STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
  $(SCOTCH_LIB) $(SCALAPACK_LIB)
 
+# CUDSS library configuration
+CUDSS_DIR         = @MFEM_DIR@/../cudss
+CUDSS_INCLUDE_DIR = $(CUDSS_DIR)/include
+CUDSS_LIBRARY_DIR = $(CUDSS_DIR)/lib
+CUDSS_OPT         = -I$(CUDSS_INCLUDE_DIR)
+CUDSS_LIB         = \
+ $(XLINKER)-rpath,$(CUDSS_LIBRARY_DIR) -L$(CUDSS_LIBRARY_DIR) -lcudss
+# The cuDSS communication and threading libraries. 
+MFEM_CUDSS_COMM_LIB = $(abspath $(wildcard $(or $(CUDSS_COMM_LIB),\
+   $(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so))))
+MFEM_CUDSS_THREADING_LIB = $(abspath $(wildcard $(or $(CUDSS_THREADING_LIB),\
+   $(subst @MFEM_DIR@,$(MFEM_DIR),$(CUDSS_LIBRARY_DIR)/libcudss_mtlayer_gomp.so))))
+
 # Ginkgo library configuration
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
 GINKGO_SEARCH_DIR = $(subst @MFEM_DIR@,$(MFEM_DIR),$(GINKGO_DIR))
@@ -621,7 +642,7 @@ PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG
 AXOM_DIR = @MFEM_DIR@/../axom
 TRIBOL_DIR = @MFEM_DIR@/../tribol
 TRIBOL_OPT = -I$(TRIBOL_DIR)/include -I$(AXOM_DIR)/include
-TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
+TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -ltribol_shared -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
    -laxom_slam -laxom_slic -laxom_core
 
 # Enzyme configuration

config/get_mumps_version.cpp

@@ -10,10 +10,18 @@
 // CONTRIBUTING.md for details.
 
 #ifdef MFEM_USE_SINGLE
+#ifdef MFEM_USE_COMPLEX_MUMPS
+#include "cmumps_c.h"
+#else
 #include "smumps_c.h"
+#endif
+#else
+#ifdef MFEM_USE_COMPLEX_MUMPS
+#include "zmumps_c.h"
 #else
 #include "dmumps_c.h"
 #endif
+#endif
 #include <string>
 #include <iostream>
 #include <algorithm>

config/githooks/README.md

@@ -39,3 +39,8 @@ when a picture was added for documentation.
 If that is the case, make sure the failure is indeed justified, and rerun the
 push command with the `--no-verify` option. This will skip the hooks, allowing
 you to push those changes.
+
+The `branch-history` check is run automatically through GitHub Actions. If a
+branch is known to have a large number of changes that are legitimate, the
+check can be overridden by setting  the label 'branch-history-override' on the
+pull request.

config/makefile

@@ -42,7 +42,7 @@ GHV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..)
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
 SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
 SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
-MUMPS = $(MFEM_USE_MUMPS:NO=)
+MUMPS = $(if $(filter YES,$(MFEM_USE_MUMPS) $(MFEM_USE_COMPLEX_MUMPS)),YES,)
 GMV_CXX ?= $(MFEM_CXX)
 GMV = get_mumps_version
 GMV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))

examples/ex1.cpp

@@ -50,6 +50,10 @@
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
 //
+// Device simplices sample runs:
+//               ex1 -pa -d gpu -m ../data/inline-tet.mesh
+//               ex1 -pa -d gpu -m ../data/inline-tri.mesh
+//
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Poisson problem
 //               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
@@ -138,25 +142,25 @@ int main(int argc, char *argv[])
    }
 
    // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
+   //    Lagrange finite elements of the specified order.
+   //    - If order < 1, we instead use an isoparametric/isogeometric space.
+   //    - If the mesh is simplicial and partial assembly is requested,
+   //      we use the positive basis, which supports device execution.
    FiniteElementCollection *fec;
-   bool delete_fec;
+   auto basis_type = (pa && mesh.IsSimplexMesh()) ?
+                     BasisType::Positive : BasisType::GaussLobatto;
    if (order > 0)
    {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
+      fec = new H1_FECollection(order, dim, basis_type);
    }
    else if (mesh.GetNodes())
    {
       fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
       cout << "Using isoparametric FEs: " << fec->Name() << endl;
    }
    else
    {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
+      fec = new H1_FECollection(order = 1, dim, basis_type);
    }
    FiniteElementSpace fespace(&mesh, fec);
    cout << "Number of finite element unknowns: "
@@ -224,17 +228,29 @@ int main(int argc, char *argv[])
    // 11. Solve the linear system A X = B.
    if (!pa)
    {
-#ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
+#ifdef MFEM_USE_CUDSS
+      if (Device::Allows(Backend::CUDA_MASK))
+      {
+         // Use cuDSS to solve the system.
+         CuDSSSolver cudss_solver;
+         cudss_solver.SetOperator(*A);
+         cudss_solver.Mult(B, X);
+      }
+      else
 #endif
+      {
+#ifndef MFEM_USE_SUITESPARSE
+         // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
+         GSSmoother M((SparseMatrix&)(*A));
+         PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
+#else
+         // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+         UMFPackSolver umf_solver;
+         umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+         umf_solver.SetOperator(*A);
+         umf_solver.Mult(B, X);
+#endif
+      }
    }
    else
    {
@@ -273,17 +289,14 @@ int main(int argc, char *argv[])
    if (visualization)
    {
       char vishost[] = "localhost";
-      int  visport   = 19916;
+      int visport = 19916;
       socketstream sol_sock(vishost, visport);
       sol_sock.precision(8);
       sol_sock << "solution\n" << mesh << x << flush;
    }
 
    // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   if (order > 0) { delete fec; }
 
    return 0;
 }

examples/ex1p.cpp

@@ -42,7 +42,11 @@
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/beam-tet.mesh
+//
+// Device simplices sample runs:
+//               mpirun -np 4 ex1p -pa -d gpu -m ../data/inline-tet.mesh
+//               mpirun -np 4 ex1p -pa -d gpu -m ../data/inline-tri.mesh
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Poisson problem
@@ -83,6 +87,9 @@ int main(int argc, char *argv[])
    const char *device_config = "cpu";
    bool visualization = true;
    bool algebraic_ceed = false;
+#ifdef MFEM_USE_CUDSS
+   bool cudss_solver = false;
+#endif
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -102,6 +109,10 @@ int main(int argc, char *argv[])
    args.AddOption(&algebraic_ceed, "-a", "--algebraic",
                   "-no-a", "--no-algebraic",
                   "Use algebraic Ceed solver");
+#endif
+#ifdef MFEM_USE_CUDSS
+   args.AddOption(&cudss_solver, "-cudss", "--cudss-solver", "-no-cudss",
+                  "--no-cudss-solver", "Use the cuDSS Solver.");
 #endif
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
@@ -158,19 +169,20 @@ int main(int argc, char *argv[])
    }
 
    // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
+   //    use continuous Lagrange finite elements of the specified order.
+   //    - If order < 1, we instead use an isoparametric/isogeometric space.
+   //    - If the mesh is simplicial and partial assembly is requested,
+   //      we use the positive basis, which supports device execution.
    FiniteElementCollection *fec;
-   bool delete_fec;
+   auto basis_type = (pa && pmesh.IsSimplexMesh()) ?
+                     BasisType::Positive : BasisType::GaussLobatto;
    if (order > 0)
    {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
+      fec = new H1_FECollection(order, dim, basis_type);
    }
    else if (pmesh.GetNodes())
    {
       fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
       if (myid == 0)
       {
          cout << "Using isoparametric FEs: " << fec->Name() << endl;
@@ -178,8 +190,7 @@ int main(int argc, char *argv[])
    }
    else
    {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
+      fec = new H1_FECollection(order = 1, dim, basis_type);
    }
    ParFiniteElementSpace fespace(&pmesh, fec);
    HYPRE_BigInt size = fespace.GlobalTrueVSize();
@@ -248,33 +259,51 @@ int main(int argc, char *argv[])
    // 13. Solve the linear system A X = B.
    //     * With full assembly, use the BoomerAMG preconditioner from hypre.
    //     * With partial assembly, use Jacobi smoothing, for now.
-   Solver *prec = NULL;
-   if (pa)
+#ifdef MFEM_USE_CUDSS
+   if (!pa && (Device::Allows(Backend::CUDA_MASK) && cudss_solver))
    {
-      if (UsesTensorBasis(fespace))
-      {
-         if (algebraic_ceed)
-         {
-            prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
-         }
-         else
-         {
-            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-         }
-      }
+      // Solve using a direct solver with cuDSS
+      CuDSSSolver cudss_solver(MPI_COMM_WORLD);
+      cudss_solver.SetMatrixSymType(
+         CuDSSSolver::SYMMETRIC_POSITIVE_DEFINITE);
+      cudss_solver.SetMatrixViewType(CuDSSSolver::UPPER);
+      cudss_solver.SetOperator(*A);
+      cudss_solver.Mult(B, X);
    }
    else
+#endif
    {
-      prec = new HypreBoomerAMG;
+      Solver *prec = NULL;
+      if (pa)
+      {
+         if (UsesTensorBasis(fespace))
+         {
+            if (algebraic_ceed)
+            {
+               prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
+            }
+            else
+            {
+               prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+            }
+         }
+      }
+      else
+      {
+         prec = new HypreBoomerAMG;
+      }
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(1);
+      if (prec)
+      {
+         cg.SetPreconditioner(*prec);
+      }
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete prec;
    }
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   if (prec) { cg.SetPreconditioner(*prec); }
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete prec;
 
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
@@ -308,10 +337,7 @@ int main(int argc, char *argv[])
    }
 
    // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   if (order > 0) { delete fec; }
 
    return 0;
 }

examples/ex25p.cpp

@@ -200,7 +200,7 @@ int main(int argc, char *argv[])
    args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
                   "--no-superlu", "Use the SuperLU Solver.");
 #endif
-#ifdef MFEM_USE_MUMPS
+#if defined(MFEM_USE_MUMPS) || defined(MFEM_USE_COMPLEX_MUMPS)
    args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
                   "--no-mumps-solver", "Use the MUMPS Solver.");
 #endif
@@ -502,16 +502,24 @@ int main(int argc, char *argv[])
       delete A;
    }
 #endif
-#ifdef MFEM_USE_MUMPS
+#if defined(MFEM_USE_MUMPS) || defined(MFEM_USE_COMPLEX_MUMPS)
    if (!pa && mumps_solver)
    {
-      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      ComplexHypreParMatrix *Ahc = Ah.As<ComplexHypreParMatrix>();
+#ifdef MFEM_USE_COMPLEX_MUMPS
+      ComplexMUMPSSolver mumps(MPI_COMM_WORLD);
+      mumps.SetPrintLevel(0);
+      mumps.SetOperator(*Ahc);
+      mumps.Mult(B, X);
+#else
+      HypreParMatrix *A = Ahc->GetSystemMatrix();
       MUMPSSolver mumps(A->GetComm());
       mumps.SetPrintLevel(0);
       mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
       mumps.SetOperator(*A);
       mumps.Mult(B, X);
       delete A;
+#endif
    }
 #endif
    // 16a. Set up the parallel Bilinear form a(.,.) for the preconditioner

examples/ex2p.cpp

@@ -95,6 +95,15 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
+   if (amg_elast && !static_cond && reorder_space)
+   {
+      if (myid == 0)
+         cerr << "\nThe AMG elasticity solver requires ordering byVDIM! "
+              << "Ignoring the specified option -nodes/--by-nodes.\n"
+              << endl;
+      reorder_space = false;
+   }
+
    // 3. Enable hardware devices such as GPUs, and programming models such as
    //    CUDA, OCCA, RAJA and OpenMP based on command line options.
    Device device(device_config);

fem/CMakeLists.txt

@@ -133,7 +133,7 @@ set(SRCS
   tmop/assemble/diag2.cpp
   tmop/assemble/grad2_limit.cpp
   tmop/assemble/grad2.cpp
-  tmop/assemble/diag3_limit.cpp  
+  tmop/assemble/diag3_limit.cpp
   tmop/assemble/diag3.cpp
   tmop/assemble/grad3_limit.cpp
   tmop/assemble/grad3.cpp
@@ -171,8 +171,12 @@ set(SRCS
   tmop_tools.cpp
   tmop_amr.cpp
   gslib.cpp
+  gslib/findptsedge_local_2.cpp
+  gslib/findptsedge_local_3.cpp
+  gslib/findptssurf_local_3.cpp
   gslib/findpts_local_2.cpp
   gslib/findpts_local_3.cpp
+  gslib/interpolate_local_1.cpp
   gslib/interpolate_local_2.cpp
   gslib/interpolate_local_3.cpp
   transfer.cpp
@@ -191,12 +195,14 @@ set(HDRS
   integ/bilininteg_dgtrace_kernels.hpp
   integ/bilininteg_vecdiffusion_kernels.hpp
   integ/bilininteg_convection_kernels.hpp
+  integ/bilininteg_diffusion_pa_simplices.hpp
   integ/bilininteg_diffusion_kernels.hpp
   integ/bilininteg_elasticity_kernels.hpp
   integ/bilininteg_hcurl_kernels.hpp
   integ/bilininteg_hdiv_kernels.hpp
   integ/bilininteg_hcurlhdiv_kernels.hpp
   integ/bilininteg_mass_kernels.hpp
+  integ/bilininteg_mass_pa_simplices.hpp
   integ/bilininteg_vecdiffusion_pa.hpp
   integ/bilininteg_vecmass_pa.hpp
   coefficient.hpp
@@ -305,6 +311,7 @@ set(HDRS
   tmop_tools.hpp
   tmop_amr.hpp
   gslib.hpp
+  gslib/gslib_kernel_helpers.hpp
   transfer.hpp
   hyperbolic.hpp
   integrator.hpp

fem/bilininteg.cpp

@@ -1345,7 +1345,8 @@ real_t DiffusionIntegrator::ComputeFluxEnergy
 }
 
 const IntegrationRule &DiffusionIntegrator::GetRule(
-   const FiniteElement &trial_fe, const FiniteElement &test_fe)
+   const FiniteElement &trial_fe, const FiniteElement &test_fe,
+   const bool stroud)
 {
    int order;
    if (trial_fe.Space() == FunctionSpace::Pk)
@@ -1362,7 +1363,15 @@ const IntegrationRule &DiffusionIntegrator::GetRule(
    {
       return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
    }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
+
+   if (stroud)
+   {
+      return StroudIntRules.Get(trial_fe.GetGeomType(), order);
+   }
+   else
+   {
+      return IntRules.Get(trial_fe.GetGeomType(), order);
+   }
 }
 
 MassIntegrator::MassIntegrator(const IntegrationRule *ir)
@@ -1449,7 +1458,8 @@ void MassIntegrator::AssembleElementMatrix2(
 
 const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
                                                const FiniteElement &test_fe,
-                                               const ElementTransformation &Trans)
+                                               const ElementTransformation &Trans,
+                                               const bool stroud)
 {
    // int order = trial_fe.GetOrder() + test_fe.GetOrder();
    const int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
@@ -1458,7 +1468,15 @@ const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
    {
       return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
    }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
+
+   if (stroud)
+   {
+      return StroudIntRules.Get(trial_fe.GetGeomType(), order);
+   }
+   else
+   {
+      return IntRules.Get(trial_fe.GetGeomType(), order);
+   }
 }
 
 

fem/bilininteg.hpp

@@ -2184,11 +2184,22 @@ public:
                                    const Vector&, const Vector&,
                                    Vector&, const int, const int);
 
+   using ApplySimplexKernelType = void(*)(const int, const bool, const Array<int>&,
+                                          const Array<int>&,
+                                          const Array<int>&, const Array<int>&, const Array<int>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Vector&, const Vector&,
+                                          Vector&, const int, const int);
+
    using DiagonalKernelType = void(*)(const int, const bool, const Array<real_t>&,
                                       const Array<real_t>&, const Vector&, Vector&,
                                       const int, const int);
 
    MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
+   MFEM_REGISTER_KERNELS(ApplySimplexPAKernels, ApplySimplexKernelType, (int, int,
+                                                                         int));
    MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
    struct Kernels { Kernels(); };
 
@@ -2341,7 +2352,8 @@ public:
    void AddMultPatchPA(const int patch, const Vector &x, Vector &y) const;
 
    static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe);
+                                         const FiniteElement &test_fe,
+                                         const bool stroud = false);
 
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
 
@@ -2352,6 +2364,13 @@ public:
    {
       ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
       DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
+      AddSimplexSpecialization<DIM,D1D,Q1D>();
+   }
+
+   template <int DIM, int D1D, int Q1D>
+   static void AddSimplexSpecialization()
+   {
+      ApplySimplexPAKernels::Specialization<DIM,D1D,Q1D>::Add();
    }
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
@@ -2388,11 +2407,22 @@ public:
                                    const Array<real_t>&, const Vector&,
                                    const Vector&, Vector&, const int, const int);
 
+   using ApplySimplexKernelType = void(*)(const int, const Array<int>&,
+                                          const Array<int>&,
+                                          const Array<int>&, const Array<int>&, const Array<int>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Array<real_t>&, const Array<real_t>&,
+                                          const Vector&, const Vector&, Vector&,
+                                          const int, const int);
+
    using DiagonalKernelType =  void(*)(const int, const Array<real_t>&,
                                        const Vector&, Vector&, const int,
                                        const int);
 
    MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
+   MFEM_REGISTER_KERNELS(ApplySimplexPAKernels, ApplySimplexKernelType, (int, int,
+                                                                         int));
    MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
    struct Kernels { Kernels(); };
 
@@ -2441,7 +2471,8 @@ public:
 
    static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                          const FiniteElement &test_fe,
-                                         const ElementTransformation &Trans);
+                                         const ElementTransformation &Trans,
+                                         const bool stroud = false);
 
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
 
@@ -2452,6 +2483,13 @@ public:
    {
       ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
       DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
+      AddSimplexSpecialization<DIM,D1D,Q1D>();
+   }
+
+   template <int DIM, int D1D, int Q1D>
+   static void AddSimplexSpecialization()
+   {
+      ApplySimplexPAKernels::Specialization<DIM,D1D,Q1D>::Add();
    }
 
 protected:

fem/coefficient.cpp

@@ -54,6 +54,8 @@ void Coefficient::Project(QuadratureFunction &qf)
    QuadratureSpaceBase &qspace = *qf.GetSpace();
    const int ne = qspace.GetNE();
    Vector values;
+   // GetValues makes a reference, but we need it to be valid on Host
+   qf.HostWrite();
    for (int iel = 0; iel < ne; ++iel)
    {
       qf.GetValues(iel, values);
@@ -327,6 +329,8 @@ void VectorCoefficient::Project(QuadratureFunction &qf)
    const int ne = qspace.GetNE();
    DenseMatrix values;
    Vector col;
+   // GetValues makes a reference, but we need it to be valid on Host
+   qf.HostWrite();
    for (int iel = 0; iel < ne; ++iel)
    {
       qf.GetValues(iel, values);
@@ -695,6 +699,8 @@ void MatrixCoefficient::Project(QuadratureFunction &qf, bool transpose)
    QuadratureSpaceBase &qspace = *qf.GetSpace();
    const int ne = qspace.GetNE();
    DenseMatrix values, matrix;
+   // GetValues makes a reference, but we need it to be valid on Host
+   qf.HostWrite();
    for (int iel = 0; iel < ne; ++iel)
    {
       qf.GetValues(iel, values);

fem/complex_fem.cpp

@@ -237,6 +237,81 @@ ComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
    gfi->SyncAliasMemory(*this);
 }
 
+real_t
+ComplexGridFunction::ComputeLpError(const real_t p,
+                                    Coefficient &exsolr,
+                                    Coefficient &exsoli,
+                                    Coefficient *weight,
+                                    const IntegrationRule *irs[],
+                                    const Array<int> *elems) const
+{
+   real_t error = 0.0;
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Vector valsr;
+   Vector valsi;
+
+   const GridFunction& gf_r = real();
+   const GridFunction& gf_i = imag();
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      if (elems != NULL && (*elems)[i] == 0) { continue; }
+      fe = fes->GetFE(i);
+      const IntegrationRule *ir;
+      if (irs)
+      {
+         ir = irs[fe->GetGeomType()];
+      }
+      else
+      {
+         int intorder = 2*fe->GetOrder() + 3;
+         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
+      }
+      real_t elem_error = 0.0;
+      gf_r.GetValues(i, *ir, valsr);
+      gf_i.GetValues(i, *ir, valsi);
+      T = fes->GetElementTransformation(i);
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+         T->SetIntPoint(&ip);
+         real_t diffr = valsr(j) - exsolr.Eval(*T, ip);
+         real_t diffi = valsi(j) - exsoli.Eval(*T, ip);
+         real_t diff = hypot(diffr, diffi);
+         if (p < infinity())
+         {
+            diff = pow(diff, p);
+            if (weight)
+            {
+               diff *= weight->Eval(*T, ip);
+            }
+            elem_error += ip.weight * T->Weight() * diff;
+         }
+         else
+         {
+            if (weight)
+            {
+               diff *= weight->Eval(*T, ip);
+            }
+            error = std::max(error, diff);
+         }
+      }
+      if (p < infinity())
+      {
+         // negative quadrature weights may cause the error to be negative
+         error += fabs(elem_error);
+      }
+   }
+
+   if (p < infinity())
+   {
+      error = pow(error, 1./p);
+   }
+
+   return error;
+}
+
 void ComplexGridFunction::Save(std::ostream &os) const
 {
    os << "ComplexGridFunction\n";
@@ -830,15 +905,9 @@ ParComplexGridFunction::ParComplexGridFunction(ParMesh *m, std::istream &input)
       int vsize = pfes->GetVSize();
       Vector::Load(input, 2*vsize);
 
-      real_t *data_  = const_cast<real_t*>(HostRead());
-      for (int i = 0; i < vsize; i++)
-      {
-         if (pfes->GetDofSign(i) < 0)
-         {
-            data_[i] = -data_[i];
-            data_[i+vsize] = -data_[i+vsize];
-         }
-      }
+      real_t *h_data = HostReadWrite();
+      pfes->ApplyDofSigns(h_data);
+      pfes->ApplyDofSigns(h_data + vsize);
 
 
       // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering
@@ -1051,15 +1120,14 @@ void ParComplexGridFunction::Save(std::ostream &os) const
    os << '\n';
 
    int vsize = pfes->GetVSize();
-   real_t *data_  = const_cast<real_t*>(HostRead());
-   for (int i = 0; i < vsize; i++)
-   {
-      if (pfes->GetDofSign(i) < 0)
-      {
-         data_[i] = -data_[i];
-         data_[i+vsize] = -data_[i+vsize];
-      }
-   }
+   // We use const_cast + HostRead (instead of HostReadWrite) because we only
+   // need to change the host data temporarily and this way we do not invalidate
+   // the data if it is on device. If we use HostReadWrite here, later calls to
+   // Read or ReadWrite will need to copy the data from host to device. With the
+   // approach used here, the host-to-device copy is avoided.
+   real_t *h_data = const_cast<real_t*>(HostRead());
+   pfes->ApplyDofSigns(h_data);
+   pfes->ApplyDofSigns(h_data + vsize);
 
    if (pfes->GetOrdering() == Ordering::byNODES)
    {
@@ -1070,14 +1138,8 @@ void ParComplexGridFunction::Save(std::ostream &os) const
       Vector::Print(os, pfes->GetVDim());
    }
 
-   for (int i = 0; i < vsize; i++)
-   {
-      if (pfes->GetDofSign(i) < 0)
-      {
-         data_[i] = -data_[i];
-         data_[i+vsize] = -data_[i+vsize];
-      }
-   }
+   pfes->ApplyDofSigns(h_data);
+   pfes->ApplyDofSigns(h_data + vsize);
 
    os.flush();
 }

fem/complex_fem.hpp

@@ -166,6 +166,75 @@ public:
       return sqrt(err_r * err_r + err_i * err_i);
    }
 
+   /// @brief Returns Max|u_ex - u_h| error for complex-valued H1 or L2 elements
+   ///
+   /// Compute the $L_\infty$ error across the entire domain.
+   ///
+   /// @param[in] exsolr  Coefficient object reproducing the real part of the
+   ///                    anticipated values of the scalar field, Re(u_ex).
+   /// @param[in] exsoli  Coefficient object reproducing the imaginary part of
+   ///                    the anticipated values of the scalar field, Im(u_ex).
+   /// @param[in] irs        Optional pointer to an array of custom integration
+   ///                       rules e.g. higher order than the default rules. If
+   ///                       present the array will be indexed by
+   ///                       Geometry::Type.
+   ///
+   /// @note Uses ComputeLpError internally. See the ComputeLpError
+   ///       documentation for generalizations of this error computation.
+   ///
+   /// @note If an array of integration rules is provided through @a irs, be
+   ///       sure to include valid rules for each element type that may occur
+   ///       in the list of elements.
+   ///
+   virtual real_t ComputeMaxError(Coefficient &exsolr,
+                                  Coefficient &exsoli,
+                                  const IntegrationRule *irs[] = NULL) const
+   {
+      return ComputeLpError(infinity(), exsolr, exsoli, NULL, irs);
+   }
+
+   /// @brief Returns ||u_ex - u_h||_Lp for complex-valued H1 or L2 elements
+   ///
+   /// Computes:
+   ///    $$(\sum_{elems} \int_{elem} w \, |u_{ex} - u_h|^p)^{1/p}$$
+   /// Where:
+   ///    $$|u_{ex} - u_h| = \sqrt{Re(u_{ex} - u_h)^2 + Im(u_{ex} - u_h)^2}$$
+   ///
+   /// @param[in] p       Real value indicating the exponent of the $L^p$ norm.
+   ///                    To avoid domain errors p should have a positive value,
+   ///                    either finite or infinite.
+   /// @param[in] exsolr  Coefficient object reproducing the real part of the
+   ///                    anticipated values of the scalar field, Re(u_ex).
+   /// @param[in] exsoli  Coefficient object reproducing the imaginary part of
+   ///                    the anticipated values of the scalar field, Im(u_ex).
+   /// @param[in] weight  Optional pointer to a Coefficient object reproducing
+   ///                    a weighting function, w.
+   /// @param[in] irs     Optional pointer to an array of custom integration
+   ///                    rules e.g. higher order than the default rules. If
+   ///                    present the array will be indexed by Geometry::Type.
+   /// @param[in] elems   Optional pointer to a marker array, with a length
+   ///                    equal to the number of local elements, indicating
+   ///                    which elements to integrate over. Only those elements
+   ///                    corresponding to non-zero entries in @a elems will
+   ///                    contribute to the computed L2 error.
+   ///
+   /// @note If an array of integration rules is provided through @a irs, be
+   ///       sure to include valid rules for each element type that may occur
+   ///       in the list of elements.
+   ///
+   /// @note Quadratures with negative weights (as in some simplex integration
+   ///       rules in MFEM) can produce negative integrals even with
+   ///       non-negative integrands. To avoid returning negative errors this
+   ///       function uses the absolute values of the element-wise integrals.
+   ///       This may lead to results which are not entirely consistent with
+   ///       such integration rules.
+   virtual real_t ComputeLpError(const real_t p,
+                                 Coefficient &exsolr,
+                                 Coefficient &exsoli,
+                                 Coefficient *weight = NULL,
+                                 const IntegrationRule *irs[] = NULL,
+                                 const Array<int> *elems = NULL) const;
+
    /// Save the ComplexGridFunction to an output stream.
    virtual void Save(std::ostream &out) const;
 

fem/conduitdatacollection.cpp

@@ -114,6 +114,10 @@ void ConduitDataCollection::Save()
                                          n_mesh["fields"][name]);
    }
 
+   // TODO: in parallel, we need to call ParFiniteElementSpace::ApplyDofSigns
+   // for all ParGridFunction objects before and after saving, see
+   // ParGridFunction::Save.
+
    // save mesh data
    SaveMeshAndFields(myid,
                      n_mesh,

fem/datacollection.cpp

@@ -1181,12 +1181,14 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,
    DenseMatrix vval, pmat;
    std::vector<char> buf;
    int vec_dim = it->second->VectorDim();
+   int map_type = it->second->FESpace()->GetTypicalFE()->GetMapType();
    os << "<DataArray type=\"" << GetDataTypeString()
       << "\" Name=\"" << it->first
       << "\" NumberOfComponents=\"" << vec_dim << "\" "
       << VTKComponentLabels(vec_dim) << " "
       << "format=\"" << GetDataFormatString() << "\" >" << '\n';
-   if (vec_dim == 1)
+   if (vec_dim == 1 && (map_type == FiniteElement::VALUE ||
+                        map_type == FiniteElement::INTEGRAL))
    {
       for (int i = 0; i < mesh->GetNE(); i++)
       {

fem/fe/fe_base.hpp

@@ -167,7 +167,15 @@ public:
       /** @brief Full multidimensional representation which does not use tensor
           product structure. The ordering of the degrees of freedom is the
           same as TENSOR, but the sizes of B and G are the same as FULL.*/
-      LEXICOGRAPHIC_FULL
+      LEXICOGRAPHIC_FULL,
+
+      /** @brief Ragged tensor product representation using 1D matrices/tensors
+          with dimensions using 1D number of quadrature points and ragged tensor degrees of
+          freedom. */
+      /** Used only for partial assembly of the H1 positive basis. The
+          size of B is d1d x qnpt x dim. Since different Gauss-Jacobi quadrature rules
+          are employed in each dimension, we need to store dim arrays. */
+      RAGGED_TENSOR
    };
 
    /// Describes the contents of the #B, #Bt, #G, and #Gt arrays, see #Mode.
@@ -228,6 +236,39 @@ public:
       const Array<DofToQuad*> &dof2quad_array,
       const IntegrationRule &ir,
       DofToQuad::Mode mode);
+
+   virtual ~DofToQuad() = default;
+};
+
+/** @brief Structure representing the matrices/tensors needed to evaluate (in
+    reference space) the values, gradients, divergences, or curls of a positive
+    FiniteElement on simplices at the quadrature points of Stroud conical quadrature. */
+class RaggedDofToQuad : public DofToQuad
+{
+public:
+   /** @brief Special basis function structures for positive (Bernstein) basis with
+      partial assembly. The storage layout of Ba1 is ndof x nqpt for scalar elements.
+      The storage layout of Ba2 is ndof x ndof x nqpt. In particular, we have
+            Ba2(iqpt, a1, a2) = B^{p-a1}_{a2}(x_{iqpt}). */
+   Array<real_t> Ba1, Ba2, Ba3;
+   Array<real_t> Ba1t, Ba2t, Ba3t;
+
+   /** @brief Special structures for gradients of positive basis with partial assembly.
+      The gradient arrays exploit properties of the Bernstein basis which allow grad(B^p_alpha)
+      to be expressed as the sum of products of B^{p-1}_alpha and the barycentric coordinates.
+      Thus, Ga1 and Ga2 simply contain the ragged tensor product components of B^{p-1}_alpha */
+   Array<real_t> Ga1, Ga2, Ga3;
+   Array<real_t> Ga1t, Ga2t, Ga3t;
+
+   /** @brief Mapping from the Bernstein multi-index (a_1, ..., a_d) to the lexicographic
+       dof index. */
+   Array<int> lex_map;
+
+   Array<int> forward_map2d_diff, forward_map3d_diff;
+   Array<int> inverse_map2d_diff, inverse_map3d_diff;
+
+   Array<int> forward_map2d_mass, forward_map3d_mass;
+   Array<int> inverse_map2d_mass, inverse_map3d_mass;
 };
 
 /// Describes the function space on each element

fem/fe/fe_pos.cpp

@@ -557,6 +557,101 @@ H1Pos_TriangleElement::H1Pos_TriangleElement(const int p)
       }
 }
 
+const DofToQuad &H1Pos_TriangleElement::GetRaggedTensorDofToQuad(
+   const FiniteElement &fe, const IntegrationRule &ir,
+   DofToQuad::Mode mode,
+   Array<DofToQuad*> &dof2quad_array)
+{
+   DofToQuad *d2q = nullptr;
+   MFEM_VERIFY(mode == DofToQuad::RAGGED_TENSOR, "invalid mode requested");
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
+#endif
+   {
+      for (int i = 0; i < dof2quad_array.Size(); i++)
+      {
+         d2q = dof2quad_array[i];
+         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
+      }
+      if (!d2q)
+      {
+         d2q = new RaggedDofToQuad;
+         const int ndof = fe.GetOrder() + 1; // verify
+         const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
+         d2q->FE = &fe;
+         d2q->IntRule = &ir;
+         d2q->mode = mode;
+         d2q->ndof = ndof;
+         d2q->nqpt = nqpt;
+
+         RaggedDofToQuad *rd2q = static_cast<RaggedDofToQuad*>(d2q);
+         rd2q->Ba1.SetSize(nqpt*ndof);
+         // second component of ragged tensor basis, technically dof*(dof-1)/2 entries
+         rd2q->Ba2.SetSize((int)nqpt*ndof*ndof);
+         rd2q->Ba1t.SetSize(nqpt*ndof);
+         rd2q->Ba2t.SetSize((int)nqpt*ndof*ndof);
+         // stores first component of ragged tensor basis with order p-1, for gradients only
+         rd2q->Ga1.SetSize(nqpt*(ndof -1));
+         // stores second component of ragged tensor basis with order p-1
+         rd2q->Ga2.SetSize(nqpt*(ndof-1)*(ndof -1));
+         rd2q->Ga1t.SetSize(nqpt*(ndof -1));
+         rd2q->Ga2t.SetSize(nqpt*(ndof-1)*(ndof -1));
+         rd2q->lex_map.SetSize(ndof * ndof);
+         Vector shape_a1(ndof), shape_a2(ndof * ndof);
+         Vector shape_Ga1(ndof-1), shape_Ga2((ndof-1) * (ndof-1));
+         for (int i = 0; i < nqpt; i++)
+         {
+            // The first 'nqpt' points in the first dimension 'ir' have the same x-coordinates as those
+            // of the 1D rule (ie. (2,0) Gauss-Jacobi rule). The first 'nqpt' points in the second dimension
+            // 'ir' have the same y-coordinates as those of the 1D rule for second dimension (i.e. (1,0)
+            // Gauss-Jacobi rule). Additionally, the Bernstein PA algorithms expect evaluation of the
+            // component 1D bases at the Stroud nodes pulled back to the unit square, so perform the pullback
+            // on the fly.
+            const real_t x = ir.IntPoint(i).x;
+            const real_t y = ir.IntPoint(nqpt*i).y / (1.0 - ir.IntPoint(nqpt*i).x);
+
+            Poly_1D::CalcBernstein(ndof-1, x, shape_a1);
+            Poly_1D::CalcBernstein(ndof-2, x, shape_Ga1);
+            for (int j = 0; j < ndof; j++)
+            {
+               rd2q->Ba1t[i+nqpt*j] = rd2q->Ba1[j+ndof*i] = shape_a1(j);
+               if (j < ndof-1)
+               {
+                  rd2q->Ga1t[i+nqpt*j] = rd2q->Ga1[j+(ndof-1)*i] = shape_Ga1(j);
+                  Poly_1D::CalcBernstein(ndof-2-j, y, shape_Ga2);
+               }
+
+               Poly_1D::CalcBernstein(ndof-1-j, y, shape_a2);
+               for (int k = 0; k < ndof-j; k++)
+               {
+                  rd2q->Ba2t[i + nqpt*(j + ndof*k)] = rd2q->Ba2[k + ndof*(j + ndof*i)] = shape_a2(
+                                                                                            k);
+                  if (j < ndof-1 && k < ndof-j-1)
+                  {
+                     rd2q->Ga2t[i + nqpt*(j + (ndof-1)*k)] = rd2q->Ga2[k + (ndof-1)*(j +
+                                                                                     (ndof-1)*i)] = shape_Ga2(k);
+                  }
+               }
+            }
+         }
+
+         // stores the mapping from 2D Bernstein multi-index (i,j,p-i-j) to the
+         // lexicographic DOF ordering
+         for (int i = 0; i < ndof; i++)
+         {
+            for (int j = 0; j < ndof-i; j++)
+            {
+               int idx = ((2 * (ndof-1) + 3) - j) * j / 2 + i;
+               rd2q->lex_map[j + ndof*i] = idx;
+            }
+         }
+         dof2quad_array.Append(d2q);
+      }
+   }
+   return *d2q;
+}
+
 // static method
 void H1Pos_TriangleElement::CalcShape(
    const int p, const real_t l1, const real_t l2, real_t *shape)
@@ -749,6 +844,213 @@ H1Pos_TetrahedronElement::H1Pos_TetrahedronElement(const int p)
          }
 }
 
+const DofToQuad &H1Pos_TetrahedronElement::GetRaggedTensorDofToQuad(
+   const FiniteElement &fe, const IntegrationRule &ir,
+   DofToQuad::Mode mode,
+   Array<DofToQuad*> &dof2quad_array)
+{
+   DofToQuad *d2q = nullptr;
+   MFEM_VERIFY(mode == DofToQuad::RAGGED_TENSOR, "invalid mode requested");
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
+#endif
+   {
+      for (int i = 0; i < dof2quad_array.Size(); i++)
+      {
+         d2q = dof2quad_array[i];
+         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
+      }
+      if (!d2q)
+      {
+         d2q = new RaggedDofToQuad;
+         const int ndof = fe.GetOrder() + 1; // verify
+         const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
+         const int basis_dim2d = ndof*(ndof+1) / 2;
+         const int basis_dim3d = ndof*(ndof+1)*(ndof+2) / 6;
+         const int basis_dim2d_diff = (ndof-1)*(ndof) / 2;
+         const int basis_dim3d_diff = (ndof-1)*(ndof)*(ndof+1) / 6;
+         d2q->FE = &fe;
+         d2q->IntRule = &ir;
+         d2q->mode = mode;
+         d2q->ndof = ndof;
+         d2q->nqpt = nqpt;
+
+         RaggedDofToQuad *rd2q = static_cast<RaggedDofToQuad*>(d2q);
+         rd2q->Ba1.SetSize(nqpt * ndof);
+         // second component of ragged tensor basis, technically dof*(dof-1)/2 entries
+         rd2q->Ba2.SetSize(nqpt * basis_dim2d);
+         // third component of ragged tensor basis, technically dof*(dof-1)/2 entries
+         rd2q->Ba3.SetSize(nqpt * basis_dim3d);
+         rd2q->Ba1t.SetSize(nqpt * ndof);
+         rd2q->Ba2t.SetSize(nqpt * basis_dim2d);
+         rd2q->Ba3t.SetSize(nqpt * basis_dim3d);
+         // stores first component of ragged tensor basis with order p-1, for gradients only
+         rd2q->Ga1.SetSize(nqpt * (ndof-1));
+         // stores second component of ragged tensor basis with order p-1
+         rd2q->Ga2.SetSize(nqpt * basis_dim2d_diff);
+         // stores third component of ragged tensor basis with order p-1
+         rd2q->Ga3.SetSize(nqpt * basis_dim3d_diff);
+         rd2q->Ga1t.SetSize(nqpt * (ndof-1));
+         rd2q->Ga2t.SetSize(nqpt * basis_dim2d_diff);
+         rd2q->Ga3t.SetSize(nqpt * basis_dim3d_diff);
+         rd2q->lex_map.SetSize(ndof * ndof * ndof);
+
+         rd2q->forward_map2d_diff.SetSize((ndof-1) * (ndof-1));
+         rd2q->forward_map3d_diff.SetSize((ndof-1) * (ndof-1) * (ndof-1));
+         rd2q->inverse_map2d_diff.SetSize(2 * basis_dim2d_diff);
+         rd2q->inverse_map3d_diff.SetSize(3 * basis_dim3d_diff);
+
+         rd2q->forward_map2d_mass.SetSize(ndof * ndof);
+         rd2q->forward_map3d_mass.SetSize(ndof * ndof * ndof);
+         rd2q->inverse_map2d_mass.SetSize(2 * basis_dim2d);
+         rd2q->inverse_map3d_mass.SetSize(2 * basis_dim3d);
+
+         // forward and inverse maps for multi-index to collpased 1d index for diffusion, can combine
+         // these four loops, but need four idx's and clause for shorter diff loops
+         int idx = 0;
+         for (int i = 0; i < ndof-1; i++)
+         {
+            for (int j = 0; j < ndof-i-1; j++)
+            {
+               rd2q->forward_map2d_diff[j + (ndof-1)*i] = idx;
+               rd2q->inverse_map2d_diff[2*idx] = i;
+               rd2q->inverse_map2d_diff[1 + 2*idx] = j;
+               idx++;
+            }
+         }
+
+         idx = 0;
+         for (int k = 0; k < ndof-1; k++)
+         {
+            for (int j = 0; j < ndof-k-1; j++)
+            {
+               for (int i = 0; i < ndof-k-j-1; i++)
+               {
+                  rd2q->forward_map3d_diff[k + (ndof-1)*(j + (ndof-1)*i)] = idx;
+                  rd2q->inverse_map3d_diff[3*idx] = i;
+                  rd2q->inverse_map3d_diff[1 + 3*idx] = j;
+                  rd2q->inverse_map3d_diff[2 + 3*idx] = k;
+                  idx++;
+               }
+            }
+         }
+
+         // forward and inverse maps for multi-index to collpased 1d index for mass
+         idx = 0;
+         for (int j = 0; j < ndof; j++)
+         {
+            for (int i = 0; i < ndof-j; i++)
+            {
+               rd2q->forward_map2d_mass[j + ndof*i] = idx;
+               rd2q->inverse_map2d_mass[2*idx] = i;
+               rd2q->inverse_map2d_mass[1 + 2*idx] = j;
+               idx++;
+            }
+         }
+
+         idx = 0;
+         for (int k = 0; k < ndof; k++)
+         {
+            for (int j = 0; j < ndof-k; j++)
+            {
+               for (int i = 0; i < ndof-k-j; i++)
+               {
+                  rd2q->forward_map3d_mass[k + ndof*(j + ndof*i)] = idx;
+                  rd2q->inverse_map3d_mass[2*idx] = i;
+                  rd2q->inverse_map3d_mass[1 + 2*idx] = j;
+                  // d2q->inverse_map3d_mass[2 + 3*idx] = k;
+                  idx++;
+               }
+            }
+         }
+
+         Vector shape_a1(ndof), shape_a2(ndof * ndof), shape_a3(ndof * ndof * ndof);
+         Vector shape_Ga1(ndof-1), shape_Ga2(ndof-1), shape_Ga3(ndof-1);
+         for (int i = 0; i < nqpt; i++)
+         {
+            // The first 'nqpt' points in the first dimension 'ir' have the same x-coordinates as those
+            // of the 1D rule (ie. (2,0) Gauss-Jacobi rule). The first 'nqpt' points in the second dimension
+            // 'ir' have the same y-coordinates as those of the 1D rule for second dimension (i.e. (1,0)
+            // Gauss-Jacobi rule). The first 'nqpt' points in the third dimension have the same z-coordinates
+            // as those of the 1D rule for the third dimension (i.e. Gauss-Legendre rule). Additionally,
+            // the Bernstein PA algorithms expect evaluation of the component 1D bases at the Stroud nodes
+            // pulled back to the unit cube, so perform the pullback on the fly.
+            const real_t x = ir.IntPoint(i).x;
+            const real_t y = ir.IntPoint(nqpt*i).y / (1.0 - ir.IntPoint(nqpt*i).x);
+            const real_t z = ir.IntPoint(nqpt*nqpt*i).z / (1.0 - ir.IntPoint(
+                                                              nqpt*nqpt*i).x - ir.IntPoint(nqpt*nqpt*i).y);
+            Poly_1D::CalcBernstein(ndof-1, x, shape_a1);
+            Poly_1D::CalcBernstein(ndof-2, x, shape_Ga1);
+            for (int j = 0; j < ndof; j++)
+            {
+               rd2q->Ba1t[i+nqpt*j] = rd2q->Ba1[j+ndof*i] = shape_a1(j);
+               if (j < ndof-1)
+               {
+                  rd2q->Ga1t[i+nqpt*j] = rd2q->Ga1[j+(ndof-1)*i] = shape_Ga1(j);
+                  Poly_1D::CalcBernstein(ndof-2-j, y, shape_Ga2);
+               }
+
+               Poly_1D::CalcBernstein(ndof-1-j, y, shape_a2);
+               for (int k = 0; k < ndof-j; k++)
+               {
+                  const int a_2d_mass = rd2q->forward_map2d_mass[k + ndof*j];
+                  rd2q->Ba2t[i + nqpt*a_2d_mass] = rd2q->Ba2[a_2d_mass + basis_dim2d*i] =
+                                                      shape_a2(
+                                                         k);
+                  if (j < ndof-1 && k < ndof-j-1)
+                  {
+                     const int a_2d_diff = rd2q->forward_map2d_diff[k + (ndof-1)*j];
+                     rd2q->Ga2t[i + nqpt*a_2d_diff] = rd2q->Ga2[a_2d_diff + basis_dim2d_diff*i] =
+                                                         shape_Ga2(k);
+                     Poly_1D::CalcBernstein(ndof-2-j-k, z, shape_Ga3);
+                  }
+
+                  Poly_1D::CalcBernstein(ndof-1-j-k, z, shape_a3);
+                  for (int m = 0; m < ndof-j-k; m++)
+                  {
+                     const int a_3d_mass = rd2q->forward_map3d_mass[m + ndof*(k + ndof*j)];
+                     rd2q->Ba3t[i + nqpt*a_3d_mass] = rd2q->Ba3[a_3d_mass + basis_dim3d*i] =
+                                                         shape_a3(
+                                                            m);
+                     if (j < ndof-1 && k < ndof-j-1 && m < ndof-j-k-1)
+                     {
+                        // // collapsed 1D access
+                        // d2q->Ga3[i + nqpt*(m + d2q->offset3d[k + (ndof-1)*j])] = shape_Ga3(m);
+                        // collapsed 1D access with forward mapping
+                        const int a_3d_diff = rd2q->forward_map3d_diff[m + (ndof-1)*(k + (ndof-1)*j)];
+                        rd2q->Ga3t[i + nqpt*a_3d_diff] = rd2q->Ga3[a_3d_diff + basis_dim3d_diff*i] =
+                                                            shape_Ga3(m);
+                     }
+                  }
+               }
+            }
+         }
+
+         // stores the mapping from 3D Bernstein multi-index (i,j,k,p-i-j-k) to the
+         // lexicographic DOF ordering
+         int p = ndof - 1;
+         for (int i = 0; i < ndof; i++)
+         {
+            for (int j = 0; j < ndof-i; j++)
+            {
+               for (int k = 0; k < ndof-i-j; k++)
+               {
+                  int dof = (p+1)*(p+2)*(p+3) / 6;
+                  int tet = (p-k)*(p-k+1)*(p-k+2) / 6;
+                  int tri = (p+1-k-j)*(p+2-k-j)/2;
+                  int multi_idx = dof - tet - tri + i;
+                  rd2q->lex_map[k + ndof*(j + ndof*i)] = multi_idx;
+               }
+            }
+         }
+
+         dof2quad_array.Append(d2q);
+      }
+   }
+   return *d2q;
+}
+
 // static method
 void H1Pos_TetrahedronElement::CalcShape(
    const int p, const real_t l1, const real_t l2, const real_t l3,

fem/fe/fe_pos.hpp

@@ -191,6 +191,21 @@ public:
    /// Construct the H1Pos_TriangleElement of order @a p
    H1Pos_TriangleElement(const int p);
 
+   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
+                                 DofToQuad::Mode mode) const override
+   {
+      return (mode == DofToQuad::RAGGED_TENSOR) ?
+             GetRaggedTensorDofToQuad(*this, ir, mode, dof2quad_array) :
+             FiniteElement::GetDofToQuad(ir, mode);
+   }
+
+   static const DofToQuad &GetRaggedTensorDofToQuad(
+      const FiniteElement &fe, const IntegrationRule &ir,
+      DofToQuad::Mode mode,
+      Array<DofToQuad*> &dof2quad_array);
+
+   const Array<int> &GetDofMap() const { return dof_map; }
+
    // The size of shape is (p+1)(p+2)/2 (dof).
    static void CalcShape(const int p, const real_t x, const real_t y,
                          real_t *shape);
@@ -220,6 +235,21 @@ public:
    /// Construct the H1Pos_TetrahedronElement of order @a p
    H1Pos_TetrahedronElement(const int p);
 
+   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
+                                 DofToQuad::Mode mode) const override
+   {
+      return (mode == DofToQuad::RAGGED_TENSOR) ?
+             GetRaggedTensorDofToQuad(*this, ir, mode, dof2quad_array) :
+             FiniteElement::GetDofToQuad(ir, mode);
+   }
+
+   static const DofToQuad &GetRaggedTensorDofToQuad(
+      const FiniteElement &fe, const IntegrationRule &ir,
+      DofToQuad::Mode mode,
+      Array<DofToQuad*> &dof2quad_array);
+
+   const Array<int> &GetDofMap() const { return dof_map; }
+
    // The size of shape is (p+1)(p+2)(p+3)/6 (dof).
    static void CalcShape(const int p, const real_t x, const real_t y,
                          const real_t z, real_t *shape);

fem/fe_coll.hpp

@@ -250,6 +250,14 @@ public:
        its GetOrder() method. */
    virtual FiniteElementCollection *Clone(int p) const;
 
+   /** @brief Return the order parameter used to construct this collection.
+    *  This differs from GetOrder() depending on the collection type. */
+   virtual int GetConstructorOrder() const
+   {
+      MFEM_ABORT("Collection " << Name() << " does not support GetConstructorOrder");
+      return -1;
+   }
+
 protected:
    const int base_p; ///< Order as returned by GetOrder().
 
@@ -314,6 +322,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new H1_FECollection(p, dim, b_type); }
 
+   int GetConstructorOrder() const override
+   { return base_p; }
+
    virtual ~H1_FECollection();
 };
 
@@ -343,6 +354,10 @@ class H1_Trace_FECollection : public H1_FECollection
 public:
    H1_Trace_FECollection(const int p, const int dim,
                          const int btype = BasisType::GaussLobatto);
+
+   FiniteElementCollection *Clone(int p) const override
+   { return new H1_Trace_FECollection(p, dim+1, b_type); }
+
 };
 
 /// Arbitrary order "L2-conforming" discontinuous finite elements.
@@ -396,6 +411,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new L2_FECollection(p, dim, b_type, m_type); }
 
+   int GetConstructorOrder() const override
+   { return base_p; }
+
    virtual ~L2_FECollection();
 };
 
@@ -456,6 +474,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new RT_FECollection(p, dim, cb_type, ob_type); }
 
+   int GetConstructorOrder() const override
+   { return base_p-1; }
+
    virtual ~RT_FECollection();
 };
 
@@ -536,6 +557,9 @@ public:
    FiniteElementCollection *Clone(int p) const override
    { return new ND_FECollection(p, dim, cb_type, ob_type); }
 
+   int GetConstructorOrder() const override
+   { return dim>1 ? base_p : base_p+1; }
+
    virtual ~ND_FECollection();
 };
 
@@ -548,6 +572,9 @@ public:
    ND_Trace_FECollection(const int p, const int dim,
                          const int cb_type = BasisType::GaussLobatto,
                          const int ob_type = BasisType::GaussLegendre);
+
+   FiniteElementCollection *Clone(int p) const override
+   { return new ND_Trace_FECollection(p, dim+1, cb_type, ob_type); }
 };
 
 /// Arbitrary order 3D H(curl)-conforming Nedelec finite elements in 1D.

fem/fespace.cpp

@@ -4631,9 +4631,8 @@ FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)
 
 ElementDofOrdering GetEVectorOrdering(const FiniteElementSpace& fes)
 {
-   return UsesTensorBasis(fes)?
+   return (UsesTensorBasis(fes) || fes.UsesRaggedTensorBasis()) ?
           ElementDofOrdering::LEXICOGRAPHIC:
           ElementDofOrdering::NATIVE;
 }
-
 } // namespace mfem

fem/fespace.hpp

@@ -1514,6 +1514,18 @@ public:
       return dynamic_cast<const L2_FECollection*>(fec) != NULL;
    }
 
+   /// @brief Return true if the mesh contains only one topology, the elements are
+   /// all triangles or tetrahedrons, and the elements are ragged tensor elements
+   /// i.e. Bernstein/positive basis.
+   bool UsesRaggedTensorBasis() const
+   {
+      bool simplex = this->GetMesh()->IsSimplexMesh();
+      bool positive =
+         dynamic_cast<const mfem::H1Pos_TriangleElement *>(this->GetTypicalFE()) ||
+         dynamic_cast<const mfem::H1Pos_TetrahedronElement *>(this->GetTypicalFE());
+      return simplex && positive;
+   }
+
    /** In variable-order spaces on nonconforming (NC) meshes, this function
        controls whether strict conformity is enforced in cases where coarse
        edges/faces have higher polynomial order than their fine NC neighbors.

fem/gridfunc.cpp

@@ -2256,6 +2256,104 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
    }
 }
 
+void GridFunction::AccumulateAndCountTraceValues(
+   Coefficient *coeff[], VectorCoefficient *vcoeff,
+   Array<int> &values_counter)
+{
+   if (vcoeff)
+   {
+      MFEM_VERIFY(fes->GetVDim() == vcoeff->GetVDim(),
+                  "vcoeff vdim != fes VDim");
+      MFEM_VERIFY(fes->GetTypicalTraceElement()->GetMapType() ==
+                  FiniteElement::VALUE &&
+                  fes->GetTypicalTraceElement()->GetRangeType() ==
+                  FiniteElement::SCALAR,
+                  "Can only call ProjectTraceCoefficient on scalar value-type "
+                  "trace elements. "
+                  "Use ProjectTraceCoefficientNormal for RT and "
+                  "ProjectTraceCoefficientTangent for ND finite elements.");
+   }
+
+   Array<int> vdofs;
+   Vector vc;
+
+   values_counter.SetSize(Size());
+   values_counter = 0;
+
+   const int vdim = fes->GetVDim();
+   HostReadWrite();
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+
+      const FiniteElement *fe = fes->GetFaceElement(i);
+      const int fdof = fe->GetDof();
+      ElementTransformation *transf = fes->GetMesh()->GetFaceTransformation(i);
+      const IntegrationRule &ir = fe->GetNodes();
+      fes->GetFaceVDofs(i, vdofs);
+
+      for (int j = 0; j < fdof; j++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(j);
+         transf->SetIntPoint(&ip);
+         if (vcoeff) { vcoeff->Eval(vc, *transf, ip); }
+         for (int d = 0; d < vdim; d++)
+         {
+            if (!vcoeff && !coeff[d]) { continue; }
+
+            real_t val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
+            int ind = vdofs[fdof*d+j];
+            if ( ind < 0 )
+            {
+               val = -val, ind = -1-ind;
+            }
+            if (++values_counter[ind] == 1)
+            {
+               (*this)(ind) = val;
+            }
+            else
+            {
+               (*this)(ind) += val;
+            }
+         }
+      }
+   }
+}
+
+void GridFunction::AccumulateAndCountTraceTangentValues(
+   VectorCoefficient &vcoeff, Array<int> &values_counter)
+{
+   MFEM_VERIFY(fes->GetVDim() == 1, "fespace VDim != 1");
+   MFEM_VERIFY(fes->GetTypicalTraceElement()
+               ->GetRangeType() == FiniteElement::VECTOR &&
+               fes->GetTypicalTraceElement()
+               ->GetMapType() == FiniteElement::H_CURL,
+               "Not an ND FE space!");
+   MFEM_VERIFY(fes->GetTypicalTraceElement()->GetPhysRangeDim(
+                  fes->GetMesh()->SpaceDimension()) == vcoeff.GetVDim(),
+               "vcoeff vdim != PhysRangeDim");
+
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Array<int> dofs;
+   Vector lvec;
+
+   values_counter.SetSize(Size());
+   values_counter = 0;
+
+   HostReadWrite();
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+      fe = fes->GetFaceElement(i);
+      T = fes->GetMesh()->GetFaceTransformation(i);
+      fes->GetFaceVDofs(i, dofs);
+      lvec.SetSize(fe->GetDof());
+      fe->Project(vcoeff, *T, lvec);
+      accumulate_dofs(dofs, lvec, *this, values_counter);
+   }
+}
+
 void GridFunction::ComputeMeans(AvgType type, Array<int> &zones_per_vdof)
 {
    switch (type)
@@ -2698,6 +2796,74 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff,
    }
 }
 
+void GridFunction::ProjectTraceCoefficient(Coefficient *coeff[])
+{
+   Array<int> values_counter;
+   AccumulateAndCountTraceValues(coeff, NULL, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
+void GridFunction::ProjectTraceCoefficient(Coefficient &coeff)
+{
+   MFEM_VERIFY(FESpace()->GetVDim() == 1, "ProjectTraceCoefficient(Coefficient&)"
+               "is only valid for scalar GridFunction");
+   Coefficient *coeff_p = &coeff;
+   ProjectTraceCoefficient(&coeff_p);
+}
+
+void GridFunction::ProjectTraceCoefficient(VectorCoefficient &vcoeff)
+{
+   MFEM_VERIFY(FESpace()->GetVDim() == vcoeff.GetVDim(),
+               "Incompatible vcoeff vdim and fes vdim");
+   Array<int> values_counter;
+   AccumulateAndCountTraceValues(NULL, &vcoeff, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
+void GridFunction::ProjectTraceCoefficientNormal(VectorCoefficient &vcoeff)
+{
+   MFEM_VERIFY(fes->GetVDim() == 1, "fespace VDim != 1");
+   MFEM_VERIFY(fes->GetTypicalTraceElement()->GetRangeType() ==
+               FiniteElement::SCALAR &&
+               fes->GetTypicalTraceElement()->GetMapType() ==
+               FiniteElement::INTEGRAL, "Not an RT FE space!");
+   MFEM_VERIFY(vcoeff.GetVDim() == fes->GetMesh()->SpaceDimension(),
+               "vcoeff vdim (" << vcoeff.GetVDim()
+               << ") != SpaceDimension ("
+               << fes->GetMesh()->SpaceDimension() << ")");
+
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Array<int> dofs;
+   int dim = vcoeff.GetVDim();
+   Vector vc(dim), nor(dim), lvec;
+
+   for (int i = 0; i < fes->GetMesh()->GetNumFaces(); i++)
+   {
+      fe = fes->GetFaceElement(i);
+      T = fes->GetMesh()->GetFaceTransformation(i);
+      const IntegrationRule &ir = fe->GetNodes();
+      lvec.SetSize(fe->GetDof());
+      for (int j = 0; j < ir.GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(j);
+         T->SetIntPoint(&ip);
+         vcoeff.Eval(vc, *T, ip);
+         CalcOrtho(T->Jacobian(), nor);
+         lvec(j) = (vc * nor);
+      }
+      fes->GetFaceVDofs(i, dofs);
+      SetSubVector(dofs, lvec);
+   }
+}
+
+void GridFunction::ProjectTraceCoefficientTangent(VectorCoefficient &vcoeff)
+{
+   Array<int> values_counter;
+   AccumulateAndCountTraceTangentValues(vcoeff, values_counter);
+   ComputeMeans(ARITHMETIC, values_counter);
+}
+
 void GridFunction::ProjectCoefficientGlobalL2(VectorCoefficient &vcoeff,
                                               real_t rtol, int iter)
 {
@@ -5286,6 +5452,7 @@ PLBound GridFunction::GetBounds(Vector &lower, Vector &upper,
 {
    int max_order = fes->GetMaxElementOrder();
    PLBound plb(fes, ref_factor*(max_order+1));
+
    Vector lel, uel;
    GetElementBounds(plb, lel, uel, vdim);
 

fem/gridfunc.hpp

@@ -578,6 +578,13 @@ protected:
                                            const Array<int> &bdr_attr,
                                            Array<int> &values_counter);
 
+   void AccumulateAndCountTraceValues(Coefficient *coeff[],
+                                      VectorCoefficient *vcoeff,
+                                      Array<int> &values_counter);
+
+   void AccumulateAndCountTraceTangentValues(VectorCoefficient &vcoeff,
+                                             Array<int> &values_counter);
+
    // Complete the computation of averages; called e.g. after
    // AccumulateAndCountZones().
    void ComputeMeans(AvgType type, Array<int> &zones_per_vdof);
@@ -663,6 +670,23 @@ public:
       ProjectBdrCoefficient(&coeff_p, attr);
    }
 
+   /// Project a Coefficient on a GridFunction defined on H1 trace space
+   void ProjectTraceCoefficient(Coefficient *coeff[]);
+   void ProjectTraceCoefficient(Coefficient &coeff);
+
+   /** @brief Project a VectorCoefficient @a vcoeff on a GridFunction
+       defined on a Vector H1 trace space. Note that this also works
+       for a scalar H1 trace space, where only the first component of
+       @a vcoeff is used. */
+   void ProjectTraceCoefficient(VectorCoefficient &vcoeff);
+   /** @brief Project a VectorCoefficient on a GridFunction
+       defined on an RT trace space */
+   void ProjectTraceCoefficientNormal(VectorCoefficient &vcoeff);
+   /** @brief Project a VectorCoefficient on a GridFunction
+       defined on an ND trace space */
+   void ProjectTraceCoefficientTangent(VectorCoefficient &vcoeff);
+
+
    /** @brief Project a VectorCoefficient on the GridFunction, modifying only
        DOFs on the boundary associated with the boundary attributes marked in
        the @a attr array. */
@@ -1767,8 +1791,8 @@ public:
                              const int ref_factor=1, const int vdim=-1) const;
 
    /// Computes the \ref PLBound for the gridfunction with number of control
-   /// points based on \p ref_factor, and returns the bounds for each element
-   /// ordered byNodes:
+   /// points based on @a ref_factor, and returns the bounds for each element
+   /// ordered byNODES:
    /// lower_{0,0}, lower_{1,0}, ..., lower_{ne-1,0},
    /// lower_{0,1}, ..., lower_{ne-1,vdim-1}. We also return the
    /// PLBound object used to compute the bounds.
@@ -1802,7 +1826,7 @@ public:
                          const int vdim = -1) const;
 
    /// Compute bounds on the grid function for all the elements. The bounds
-   /// are returned in @b lower and @b upper, ordered byNodes:
+   /// are returned in @b lower and @b upper, ordered byNODES:
    /// lower_{0,0}, lower_{1,0}, ..., lower_{ne-1,0},
    /// lower_{0,1}, ..., lower_{ne-1,vdim-1}
    void GetElementBounds(const PLBound &plb, Vector &lower, Vector &upper,

fem/gslib.hpp

@@ -12,6 +12,9 @@
 #ifndef MFEM_GSLIB
 #define MFEM_GSLIB
 
+#include <map>
+#include <vector>
+
 #include "../config/config.hpp"
 #ifdef MFEM_USE_MPI
 #include "pgridfunc.hpp"
@@ -21,6 +24,45 @@
 
 #ifdef MFEM_USE_GSLIB
 
+/* gslib license and copyright statement for code adapted from gslib:
+
+Copyright (c) 2008-2024, UCHICAGO ARGONNE, LLC.
+
+The UChicago Argonne, LLC as Operator of Argonne National
+Laboratory holds copyright in the Software. The copyright holder
+reserves all rights except those expressly granted to licensees,
+and U.S. Government license rights.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the disclaimer below.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the disclaimer (as noted below)
+in the documentation and/or other materials provided with the
+distribution.
+
+3. Neither the name of ANL nor the names of its contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+UCHICAGO ARGONNE, LLC, THE U.S. DEPARTMENT OF
+ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
 namespace gslib
 {
 struct comm;
@@ -80,13 +122,18 @@ protected:
    // IntegrationRules for simplex->Quad/Hex and to project to p_max in-case of
    // p-refinement.
    Array<IntegrationRule *> ir_split;
+   /// Integration rules built at the field polynomial order (only for surface
+   /// meshes when mesh order is not the same as gridfunction order).
+   Array<IntegrationRule *> ir_split_sol;
+   /// Order at which #ir_split_sol was built; -1 means not built.
+   int ir_split_sol_order = -1;
    Array<FiniteElementSpace *> fes_rst_map; //FESpaces to map Quad/Hex->Simplex
    Array<GridFunction *> gf_rst_map; // GridFunctions to map Quad/Hex->Simplex
    FiniteElementCollection *fec_map_lin;
    void *fdataD;
    struct gslib::crystal *cr;             // gslib's internal data
    struct gslib::comm *gsl_comm;          // gslib's internal data
-   int dim, points_cnt;                   // mesh dimension and number of points
+   int dim, spacedim, points_cnt;         // mesh dimension and number of points
    Array<unsigned int> gsl_code, gsl_proc, gsl_elem, gsl_mfem_elem;
    Vector gsl_mesh, gsl_ref, gsl_dist, gsl_mfem_ref;
    Array<unsigned int> recv_proc, recv_index; // data for custom interpolation
@@ -95,6 +142,8 @@ protected:
    AvgType avgtype;             // average type used for L2 functions
    Array<int> split_element_map;
    Array<int> split_element_index;
+   // Geometry::Type (as int) of the original element for each split quad.
+   Array<int> split_element_geom;
    int        NE_split_total;   // total number of elements after mesh splitting
    int        mesh_points_cnt;  // number of mesh nodes
    // Tolerance to ignore points found beyond the mesh boundary.
@@ -102,111 +151,235 @@ protected:
    double     bdr_tol;
    // Use CPU functions for Mesh/GridFunction on device for gslib1.0.7
    bool       gpu_to_cpu_fallback = false;
+   // Check if a point is inside the oriented bounding box of an
+   // element before the Newton iteration.
+   // Note: only used in MFEM implementation (not in gslib) which currently
+   // supports GPU kernels for area meshes in 2D, volume meshes in 3D,
+   // and surface meshes in 1D/2D/3D.
+   bool       obb_check = true;
 
    // Device specific data used for FindPoints
-   struct
+   struct DEV_STRUCT
    {
       bool setup_device = false;
       bool find_device  = false;
-      int local_hash_size, dof1d, dof1d_sol, h_o_size, h_nx;
+      int local_hash_size, dof1d, dof1d_sol, lh_nx, gh_nx;
       double newt_tol; // Tolerance specified during setup for Newton solve
       struct gslib::crystal *cr;
       struct gslib::hash_data_3 *hash3;
       struct gslib::hash_data_2 *hash2;
       mutable Vector bb, wtend, gll1d, lagcoeff, gll1d_sol, lagcoeff_sol;
-      mutable Array<unsigned int> loc_hash_offset;
-      mutable Vector loc_hash_min, loc_hash_fac;
+      mutable Array<unsigned int> lh_offset, gh_offset;
+      mutable Vector lh_min, lh_fac, gh_min, gh_fac;
+      // Tolerance to mark points found on the surface as CODE_INTERNAL
+      // or CODE_BORDER. This is needed because we cannot only use reference
+      // space coordinates to determine if a point is located inside the
+      // element or not.
+      mutable double surf_dist_tol;
    } DEV;
 
-   /// Use GSLIB for communication and interpolation
+   // Helper function to setup and free gslib's crystal router.
+   void SetupCrystal(); // Called inside Setup and SetupSurf_base
+   void FreeCrystal();  // Called inside FreeData
+
+   /// Use GSLIB for communication and interpolation. Updates field_out on
+   /// host.
    virtual void InterpolateH1(const GridFunction &field_in, Vector &field_out,
                               const int field_out_ordering);
    /// Uses GSLIB Crystal Router for communication followed by MFEM's
-   /// interpolation functions
+   /// interpolation functions. Updates field_out on host.
    virtual void InterpolateGeneral(const GridFunction &field_in,
                                    Vector &field_out,
                                    const int field_out_ordering);
 
-   /// Since GSLIB is designed to work with quads/hexes, we split every
-   /// triangle/tet/prism/pyramid element into quads/hexes.
+   /** @brief Since GSLIB is designed to work with quads/hexes, we split every
+    *  triangle/tet/prism/pyramid element into quads/hexes. */
    virtual void SetupSplitMeshes();
 
-   /// Setup integration points that will be used to interpolate the nodal
-   /// location at points expected by GSLIB.
+   /** @brief Setup integration points that will be used to interpolate the
+    *  nodal location at points expected by GSLIB. */
    virtual void SetupIntegrationRuleForSplitMesh(Mesh *mesh,
                                                  IntegrationRule *irule,
                                                  int order);
 
-   /// Helper function that calls \ref SetupSplitMeshes and
-   /// \ref SetupIntegrationRuleForSplitMesh.
+   /** @brief Build integration rules at the given @a order for each split mesh
+    *  and store them in @a ir_out. Requires that \ref SetupSplitMeshes has
+    *  already been called. */
+   virtual void SetupIntegrationRules(const int order,
+                                      Array<IntegrationRule *> &ir_out);
+
+   /** @brief Helper function that calls \ref SetupSplitMeshes and
+    * \ref SetupIntegrationRules. */
    virtual void SetupSplitMeshesAndIntegrationRules(const int order);
 
-   /// Get GridFunction value at the points expected by GSLIB.
-   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals) const;
+   /** @brief Get GridFunction value at the points expected by GSLIB.
+    *  @param[in]  gf_in       Grid function to evaluate.
+    *  @param[out] node_vals   Output values.
+    *  @param[in]  ir_in       If non-null, use these rules instead of #ir_split.
+    *  @param[in]  by_element  If true, output has element-major layout
+    *                          [nel][vdim][ndofs]; otherwise component-major
+    *                          layout [vdim][total_pts]. */
+   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals,
+                               const Array<IntegrationRule *> *ir_in = nullptr,
+                               bool by_element = false) const;
 
-   /// Map {r,s,t} coordinates from [-1,1] to [0,1] for MFEM. For simplices,
-   /// find the original element number (that was split into micro quads/hexes)
-   /// during the setup phase.
+   /** @brief Map {r,s,t} coordinates from [-1,1] to [0,1] for MFEM. For
+    *  simplices, find the original element number (that was split into
+    *  micro quads/hexes) during the setup phase. */
    virtual void MapRefPosAndElemIndices();
 
-   // Device functions
-   // FindPoints locally on device for 3D.
+   /// FindPoints locally on device for 3D.
    void FindPointsLocal3(const Vector &point_pos, int point_pos_ordering,
                          Array<unsigned int> &gsl_code_dev_l,
                          Array<unsigned int> &gsl_elem_dev_l, Vector &gsl_ref_l,
                          Vector &gsl_dist_l, int npt);
 
-   // FindPoints locally on device for 2D.
+   /// FindPoints locally on device for 2D.
    void FindPointsLocal2(const Vector &point_pos, int point_pos_ordering,
                          Array<unsigned int> &gsl_code_dev_l,
                          Array<unsigned int> &gsl_elem_dev_l, Vector &gsl_ref_l,
                          Vector &gsl_dist_l, int npt);
 
-   // Interpolate on device for 3D.
+   /// FindPoints locally on device for 3D surface elements.
+   void FindPointsSurfLocal3(const Vector &point_pos,
+                             int point_pos_ordering,
+                             Array<unsigned int> &gsl_code_dev_l,
+                             Array<unsigned int> &gsl_elem_dev_l,
+                             Vector &gsl_ref_l,
+                             Vector &gsl_dist_l,
+                             int npt);
+
+   /// FindPoints locally on device for 3D edge elements.
+   void FindPointsEdgeLocal3(const Vector &point_pos,
+                             int point_pos_ordering,
+                             Array<unsigned int> &gsl_code_dev_l,
+                             Array<unsigned int> &gsl_elem_dev_l,
+                             Vector &gsl_ref_l,
+                             Vector &gsl_dist_l,
+                             int npt);
+
+   /// FindPoints locally on device for 2D edge elements.
+   void FindPointsEdgeLocal2(const Vector &point_pos,
+                             int point_pos_ordering,
+                             Array<unsigned int> &gsl_code_dev_l,
+                             Array<unsigned int> &gsl_elem_dev_l,
+                             Vector &gsl_ref_l,
+                             Vector &gsl_dist_l,
+                             int npt);
+
+   /// Interpolate on device for 3D.
    void InterpolateLocal3(const Vector &field_in,
                           Array<int> &gsl_elem_dev_l,
                           Vector &gsl_ref_l,
                           Vector &field_out,
                           int npt, int ncomp,
-                          int nel, int dof1dsol);
-   // Interpolate on device for 2D.
+                          int dof1dsol);
+
+   /// Interpolate on device for 2D.
    void InterpolateLocal2(const Vector &field_in,
                           Array<int> &gsl_elem_dev_l,
                           Vector &gsl_ref_l,
                           Vector &field_out,
                           int npt, int ncomp,
-                          int nel, int dof1dsol);
+                          int dof1dsol);
 
-   // Prepare data for device functions.
+   /// Interpolate on device for 1D.
+   void InterpolateLocal1(const Vector &field_in,
+                          Array<int> &gsl_elem_dev_l,
+                          Vector &gsl_ref_l,
+                          Vector &field_out,
+                          int npt, int ncomp, int dof1dsol);
+
+   /// Prepare data for device execution for volume meshes.
    void SetupDevice();
 
-   /** Searches positions given in physical space by @a point_pos.
+   /** @brief Searches positions given in physical space by @a point_pos.
        These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
        byVDim: (XYZ,XYZ,....XYZ) specified by @a point_pos_ordering. */
    void FindPointsOnDevice(const Vector &point_pos,
                            const int point_pos_ordering = Ordering::byNODES);
 
-   /** Interpolation of field values at prescribed reference space positions.
-       @param[in] field_in_evec E-vector of grid function to be interpolated.
-                                Assumed ordering is NDOFSxVDIMxNEL
-       @param[in] nel           Number of elements in the mesh.
-       @param[in] ncomp         Number of components in the field.
-       @param[in] dof1dsol      Number of degrees of freedom in each reference
-                                space direction.
-       @param[in] ordering      Ordering of the out field values: byNodes/byVDIM
-
-       @param[out] field_out  Interpolated values. For points that are not found
-                              the value is set to #default_interp_value. */
+   /** @brief Interpolation of field values at prescribed reference space
+    *         positions.
+    *  @param[in] field_in_evec E-vector of grid function to be interpolated.
+    *                           Assumed ordering is NDOFSxVDIMxNEL
+    *  @param[in] nel           Number of elements in the mesh.
+    *  @param[in] ncomp         Number of components in the field.
+    *  @param[in] dof1dsol      Number of degrees of freedom in each reference
+    *                           space direction.
+    *  @param[in] ordering      Ordering of the out field values: byNodes/byVDIM
+    *
+    *  @param[out] field_out    Interpolated values. For points that are not
+    *                           found the value is set to
+    *                           #default_interp_value. */
    void InterpolateOnDevice(const Vector &field_in_evec, Vector &field_out,
                             const int nel, const int ncomp,
                             const int dof1dsol, const int ordering);
 
+   /** @brief Interpolation of field values at prescribed reference space
+    *         positions for surface meshes. */
+   void InterpolateSurfBase(const Vector &field_in, Vector &field_out,
+                            const int nel, const int ncomp,
+                            const int dof1dsol, const int field_out_ordering);
+
+   /// Preprocess 2D surface mesh needed for FindPoints.
+   void findptsedge_setup_2(DEV_STRUCT &devs,
+                            const double *const elx[2],
+                            const unsigned n,
+                            const unsigned int nel,
+                            const unsigned m,
+                            const double bbox_rel_size_inc,
+                            const unsigned int local_hash_size,
+                            const unsigned int global_hash_size,
+                            const Vector *aabb_sz_inc);
+
+   /// Preprocess 3D surface mesh needed for FindPoints.
+   void findptssurf_setup_3(DEV_STRUCT &devs,
+                            const double *const elx[3],
+                            const unsigned n,
+                            const unsigned int nel,
+                            const unsigned m,
+                            const double bbox_rel_size_inc,
+                            const unsigned int local_hash_size,
+                            const unsigned int global_hash_size,
+                            const int rD,
+                            const Vector *aabb_sz_inc);
+
+   /** @brief Shared implementation for the public surface-setup methods.
+    *
+    *  @details Initializes the surface-search data structures, builds the
+    *  split-element representation expected by gslib, and constructs the
+    *  element bounding boxes used by the MFEM surface kernels.
+    *
+    *  If @a aabb_sz_inc is null, the setup stores the default oriented
+    *  bounding boxes and uses @a bbox_rel_size_inc as their relative size
+    *  increase factor.
+    *
+    *  If @a aabb_sz_inc is non-null, the setup stores axis-aligned bounding
+    *  boxes only, applies the requested absolute AABB expansion in each
+    *  physical direction, and adjusts the tolerance @a bdr_tol so points
+    *  found in the expanded region are classified as border points.
+    *
+    *  @param[in] m                  Input surface mesh.
+    *  @param[in] bbox_rel_size_inc  Relative size increase applied when
+    *                                expanding each element bounding box during
+    *                                setup.
+    *  @param[in] aabb_sz_inc        Optional total absolute AABB expansion
+    *                                applied to the stored axis-aligned
+    *                                bounding boxes after construction.
+    *  @param[in] newt_tol           Newton tolerance for the point-search
+    *                                kernels.
+    */
+   void SetupSurf_Base(Mesh &m,
+                       const double bbox_rel_size_inc,
+                       const Vector *aabb_sz_inc,
+                       const double newt_tol);
 public:
    /// Serial constructor
    FindPointsGSLIB();
 
    /// Serial constructor + setup with given Mesh (see \ref Setup)
-   FindPointsGSLIB(Mesh &mesh_in, const double bb_t = 0.1,
+   FindPointsGSLIB(Mesh &mesh_in, const double bbox_rel_size_inc = 0.1,
                    const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 
@@ -215,7 +388,7 @@ public:
    FindPointsGSLIB(MPI_Comm comm_);
 
    /// Constructor + setup with given ParMesh (see \ref Setup)
-   FindPointsGSLIB(ParMesh &mesh_in, const double bb_t = 0.1,
+   FindPointsGSLIB(ParMesh &mesh_in, const double bbox_rel_size_inc = 0.1,
                    const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 #endif
@@ -224,25 +397,72 @@ public:
    FindPointsGSLIB(const FindPointsGSLIB&) = delete;
    FindPointsGSLIB& operator=(const FindPointsGSLIB&) = delete;
 
-   /** Initializes the internal mesh in gslib, by sending the positions of the
-       Gauss-Lobatto nodes of the input Mesh object \p m.
+   /** @brief Preprocess the internal mesh in gslib.
+
+       @details Initializes the internal mesh in gslib, by sending the
+       positions of the Gauss-Lobatto nodes of the input Mesh object \p m.
        Note: not tested with periodic (L2).
        Note: the input mesh \p m must have Nodes set.
 
-       @param[in] m         Input mesh.
-       @param[in] bb_t      (Optional) Relative size of bounding box around
-                            each element.
-       @param[in] newt_tol  (Optional) Newton tolerance for the gslib
-                            search methods.
-       @param[in] npt_max   (Optional) Number of points for simultaneous
-                            iteration. This alters performance and
-                            memory footprint.*/
-
-   void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12,
+       @param[in] m                  Input mesh.
+       @param[in] bbox_rel_size_inc  (Optional) Relative size increase applied
+                                     when expanding each element bounding box.
+       @param[in] newt_tol           (Optional) Newton tolerance for the gslib
+                                     search methods.
+       @param[in] npt_max            (Optional) Number of points for
+                                     simultaneous iteration. This alters
+                                     performance and memory footprint.
+   */
+   void Setup(Mesh &m, const double bbox_rel_size_inc = 0.1,
+              const double newt_tol = 1.0e-12,
               const int npt_max = 256);
-   /** Searches positions given in physical space by \p point_pos.
-       These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
+
+   /// Preprocess the surface mesh to compute data for FindPoints.
+   void SetupSurf(Mesh &m,
+                  const double bbox_rel_size_inc = 0.1,
+                  const double newt_tol = 1.0e-12);
+
+   /** @brief Preprocess the surface mesh to compute data for FindPoints using
+    *  absolute AABB expansion.
+    *
+    *  @details This method computes only axis-aligned bounding boxes and
+    *  increases their total length by a user-specified amount in each
+    *  physical direction. The absolute AABB expansion is applied
+    *  symmetrically to the lower and upper bounds.
+    *
+    *  The size of @a aabb_sz_inc determines how the expansion values are
+    *  interpreted:
+    *  - `1`: one expansion value used in every direction for every element
+    *  - `NElements`: one expansion value per element, reused in x/y/z
+    *    directions
+    *  - `SpaceDim`: one expansion value per physical direction, reused for
+    *    every element
+    *  - `NElements*SpaceDim`: one expansion value per element and direction,
+    *    ordered as `(dx1,dy1,dz1, ... dxN,dyN,dzN)`
+    *
+    *  This method disables the oriented bounding-box precheck because the
+    *  stored boxes are modified only in their axis-aligned representation.
+    *
+    *  @param[in] m                  Input surface mesh.
+    *  @param[in] aabb_sz_inc        Total absolute AABB expansion applied in
+    *                                each physical direction to the stored
+    *                                axis-aligned bounding boxes.
+    *  @param[in] newt_tol           Newton tolerance for the point-search
+    *                                kernels.
+    *
+    *  @note We disable the oriented bounding box check with this setup.
+    *        @a bdr_tol is also adjusted so that all points in the AABBs can
+    *        be found.
+    */
+   void SetupSurfWithAABBExpansion(Mesh &m, const Vector &aabb_sz_inc,
+                                   const double newt_tol = 1.0e-12);
+
+
+   /** @brief Searches positions given in physical space by \p point_pos.
+
+       @details These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
        byVDim: (XYZ,XYZ,....XYZ) specified by \p point_pos_ordering.
+
        This function populates the following member variables:
        #gsl_code        Return codes for each point: inside element (0),
                         element boundary (1), not found (2).
@@ -261,40 +481,77 @@ public:
        #gsl_dist        Distance between the sought and the found point
                         in physical space. */
    void FindPoints(const Vector &point_pos,
-                   const int point_pos_ordering = Ordering::byNODES);
+                   int point_pos_ordering = Ordering::byNODES);
+
    /// Convenience function when point positions are in a ParticleVector
    void FindPoints(const ParticleVector &point_pos)
    {
       FindPoints(point_pos, point_pos.GetOrdering());
    }
+
+   /** @brief Searches positions given in physical space by \p point_pos on
+    *  surface mesh. */
+   void FindPointsSurf(const Vector &point_pos,
+                       int point_pos_ordering = Ordering::byNODES);
+
+   /// Convenience function when point positions are in a ParticleVector
+   void FindPointsSurf(const ParticleVector &point_pos)
+   {
+      FindPointsSurf(point_pos, point_pos.GetOrdering());
+   }
+
    /// Setup FindPoints and search positions
    void FindPoints(Mesh &m, const Vector &point_pos,
                    const int point_pos_ordering = Ordering::byNODES,
-                   const double bb_t = 0.1, const double newt_tol = 1.0e-12,
+                   const double bbox_rel_size_inc = 0.1,
+                   const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 
-   /** Interpolation of field values at prescribed reference space positions.
+   /** @brief Interpolation of field values at prescribed reference space
+    *  positions.
+
        @param[in] field_in    Function values that will be interpolated on the
                               reference positions. Note: it is assumed that
                               \p field_in is in H1 and in the same space as the
                               mesh that was given to Setup().
        @param[out] field_out  Interpolated values. For points that are not found
                               the value is set to #default_interp_value.
-                              The output ordering is determined from field_in.*/
+                              The output ordering is determined from field_in.
+
+       @note: field_out is moved to device if field_in is on device. Otherwise,
+              field_out memory allocation is not changed.
+   */
    virtual void Interpolate(const GridFunction &field_in, Vector &field_out);
+
    /// Interpolation of field values, with output ordering specification.
    virtual void Interpolate(const GridFunction &field_in, Vector &field_out,
                             const int field_out_ordering);
-   /** Search positions and interpolate. The ordering (byNODES or byVDIM) of
-       the output values in \p field_out corresponds to the ordering used
-       in the input GridFunction \p field_in. */
+
+   /** @brief Same as Interpolate but for surface meshes */
+   virtual void InterpolateSurf(const GridFunction &field_in,
+                                Vector &field_out);
+
+   /** @brief Same as Interpolate but for surface meshes with specified output
+       ordering */
+   virtual void InterpolateSurf(const GridFunction &field_in,
+                                Vector &field_out,
+                                const int field_out_ordering);
+
+   /** @brief Search positions and interpolate.
+    *
+    *  @details The ordering (byNODES or byVDIM) of the output values in
+    *  \p field_out corresponds to the ordering used in the input
+    *  GridFunction \p field_in.
+    */
    void Interpolate(const Vector &point_pos, const GridFunction &field_in,
                     Vector &field_out,
-                    const int point_pos_ordering = Ordering::byNODES);
+                    int point_pos_ordering = Ordering::byNODES);
+
    /// Search positions and interpolate with given point and output ordering.
    void Interpolate(const Vector &point_pos, const GridFunction &field_in,
                     Vector &field_out, const int point_pos_ordering,
                     const int field_out_ordering);
+
    /** Setup FindPoints, search positions and interpolate. The ordering (byNODES
        or byVDIM) of the output values in \p field_out corresponds to the
        ordering used in the input GridFunction \p field_in. */
@@ -302,32 +559,41 @@ public:
                     const GridFunction &field_in, Vector &field_out,
                     const int point_pos_ordering = Ordering::byNODES);
 
-   /// Average type to be used for L2 functions in-case a point is located at
-   /// an element boundary where the function might be multi-valued.
+   /** @brief Average type to be used for L2 functions in-case a point is
+    *  located at an element boundary where the function might be multi-valued.
+    */
    virtual void SetL2AvgType(AvgType avgtype_) { avgtype = avgtype_; }
 
-   /// Set the default interpolation value for points that are not found in the
-   /// mesh.
+   /** @brief Set the default interpolation value for points that are not found in the mesh. */
    virtual void SetDefaultInterpolationValue(double interp_value_)
    {
       default_interp_value = interp_value_;
    }
 
-   /// Set the tolerance for detecting points outside the 'curvilinear' boundary
-   /// that gslib may return as found on the boundary. Points found on boundary
-   /// with distance greater than @ bdr_tol are marked as not found.
+   /** @brief Tolerance for detecting points outside the 'curvilinear' boundary.
+    *
+    *  @details When using FindPoints, gslib may return points as found on the
+    *  boundary even when they are slightly outside the domain. This tolerance
+    *  is used to filter such points based on the distance^2 value and mark them
+    *  as not found.
+    *
+    *  @note When the SetupSurfWithAABBExpansion method is used for surface
+    *  meshes, this tolerance is automatically computed based on the size of
+    *  expanded AABBs. Using this method will override that computed tolerance.
+    *  */
    virtual void SetDistanceToleranceForPointsFoundOnBoundary(double bdr_tol_)
    {
       bdr_tol = bdr_tol_;
    }
 
-   /// Enable/Disable use of CPU functions for GPU data if the gslib version
-   /// is older.
+   /** @brief Enable/Disable use of CPU functions for GPU data if the gslib
+    *  version is older. */
    virtual void SetGPUtoCPUFallback(bool mode) { gpu_to_cpu_fallback = mode; }
 
-   /** Cleans up memory allocated internally by gslib.
-       Note that in parallel, this must be called before MPI_Finalize(), as it
-       calls MPI_Comm_free() for internal gslib communicators. FreeData is
+   /** @brief Cleans up memory allocated internally by gslib.
+
+       @details Note that in parallel, this must be called before MPI_Finalize,
+       as it calls MPI_Comm_free() for internal gslib communicators. FreeData is
        also called by the class destructor and there are no memory leaks if the
        destructor is called before MPI_Finalize(). If the destructor is called
        after MPI_Finalize(), there will be an error because gslib will try to
@@ -335,8 +601,8 @@ public:
    */
    virtual void FreeData();
 
-   /// Return code for each point searched by FindPoints: inside element (0), on
-   /// element boundary (1), or not found (2).
+   /** @brief Return code for each point searched by FindPoints:
+    *  inside element (0), element boundary (1), or not found (2). */
    virtual const Array<unsigned int> &GetCode() const { return gsl_code; }
    /// Return element number for each point found by FindPoints.
    virtual const Array<unsigned int> &GetElem() const { return gsl_mfem_elem; }
@@ -344,15 +610,15 @@ public:
    virtual const Array<unsigned int> &GetProc() const { return gsl_proc; }
    /// Return reference coordinates for each point found by FindPoints.
    virtual const Vector &GetReferencePosition() const { return gsl_mfem_ref;  }
-   /// Return distance between the sought and the found point in physical space,
-   /// for each point found by FindPoints.
+   /// Return distance between the sought and the found point in physical space.
    virtual const Vector &GetDist()              const { return gsl_dist; }
 
-   /// Return element number for each point found by FindPoints corresponding to
-   /// GSLIB mesh. gsl_mfem_elem != gsl_elem for mesh with simplices.
+   /** @brief Return element number for each point found by FindPoints
+    *  corresponding to GSLIB mesh. gsl_mfem_elem != gsl_elem for mesh with
+    *  simplices. */
    virtual const Array<unsigned int> &GetGSLIBElem() const { return gsl_elem; }
-   /// Return reference coordinates in [-1,1] (internal range in GSLIB) for each
-   /// point found by FindPoints.
+   /** @brief Return reference coordinates in [-1,1] (internal range in GSLIB)
+    *  for each point found by FindPoints. */
    virtual const Vector &GetGSLIBReferencePosition() const { return gsl_ref; }
 
    /// Get array of indices of not-found points.
@@ -395,7 +661,7 @@ public:
 
    /// Return the axis-aligned bounding boxes (AABB) computed during \ref Setup.
    /// The size of the returned vector is (nel x nverts x dim), where nel is the
-   /// number of elements (after splitting for simplcies), nverts is number of
+   /// number of elements (after splitting for simplicies), nverts is number of
    /// vertices (4 in 2D, 8 in 3D), and dim is the spatial dimension.
    void GetAxisAlignedBoundingBoxes(Vector &aabb) const;
 
@@ -409,6 +675,18 @@ public:
    /// \p obbV, a vector of size (nel x nverts x dim) .
    void GetOrientedBoundingBoxes(DenseTensor &obbA, Vector &obbC,
                                  Vector &obbV) const;
+
+   /** @brief Return the bounding boxes as a mesh on rank 0.
+    *
+    *  @param[in] type  Bounding-box type: 0 - AABB, 1 - OBB.
+    *
+    *  @return On rank 0, returns a newly allocated mesh containing the
+    *  bounding boxes. The caller owns the returned pointer and is responsible
+    *  for deleting it. On other ranks, returns nullptr.
+    */
+   Mesh *GetBoundingBoxMesh(int type);
+
+   virtual const Vector &GetGLLMesh()           const { return gsl_mesh; }
 };
 
 /** \brief OversetFindPointsGSLIB enables use of findpts for arbitrary number of
@@ -437,25 +715,28 @@ public:
        Note: not tested with periodic meshes (L2).
        Note: the input mesh \p m must have Nodes set.
 
-       @param[in] m         Input mesh.
-       @param[in] meshid    A unique # for each overlapping mesh. This id is
-                            used to make sure that points being searched are not
-                            looked for in the mesh that they belong to.
-       @param[in] gfmax     (Optional) GridFunction in H1 that is used as a
-                            discriminator when one point is located in multiple
-                            meshes. The mesh that maximizes gfmax is chosen.
-                            For example, using the distance field based on the
-                            overlapping boundaries is helpful for convergence
-                            during Schwarz iterations.
-       @param[in] bb_t      (Optional) Relative size of bounding box around
-                            each element.
-       @param[in] newt_tol  (Optional) Newton tolerance for the gslib
-                            search methods.
-       @param[in] npt_max   (Optional) Number of points for simultaneous
-                            iteration. This alters performance and
-                            memory footprint.*/
-   void Setup(Mesh &m, const int meshid, GridFunction *gfmax = NULL,
-              const double bb_t = 0.1, const double newt_tol = 1.0e-12,
+       @param[in] m                  Input mesh.
+       @param[in] meshid             A unique # for each overlapping mesh.
+                                     This id is used to make sure that points
+                                     being searched are not looked for in the
+                                     mesh that they belong to.
+       @param[in] gfmax              (Optional) GridFunction in H1 that is used
+                                     as a discriminator when one point is
+                                     located in multiple meshes. The mesh that
+                                     maximizes gfmax is chosen. For example,
+                                     using the distance field based on the
+                                     overlapping boundaries is helpful for
+                                     convergence during Schwarz iterations.
+       @param[in] bbox_rel_size_inc  (Optional) Relative size increase applied
+                                     when expanding each element bounding box.
+       @param[in] newt_tol           (Optional) Newton tolerance for the gslib
+                                     search methods.
+       @param[in] npt_max            (Optional) Number of points for
+                                     simultaneous iteration. This alters
+                                     performance and memory footprint.*/
+   void Setup(Mesh &m, const int meshid, GridFunction *gfmax = nullptr,
+              const double bbox_rel_size_inc = 0.1,
+              const double newt_tol = 1.0e-12,
               const int npt_max = 256);
 
    /** Searches positions given in physical space by \p point_pos. All output
@@ -511,7 +792,7 @@ class GSOPGSLIB
 protected:
    struct gslib::crystal *cr;               // gslib's internal data
    struct gslib::comm *gsl_comm;            // gslib's internal data
-   struct gslib::gs_data *gsl_data = NULL;
+   struct gslib::gs_data *gsl_data = nullptr;
    int num_ids;
 
 public:
@@ -536,6 +817,116 @@ public:
    void GS(Vector &senddata, GSOp op);
 };
 
+#if defined(MFEM_USE_MPI)
+/** \brief Class to map a point in physical space to candidate ranks.
+ *
+ *  This class builds a Cartesian-aligned tensor grid that covers the entire
+ *  domain and precomputes which ranks have elements intersecting each
+ *  grid cell. Given a point in physical space, the grid cell containing
+ *  the point is determined, and the list of candidate ranks whose
+ *  elements intersect that cell is returned. This yields a fast, conservative
+ *  point-to-rank candidate query. This is used internally by FindPointsGSLIB
+ *  to speed up point searches in parallel.
+ *
+ *  See Mittal et al., "General Field Evaluation in High-Order Meshes on GPUs".
+ *  (2025). Computers & Fluids. for technical details.
+ *
+ */
+class GlobalBBoxTensorGridMap
+{
+private:
+   struct gslib::crystal *cr = nullptr;               // gslib's internal data
+   struct gslib::comm *gsl_comm = nullptr;            // gslib's internal data
+   int sdim, n_local_cells, num_procs;
+   Array<int> gmap_n;
+   Vector gmap_bnd_min, gmap_bnd_max;
+   Vector gmap_fac;
+   Array<int> ggrid_map;
+
+   void SetupCrystal(const MPI_Comm &comm);
+public:
+   /// Constructor for a given mesh and number of tensor grid divisions
+   GlobalBBoxTensorGridMap(ParMesh &pmesh, int nx);
+
+   /** @brief Constructor for given element bounds and spatial dimension.
+    *
+    *  @details This constructor must be called collectively on \a comm.
+    *  Supports spatial dimensions 1, 2, and 3, and accepts nel == 0 on a rank.
+    *
+    *  Assumes elmin, elmax Ordering::byNodes:
+    *  elmin -> [x_{0,min},x_{1,min},... ,y_{0,min},y_{1,min},..,z_{nel-1,min}]
+    *  elmax -> [x_{0,max},x_{1,max},... ,y_{0,max},y_{1,max},..,z_{nel-1,max}]
+    *  Note elmin, elmax can be obtained using GridFunction::GetElementBounds()
+    *
+    *  When by_max_size=false, n gives the number of tensor-grid divisions in
+    *  each direction. When by_max_size=true, n is a per-rank size hint used to
+    *  derive a uniform global resolution. The communicator-wide sum of n is
+    *  converted to nx = ceil(pow(sum(n), 1./sdim)) in each direction, so n is
+    *  not a hard cap on ggrid_map.Size().
+    */
+   GlobalBBoxTensorGridMap(const MPI_Comm &comm, Vector &elmin,
+                           Vector &elmax, int nel, int sdim, int n,
+                           bool by_max_size);
+
+   /** @brief Constructor for given element bounds, spatial dimension, and
+    *  tensor-grid divisions in each direction.
+    *
+    *  @details This constructor must be called collectively on \a comm.
+    *  Supports spatial dimensions 1, 2, and 3, and accepts nel == 0 on a rank.
+    *  Requires nx.Size() == sdim and positive entries in nx.
+    *
+    *  Assumes elmin, elmax Ordering::byNodes:
+    *  elmin -> [x_{0,min},x_{1,min},... ,y_{0,min},y_{1,min},..,z_{nel-1,min}]
+    *  elmax -> [x_{0,max},x_{1,max},... ,y_{0,max},y_{1,max},..,z_{nel-1,max}]
+    *  Note elmin, elmax can be obtained using GridFunction::GetElementBounds()
+    */
+   GlobalBBoxTensorGridMap(const MPI_Comm &comm, Vector &elmin,
+                           Vector &elmax, int nel, int sdim, Array<int> &nx);
+
+   ~GlobalBBoxTensorGridMap();
+
+   /** @brief Get list of procs corresponding to the list of points.
+    *
+    *  @details This method must be called collectively on the communicator
+    *  used to construct the map. The input points can be ordered byNodes:
+    *  (XXX...,YYY...,ZZZ) or byVDIM: (XYZ,XYZ,...), as specified by
+    *  \a ordering.
+    *
+    *  The output map contains one entry for each input point, keyed by the
+    *  point's local index in \a xyz. Points with no candidate ranks, including
+    *  points outside the global bounding box, have an empty list of candidate
+    *  ranks.
+    */
+   void MapPointsToProcs(Vector &xyz, int ordering,
+                         std::map<int, std::vector<int>> &pt_to_procs) const;
+
+   // Some getters
+   const Array<int> &GetGridMap() const { return ggrid_map; }
+   const Vector &GetGridFac() const { return gmap_fac; }
+   const Vector &GetGridMin() const { return gmap_bnd_min; }
+   const Vector &GetGridMax() const { return gmap_bnd_max; }
+   const Array<int> &GetGridN() const { return gmap_n; }
+
+private:
+   /// Setup the map given element bounds and number of tensor grid divisions.
+   void Setup(const MPI_Comm &comm, Vector &elmin, Vector &elmax,
+              int nel, Array<int> &nx);
+
+   /// Get global hash cell index for a given point.
+   int GetGlobalGridCellFromPoint(Vector &xyz) const;
+
+   /** @brief Get owning proc and local index on that proc for given global
+    *  grid cell index. */
+   void GlobalGridCellToProcAndLocalIndex(int i, int &proc, int &idx) const;
+
+   /// Map a point to proc and local index of the corresponding grid cell
+   void GetProcAndLocalIndexFromPoint(Vector &xyz, int &proc, int &idx) const;
+
+   /// Given local cell index, return list of procs saved in the map
+   Array<int> MapCellToProcs(int l_idx) const;
+};
+#endif // MFEM_USE_MPI
+
 } // namespace mfem
 
 #endif // MFEM_USE_GSLIB

fem/gslib/findpts_local_2.cpp

@@ -11,7 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "../../linalg/kernels.hpp"
+#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -27,8 +27,6 @@
 #pragma GCC diagnostic pop
 #endif
 
-#include <climits>
-
 namespace mfem
 {
 #if GSLIB_RELEASE_VERSION >= 10009
@@ -54,127 +52,14 @@ struct findptsElementGPT_t
    double x[DIM], jac[DIM * DIM], hes[4];
 };
 
-struct dbl_range_t
-{
-   double min, max;
-};
-struct obbox_t
-{
-   double c0[DIM], A[DIM * DIM];
-   dbl_range_t x[DIM];
-};
-
-struct findptsLocalHashData_t
-{
-   int hash_n;
-   dbl_range_t bnd[DIM];
-   double fac[DIM];
-   unsigned int *offset;
-   int max;
-};
-
-// Eval the ith Lagrange interpolant and its first derivative at x.
-// Note: lCoeff stores pre-computed coefficients for fast evaluation.
-static MFEM_HOST_DEVICE inline void lag_eval_first_der(double *p0, double x,
-                                                       int i, const double *z,
-                                                       const double *lCoeff,
-                                                       int pN)
-{
-   double u0 = 1, u1 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         double d_j = 2 * (x - z[j]);
-         u1 = d_j * u1 + u0;
-         u0 = d_j * u0;
-      }
-   }
-   p0[i] = lCoeff[i] * u0;
-   p0[pN+i] = 2.0 * lCoeff[i] * u1;
-}
-
-// Eval the ith Lagrange interpolant and its first and second derivative at x.
-// Note: lCoeff stores pre-computed coefficients for fast evaluation.
-static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
-                                                        int i, const double *z,
-                                                        const double *lCoeff,
-                                                        int pN)
-{
-   double u0 = 1, u1 = 0, u2 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         double d_j = 2 * (x - z[j]);
-         u2 = d_j * u2 + u1;
-         u1 = d_j * u1 + u0;
-         u0 = d_j * u0;
-      }
-   }
-   p0[i] = lCoeff[i] * u0;
-   p0[pN+i] = 2.0 * lCoeff[i] * u1;
-   p0[2*pN+i] = 8.0 * lCoeff[i] * u2;
-}
-
-// Axis-aligned bounding box test.
-static MFEM_HOST_DEVICE inline double AABB_test(const obbox_t *const b,
-                                                const double x[2])
-{
-   double test = 1;
-   for (int d = 0; d < 2; ++d)
-   {
-      double b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
-      test = test < 0 ? test : b_d;
-   }
-   return test;
-}
-
-// Axis-aligned bounding box test followed by oriented bounding-box test.
-static MFEM_HOST_DEVICE inline double bbox_test(const obbox_t *const b,
-                                                const double x[2])
-{
-   const double bxyz = AABB_test(b, x);
-   if (bxyz < 0)
-   {
-      return bxyz;
-   }
-   else
-   {
-      double dxyz[2];
-      for (int d = 0; d < 2; ++d)
-      {
-         dxyz[d] = x[d] - b->c0[d];
-      }
-      double test = 1;
-      for (int d = 0; d < 2; ++d)
-      {
-         double rst = 0;
-         for (int e = 0; e < 2; ++e)
-         {
-            rst += b->A[d * 2 + e] * dxyz[e];
-         }
-         double brst = (rst + 1) * (1 - rst);
-         test = test < 0 ? test : brst;
-      }
-      return test;
-   }
-}
-
-// Element index corresponding to hash mesh that the point is located in.
-static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
-                                              const double x[2])
-{
-   const int n = p->hash_n;
-   int sum = 0;
-   for (int d = 2 - 1; d >= 0; --d)
-   {
-      sum *= n;
-      int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
-      sum += i < 0 ? 0 : (n - 1 < i ? n - 1 : i);
-   }
-   return sum;
-}
+using dbl_range_t = gslib::dbl_range_t;
+using obbox_t = gslib::obbox_t<DIM>;
+using findptsLocalHashData_t = gslib::findptsLocalHashData_t<DIM>;
+using gslib::bbox_test;
+using gslib::hash_index;
+using gslib::l2norm2;
+using gslib::lag_eval_first_der;
+using gslib::lag_eval_second_der;
 
 /*Solve Ax=y. A is row-major */
 static MFEM_HOST_DEVICE inline void lin_solve_2(double x[2], const double A[4],
@@ -185,12 +70,6 @@ static MFEM_HOST_DEVICE inline void lin_solve_2(double x[2], const double A[4],
    x[1] = idet*(A[0]*y[1] - A[2]*y[0]);
 }
 
-/* L2 norm squared. */
-static MFEM_HOST_DEVICE inline double l2norm2(const double x[2])
-{
-   return x[0] * x[0] + x[1] * x[1];
-}
-
 /* the bit structure of flags is CSSRR
    the C bit --- 1<<4 --- is set when the point is converged
    RR is 0 = 00b if r is unconstrained,
@@ -352,7 +231,7 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *res,
                                                  const findptsElementPoint_t *p,
                                                  const double tol)
 {
-   const double dist2 = l2norm2(resid);
+   const double dist2 = l2norm2<2>(resid);
    const double decr = p->dist2 - dist2;
    const double pred = p->dist2p;
    for (int d = 0; d < 2; ++d)
@@ -562,7 +441,7 @@ newton_area_fin:
       int f = flags >> (2 * dd) & 3u;
       res->r[dd] = f == 0 ? r0[dd] + dr[dd] : (f == 1 ? -1 : 1);
    }
-   res->flags = flags | (p->flags << 5);
+   res->flags = flags | ((p->flags & FLAG_MASK) << 5);
 }
 
 // Full Newton solve on the face. One of r/s/t is constrained.
@@ -635,7 +514,8 @@ newton_edge_fin:
    res->r[de] = nr;
    res->r[dn]=p->r[dn];
    res->dist2p = -v;
-   res->flags = flags | new_flags | (p->flags << 5);
+   res->flags = flags | new_flags | ((p->flags & FLAG_MASK) << 5);
+#undef EVAL
 }
 
 // Find closest mesh node to the sought point.
@@ -694,27 +574,26 @@ static MFEM_HOST_DEVICE double tensor_ig2_j(double *g_partials,
 }
 
 template<int T_D1D = 0>
-static void FindPointsLocal2D_Kernel(const int npt,
-                                     const double tol,
-                                     const double *x,
-                                     const int point_pos_ordering,
-                                     const double *xElemCoord,
-                                     const int nel,
-                                     const double *wtend,
-                                     const double *boxinfo,
-                                     const int hash_n,
-                                     const double *hashMin,
-                                     const double *hashFac,
-                                     unsigned int *hashOffset,
-                                     unsigned int *const code_base,
-                                     unsigned int *const el_base,
-                                     double *const r_base,
-                                     double *const dist2_base,
-                                     const double *gll1D,
-                                     const double *lagcoeff,
-                                     const int pN = 0)
+static void FindPointsLocal2DKernel(const int npt,
+                                    const double tol,
+                                    const double *x,
+                                    const int point_pos_ordering,
+                                    const double *xElemCoord,
+                                    const int nel,
+                                    const double *wtend,
+                                    const double *boxinfo,
+                                    const int hash_n,
+                                    const double *hashMin,
+                                    const double *hashFac,
+                                    unsigned int *hashOffset,
+                                    unsigned int *const code_base,
+                                    unsigned int *const el_base,
+                                    double *const r_base,
+                                    double *const dist2_base,
+                                    const double *gll1D,
+                                    const double *lagcoeff,
+                                    const int pN = 0)
 {
-#define MAX_CONST(a, b) (((a) > (b)) ? (a) : (b))
    const int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
    const int D1D = T_D1D ? T_D1D : pN;
    const int p_NE = D1D*D1D;
@@ -729,7 +608,7 @@ static void FindPointsLocal2D_Kernel(const int npt,
       // 3D1D for seed, 10D1D+6 for area, 3D1D+9 for edge
       constexpr int size1 = 10*MD1 + 6;
       constexpr int size2 = MD1*4;            // edge constraints
-      constexpr int size3 = MD1*MD1*MD1*DIM;  // local element coordinates
+      constexpr int size3 = MD1*MD1*DIM;  // local element coordinates
 
       MFEM_SHARED double r_workspace[size1];
       MFEM_SHARED findptsElementPoint_t el_pts[2];
@@ -1162,9 +1041,9 @@ void FindPointsGSLIB::FindPointsLocal2(const Vector &point_pos,
    auto pgslm = gsl_mesh.Read();
    auto pwt = DEV.wtend.Read();
    auto pbb = DEV.bb.Read();
-   auto plhm = DEV.loc_hash_min.Read();
-   auto plhf = DEV.loc_hash_fac.Read();
-   auto plho = DEV.loc_hash_offset.ReadWrite();
+   auto plhm = DEV.lh_min.Read();
+   auto plhf = DEV.lh_fac.Read();
+   auto plho = DEV.lh_offset.ReadWrite();
    auto pcode = code.Write();
    auto pelem = elem.Write();
    auto pref = ref.Write();
@@ -1175,32 +1054,49 @@ void FindPointsGSLIB::FindPointsLocal2(const Vector &point_pos,
    switch (DEV.dof1d)
    {
       case 2:
-         return FindPointsLocal2D_Kernel<2>(
-                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.h_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
-                   pgll1d, plc);
+         FindPointsLocal2DKernel<2>(npt, DEV.newt_tol, pp,
+                                    point_pos_ordering, pgslm,
+                                    NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist,
+                                    pgll1d, plc);
+         break;
       case 3:
-         return FindPointsLocal2D_Kernel<3>(
-                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.h_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
-                   pgll1d, plc);
+         FindPointsLocal2DKernel<3>(npt, DEV.newt_tol, pp,
+                                    point_pos_ordering, pgslm,
+                                    NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist,
+                                    pgll1d, plc);
+         break;
       case 4:
-         return FindPointsLocal2D_Kernel<4>(
-                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.h_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
-                   pgll1d, plc);
+         FindPointsLocal2DKernel<4>(npt, DEV.newt_tol, pp,
+                                    point_pos_ordering, pgslm,
+                                    NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist,
+                                    pgll1d, plc);
+         break;
       case 5:
-         return FindPointsLocal2D_Kernel<5>(
-                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.h_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
-                   pgll1d, plc);
+         FindPointsLocal2DKernel<5>(npt, DEV.newt_tol, pp,
+                                    point_pos_ordering, pgslm,
+                                    NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist,
+                                    pgll1d, plc);
+         break;
       default:
-         return FindPointsLocal2D_Kernel(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                         pgslm, NE_split_total, pwt, pbb, DEV.h_nx,
-                                         plhm, plhf, plho, pcode, pelem,
-                                         pref, pdist, pgll1d, plc, DEV.dof1d);
+         FindPointsLocal2DKernel(npt, DEV.newt_tol, pp,
+                                 point_pos_ordering, pgslm,
+                                 NE_split_total, pwt, pbb,
+                                 DEV.lh_nx, plhm, plhf, plho,
+                                 pcode, pelem, pref, pdist,
+                                 pgll1d, plc, DEV.dof1d);
+         break;
    }
 }
+#undef DIM2
+#undef DIM
 #undef CODE_INTERNAL
 #undef CODE_BORDER
 #undef CODE_NOT_FOUND

fem/gslib/findpts_local_3.cpp

@@ -11,9 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "../../linalg/kernels.hpp"
-
-#include <climits>
+#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -59,128 +57,15 @@ struct findptsElemPt
    double x[DIM], jac[DIM * DIM], hes[18];
 };
 
-struct dbl_range_t
-{
-   double min, max;
-};
-
-struct obbox_t
-{
-   double c0[DIM], A[DIM * DIM];
-   dbl_range_t x[DIM];
-};
-
-struct findptsLocalHashData_t
-{
-   int hash_n;
-   dbl_range_t bnd[DIM];
-   double fac[DIM];
-   unsigned int *offset;
-   // int max;
-};
-
-// Eval the ith Lagrange interpolant and its first derivative at x.
-// Note: lCoeff stores pre-computed coefficients for fast evaluation.
-static MFEM_HOST_DEVICE inline void lag_eval_first_der(double *p0, double x,
-                                                       int i, const double *z,
-                                                       const double *lCoeff,
-                                                       int pN)
-{
-   double u0 = 1, u1 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         double d_j = 2*(x-z[j]);
-         u1 = d_j*u1+u0;
-         u0 = d_j*u0;
-      }
-   }
-   p0[i] = lCoeff[i]*u0;
-   p0[pN+i] = 2.0*lCoeff[i]*u1;
-}
-
-// Eval the ith Lagrange interpolant and its first and second derivative at x.
-// Note: lCoeff stores pre-computed coefficients for fast evaluation.
-static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
-                                                        int i, const double *z,
-                                                        const double *lCoeff,
-                                                        int pN)
-{
-   double u0 = 1, u1 = 0, u2 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         double d_j = 2*(x-z[j]);
-         u2 = d_j*u2+u1;
-         u1 = d_j*u1+u0;
-         u0 = d_j*u0;
-      }
-   }
-   p0[i] = lCoeff[i]*u0;
-   p0[pN+i] = 2.0*lCoeff[i]*u1;
-   p0[2*pN+i] = 8.0*lCoeff[i]*u2;
-}
-
-// Axis-aligned bounding box test.
-static MFEM_HOST_DEVICE inline double AABB_test(const obbox_t *const b,
-                                                const double x[3])
-{
-   double b_d;
-   for (int d = 0; d < 3; ++d)
-   {
-      b_d = (x[d]-b->x[d].min)*(b->x[d].max-x[d]);
-      if (b_d < 0) { return b_d; }
-   }
-   return b_d;
-}
-
-// Axis-aligned bounding box test followed by oriented bounding-box test.
-static MFEM_HOST_DEVICE inline double bbox_test(const obbox_t *const b,
-                                                const double x[3])
-{
-   const double bxyz = AABB_test(b, x);
-   if (bxyz < 0)
-   {
-      return bxyz;
-   }
-   else
-   {
-      double dxyz[3];
-      for (int d = 0; d < 3; ++d)
-      {
-         dxyz[d] = x[d]-b->c0[d];
-      }
-      double test = 1;
-      for (int d = 0; d < 3; ++d)
-      {
-         double rst = 0;
-         for (int e = 0; e < 3; ++e)
-         {
-            rst += b->A[d*3+e]*dxyz[e];
-         }
-         double brst = (rst+1)*(1-rst);
-         test = test < 0 ? test : brst;
-      }
-      return test;
-   }
-}
-
-// Element index corresponding to hash mesh that the point is located in.
-static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
-                                              const double x[3])
-{
-   const int n = p->hash_n;
-   int sum = 0;
-   for (int d = 3-1; d >= 0; --d)
-   {
-      sum *= n;
-      int i = (int)floor((x[d]-p->bnd[d].min)*p->fac[d]);
-      sum += i < 0 ? 0 : (n-1 < i ? n-1 : i);
-   }
-   return sum;
-}
+using dbl_range_t = gslib::dbl_range_t;
+using obbox_t = gslib::obbox_t<DIM>;
+using findptsLocalHashData_t = gslib::findptsLocalHashData_t<DIM>;
+using gslib::bbox_test;
+using gslib::hash_index;
+using gslib::l2norm2;
+using gslib::lag_eval_first_der;
+using gslib::lag_eval_second_der;
+using gslib::lin_solve_sym_2;
 
 // Solve Ax=y. A is row-major.
 static MFEM_HOST_DEVICE inline void lin_solve_3(double x[3], const double A[9],
@@ -199,22 +84,6 @@ static MFEM_HOST_DEVICE inline void lin_solve_3(double x[3], const double A[9],
    x[2] = idet*(inv6*y[0]+inv7*y[1]+inv8*y[2]);
 }
 
-// Solve Ax=y. A is a symmetric 2x2 matrix.
-static MFEM_HOST_DEVICE inline void lin_solve_sym_2(double x[2],
-                                                    const double A[3],
-                                                    const double y[2])
-{
-   const double idet = 1 / (A[0]*A[2]-A[1]*A[1]);
-   x[0] = idet*(A[2]*y[0]-A[1]*y[1]);
-   x[1] = idet*(A[0]*y[1]-A[1]*y[0]);
-}
-
-// L2 norm.
-static MFEM_HOST_DEVICE inline double l2norm2(const double x[3])
-{
-   return x[0]*x[0]+x[1]*x[1]+x[2]*x[2];
-}
-
 /* the bit structure of flags is CTTSSRR
    the C bit --- 1<<6 --- is set when the point is converged
    RR is 0 = 00b if r is unconstrained,
@@ -459,7 +328,7 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsPt *res,
                                                  const findptsPt *p,
                                                  const double tol)
 {
-   const double dist2 = l2norm2(resid);
+   const double dist2 = l2norm2<3>(resid);
    const double decr = p->dist2-dist2;
    const double pred = p->dist2p;
    for (int d = 0; d < 3; ++d)
@@ -706,7 +575,7 @@ newton_vol_fin:
       int f = flags >> (2*dd) & 3u;
       res->r[dd] = f == 0 ? r0[dd]+dr[dd] : (f == 1 ? -1 : 1);
    }
-   res->flags = flags | (p->flags << 7);
+   res->flags = flags | ((p->flags & FLAG_MASK) << 7);
 }
 
 // Full Newton solve on the face. One of r/s/t is constrained.
@@ -889,7 +758,7 @@ newton_face_fin:
    res->r[dn] = p->r[dn];
    res->r[d1] = r[0];
    res->r[d2] = r[1];
-   res->flags = new_flags | (p->flags << 7);
+   res->flags = new_flags | ((p->flags & FLAG_MASK) << 7);
 }
 
 // Full Newton solve on the edge. Two of r/s/t are constrained.
@@ -973,7 +842,8 @@ newton_edge_fin:
    res->r[dn1] = p->r[dn1];
    res->r[dn2] = p->r[dn2];
    res->dist2p = -v;
-   res->flags = flags | new_flags | (p->flags << 7);
+   res->flags = flags | new_flags | ((p->flags & FLAG_MASK) << 7);
+#undef EVAL
 }
 
 // Find closest mesh node to the sought point.
@@ -1252,7 +1122,6 @@ static void FindPointsLocal3DKernel(const int npt,
                      case 0: // findpt_vol
                      {
                         double *wtr = r_workspace_ptr;
-
                         double *resid = wtr+6*D1D;
                         double *jac = resid+3;
                         double *resid_temp = jac+9;
@@ -1503,7 +1372,7 @@ static void FindPointsLocal3DKernel(const int npt,
                               // Hes_T is transposed version (i.e. in col major)
                               // n1*[2, 1, 1, 0, 0]
                               // j==1 => wt_j = wt+n1
-                              double *wt_j = wt+D1D*(2-(row+1) / 2);
+                              double *wt_j = wt+D1D*(2 - (row+1)/2);
                               const double *x = e_x[row+1][d];
                               hes_T[j] = 0.0;
                               for (int k = 0; k < D1D; ++k)
@@ -1522,7 +1391,6 @@ static void FindPointsLocal3DKernel(const int npt,
                               hes[j] += resid[d]*hes_T[j*3+d];
                            }
                         }
-
                         MFEM_SYNC_THREAD;
 
                         MFEM_FOREACH_THREAD(l,x,1)
@@ -1780,6 +1648,7 @@ static void FindPointsLocal3DKernel(const int npt,
          } //findpts_local
       } //elp
    });
+#undef MAXC
 }
 
 void FindPointsGSLIB::FindPointsLocal3(const Vector &point_pos,
@@ -1796,9 +1665,9 @@ void FindPointsGSLIB::FindPointsLocal3(const Vector &point_pos,
    auto pgslm = gsl_mesh.Read();
    auto pwt = DEV.wtend.Read();
    auto pbb = DEV.bb.Read();
-   auto plhm = DEV.loc_hash_min.Read();
-   auto plhf = DEV.loc_hash_fac.Read();
-   auto plho = DEV.loc_hash_offset.ReadWrite();
+   auto plhm = DEV.lh_min.Read();
+   auto plhf = DEV.lh_fac.Read();
+   auto plho = DEV.lh_offset.ReadWrite();
    auto pcode = code.Write();
    auto pelem = elem.Write();
    auto pref = ref.Write();
@@ -1809,33 +1678,36 @@ void FindPointsGSLIB::FindPointsLocal3(const Vector &point_pos,
    {
       case 2:
          FindPointsLocal3DKernel<2>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.h_nx, plhm,
-                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
-                                    plc);
+                                    pgslm, NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist, pgll1d, plc);
          break;
       case 3:
          FindPointsLocal3DKernel<3>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.h_nx, plhm,
-                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
-                                    plc);
+                                    pgslm, NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist, pgll1d, plc);
          break;
       case 4:
          FindPointsLocal3DKernel<4>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.h_nx, plhm,
-                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
-                                    plc);
+                                    pgslm, NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist, pgll1d, plc);
          break;
       case 5:
          FindPointsLocal3DKernel<5>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.h_nx, plhm,
-                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
-                                    plc);
+                                    pgslm, NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist, pgll1d, plc);
          break;
       default:
-         FindPointsLocal3DKernel(npt, DEV.newt_tol, pp, point_pos_ordering, pgslm,
-                                 NE_split_total, pwt, pbb, DEV.h_nx, plhm, plhf,
-                                 plho, pcode, pelem, pref, pdist, pgll1d, plc,
+         FindPointsLocal3DKernel(npt, DEV.newt_tol, pp,
+                                 point_pos_ordering, pgslm,
+                                 NE_split_total, pwt, pbb,
+                                 DEV.lh_nx, plhm, plhf, plho,
+                                 pcode, pelem, pref, pdist, pgll1d, plc,
                                  DEV.dof1d);
+         break;
    }
 }
 #undef pMax

fem/gslib/findptsedge_local_2.cpp

@@ -0,0 +1,656 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../gslib.hpp"
+#include "../../general/forall.hpp"
+#include "gslib_kernel_helpers.hpp"
+
+#ifdef MFEM_USE_GSLIB
+
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+#include "gslib.h"
+#ifndef GSLIB_RELEASE_VERSION //gslib v1.0.7
+#define GSLIB_RELEASE_VERSION 10007
+#endif
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic pop
+#endif
+
+namespace mfem
+{
+#if GSLIB_RELEASE_VERSION >= 10009
+#define CODE_INTERNAL 0
+#define CODE_BORDER 1
+#define CODE_NOT_FOUND 2
+#define sDIM 2
+#define sDIM2 4
+#define rDIM 1
+
+struct findptsElementPoint_t
+{
+   double x[sDIM], r, oldr, dist2, dist2p, tr;
+   int flags;
+};
+
+struct findptsElementGEdge_t
+{
+   double *x[sDIM];
+};
+
+struct findptsElementGPT_t
+{
+   double x[sDIM], jac[sDIM*rDIM], hes[sDIM*rDIM];
+};
+
+using dbl_range_t = gslib::dbl_range_t;
+using obbox_t = gslib::obbox_t<sDIM>;
+using findptsLocalHashData_t = gslib::findptsLocalHashData_t<sDIM>;
+using gslib::AABB_test;
+using gslib::bbox_test;
+using gslib::hash_index;
+using gslib::l2norm2;
+using gslib::lag_eval_second_der;
+
+/* the bit structure of flags is CRR
+   the C bit --- 1<<2 --- is set when the point is converged
+   RR is 0 = 00b if r is unconstrained,
+         1 = 01b if r is constrained at -1, i.e., rmin
+         2 = 10b if r is constrained at +1, i.e., rmax
+*/
+
+#define CONVERGED_FLAG (1u<<2)
+#define FLAG_MASK 0x07u // = 111b
+
+/* returns 1 if r direction (the only free direction in 2D) is constrained.
+   returns 1 if either 1st or 2nd bit of flags is set.
+*/
+static MFEM_HOST_DEVICE inline int num_constrained(const int flags)
+{
+   return ((flags | flags>>1) & 1u);
+}
+
+/* pi=0, r=-1; pi=1, r=+1 */
+static MFEM_HOST_DEVICE inline int point_index(const int x)
+{
+   return ((x>>1) & 1u);
+}
+
+/* check reduction in objective against prediction, and adjust
+   trust region radius (p->tr) accordingly;
+   may reject the prior step, returning 1; otherwise returns 0
+   sets out_pt->dist2, out_pt->index, out_pt->x, out_pt->oldr in any event,
+   leaving out_pt->r, out_pt->dr, out_pt->flags to be set when returning 0 */
+static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
+                                                 const double resid[2],
+                                                 const findptsElementPoint_t *p,
+                                                 const double tol)
+{
+   const double dist2 = l2norm2<2>(resid);
+   const double decr = p->dist2 - dist2;
+   const double pred = p->dist2p;
+   out_pt->x[0] = p->x[0];
+   out_pt->x[1] = p->x[1];
+   out_pt->oldr = p->r;
+   out_pt->dist2 = dist2;
+   if (decr >= 0.01*pred)
+   {
+      if (decr >= 0.9*pred)   // very good iteration
+      {
+         out_pt->tr = p->tr*2;
+      }
+      else                    // somewhat good iteration
+      {
+         out_pt->tr = p->tr;
+      }
+      return false;
+   }
+   else
+   {
+      /* reject step; note: the point will pass through this routine
+         again, and we set things up here so it gets classed as a
+         "very good iteration" --- this doubles the trust radius,
+         which is why we divide by 4 below */
+      double v0 = fabs(p->r - p->oldr);
+      out_pt->tr = v0/4.0;
+      out_pt->dist2 = p->dist2;
+      out_pt->r = p->oldr;
+      out_pt->flags = p->flags>>3;
+      out_pt->dist2p = -HUGE_VAL;
+      if (pred < dist2*tol)
+      {
+         out_pt->flags |= CONVERGED_FLAG;
+      }
+      return true;
+   }
+}
+
+static MFEM_HOST_DEVICE inline void newton_edge( findptsElementPoint_t *const
+                                                 out_pt,
+                                                 const double jac[2],
+                                                 const double rhess,
+                                                 const double resid[2],
+                                                 int flags,
+                                                 const findptsElementPoint_t *const p,
+                                                 const double tol )
+{
+   const double tr = p->tr;
+   const double A = jac[0] * jac[0] + jac[1] * jac[1] -
+                    rhess; // A = J^T J - resid_d H_d
+   const double y = jac[0]*resid[0] + jac[1]*resid[1];        // y = J^T resid
+
+   const double oldr = p->r;
+   double dr, newr, tdr, tnewr, v, tv;
+   int new_flags=0, tnew_flags=0;
+
+#define EVAL(dr) ( (dr*A - 2*y) * dr )
+   if (A>0)
+   {
+      dr = y/A;
+      if (fabs(dr)<tol)
+      {
+         dr=0.0;
+         newr = oldr;
+      }
+      else
+      {
+         newr = oldr+dr;
+      }
+
+      if (fabs(dr)<tr && fabs(newr)<1)
+      {
+         v = EVAL(dr);
+         goto newton_edge_fin;
+      }
+   }
+
+   if ((newr=oldr-tr) > -1)
+   {
+      dr = -tr;
+   }
+   else
+   {
+      newr = -1, dr = -1-oldr, new_flags = flags|1u;
+   }
+   v = EVAL(dr);
+
+   if ((tnewr=oldr+tr) < 1)
+   {
+      tdr = tr;
+   }
+   else
+   {
+      tnewr = 1, tdr = 1-oldr, tnew_flags = flags|2u;
+   }
+   tv = EVAL(tdr);
+
+   if (tv<v)
+   {
+      newr = tnewr, dr = tdr, v = tv, new_flags = tnew_flags;
+   }
+#undef EVAL
+
+newton_edge_fin:
+   // check convergence by testing if change in r is less than tol
+   if (fabs(dr)<tol)
+   {
+      new_flags |= CONVERGED_FLAG;
+   }
+   out_pt->r = newr;
+   out_pt->dist2p = -v;
+   out_pt->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
+}
+
+static MFEM_HOST_DEVICE void seed_j( const double *elx[sDIM],
+                                     const double x[sDIM],
+                                     const double *z,
+                                     double       *dist2,
+                                     double       *r,
+                                     const int    ir,
+                                     const int    pN )
+{
+   double dx[sDIM];
+   for (int d=0; d<sDIM; ++d)
+   {
+      dx[d] = x[d] - elx[d][ir];
+   }
+   dist2[ir] = HUGE_VAL;
+   const double dist2_rs = l2norm2(dx);
+   if (dist2[ir]>dist2_rs)
+   {
+      dist2[ir] = dist2_rs;
+      r[ir] = z[ir];
+   }
+}
+
+template<int T_D1D = 0>
+static void FindPointsEdgeLocal2DKernel( const int     npt,
+                                         const double  tol,
+                                         const double  dist2tol,
+                                         const double  *x,
+                                         const int     point_pos_ordering,
+                                         const double  *xElemCoord,
+                                         const int     nel,
+                                         const double  *wtend,
+                                         const double  *boxinfo,
+                                         const bool    obb_check,
+                                         const int     hash_n,
+                                         const double  *hashMin,
+                                         const double  *hashFac,
+                                         unsigned int  *hashOffset,
+                                         unsigned int *const code_base,
+                                         unsigned int *const el_base,
+                                         double *const r_base,
+                                         double *const dist2_base,
+                                         const double  *gll1D,
+                                         const double  *lagcoeff,
+                                         const int     pN = 0 )
+{
+   const int MD1   = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+   const int D1D   = T_D1D ? T_D1D : pN;
+   const int p_NEL = nel*D1D;
+   MFEM_VERIFY(MD1<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(pN<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(D1D!=0, "Polynomial order not specified.");
+   const int nThreads = D1D*sDIM;
+
+   mfem::forall_2D(npt, nThreads, 1, [=] MFEM_HOST_DEVICE (int i)
+   {
+      // 2D1D for seed, 3D1D + 7 for edge
+      constexpr int size1 = 3*MD1 + 7;
+      // edge coordinates = D1D*2
+      constexpr int size2 = 2*MD1;
+      // local element coordinates in shared memory
+      constexpr int size3 = MD1*sDIM;
+
+      MFEM_SHARED findptsElementPoint_t el_pts[2];
+      MFEM_SHARED double r_workspace[size1];
+
+      MFEM_SHARED double constraint_workspace[size2];
+
+      MFEM_SHARED double elem_coords[MD1 <= 6 ? size3 : 1];
+
+      double *r_workspace_ptr = r_workspace;
+      findptsElementPoint_t *fpt, *tmp;
+      fpt = el_pts + 0;
+      tmp = el_pts + 1;
+
+      // x and y coord index within point_pos for point i
+      int id_x = point_pos_ordering == 0 ? i     : i*sDIM;
+      int id_y = point_pos_ordering == 0 ? i+npt : i*sDIM+1;
+      double x_i[2] = {x[id_x], x[id_y]};
+
+      unsigned int *code_i = code_base  + i;
+      double *dist2_i      = dist2_base + i;
+
+      //---------------- map_points_to_els --------------------
+      findptsLocalHashData_t hash;
+      for (int d=0; d<sDIM; ++d)
+      {
+         hash.bnd[d].min = hashMin[d];
+         hash.fac[d] = hashFac[d];
+      }
+      hash.hash_n = hash_n;
+      hash.offset = hashOffset;
+
+      const int hi                  = hash_index(&hash, x_i);
+      const unsigned int *elp       = hash.offset + hash.offset[hi];
+      const unsigned int *const ele = hash.offset + hash.offset[hi+1];
+      *code_i  = CODE_NOT_FOUND;
+      *dist2_i = HUGE_VAL;
+
+      for (; elp!=ele; ++elp)
+      {
+         const unsigned int el = *elp;
+
+         const int n_box_ents = obb_check ? (3*sDIM + sDIM2) : (2*sDIM);
+         bool pass_bb = true;
+         obbox_t box;
+         if (obb_check)
+         {
+            for (int idx = 0; idx < sDIM; ++idx)
+            {
+               box.c0[idx] = boxinfo[n_box_ents*el + idx];
+               box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
+               box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
+            }
+            for (int idx = 0; idx < sDIM2; ++idx)
+            {
+               box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
+            }
+            pass_bb = (bbox_test(&box, x_i) >= 0);
+         }
+         else
+         {
+            for (int d = 0; d < sDIM; ++d)
+            {
+               box.x[d].min = boxinfo[n_box_ents*el + d];
+               box.x[d].max = boxinfo[n_box_ents*el + sDIM + d];
+            }
+            pass_bb = (AABB_test(&box, x_i) >= 0);
+         }
+
+         if (pass_bb)
+         {
+            //------------ findpts_local ------------------
+            {
+               if (MD1 <= 6)
+               {
+                  MFEM_FOREACH_THREAD(j,x,D1D*sDIM)
+                  {
+                     const int qp = j % D1D;
+                     const int d = j / D1D;
+                     elem_coords[qp + d*D1D] =
+                        xElemCoord[qp + el*D1D + d*p_NEL];
+                  }
+                  MFEM_SYNC_THREAD;
+               }
+
+               const double *elx[sDIM];
+               for (int d=0; d<sDIM; d++)
+               {
+                  elx[d] = MD1<= 6 ? &elem_coords[d*D1D] :
+                           xElemCoord + d*p_NEL + el*D1D;
+               }
+               MFEM_SYNC_THREAD;
+               //// findpts_el ////
+               {
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     fpt->dist2 = HUGE_VAL;
+                     fpt->dist2p = 0;
+                     fpt->tr = 1;
+                  }
+                  MFEM_FOREACH_THREAD(j,x,sDIM)
+                  {
+                     fpt->x[j] = x_i[j];
+                  }
+                  MFEM_SYNC_THREAD;
+
+                  {
+                     double *dist2_temp = r_workspace_ptr;
+                     double *r_temp = dist2_temp + D1D;
+                     MFEM_FOREACH_THREAD(j,x,D1D)
+                     {
+                        seed_j(elx, x_i, gll1D, dist2_temp, r_temp, j, D1D);
+                     }
+                     MFEM_SYNC_THREAD;
+
+                     MFEM_FOREACH_THREAD(j,x,1)
+                     {
+                        for (int ir=0; ir<D1D; ++ir)
+                        {
+                           if (dist2_temp[ir]<fpt->dist2)
+                           {
+                              fpt->dist2 = dist2_temp[ir];
+                              fpt->r = r_temp[ir];
+                           }
+                        }
+                     }
+                     MFEM_SYNC_THREAD;
+                  } //seed done
+
+                  // Initialize tmp struct with fpt values before starting Newton iterations
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     tmp->dist2 = HUGE_VAL;
+                     tmp->dist2p = 0;
+                     tmp->tr = 1;
+                     tmp->flags = 0;
+                     tmp->r = fpt->r;
+                  }
+                  MFEM_FOREACH_THREAD(j,x,sDIM)
+                  {
+                     tmp->x[j] = fpt->x[j];
+                  }
+                  MFEM_SYNC_THREAD;
+
+
+                  for (int step=0; step<50; step++)
+                  {
+                     int nc = num_constrained(tmp->flags & FLAG_MASK);
+                     switch (nc)
+                     {
+                        case 0:
+                        {
+                           double *wt = r_workspace_ptr;
+                           double *resid = wt + 3*D1D;
+                           double *jac = resid + sDIM;
+                           double *hess = jac + sDIM*rDIM;
+
+                           findptsElementGEdge_t edge;
+                           for (int d=0; d<sDIM; ++d)
+                           {
+                              edge.x[d] = constraint_workspace + d*D1D;
+                           }
+                           MFEM_FOREACH_THREAD(j,x,D1D)
+                           {
+                              for (int d=0; d<sDIM; ++d)
+                              {
+                                 edge.x[d][j] = elx[d][j];
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           // compute basis function info upto 2nd derivative
+                           MFEM_FOREACH_THREAD(j,x,D1D)
+                           {
+                              lag_eval_second_der(wt, tmp->r, j, gll1D,
+                                                  lagcoeff, D1D);
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,sDIM)
+                           {
+                              resid[j] = tmp->x[j];
+                              jac[j] = 0.0;
+                              hess[j] = 0.0;
+                              for (int k=0; k<D1D; ++k)
+                              {
+                                 resid[j] -= wt[      k]*edge.x[j][k];
+                                 jac[j]   += wt[D1D+k]*edge.x[j][k];
+                                 hess[j]  += wt[2*D1D+k]*edge.x[j][k];
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,1)
+                           {
+                              hess[2] = resid[0]*hess[0] + resid[1]*hess[1];
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,1)
+                           {
+                              if (!reject_prior_step_q(fpt, resid, tmp, tol))
+                              {
+                                 newton_edge(fpt, jac, hess[2], resid,
+                                             tmp->flags & FLAG_MASK, tmp, tol);
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+                           break;
+                        }
+                        case 1:    // r is constrained to either -1 or 1
+                        {
+                           MFEM_FOREACH_THREAD(j,x,1)
+                           {
+                              const int pi = point_index(tmp->flags &
+                                                         FLAG_MASK);
+                              const double *wt   = wtend + pi*3*D1D;
+                              findptsElementGPT_t gpt;
+                              for (int d=0; d<sDIM; ++d)
+                              {
+                                 gpt.x[d] = elx[d][pi*(D1D-1)];
+                                 gpt.jac[d] = 0.0;
+                                 gpt.hes[d] = 0.0;
+                                 for (int k=0; k<D1D; ++k)
+                                 {
+                                    gpt.jac[d] += wt[D1D  +k]*elx[d][k];
+                                    gpt.hes[d] += wt[2*D1D+k]*elx[d][k];
+                                 }
+                              }
+
+                              const double *const pt_x = gpt.x;
+                              const double *const jac = gpt.jac;
+                              const double *const hes = gpt.hes;
+                              double resid[sDIM], steep, sr;
+                              resid[0] = fpt->x[0] - pt_x[0];
+                              resid[1] = fpt->x[1] - pt_x[1];
+                              steep = jac[0]*resid[0] + jac[1]*resid[1];
+                              sr = steep*tmp->r;
+                              if ( !reject_prior_step_q(fpt, resid, tmp, tol) )
+                              {
+                                 if (sr<0)
+                                 {
+                                    const double rhess = resid[0]*hes[0] +
+                                                         resid[1]*hes[1];
+                                    newton_edge(fpt, jac, rhess,
+                                                resid, 0, tmp, tol);
+                                 }
+                                 else // sr==0
+                                 {
+                                    fpt->r = tmp->r;
+                                    fpt->dist2p = 0;
+                                    fpt->flags = tmp->flags | CONVERGED_FLAG;
+                                 }
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+                           break;
+                        } // case 1
+                     } //switch
+                     if (fpt->flags & CONVERGED_FLAG)
+                     {
+                        break;
+                     }
+                     MFEM_SYNC_THREAD;
+                     MFEM_FOREACH_THREAD(j,x,1)
+                     {
+                        *tmp = *fpt;
+                     }
+                     MFEM_SYNC_THREAD;
+                  } //for int step<50
+               } //findpts_el
+
+               bool converged_internal =
+                  ((fpt->flags&FLAG_MASK) == CONVERGED_FLAG) &&
+                  (fpt->dist2<dist2tol);
+
+               if (*code_i == CODE_NOT_FOUND || converged_internal ||
+                   fpt->dist2 < *dist2_i)
+               {
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     *(el_base+i) = el;
+                     *code_i = converged_internal ? CODE_INTERNAL : CODE_BORDER;
+                     *dist2_i = fpt->dist2;
+                     *(r_base+i) = fpt->r;
+                  }
+                  MFEM_SYNC_THREAD;
+                  if (converged_internal)
+                  {
+                     break;
+                  }
+               }
+            } //findpts_local
+         } //obbox_test
+      } //elp
+   });
+}
+
+void FindPointsGSLIB::FindPointsEdgeLocal2( const Vector &point_pos,
+                                            int point_pos_ordering,
+                                            Array<unsigned int> &code,
+                                            Array<unsigned int> &elem,
+                                            Vector &ref,
+                                            Vector &dist,
+                                            int npt )
+{
+   if (npt==0)
+   {
+      return;
+   }
+   MFEM_VERIFY(dim==1 && spacedim==2,"Function for 2D edges only");
+   bool use_dev = point_pos.UseDevice();
+   auto pp = point_pos.Read(use_dev);
+   auto pgslm = gsl_mesh.Read(use_dev);
+   auto pwt = DEV.wtend.Read(use_dev);
+   auto pbb = DEV.bb.Read(use_dev);
+   auto plhm = DEV.lh_min.Read(use_dev);
+   auto plhf = DEV.lh_fac.Read(use_dev);
+   auto plho = DEV.lh_offset.ReadWrite(use_dev);
+   auto pcode = code.Write(use_dev);
+   auto pelem = elem.Write(use_dev);
+   auto pref = ref.Write(use_dev);
+   auto pdist = dist.Write(use_dev);
+   auto pgll1d = DEV.gll1d.ReadWrite(use_dev);
+   auto plc = DEV.lagcoeff.Read(use_dev);
+   double dist2tol = DEV.surf_dist_tol;
+   const bool obb_chk = obb_check;
+   switch (DEV.dof1d)
+   {
+      case 2:
+         FindPointsEdgeLocal2DKernel<2>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
+      case 3:
+         FindPointsEdgeLocal2DKernel<3>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
+      case 4:
+         FindPointsEdgeLocal2DKernel<4>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
+      default:
+         FindPointsEdgeLocal2DKernel(npt, DEV.newt_tol, dist2tol, pp,
+                                     point_pos_ordering, pgslm,
+                                     NE_split_total, pwt, pbb, obb_chk,
+                                     DEV.lh_nx, plhm, plhf, plho,
+                                     pcode, pelem, pref, pdist,
+                                     pgll1d, plc, DEV.dof1d);
+         break;
+   }
+}
+#undef sDIM
+#undef rDIM
+#undef sDIM2
+#undef CODE_INTERNAL
+#undef CODE_BORDER
+#undef CODE_NOT_FOUND
+#else
+void FindPointsGSLIB::FindPointsEdgeLocal2( const Vector &point_pos,
+                                            int point_pos_ordering,
+                                            Array<unsigned int> &code,
+                                            Array<unsigned int> &elem,
+                                            Vector &ref,
+                                            Vector &dist,
+                                            int npt ) {} ;
+#endif
+} // namespace mfem
+
+#endif //ifdef MFEM_USE_GSLIB

fem/gslib/findptsedge_local_3.cpp

@@ -0,0 +1,661 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../gslib.hpp"
+#include "../../general/forall.hpp"
+#include "gslib_kernel_helpers.hpp"
+
+#ifdef MFEM_USE_GSLIB
+
+
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+#include "gslib.h"
+#ifndef GSLIB_RELEASE_VERSION //gslib v1.0.7
+#define GSLIB_RELEASE_VERSION 10007
+#endif
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic pop
+#endif
+
+namespace mfem
+{
+#if GSLIB_RELEASE_VERSION >= 10009
+#define CODE_INTERNAL 0
+#define CODE_BORDER 1
+#define CODE_NOT_FOUND 2
+#define sDIM 3
+#define rDIM 1
+#define sDIM2 (sDIM*sDIM)
+#define rDIM2 (rDIM*rDIM)
+
+struct findptsElementPoint_t
+{
+   double x[sDIM], r, oldr, dist2, dist2p, tr;
+   int flags;
+};
+
+struct findptsElementGEdge_t
+{
+   double *x[sDIM], *dxdn[sDIM], *d2xdn[sDIM];
+};
+
+struct findptsElementGPT_t
+{
+   double x[sDIM], jac[sDIM], hes[sDIM*(1+1)];
+};
+
+using dbl_range_t = gslib::dbl_range_t;
+using obbox_t = gslib::obbox_t<sDIM>;
+using findptsLocalHashData_t = gslib::findptsLocalHashData_t<sDIM>;
+using gslib::AABB_test;
+using gslib::bbox_test;
+using gslib::hash_index;
+using gslib::l2norm2;
+using gslib::lag_eval_second_der;
+
+/* the bit structure of flags is CRR
+   the C bit --- 1<<2 --- is set when the point is converged
+   RR is 0 = 00b if r is unconstrained,
+         1 = 01b if r is constrained at -1, i.e., rmin
+         2 = 10b if r is constrained at +1, i.e., rmax
+*/
+#define CONVERGED_FLAG (1u<<2)
+#define FLAG_MASK 0x07u
+
+/* returns the number of constrained reference coordinates, max 1
+*/
+static MFEM_HOST_DEVICE inline int num_constrained(const int flags)
+{
+   return ((flags | flags>>1) & 1u);
+}
+
+static MFEM_HOST_DEVICE inline int point_index(const int x)
+{
+   return ((x>>1)&1u);
+}
+
+/* check reduction in objective against prediction, and adjust
+   trust region radius (p->tr) accordingly;
+   may reject the prior step, returning 1; otherwise returns 0
+   sets out_pt->dist2, out_pt->index, out_pt->x, out_pt->oldr in any event,
+   leaving out_pt->r, out_pt->dr, out_pt->flags to be set when returning 0 */
+static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
+                                                 const double resid[3],
+                                                 const findptsElementPoint_t *p,
+                                                 const double tol)
+{
+   const double dist2 = l2norm2<sDIM>(resid);
+   const double decr  = p->dist2 - dist2;
+   const double pred  = p->dist2p;
+   for (int d=0; d<sDIM; ++d)
+   {
+      out_pt->x[d] = p->x[d];
+   }
+   out_pt->oldr = p->r;
+   out_pt->dist2 = dist2;
+   if (decr>=0.01*pred)
+   {
+      if (decr>=0.9*pred)   // very good iteration
+      {
+         out_pt->tr = 2*p->tr;
+      }
+      else   // good iteration
+      {
+         out_pt->tr = p->tr;
+      }
+      return false;
+   }
+   else   // if the iteration in not good
+   {
+      /* reject step; note: the point will pass through this routine
+         again, and we set things up here so it gets classed as a
+         "very good iteration" --- this doubles the trust radius,
+         which is why we divide by 4 below */
+      double v0 = fabs(p->r - p->oldr);
+      out_pt->tr = v0/4.0;
+      out_pt->dist2 = p->dist2;
+      out_pt->r = p->oldr;
+      out_pt->flags = p->flags>>3;
+      out_pt->dist2p = -HUGE_VAL;
+      if (pred<dist2*tol)
+      {
+         out_pt->flags |= CONVERGED_FLAG;
+      }
+      return true;
+   }
+}
+
+static MFEM_HOST_DEVICE inline void newton_edge(findptsElementPoint_t *const
+                                                out_pt,
+                                                const double jac[sDIM*rDIM],
+                                                const double rhes,
+                                                const double resid[sDIM],
+                                                int flags,
+                                                const findptsElementPoint_t *const p,
+                                                const double tol)
+{
+   const double tr = p->tr;
+   /* A = J^T J - resid_d H_d */
+   const double A = jac[0]*jac[0]+ jac[1] * jac[1] + jac[2] * jac[2]
+                    - rhes;
+   /* y = J^T r */
+   const double y = jac[0]*resid[0] + jac[1]*resid[1] + jac[0+2]*resid[2];
+
+   const double oldr = p->r;
+   double dr, nr, tdr, tnr;
+   double v, tv;
+   int new_flags = 0, tnew_flags = 0;
+
+#define EVAL(dr) (dr*A - 2*y)*dr
+
+   /* if A is not SPD, quadratic model has no minimum */
+   if (A>0)
+   {
+      dr = y/A;
+
+      if (fabs(dr)<tol)
+      {
+         dr=0.0;
+         nr = oldr;
+      }
+      else
+      {
+         nr = oldr+dr;
+      }
+      if ( fabs(dr)<tr && fabs(nr)<1 )
+      {
+         v = EVAL(dr);
+         goto newton_edge_fin;
+      }
+   }
+
+   if ( (nr=oldr-tr)>-1 )
+   {
+      dr = -tr;
+   }
+   else
+   {
+      nr = -1, dr = -1-oldr, new_flags = flags | 1u;
+   }
+   v = EVAL(dr);
+
+   if ( (tnr = oldr+tr)<1 )
+   {
+      tdr = tr;
+   }
+   else
+   {
+      tnr = 1, tdr = 1-oldr, tnew_flags = flags | 2u;
+   }
+   tv = EVAL(tdr);
+
+   if (tv<v)
+   {
+      nr = tnr, dr = tdr, v = tv, new_flags = tnew_flags;
+   }
+
+newton_edge_fin:
+   /* check convergence */
+   if ( fabs(dr)<tol )
+   {
+      new_flags |= CONVERGED_FLAG;
+   }
+   out_pt->r = nr;
+   out_pt->dist2p = -v;
+   out_pt->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
+#undef EVAL
+}
+
+static MFEM_HOST_DEVICE void seed_j(const double *elx[sDIM],
+                                    const double x[sDIM],
+                                    const double *z,
+                                    double *dist2,
+                                    double *r,
+                                    const int ir,
+                                    const int pN)
+{
+   if (ir>=pN)
+   {
+      return;
+   }
+
+   double dx[sDIM];
+   for (int d=0; d<sDIM; ++d)
+   {
+      dx[d] = x[d] - elx[d][ir];
+   }
+   dist2[ir] = l2norm2(dx);
+   r[ir] = z[ir];
+}
+
+template<int T_D1D = 0>
+static void FindPointsEdgeLocal3DKernel(const int npt,
+                                        const double tol,
+                                        const double dist2tol,
+                                        const double *x,
+                                        const int point_pos_ordering,
+                                        const double *xElemCoord,
+                                        const int nel,
+                                        const double *wtend,
+                                        const double *boxinfo,
+                                        const bool obb_check,
+                                        const int hash_n,
+                                        const double *hashMin,
+                                        const double *hashFac,
+                                        unsigned int *hashOffset,
+                                        unsigned int *const code_base,
+                                        unsigned int *const el_base,
+                                        double *const r_base,
+                                        double *const dist2_base,
+                                        const double *gll1D,
+                                        const double *lagcoeff,
+                                        const int pN = 0)
+{
+   const int MD1   = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+   const int D1D   = T_D1D ? T_D1D : pN;
+   const int p_NEL = nel*D1D;
+   MFEM_VERIFY(MD1<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(pN<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(D1D!=0, "Polynomial order not specified.");
+   const int nThreads = D1D*sDIM;
+
+   mfem::forall_2D(npt, nThreads, 1, [=] MFEM_HOST_DEVICE (int i)
+   {
+      constexpr int size1 = 3*MD1 + 13;
+      constexpr int size2 = 3*MD1;
+      constexpr int size3 = MD1*sDIM;
+
+      MFEM_SHARED findptsElementPoint_t el_pts[2];
+      MFEM_SHARED double r_workspace[size1];
+
+      MFEM_SHARED double constraint_workspace[size2];
+
+      MFEM_SHARED double elem_coords[MD1 <= 6 ? size3 : 1];
+
+      double *r_workspace_ptr = r_workspace;
+      findptsElementPoint_t *fpt, *tmp;
+      fpt = el_pts + 0;
+      tmp = el_pts + 1;
+
+      int id_x = point_pos_ordering==0 ?       i :   i*sDIM;
+      int id_y = point_pos_ordering==0 ?   npt+i : 1+i*sDIM;
+      int id_z = point_pos_ordering==0 ? 2*npt+i : 2+i*sDIM;
+      double x_i[3] = {x[id_x], x[id_y], x[id_z]};
+
+      unsigned int *code_i = code_base  + i;
+      double *dist2_i      = dist2_base + i;
+
+      //// map_points_to_els ////
+      findptsLocalHashData_t hash;
+      for (int d=0; d<sDIM; ++d)
+      {
+         hash.bnd[d].min = hashMin[d];
+         hash.fac[d]     = hashFac[d];
+      }
+      hash.hash_n = hash_n;
+      hash.offset = hashOffset;
+
+      const unsigned int hi   = hash_index(&hash, x_i);
+      const unsigned int *elp       = hash.offset + hash.offset[hi];
+      const unsigned int *const ele = hash.offset + hash.offset[hi+1];
+      *code_i  = CODE_NOT_FOUND;
+      *dist2_i = HUGE_VAL;
+
+      for (; elp!=ele; ++elp)
+      {
+         const unsigned int el = *elp;
+
+         const int n_box_ents = obb_check ? (3*sDIM + sDIM2) : (2*sDIM);
+         bool pass_bb = true;
+         obbox_t box;
+         if (obb_check)
+         {
+            for (int idx = 0; idx < sDIM; ++idx)
+            {
+               box.c0[idx] = boxinfo[n_box_ents*el + idx];
+               box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
+               box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
+            }
+            for (int idx = 0; idx < sDIM2; ++idx)
+            {
+               box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
+            }
+            pass_bb = (bbox_test(&box, x_i) >= 0);
+         }
+         else
+         {
+            for (int d = 0; d < sDIM; ++d)
+            {
+               box.x[d].min = boxinfo[n_box_ents*el + d];
+               box.x[d].max = boxinfo[n_box_ents*el + sDIM + d];
+            }
+            pass_bb = (AABB_test(&box, x_i) >= 0);
+         }
+
+         if (pass_bb)
+         {
+            //// findpts_local ////
+            {
+               if (MD1 <= 6)
+               {
+                  MFEM_FOREACH_THREAD(j,x,D1D*sDIM)
+                  {
+                     const int qp = j % D1D;
+                     const int d = j / D1D;
+                     elem_coords[qp + d*D1D] =
+                        xElemCoord[qp + el*D1D + d*p_NEL];
+                  }
+                  MFEM_SYNC_THREAD;
+               }
+
+               const double *elx[sDIM];
+               for (int d=0; d<sDIM; d++)
+               {
+                  elx[d] = MD1<= 6 ? &elem_coords[d*D1D] :
+                           xElemCoord + d*p_NEL + el*D1D;
+               }
+               MFEM_SYNC_THREAD;
+               //// findpts_el ////
+               {
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     fpt->dist2 = HUGE_VAL;
+                     fpt->dist2p = 0;
+                     fpt->tr = 1.0;
+                  }
+                  MFEM_FOREACH_THREAD(j,x,sDIM)
+                  {
+                     fpt->x[j] = x_i[j];
+                  }
+                  MFEM_SYNC_THREAD;
+
+                  //// seed ////
+                  {
+                     double *dist2_temp = r_workspace_ptr;
+                     double *r_temp = dist2_temp + D1D;
+                     MFEM_FOREACH_THREAD(j,x,nThreads)
+                     {
+                        seed_j(elx, x_i, gll1D, dist2_temp, r_temp, j, D1D);
+                     }
+                     MFEM_SYNC_THREAD;
+
+                     MFEM_FOREACH_THREAD(j,x,1)
+                     {
+                        fpt->dist2 = HUGE_VAL;
+                        for (int ir=0; ir<D1D; ++ir)
+                        {
+                           if (dist2_temp[ir] < fpt->dist2)
+                           {
+                              fpt->dist2 = dist2_temp[ir];
+                              fpt->r = r_temp[ir];
+                           }
+                        }
+                     }
+                     MFEM_SYNC_THREAD;
+                  } //seed done
+
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     tmp->dist2 = HUGE_VAL;
+                     tmp->dist2p = 0;
+                     tmp->tr = 1;
+                     tmp->flags = 0;
+                     tmp->r = fpt->r;
+                  }
+                  MFEM_FOREACH_THREAD(j,x,sDIM)
+                  {
+                     tmp->x[j] = fpt->x[j];
+                  }
+                  MFEM_SYNC_THREAD;
+
+                  for (int step=0; step<50; step++)
+                  {
+                     switch (num_constrained(tmp->flags & FLAG_MASK))
+                     {
+                        case 0:
+                        {
+                           double *wt = r_workspace_ptr;
+                           double *resid = wt + 3*D1D;
+                           double *jac = resid + sDIM;
+                           double *hess = jac + sDIM*rDIM;
+
+                           findptsElementGEdge_t edge;
+                           for (int d=0; d<sDIM; ++d)
+                           {
+                              edge.x[d] = constraint_workspace + d*D1D;
+                           }
+                           MFEM_FOREACH_THREAD(j,x,D1D)
+                           {
+                              for (int d=0; d<sDIM; ++d)
+                              {
+                                 edge.x[d][j] = elx[d][j];
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,D1D)
+                           {
+                              lag_eval_second_der(wt, tmp->r, j, gll1D,
+                                                  lagcoeff, D1D);
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,sDIM)
+                           {
+                              resid[j] = tmp->x[j];
+                              jac[j] = 0.0;
+                              hess[j] = 0.0;
+                              for (int k=0; k<D1D; ++k)
+                              {
+                                 resid[j] -= wt[      k]*edge.x[j][k];
+                                 jac[j]   += wt[D1D+k]*edge.x[j][k];
+                                 hess[j]  += wt[2*D1D+k]*edge.x[j][k];
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+
+                           MFEM_FOREACH_THREAD(j,x,1)
+                           {
+                              hess[3] = resid[0]*hess[0] + resid[1]*hess[1] +
+                                        resid[2]*hess[2];
+                           }
+
+                           MFEM_FOREACH_THREAD(l,x,1)
+                           {
+                              if (!reject_prior_step_q(fpt,resid,tmp,tol))
+                              {
+                                 newton_edge(fpt,jac,hess[3],resid,
+                                             tmp->flags&FLAG_MASK,tmp,tol);
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+                           break;
+                        }
+                        case 1:
+                        {
+                           MFEM_FOREACH_THREAD(j,x,1)
+                           {
+                              const int pi = point_index(tmp->flags &
+                                                         FLAG_MASK);
+                              const double *wt = wtend + pi*3*D1D;
+                              findptsElementGPT_t gpt;
+                              for (int d=0; d<sDIM; ++d)
+                              {
+                                 gpt.x[d] = elx[d][pi*(D1D-1)];
+                                 gpt.jac[d] = 0.0;
+                                 gpt.hes[d] = 0.0;
+                                 for (int k=0; k<D1D; ++k)
+                                 {
+                                    gpt.jac[d] += wt[D1D  +k]*elx[d][k];
+                                    gpt.hes[d] += wt[2*D1D+k]*elx[d][k];
+                                 }
+                              }
+
+                              const double *const pt_x = gpt.x;
+                              const double *const jac = gpt.jac;
+                              const double *const hes = gpt.hes;
+                              double resid[sDIM], steep, sr;
+                              resid[0] = fpt->x[0] - pt_x[0];
+                              resid[1] = fpt->x[1] - pt_x[1];
+                              resid[2] = fpt->x[2] - pt_x[2];
+                              steep = jac[0]*resid[0] + jac[1]*resid[1] +
+                                      jac[2]*resid[2];
+                              sr = steep*tmp->r;
+                              if (!reject_prior_step_q(fpt, resid, tmp, tol))
+                              {
+                                 if (sr<0)
+                                 {
+                                    const double rhess = resid[0]*hes[0] +
+                                                         resid[1]*hes[1] +
+                                                         resid[2]*hes[2];
+                                    newton_edge(fpt, jac, rhess,
+                                                resid, 0, tmp, tol);
+                                 }
+                                 else   // sr==0
+                                 {
+                                    fpt->r = tmp->r;
+                                    fpt->dist2p = 0;
+                                    fpt->flags = tmp->flags | CONVERGED_FLAG;
+                                 }
+                              }
+                           }
+                           MFEM_SYNC_THREAD;
+                           break;
+                        } // case 1
+                     } //switch
+                     if (fpt->flags & CONVERGED_FLAG)
+                     {
+                        break;
+                     }
+                     MFEM_SYNC_THREAD;
+
+                     MFEM_FOREACH_THREAD(j,x,1)
+                     {
+                        *tmp = *fpt;
+                     }
+                     MFEM_SYNC_THREAD;
+                  } // for step<50
+               } // findpts_el
+
+               bool converged_internal =
+                  ((fpt->flags&FLAG_MASK) == CONVERGED_FLAG) &&
+                  (fpt->dist2<dist2tol);
+               if (*code_i==CODE_NOT_FOUND || converged_internal ||
+                   fpt->dist2<*dist2_i)
+               {
+                  MFEM_FOREACH_THREAD(j,x,1)
+                  {
+                     *(el_base+i) = el;
+                     *code_i  = converged_internal?CODE_INTERNAL:CODE_BORDER;
+                     *dist2_i = fpt->dist2;
+                     *(r_base+i) = fpt->r;
+                  }
+                  MFEM_SYNC_THREAD;
+                  if (converged_internal)
+                  {
+                     break;
+                  }
+               }
+            } // findpts_local
+         } // obbox_test
+      } // elp
+   });
+}
+
+void FindPointsGSLIB::FindPointsEdgeLocal3(const Vector &point_pos,
+                                           int point_pos_ordering,
+                                           Array<unsigned int> &code,
+                                           Array<unsigned int> &elem,
+                                           Vector &ref,
+                                           Vector &dist,
+                                           int npt)
+{
+   if (npt == 0)
+   {
+      return;
+   }
+   MFEM_VERIFY(spacedim==3 && dim == 1,"Function for 3D edges only");
+   bool use_dev = point_pos.UseDevice();
+   auto pp = point_pos.Read(use_dev);
+   auto pgslm = gsl_mesh.Read(use_dev);
+   auto pwt = DEV.wtend.Read(use_dev);
+   auto pbb = DEV.bb.Read(use_dev);
+   auto plhm = DEV.lh_min.Read(use_dev);
+   auto plhf = DEV.lh_fac.Read(use_dev);
+   auto plho = DEV.lh_offset.ReadWrite(use_dev);
+   auto pcode = code.Write(use_dev);
+   auto pelem = elem.Write(use_dev);
+   auto pref = ref.Write(use_dev);
+   auto pdist = dist.Write(use_dev);
+   auto pgll1d = DEV.gll1d.ReadWrite(use_dev);
+   auto plc = DEV.lagcoeff.Read(use_dev);
+   double dist2tol = DEV.surf_dist_tol;
+   const bool obb_chk = obb_check;
+   switch (DEV.dof1d)
+   {
+      case 2:
+         FindPointsEdgeLocal3DKernel<2>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
+      case 3:
+         FindPointsEdgeLocal3DKernel<3>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
+      case 4:
+         FindPointsEdgeLocal3DKernel<4>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
+      default:
+         FindPointsEdgeLocal3DKernel(npt, DEV.newt_tol, dist2tol, pp,
+                                     point_pos_ordering, pgslm,
+                                     NE_split_total, pwt, pbb, obb_chk,
+                                     DEV.lh_nx, plhm, plhf, plho,
+                                     pcode, pelem, pref, pdist,
+                                     pgll1d, plc, DEV.dof1d);
+         break;
+   }
+}
+#undef rDIM2
+#undef sDIM2
+#undef rDIM
+#undef sDIM
+#undef CODE_INTERNAL
+#undef CODE_BORDER
+#undef CODE_NOT_FOUND
+#else
+void FindPointsGSLIB::FindPointsEdgeLocal3( const Vector &point_pos,
+                                            int point_pos_ordering,
+                                            Array<unsigned int> &code,
+                                            Array<unsigned int> &elem,
+                                            Vector &ref,
+                                            Vector &dist,
+                                            int npt ) {} ;
+#endif
+} // namespace mfem
+
+#endif //ifdef MFEM_USE_GSLIB

fem/gslib/gslib_kernel_helpers.hpp

@@ -0,0 +1,190 @@
+#ifndef MFEM_GSLIB_KERNEL_HELPERS_HPP
+#define MFEM_GSLIB_KERNEL_HELPERS_HPP
+
+#include "../../config/config.hpp"
+
+#include <cmath>
+
+namespace mfem
+{
+
+namespace gslib
+{
+
+struct dbl_range_t
+{
+   double min, max;
+};
+
+template <int SDIM>
+struct obbox_t
+{
+   double c0[SDIM], A[SDIM * SDIM];
+   dbl_range_t x[SDIM];
+};
+
+template <int SDIM>
+struct findptsLocalHashData_t
+{
+   int hash_n;
+   dbl_range_t bnd[SDIM];
+   double fac[SDIM];
+   unsigned int *offset;
+};
+
+// Eval the ith Lagrange interpolant at x.
+MFEM_HOST_DEVICE inline void lagrange_eval(double *p0, double x,
+                                           int i, int p_Nq,
+                                           double *z, double *lagrangeCoeff)
+{
+   double p_i = (1 << (p_Nq - 1));
+   for (int j = 0; j < p_Nq; ++j)
+   {
+      const double d_j = x - z[j];
+      p_i *= j == i ? 1 : d_j;
+   }
+   p0[i] = lagrangeCoeff[i] * p_i;
+}
+
+// Eval the ith Lagrange interpolant and its first derivative at x.
+MFEM_HOST_DEVICE inline void lag_eval_first_der(double *p0, double x,
+                                                int i, const double *z,
+                                                const double *lCoeff,
+                                                int pN)
+{
+   double u0 = 1, u1 = 0;
+   for (int j = 0; j < pN; ++j)
+   {
+      if (i != j)
+      {
+         const double d_j = 2 * (x - z[j]);
+         u1 = d_j * u1 + u0;
+         u0 = d_j * u0;
+      }
+   }
+   p0[i] = lCoeff[i] * u0;
+   p0[pN + i] = 2.0 * lCoeff[i] * u1;
+}
+
+// Eval the ith Lagrange interpolant and its first and second derivative at x.
+MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
+                                                 int i, const double *z,
+                                                 const double *lCoeff,
+                                                 int pN)
+{
+   double u0 = 1, u1 = 0, u2 = 0;
+   for (int j = 0; j < pN; ++j)
+   {
+      if (i != j)
+      {
+         const double d_j = 2 * (x - z[j]);
+         u2 = d_j * u2 + u1;
+         u1 = d_j * u1 + u0;
+         u0 = d_j * u0;
+      }
+   }
+   p0[i] = lCoeff[i] * u0;
+   p0[pN + i] = 2.0 * lCoeff[i] * u1;
+   p0[2 * pN + i] = 8.0 * lCoeff[i] * u2;
+}
+
+// Solve Ax=y where A is a symmetric 2x2 matrix packed as {a00, a01, a11}.
+MFEM_HOST_DEVICE inline void lin_solve_sym_2(double x[2],
+                                             const double A[3],
+                                             const double y[2])
+{
+   const double idet = 1 / (A[0] * A[2] - A[1] * A[1]);
+   x[0] = idet * (A[2] * y[0] - A[1] * y[1]);
+   x[1] = idet * (A[0] * y[1] - A[1] * y[0]);
+}
+
+// Positive when the point is inside the axis-aligned bounding box.
+template <int SDIM>
+MFEM_HOST_DEVICE inline double AABB_test(const obbox_t<SDIM> *const b,
+                                         const double (&x)[SDIM])
+{
+   double test = 1.0;
+   for (int d = 0; d < SDIM; ++d)
+   {
+      const double b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
+      test = test < 0.0 ? test : b_d;
+   }
+   return test;
+}
+
+// Positive when the point is inside the oriented bounding box.
+template <int SDIM>
+MFEM_HOST_DEVICE inline double bbox_test(const obbox_t<SDIM> *const b,
+                                         const double (&x)[SDIM])
+{
+   const double bxyz = AABB_test(b, x);
+   if (bxyz < 0.0)
+   {
+      return bxyz;
+   }
+
+   double dxyz[SDIM];
+   for (int d = 0; d < SDIM; ++d)
+   {
+      dxyz[d] = x[d] - b->c0[d];
+   }
+
+   double test = 1.0;
+   for (int d = 0; d < SDIM; ++d)
+   {
+      double rst = 0.0;
+      for (int e = 0; e < SDIM; ++e)
+      {
+         rst += b->A[d * SDIM + e] * dxyz[e];
+      }
+      const double brst = (rst + 1.0) * (1.0 - rst);
+      test = test < 0.0 ? test : brst;
+   }
+   return test;
+}
+
+// Hash index in the hash table for the point x.
+template <int SDIM>
+MFEM_HOST_DEVICE inline int hash_index(
+   const findptsLocalHashData_t<SDIM> *const p,
+   const double (&x)[SDIM])
+{
+   const int n = p->hash_n;
+   int sum = 0;
+   for (int d = SDIM - 1; d >= 0; --d)
+   {
+      sum *= n;
+      const int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
+      sum += i < 0 ? 0 : (n - 1 < i ? n - 1 : i);
+   }
+   return sum;
+}
+
+// Squared Euclidean norm.
+template <int SDIM>
+MFEM_HOST_DEVICE inline double l2norm2(const double (&x)[SDIM])
+{
+   double sum = 0.0;
+   for (int d = 0; d < SDIM; ++d)
+   {
+      sum += x[d] * x[d];
+   }
+   return sum;
+}
+
+template <int SDIM>
+MFEM_HOST_DEVICE inline double l2norm2(const double *x)
+{
+   double sum = 0.0;
+   for (int d = 0; d < SDIM; ++d)
+   {
+      sum += x[d] * x[d];
+   }
+   return sum;
+}
+
+} // namespace gslib
+
+} // namespace mfem
+
+#endif

fem/gslib/interpolate_local_1.cpp

@@ -0,0 +1,152 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../gslib.hpp"
+#include "../../general/forall.hpp"
+#include "gslib_kernel_helpers.hpp"
+
+#ifdef MFEM_USE_GSLIB
+
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+#include "gslib.h"
+#ifndef GSLIB_RELEASE_VERSION //gslib v1.0.7
+#define GSLIB_RELEASE_VERSION 10007
+#endif
+#ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
+#pragma GCC diagnostic pop
+#endif
+namespace mfem
+{
+#if GSLIB_RELEASE_VERSION >= 10009
+#define CODE_INTERNAL 0
+#define CODE_BORDER 1
+#define CODE_NOT_FOUND 2
+
+using gslib::lagrange_eval;
+
+template<int T_D1D = 0>
+static void InterpolateLocal1DKernel(const double *const gf_in,
+                                     int *const el,
+                                     double *const r,
+                                     double *const int_out,
+                                     const int npt,
+                                     const int nfields,
+                                     double *gll1D,
+                                     double *lagcoeff,
+                                     const int pN = 0)
+{
+   const int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+   const int D1D = T_D1D ? T_D1D : pN;
+   const int p_Nq = D1D;
+   MFEM_VERIFY(MD1 <= DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(pN<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(D1D != 0, "Polynomial order not specified.");
+   // for each point of the npt points, create a thread block of size dof1Dsol
+   mfem::forall_2D(npt, D1D, 1, [=] MFEM_HOST_DEVICE (int i)
+   {
+      MFEM_SHARED double wtr[MD1];
+      MFEM_SHARED double sums[MD1];
+
+      // Evaluate basis functions at the reference space coordinates
+      MFEM_FOREACH_THREAD(j,x,D1D)
+      {
+         lagrange_eval(wtr, r[i], j, p_Nq, gll1D, lagcoeff);
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int fld=0; fld<nfields; ++fld)
+      {
+         // If using GetNodalValues, ordering is NDOFS x NEL x VDIM and the
+         // offset would be `el[i] * p_Nq + fld * gf_offset`.
+         // R->Mult produces element vectors in NDOFS x VDIM x NEL layout.
+         const int elemOffset = el[i]*nfields*p_Nq + fld*p_Nq;
+         MFEM_FOREACH_THREAD(j,x,D1D)
+         {
+            sums[j] = wtr[j] * gf_in[elemOffset + j];
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(j,x,1)
+         {
+            double sumv = 0.0;
+            // sum the contributions of each lagrange polynomial
+            for (int jj=0; jj<D1D; ++jj)
+            {
+               sumv += sums[jj];
+            }
+            int_out[fld*npt + i] = sumv;
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+void FindPointsGSLIB::InterpolateLocal1( const Vector &field_in,
+                                         Array<int> &gsl_elem_dev_l,
+                                         Vector &gsl_ref_l,
+                                         Vector &field_out,
+                                         int npt,
+                                         int ncomp,
+                                         int dof1Dsol )
+{
+   MFEM_VERIFY(dim == 1, "Kernel for edges only.");
+   if (npt == 0) { return; }
+   bool use_dev = field_in.UseDevice();
+   auto pfin = field_in.Read(use_dev);
+   auto pgsl = gsl_elem_dev_l.ReadWrite(use_dev);
+   auto pgslr = gsl_ref_l.ReadWrite(use_dev);
+   auto pfout = field_out.Write(use_dev);
+   auto pgll = DEV.gll1d_sol.ReadWrite(use_dev);
+   auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
+   switch (dof1Dsol)
+   {
+      case 2:
+         InterpolateLocal1DKernel<2>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 3:
+         InterpolateLocal1DKernel<3>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 4:
+         InterpolateLocal1DKernel<4>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 5:
+         InterpolateLocal1DKernel<5>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      default:
+         InterpolateLocal1DKernel(pfin, pgsl, pgslr, pfout,
+                                  npt, ncomp, pgll, plcf, dof1Dsol);
+         break;
+   }
+}
+#undef CODE_INTERNAL
+#undef CODE_BORDER
+#undef CODE_NOT_FOUND
+#else
+void FindPointsGSLIB::InterpolateLocal1(const Vector &field_in,
+                                        Array<int> &gsl_elem_dev_l,
+                                        Vector &gsl_ref_l,
+                                        Vector &field_out,
+                                        int npt, int ncomp,
+                                        int dof1Dsol) {};
+#endif
+
+} // namespace mfem
+
+#endif //ifdef MFEM_USE_GSLIB

fem/gslib/interpolate_local_2.cpp

@@ -11,6 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
+#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -32,18 +33,7 @@ namespace mfem
 #define CODE_BORDER 1
 #define CODE_NOT_FOUND 2
 
-static MFEM_HOST_DEVICE void lagrange_eval(double *p0, double x,
-                                           int i, int p_Nq,
-                                           double *z, double *lagrangeCoeff)
-{
-   double p_i = (1 << (p_Nq - 1));
-   for (int j = 0; j < p_Nq; ++j)
-   {
-      double d_j = x - z[j];
-      p_i *= j == i ? 1 : d_j;
-   }
-   p0[i] = lagrangeCoeff[i] * p_i;
-}
+using gslib::lagrange_eval;
 
 template<int T_D1D = 0>
 static void InterpolateLocal2DKernel(const double *const gf_in,
@@ -52,8 +42,6 @@ static void InterpolateLocal2DKernel(const double *const gf_in,
                                      double *const int_out,
                                      const int npt,
                                      const int ncomp,
-                                     const int nel,
-                                     const int gf_offset,
                                      double *gll1D,
                                      double *lagcoeff,
                                      const int pN = 0)
@@ -64,6 +52,8 @@ static void InterpolateLocal2DKernel(const double *const gf_in,
    const int p_Np = D1D*D1D;
    MFEM_VERIFY(MD1 <= DofQuadLimits::MAX_D1D,
                "Increase Max allowable polynomial order.");
+   MFEM_VERIFY(pN<=DofQuadLimits::MAX_D1D,
+               "Increase Max allowable polynomial order.");
    MFEM_VERIFY(D1D != 0, "Polynomial order not specified.");
    mfem::forall_2D(npt, D1D, D1D, [=] MFEM_HOST_DEVICE (int i)
    {
@@ -82,9 +72,9 @@ static void InterpolateLocal2DKernel(const double *const gf_in,
 
       for (int fld = 0; fld < Nfields; ++fld)
       {
-         // If using GetNodalValues, ordering is NDOFSxNELxVDIM
-         // const int elemOffset = el[i] * p_Np + fld * gf_offset;
-         //if using R->Mult for L -> E-Vec use below: NDOFSxVDIMxNEL
+         // If using GetNodalValues, ordering is NDOFS x NEL x VDIM and the
+         // offset would be `el[i] * p_Np + fld * gf_offset`.
+         // R->Mult produces element vectors in NDOFS x VDIM x NEL layout.
          const int elemOffset = el[i] * p_Np * Nfields + fld * p_Np;
          MFEM_FOREACH_THREAD(j,x,D1D)
          {
@@ -120,33 +110,38 @@ void FindPointsGSLIB::InterpolateLocal2(const Vector &field_in,
                                         Vector &gsl_ref_l,
                                         Vector &field_out,
                                         int npt, int ncomp,
-                                        int nel, int dof1Dsol)
+                                        int dof1Dsol)
 {
    if (npt == 0) { return; }
-   const int gf_offset = field_in.Size()/ncomp;
-   auto pfin = field_in.Read();
-   auto pgsl = gsl_elem_dev_l.ReadWrite();
-   auto pgslr = gsl_ref_l.ReadWrite();
-   auto pfout = field_out.Write();
-   auto pgll = DEV.gll1d_sol.ReadWrite();
-   auto plcf = DEV.lagcoeff_sol.ReadWrite();
+   bool use_dev = field_in.UseDevice();
+   auto pfin = field_in.Read(use_dev);
+   auto pgsl = gsl_elem_dev_l.ReadWrite(use_dev);
+   auto pgslr = gsl_ref_l.ReadWrite(use_dev);
+   auto pfout = field_out.Write(use_dev);
+   auto pgll = DEV.gll1d_sol.ReadWrite(use_dev);
+   auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
    switch (dof1Dsol)
    {
-      case 2: return InterpolateLocal2DKernel<2>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
-                                                    pgll, plcf);
-      case 3: return InterpolateLocal2DKernel<3>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
-                                                    pgll, plcf);
-      case 4: return InterpolateLocal2DKernel<4>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
-                                                    pgll, plcf);
-      case 5: return InterpolateLocal2DKernel<5>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
-                                                    pgll, plcf);
-      default: return InterpolateLocal2DKernel(pfin, pgsl, pgslr, pfout,
-                                                  npt, ncomp, nel, gf_offset,
-                                                  pgll, plcf, dof1Dsol);
+      case 2:
+         InterpolateLocal2DKernel<2>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 3:
+         InterpolateLocal2DKernel<3>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 4:
+         InterpolateLocal2DKernel<4>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 5:
+         InterpolateLocal2DKernel<5>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      default:
+         InterpolateLocal2DKernel(pfin, pgsl, pgslr, pfout,
+                                  npt, ncomp, pgll, plcf, dof1Dsol);
+         break;
    }
 }
 
@@ -160,7 +155,7 @@ void FindPointsGSLIB::InterpolateLocal2(const Vector &field_in,
                                         Vector &gsl_ref_l,
                                         Vector &field_out,
                                         int npt, int ncomp,
-                                        int nel, int dof1Dsol) {};
+                                        int dof1Dsol) {};
 #endif
 } // namespace mfem
 

fem/gslib/interpolate_local_3.cpp

@@ -11,6 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
+#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -32,18 +33,7 @@ namespace mfem
 #define CODE_BORDER 1
 #define CODE_NOT_FOUND 2
 
-static MFEM_HOST_DEVICE void lagrange_eval(double *p0, double x,
-                                           int i, int p_Nq,
-                                           double *z, double *lagrangeCoeff)
-{
-   double p_i = (1 << (p_Nq - 1));
-   for (int j = 0; j < p_Nq; ++j)
-   {
-      double d_j = x - z[j];
-      p_i *= j == i ? 1 : d_j;
-   }
-   p0[i] = lagrangeCoeff[i] * p_i;
-}
+using gslib::lagrange_eval;
 
 template<int T_D1D = 0>
 static void InterpolateLocal3DKernel(const double *const gf_in,
@@ -52,8 +42,6 @@ static void InterpolateLocal3DKernel(const double *const gf_in,
                                      double *const int_out,
                                      const int npt,
                                      const int ncomp,
-                                     const int nel,
-                                     const int gf_offset,
                                      double *gll1D,
                                      double *lagcoeff,
                                      const int pN = 0)
@@ -84,9 +72,9 @@ static void InterpolateLocal3DKernel(const double *const gf_in,
 
       for (int fld = 0; fld < Nfields; ++fld)
       {
-         // If using GetNodalValues, ordering is NDOFSxNELxVDIM
-         // const int elemOffset = el[i] * p_Np + fld * gf_offset;
-         //if using R->Mult for L -> E-Vec use below.
+         // If using GetNodalValues, ordering is NDOFS x NEL x VDIM and the
+         // offset would be `el[i] * p_Np + fld * gf_offset`.
+         // R->Mult produces element vectors in NDOFS x VDIM x NEL layout.
          const int elemOffset = el[i] * p_Np * Nfields + fld * p_Np;
          MFEM_FOREACH_THREAD(j,x,D1D)
          {
@@ -125,37 +113,43 @@ void FindPointsGSLIB::InterpolateLocal3(const Vector &field_in,
                                         Vector &gsl_ref_l,
                                         Vector &field_out,
                                         int npt, int ncomp,
-                                        int nel, int dof1Dsol)
+                                        int dof1Dsol)
 {
    if (npt == 0) { return; }
-   const int gf_offset = field_in.Size()/ncomp;
-   auto pfin = field_in.Read();
-   auto pgsle = gsl_elem_dev_l.ReadWrite();
-   auto pgslr = gsl_ref_l.ReadWrite();
-   auto pfout = field_out.Write();
-   auto pgll = DEV.gll1d_sol.ReadWrite();
-   auto plcf = DEV.lagcoeff_sol.ReadWrite();
+   bool use_dev = field_in.UseDevice();
+   auto pfin = field_in.Read(use_dev);
+   auto pgsle = gsl_elem_dev_l.ReadWrite(use_dev);
+   auto pgslr = gsl_ref_l.ReadWrite(use_dev);
+   auto pfout = field_out.Write(use_dev);
+   auto pgll = DEV.gll1d_sol.ReadWrite(use_dev);
+   auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
    switch (dof1Dsol)
    {
-      case 2: return InterpolateLocal3DKernel<2>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
-                                                    pgll, plcf);
-      case 3: return InterpolateLocal3DKernel<3>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
-                                                    pgll, plcf);
-      case 4: return InterpolateLocal3DKernel<4>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
-                                                    pgll, plcf);
-      case 5: return InterpolateLocal3DKernel<5>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp, nel, gf_offset,
-                                                    pgll, plcf);
-      default: return InterpolateLocal3DKernel(pfin, pgsle, pgslr, pfout,
-                                                  npt, ncomp, nel, gf_offset,
-                                                  pgll, plcf, dof1Dsol);
+      case 2:
+         InterpolateLocal3DKernel<2>(pfin, pgsle, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 3:
+         InterpolateLocal3DKernel<3>(pfin, pgsle, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 4:
+         InterpolateLocal3DKernel<4>(pfin, pgsle, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 5:
+         InterpolateLocal3DKernel<5>(pfin, pgsle, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      default:
+         InterpolateLocal3DKernel(pfin, pgsle, pgslr, pfout,
+                                  npt, ncomp, pgll, plcf, dof1Dsol);
+         break;
    }
 }
 
 
+#undef MAXC
 #undef CODE_INTERNAL
 #undef CODE_BORDER
 #undef CODE_NOT_FOUND
@@ -165,7 +159,7 @@ void FindPointsGSLIB::InterpolateLocal3(const Vector &field_in,
                                         Vector &gsl_ref_l,
                                         Vector &field_out,
                                         int npt, int ncomp,
-                                        int nel, int dof1Dsol) {};
+                                        int dof1Dsol) {};
 #endif
 } // namespace mfem
 

fem/integ/bilininteg_diffusion_kernels.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "bilininteg_diffusion_kernels.hpp"
+#include "bilininteg_diffusion_pa_simplices.hpp" // IWYU pragma: keep
 
 namespace mfem
 {
@@ -19,6 +20,13 @@ namespace mfem
 DiffusionIntegrator::Kernels::Kernels()
 {
    // 2D
+   // Q = P, only for simplex
+   DiffusionIntegrator::AddSimplexSpecialization<2,2,1>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,3,2>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,4,3>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,5,4>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,6,5>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,7,6>();
    // Q = P+1
    DiffusionIntegrator::AddSpecialization<2,1,1>();
    DiffusionIntegrator::AddSpecialization<2,2,2>();
@@ -40,7 +48,18 @@ DiffusionIntegrator::Kernels::Kernels()
    DiffusionIntegrator::AddSpecialization<2,8,9>();
    DiffusionIntegrator::AddSpecialization<2,9,10>();
    // others
+   DiffusionIntegrator::AddSimplexSpecialization<2,2,5>();
+   DiffusionIntegrator::AddSimplexSpecialization<2,3,6>();
+
    // 3D
+   // Q = P, only for simplex
+   DiffusionIntegrator::AddSimplexSpecialization<3,2,1>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,3,2>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,4,3>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,5,4>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,6,5>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,7,6>();
+   DiffusionIntegrator::AddSimplexSpecialization<3,8,7>();
    // Q = P+1
    DiffusionIntegrator::AddSpecialization<3,1,1>();
    DiffusionIntegrator::AddSpecialization<3,2,2>();

fem/integ/bilininteg_diffusion_kernels.hpp

@@ -12,7 +12,6 @@
 #ifndef MFEM_BILININTEG_DIFFUSION_KERNELS_HPP
 #define MFEM_BILININTEG_DIFFUSION_KERNELS_HPP
 
-#include "../kernel_dispatch.hpp"
 #include "../../config/config.hpp"
 #include "../../general/array.hpp"
 #include "../../general/forall.hpp"
@@ -20,6 +19,8 @@
 #include "../../linalg/vector.hpp"
 #include "../bilininteg.hpp"
 
+#include "bilininteg_diffusion_pa_simplices.hpp"
+
 namespace mfem
 {
 
@@ -637,8 +638,8 @@ inline void SmemPADiffusionApply2D(const int NE,
                                    const bool symmetric,
                                    const Array<real_t> &b_,
                                    const Array<real_t> &g_,
-                                   const Array<real_t> &bt_,
-                                   const Array<real_t> &gt_,
+                                   const Array<real_t> &,
+                                   const Array<real_t> &,
                                    const Vector &d_,
                                    const Vector &x_,
                                    Vector &y_,
@@ -1218,43 +1219,47 @@ inline void SmemPADiffusionApply3D(const int NE,
 namespace
 {
 using ApplyKernelType = DiffusionIntegrator::ApplyKernelType;
+using ApplySimplexKernelType = DiffusionIntegrator::ApplySimplexKernelType;
 using DiagonalKernelType = DiffusionIntegrator::DiagonalKernelType;
 }
 
-template<int DIM, int T_D1D, int T_Q1D>
+template<int DIM, int D1D, int Q1D>
 ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Kernel()
 {
-   if constexpr (DIM == 2) { return internal::SmemPADiffusionApply2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPADiffusionApply3D<T_D1D, T_Q1D>; }
-   MFEM_ABORT("");
+   if constexpr (DIM == 2) { return internal::SmemPADiffusionApply2D<D1D, Q1D>; }
+   else if constexpr (DIM == 3) { return internal::SmemPADiffusionApply3D<D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
+   return nullptr;
 }
 
 inline
-ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
+ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int dim, int, int)
 {
-   if (DIM == 2) { return internal::PADiffusionApply2D; }
-   else if (DIM == 3) { return internal::PADiffusionApply3D; }
+   if (dim == 2) { return internal::PADiffusionApply2D; }
+   else if (dim == 3) { return internal::PADiffusionApply3D; }
    else { MFEM_ABORT(""); }
 }
 
 template<int DIM, int D1D, int Q1D>
 DiagonalKernelType DiffusionIntegrator::DiagonalPAKernels::Kernel()
 {
-   if constexpr (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D,Q1D>; }
+   if constexpr (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D, Q1D>; }
    else if constexpr (DIM == 3) { return internal::SmemPADiffusionDiagonal3D<D1D, Q1D>; }
-   MFEM_ABORT("");
+   else { MFEM_ABORT(""); }
+   return nullptr;
 }
 
 inline DiagonalKernelType
-DiffusionIntegrator::DiagonalPAKernels::Fallback(int DIM, int, int)
+DiffusionIntegrator::DiagonalPAKernels::Fallback(int dim, int, int)
 {
-   if (DIM == 2) { return internal::PADiffusionDiagonal2D; }
-   else if (DIM == 3) { return internal::PADiffusionDiagonal3D; }
+   if (dim == 2) { return internal::PADiffusionDiagonal2D; }
+   else if (dim == 3) { return internal::PADiffusionDiagonal3D; }
    else { MFEM_ABORT(""); }
+   return nullptr;
 }
+
 /// \endcond DO_NOT_DOCUMENT
 
 } // namespace mfem
 
-
 #endif

fem/integ/bilininteg_diffusion_pa.cpp

@@ -15,6 +15,7 @@
 #include "../../mesh/nurbs.hpp"
 #include "../ceed/integrators/diffusion/diffusion.hpp"
 #include "bilininteg_diffusion_kernels.hpp"
+#include "bilininteg_diffusion_pa_simplices.hpp"
 
 namespace mfem
 {
@@ -68,6 +69,24 @@ void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
       }
 #endif // MFEM_USE_OCCA
 
+      if (fespace->UsesRaggedTensorBasis())
+      {
+         const auto *rmaps = static_cast<const RaggedDofToQuad*>(maps);
+         return ApplySimplexPAKernels::Run(dim, dofs1D, quad1D, ne, symmetric,
+                                           rmaps->lex_map,
+                                           rmaps->forward_map2d_diff,
+                                           rmaps->inverse_map2d_diff,
+                                           rmaps->forward_map3d_diff,
+                                           rmaps->inverse_map3d_diff,
+                                           rmaps->Ga1,
+                                           rmaps->Ga2,
+                                           rmaps->Ga3,
+                                           rmaps->Ga1t,
+                                           rmaps->Ga2t,
+                                           rmaps->Ga3t,
+                                           Dv, x, y, dofs1D, quad1D);
+      }
+
       ApplyPAKernels::Run(dim, dofs1D, quad1D, ne, symmetric, B, G, Bt,
                           Gt, Dv, x, y, dofs1D, quad1D);
    }
@@ -94,7 +113,8 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    fespace = &fes;
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+   const bool stroud = fes.UsesRaggedTensorBasis();
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, stroud);
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
@@ -119,13 +139,22 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dim = mesh->Dimension();
    ne = fes.GetNE();
    geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
+   if (stroud)
+   {
+      maps = &el.GetDofToQuad(*ir, DofToQuad::RAGGED_TENSOR);
+   }
+   else
+   {
+      maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   }
    const int sdim = mesh->SpaceDimension();
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
 
    QuadratureSpace qs(*mesh, *ir);
    CoefficientVector coeff(qs, CoefficientStorage::COMPRESSED);
+   // QuadratureSpace expects ir defined in reference simplex for Bernstein
+   // elements with partial assembly
 
    if (MQ) { coeff.ProjectTranspose(*MQ); }
    else if (VQ) { coeff.Project(*VQ); }

fem/integ/bilininteg_mass_kernels.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "bilininteg_mass_kernels.hpp"
+#include "bilininteg_mass_pa_simplices.hpp" // IWYU pragma: keep
 
 namespace mfem
 {
@@ -39,8 +40,10 @@ MassIntegrator::Kernels::Kernels()
    MassIntegrator::AddSpecialization<2,9,10>();
    // others
    MassIntegrator::AddSpecialization<2,2,4>();
+   MassIntegrator::AddSpecialization<2,2,5>();
    MassIntegrator::AddSpecialization<2,3,6>();
    MassIntegrator::AddSpecialization<2,4,6>();
+
    // 3D
    // Q=P+1
    MassIntegrator::AddSpecialization<3,1,1>();

fem/integ/bilininteg_mass_kernels.hpp

@@ -19,6 +19,8 @@
 #include "../../linalg/vector.hpp"
 #include "../bilininteg.hpp"
 
+#include "bilininteg_mass_pa_simplices.hpp"
+
 namespace mfem
 {
 
@@ -1408,51 +1410,57 @@ using ApplyKernelType = MassIntegrator::ApplyKernelType;
 using DiagonalKernelType = MassIntegrator::DiagonalKernelType;
 }
 
-template<int DIM, int T_D1D, int T_Q1D>
+template<int DIM, int D1D, int Q1D>
 ApplyKernelType MassIntegrator::ApplyPAKernels::Kernel()
 {
    if constexpr (DIM == 1) { return internal::PAMassApply1D; }
-   else if constexpr (DIM == 2) { return internal::SmemPAMassApply2D<T_D1D,T_Q1D>; }
+   else if constexpr (DIM == 2) { return internal::SmemPAMassApply2D<D1D, Q1D>; }
    else if constexpr (DIM == 3)
    {
-      constexpr int MDQ = T_D1D >= T_Q1D ? T_D1D : T_Q1D;
+      constexpr int MDQ = D1D >= Q1D ? D1D : Q1D;
       // max 64 threads in z limit in cuda and hip
       if constexpr (MDQ > 0)
       {
-         return internal::SmemPAMassApply3D<T_D1D, T_Q1D,
+         return internal::SmemPAMassApply3D<D1D, Q1D,
                 internal::mass::NBZ3D(MDQ)>;
       }
    }
-   MFEM_ABORT("");
+   else { MFEM_ABORT(""); }
+   return nullptr;
 }
 
 inline ApplyKernelType MassIntegrator::ApplyPAKernels::Fallback(
-   int DIM, int, int)
+   int dim, int, int)
 {
-   if (DIM == 1) { return internal::PAMassApply1D; }
-   else if (DIM == 2) { return internal::PAMassApply2D; }
-   else if (DIM == 3) { return internal::PAMassApply3D; }
+   if (dim == 1) { return internal::PAMassApply1D; }
+   else if (dim == 2) { return internal::PAMassApply2D; }
+   else if (dim == 3) { return internal::PAMassApply3D; }
    else { MFEM_ABORT(""); }
+   return nullptr;
 }
 
-template<int DIM, int T_D1D, int T_Q1D>
+template<int DIM, int D1D, int Q1D>
 DiagonalKernelType MassIntegrator::DiagonalPAKernels::Kernel()
 {
    if constexpr (DIM == 1) { return internal::PAMassAssembleDiagonal1D; }
-   else if constexpr (DIM == 2) { return internal::SmemPAMassAssembleDiagonal2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPAMassAssembleDiagonal3D<T_D1D, T_Q1D>; }
-   MFEM_ABORT("");
+   else if constexpr (DIM == 2) { return internal::SmemPAMassAssembleDiagonal2D<D1D, Q1D>; }
+   else if constexpr (DIM == 3) { return internal::SmemPAMassAssembleDiagonal3D<D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
+   return nullptr;
 }
 
 inline DiagonalKernelType MassIntegrator::DiagonalPAKernels::Fallback(
-   int DIM, int, int)
+   int dim, int, int)
 {
-   if (DIM == 1) { return internal::PAMassAssembleDiagonal1D; }
-   else if (DIM == 2) { return internal::PAMassAssembleDiagonal2D; }
-   else if (DIM == 3) { return internal::PAMassAssembleDiagonal3D; }
+   if (dim == 1) { return internal::PAMassAssembleDiagonal1D; }
+   else if (dim == 2) { return internal::PAMassAssembleDiagonal2D; }
+   else if (dim == 3) { return internal::PAMassAssembleDiagonal3D; }
    else { MFEM_ABORT(""); }
+   return nullptr;
 }
+
 /// \endcond DO_NOT_DOCUMENT
+
 } // namespace mfem
 
 #endif

fem/integ/bilininteg_mass_pa.cpp

@@ -15,6 +15,7 @@
 #include "../qfunction.hpp"
 #include "../ceed/integrators/mass/mass.hpp"
 #include "bilininteg_mass_kernels.hpp"
+#include "bilininteg_mass_pa_simplices.hpp"
 
 namespace mfem
 {
@@ -29,9 +30,11 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    // Assuming the same element type
    fespace = &fes;
    Mesh *mesh = fes.GetMesh();
+   dim = mesh->Dimension();
    const FiniteElement &el = *fes.GetTypicalFE();
    ElementTransformation *T0 = mesh->GetTypicalElementTransformation();
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T0);
+   const bool stroud = fes.UsesRaggedTensorBasis();
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T0, stroud);
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
@@ -48,17 +51,25 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
       return;
    }
    int map_type = el.GetMapType();
-   dim = mesh->Dimension();
    ne = fes.GetMesh()->GetNE();
    nq = ir->GetNPoints();
    geom = mesh->GetGeometricFactors(*ir, GeometricFactors::DETERMINANTS, mt);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   if (stroud)
+   {
+      maps = &el.GetDofToQuad(*ir, DofToQuad::RAGGED_TENSOR);
+   }
+   else
+   {
+      maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   }
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(ne*nq, mt);
 
    QuadratureSpace qs(*mesh, *ir);
    CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+   // QuadratureSpace expects ir defined in reference simplex for Bernstein
+   // elements with partial assembly
    {
       const int NE = ne;
       const int NQ = nq;
@@ -147,9 +158,10 @@ void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
    {
       const int D1D = dofs1D;
       const int Q1D = quad1D;
+      const Vector &D = pa_data;
       const Array<real_t> &B = maps->B;
       const Array<real_t> &Bt = maps->Bt;
-      const Vector &D = pa_data;
+
 #ifdef MFEM_USE_OCCA
       if (DeviceCanUseOcca())
       {
@@ -164,7 +176,31 @@ void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
          MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
       }
 #endif // MFEM_USE_OCCA
-      ApplyPAKernels::Run(dim, D1D, Q1D, ne, B, Bt, D, x, y, D1D, Q1D);
+
+      if (fespace->UsesRaggedTensorBasis())
+      {
+         const auto *rmaps = static_cast<const RaggedDofToQuad*>(maps);
+
+         const Array<real_t> &Ba1 = rmaps->Ba1;
+         const Array<real_t> &Ba2 = rmaps->Ba2;
+         const Array<real_t> &Ba3 = rmaps->Ba3;
+         const Array<real_t> &Ba1t = rmaps->Ba1t;
+         const Array<real_t> &Ba2t = rmaps->Ba2t;
+         const Array<real_t> &Ba3t = rmaps->Ba3t;
+         const Array<int> &lex_map = rmaps->lex_map;
+         const Array<int> &forward_map2d = rmaps->forward_map2d_mass;
+         const Array<int> &inverse_map2d = rmaps->inverse_map2d_mass;
+         const Array<int> &forward_map3d = rmaps->forward_map3d_mass;
+         const Array<int> &inverse_map3d = rmaps->inverse_map3d_mass;
+         ApplySimplexPAKernels::Run(dim, D1D, Q1D, ne, lex_map, forward_map2d,
+                                    inverse_map2d,
+                                    forward_map3d, inverse_map3d, Ba1, Ba2, Ba3, Ba1t, Ba2t, Ba3t,
+                                    D, x, y, D1D, Q1D);
+      }
+      else
+      {
+         ApplyPAKernels::Run(dim, D1D, Q1D, ne, B, Bt, D, x, y, D1D, Q1D);
+      }
    }
 }
 
@@ -177,6 +213,8 @@ void MassIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
    }
    else
    {
+      MFEM_VERIFY(!fespace->UsesRaggedTensorBasis(),
+                  "AbsMultPA not implemented for ragged tensor basis");
       Vector abs_pa_data(pa_data);
       abs_pa_data.Abs();
       Array<real_t> absB(maps->B);

fem/integ/lininteg_domain_kernels.hpp

@@ -307,6 +307,506 @@ DomainLFIntegrator::AssembleKernels::Kernel()
    MFEM_ABORT("");
 }
 
+template <int T_D1D = 0, int T_Q1D = 0>
+static void HdivDLFAssemble2D(const int ne, const Array<int> &markers,
+                              const Vector &jac, const Array<real_t> &weights,
+                              const Array<real_t> &testBO,
+                              const Array<real_t> &testBC, const Vector &coeff,
+                              Vector &y, const int d, const int q)
+{
+   MFEM_VERIFY(T_D1D || d <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
+               "Problem size too large.");
+   MFEM_VERIFY(T_Q1D || q <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
+               "Problem size too large.");
+   MFEM_VERIFY(y.Size() == 2 * (d - 1) * d * ne, "");
+
+   constexpr int vdim = 2;
+   const auto F = coeff.Read();
+   const auto M = markers.Read();
+   const auto BO = Reshape(testBO.Read(), q, d-1);
+   const auto BC = Reshape(testBC.Read(), q, d);
+   const auto J = Reshape(jac.Read(), q, q, vdim, vdim, ne);
+   const auto W = Reshape(weights.Read(), q, q);
+   const bool cst = coeff.Size() == vdim;
+   const auto C = cst ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,ne);
+   auto Y = y.ReadWrite();
+
+   mfem::forall_3D(ne, q, q, vdim, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int vdim = 2;
+      if (M[e] == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
+
+      MFEM_SHARED real_t sBot[Q*D];
+      MFEM_SHARED real_t sBct[Q*D];
+      MFEM_SHARED real_t sQQ[vdim*Q*Q];
+      MFEM_SHARED real_t sQD[vdim*Q*D];
+
+      // Bo and Bc into shared memory
+      const DeviceMatrix Bot(sBot, d-1, q);
+      kernels::internal::LoadB<D,Q>(d-1, q, BO, sBot);
+      const DeviceMatrix Bct(sBct, d, q);
+      kernels::internal::LoadB<D,Q>(d, q, BC, sBct);
+
+      const DeviceCube QQ(sQQ, q, q, vdim);
+      const DeviceCube QD(sQD, q, d, vdim);
+
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const real_t cst_val_0 = C(0,0,0,0);
+         const real_t cst_val_1 = C(1,0,0,0);
+         MFEM_FOREACH_THREAD(y,y,q)
+         {
+            MFEM_FOREACH_THREAD(x,x,q)
+            {
+               const real_t J0 = J(x,y,0,vd,e);
+               const real_t J1 = J(x,y,1,vd,e);
+               const real_t C0 = cst ? cst_val_0 : C(0,x,y,e);
+               const real_t C1 = cst ? cst_val_1 : C(1,x,y,e);
+               QQ(x,y,vd) = W(x,y)*(J0*C0 + J1*C1);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         DeviceMatrix Btx = (vd == 0) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(qy,y,q)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               real_t qd = 0.0;
+               for (int qx = 0; qx < q; ++qx)
+               {
+                  qd += QQ(qx,qy,vd) * Btx(dx,qx);
+               }
+               QD(dx,qy,vd) = qd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         const int ny = (vd == 1) ? d : d-1;
+         DeviceMatrix Bty = (vd == 1) ? Bct : Bot;
+         DeviceTensor<4> Yxy(Y, nx, ny, vdim, ne);
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               real_t dd = 0.0;
+               for (int qy = 0; qy < q; ++qy)
+               {
+                  dd += QD(dx,qy,vd) * Bty(dy,qy);
+               }
+               Yxy(dx,dy,vd,e) += dd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+template <int T_D1D = 0, int T_Q1D = 0>
+static void HdivDLFAssemble3D(const int ne, const Array<int> &markers,
+                              const Vector &jac, const Array<real_t> &weights,
+                              const Array<real_t> &testBO,
+                              const Array<real_t> &testBC, const Vector &coeff,
+                              Vector &y, const int d, const int q)
+{
+   MFEM_VERIFY(T_D1D || d <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
+               "Problem size too large.");
+   MFEM_VERIFY(T_Q1D || q <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
+               "Problem size too large.");
+   MFEM_VERIFY(y.Size() == 3 * (d - 1) * (d - 1) * d * ne, "y wrong length");
+
+   constexpr int vdim = 3;
+   const auto F = coeff.Read();
+   const auto M = markers.Read();
+   const auto BO = Reshape(testBO.Read(), q, d-1);
+   const auto BC = Reshape(testBC.Read(), q, d);
+   const auto J = Reshape(jac.Read(), q, q, q, vdim, vdim, ne);
+   const auto W = Reshape(weights.Read(), q, q, q);
+   const bool cst = coeff.Size() == vdim;
+   const auto C = cst ? Reshape(F,vdim,1,1,1,1) : Reshape(F,vdim,q,q,q,ne);
+   auto Y = y.ReadWrite();
+
+   mfem::forall_3D(ne, q, q, vdim, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int vdim = 3;
+      if (M[e] == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
+
+      MFEM_SHARED real_t sBot[Q*D];
+      MFEM_SHARED real_t sBct[Q*D];
+
+      // Bo and Bc into shared memory
+      const DeviceMatrix Bot(sBot, d-1, q);
+      kernels::internal::LoadB<D,Q>(d-1, q, BO, sBot);
+      const DeviceMatrix Bct(sBct, d, q);
+      kernels::internal::LoadB<D,Q>(d, q, BC, sBct);
+
+      MFEM_SHARED real_t sm0[vdim*Q*Q*Q];
+      MFEM_SHARED real_t sm1[vdim*Q*Q*Q];
+      DeviceTensor<4> QQQ(sm1, q, q, q, vdim);
+      DeviceTensor<4> DQQ(sm0, d, q, q, vdim);
+      DeviceTensor<4> DDQ(sm1, d, d, q, vdim);
+
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const real_t cst_val_0 = C(0,0,0,0,0);
+         const real_t cst_val_1 = C(1,0,0,0,0);
+         const real_t cst_val_2 = C(2,0,0,0,0);
+         MFEM_FOREACH_THREAD(y,y,q)
+         {
+            MFEM_FOREACH_THREAD(x,x,q)
+            {
+               for (int z = 0; z < q; ++z)
+               {
+                  const real_t J0 = J(x,y,z,0,vd,e);
+                  const real_t J1 = J(x,y,z,1,vd,e);
+                  const real_t J2 = J(x,y,z,2,vd,e);
+                  const real_t C0 = cst ? cst_val_0 : C(0,x,y,z,e);
+                  const real_t C1 = cst ? cst_val_1 : C(1,x,y,z,e);
+                  const real_t C2 = cst ? cst_val_2 : C(2,x,y,z,e);
+                  QQQ(x,y,z,vd) = W(x,y,z)*(J0*C0 + J1*C1 + J2*C2);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply Bt operator
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         DeviceMatrix Btx = (vd == 0) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(qy,y,q)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               real_t u[Q];
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(Q)
+               for (int qx = 0; qx < q; ++qx)
+               {
+                  MFEM_UNROLL(Q)
+                  for (int qz = 0; qz < q; ++qz)
+                  {
+                     u[qz] += QQQ(qx,qy,qz,vd) * Btx(dx,qx);
+                  }
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { DQQ(dx,qy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         const int ny = (vd == 1) ? d : d-1;
+         DeviceMatrix Bty = (vd == 1) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               real_t u[Q];
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(Q)
+               for (int qy = 0; qy < q; ++qy)
+               {
+                  MFEM_UNROLL(Q)
+                  for (int qz = 0; qz < q; ++qz)
+                  {
+                     u[qz] += DQQ(dx,qy,qz,vd) * Bty(dy,qy);
+                  }
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz) { DDQ(dx,dy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,vdim)
+      {
+         const int nx = (vd == 0) ? d : d-1;
+         const int ny = (vd == 1) ? d : d-1;
+         const int nz = (vd == 2) ? d : d-1;
+         DeviceTensor<5> Yxyz(Y, nx, ny, nz, vdim, ne);
+         DeviceMatrix Btz = (vd == 2) ? Bct : Bot;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               real_t u[D];
+               MFEM_UNROLL(D)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  MFEM_UNROLL(D)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += DDQ(dx,dy,qz,vd) * Btz(dz,qz);
+                  }
+               }
+               MFEM_UNROLL(D)
+               for (int dz = 0; dz < nz; ++dz) { Yxyz(dx,dy,dz,vd,e) += u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+/// @param ne number of elements
+/// @param markers array where entry markers[e] == 0 to skip assembly over
+/// element e element
+/// @param jac Spatial Jacobians evaluated at all quadrature points
+/// @param weights 1D quadrature weights
+/// @param testBO 1D open basis test functions
+/// @param testBC 1D closed basis test functions
+/// @param coeff coefficient values evaluated at quadrature points, possibly
+/// compressed.
+/// @param d number of 1D closed dofs
+/// @param q number of 1D quadrature points
+/// @tparam T_D1D maximum number of dofs along any direction, or 0
+/// @tparam T_Q1D maximum number of quadrature points along any direction, or 0
+template <int T_D1D = 0, int T_Q1D = 0>
+static void HcurlDLFAssemble3D(const int ne, const Array<int> &markers,
+                               const Vector &jac, const Array<real_t> &weights,
+                               const Array<real_t> &testBO,
+                               const Array<real_t> &testBC, const Vector &coeff,
+                               Vector &y, const int d, const int q)
+{
+   MFEM_VERIFY(T_D1D || d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
+               "Problem size too large.");
+   MFEM_VERIFY(T_Q1D || q <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
+               "Problem size too large.");
+   MFEM_VERIFY(y.Size() == 3 * (d - 1) * d * d * ne, "y wrong length");
+
+   constexpr int vdim = 3;
+   const auto F = coeff.Read();
+   const auto M = markers.Read();
+   const auto BO = Reshape(testBO.Read(), q, d-1);
+   const auto BC = Reshape(testBC.Read(), q, d);
+   const auto J = Reshape(jac.Read(), q, q, q, vdim, vdim, ne);
+   const auto W = Reshape(weights.Read(), q, q, q);
+   const bool cst = coeff.Size() == vdim;
+   const auto C = cst ? Reshape(F,vdim,1,1,1,1) : Reshape(F,vdim,q,q,q,ne);
+   auto Y = y.ReadWrite();
+
+   mfem::forall_3D(ne, q, q, vdim, [=] MFEM_HOST_DEVICE(int e)
+   {
+      if (M[e] == 0)
+      {
+         // ignore
+         return;
+      }
+
+      constexpr int vdim = 3;
+      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
+
+      MFEM_SHARED real_t sBot[Q * D];
+      MFEM_SHARED real_t sBct[Q * D];
+
+      // Bo and Bc into shared memory
+      const DeviceMatrix Bot(sBot, d - 1, q);
+      kernels::internal::LoadB<D, Q>(d - 1, q, BO, sBot);
+      const DeviceMatrix Bct(sBct, d, q);
+      kernels::internal::LoadB<D, Q>(d, q, BC, sBct);
+
+      MFEM_SHARED real_t sm0[vdim * Q * Q * Q];
+      MFEM_SHARED real_t sm1[vdim * Q * Q * Q];
+      DeviceTensor<4> QQQ(sm1, q, q, q, vdim);
+      DeviceTensor<4> DQQ(sm0, d, q, q, vdim);
+      DeviceTensor<4> DDQ(sm1, d, d, q, vdim);
+
+      const real_t cst_val_0 = C(0, 0, 0, 0, 0);
+      const real_t cst_val_1 = C(1, 0, 0, 0, 0);
+      const real_t cst_val_2 = C(2, 0, 0, 0, 0);
+
+      MFEM_FOREACH_THREAD(vd, z, vdim)
+      {
+         MFEM_FOREACH_THREAD(y, y, q)
+         {
+            MFEM_FOREACH_THREAD(x, x, q)
+            {
+               for (int z = 0; z < q; ++z)
+               {
+                  real_t curr[3];
+                  curr[0] = cst ? cst_val_0 : C(0, x, y, z, e);
+                  curr[1] = cst ? cst_val_1 : C(1, x, y, z, e);
+                  curr[2] = cst ? cst_val_2 : C(2, x, y, z, e);
+
+                  const real_t J11 = J(x, y, z, 0, 0, e);
+                  const real_t J21 = J(x, y, z, 1, 0, e);
+                  const real_t J31 = J(x, y, z, 2, 0, e);
+                  const real_t J12 = J(x, y, z, 0, 1, e);
+                  const real_t J22 = J(x, y, z, 1, 1, e);
+                  const real_t J32 = J(x, y, z, 2, 1, e);
+                  const real_t J13 = J(x, y, z, 0, 2, e);
+                  const real_t J23 = J(x, y, z, 1, 2, e);
+                  const real_t J33 = J(x, y, z, 2, 2, e);
+                  // adj(J)
+                  const real_t A11 = (J22 * J33) - (J23 * J32);
+                  const real_t A12 = (J32 * J13) - (J12 * J33);
+                  const real_t A13 = (J12 * J23) - (J22 * J13);
+                  const real_t A21 = (J31 * J23) - (J21 * J33);
+                  const real_t A22 = (J11 * J33) - (J13 * J31);
+                  const real_t A23 = (J21 * J13) - (J11 * J23);
+                  const real_t A31 = (J21 * J32) - (J31 * J22);
+                  const real_t A32 = (J31 * J12) - (J11 * J32);
+                  const real_t A33 = (J11 * J22) - (J12 * J21);
+                  const real_t A[9] = {A11, A12, A13, A21, A22,
+                                       A23, A31, A32, A33
+                                      };
+                  QQQ(x, y, z, vd) = W(x, y, z) * (A[vd * vdim] * curr[0] +
+                                                   A[vd * vdim + 1] * curr[1] +
+                                                   A[vd * vdim + 2] * curr[2]);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply Bt operator
+      MFEM_FOREACH_THREAD(vd, z, vdim)
+      {
+         const int nx = (vd == 0) ? d - 1 : d;
+         DeviceMatrix Btx = (vd == 0) ? Bot : Bct;
+         MFEM_FOREACH_THREAD(qy, y, q)
+         {
+            MFEM_FOREACH_THREAD(dx, x, nx)
+            {
+               real_t u[Q];
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  u[qz] = 0.0;
+               }
+               MFEM_UNROLL(Q)
+               for (int qx = 0; qx < q; ++qx)
+               {
+                  MFEM_UNROLL(Q)
+                  for (int qz = 0; qz < q; ++qz)
+                  {
+                     u[qz] += QQQ(qx, qy, qz, vd) * Btx(dx, qx);
+                  }
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  DQQ(dx, qy, qz, vd) = u[qz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd, z, vdim)
+      {
+         const int nx = (vd == 0) ? d - 1 : d;
+         const int ny = (vd == 1) ? d - 1 : d;
+         DeviceMatrix Bty = (vd == 1) ? Bot : Bct;
+         MFEM_FOREACH_THREAD(dy, y, ny)
+         {
+            MFEM_FOREACH_THREAD(dx, x, nx)
+            {
+               real_t u[Q];
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  u[qz] = 0.0;
+               }
+               MFEM_UNROLL(Q)
+               for (int qy = 0; qy < q; ++qy)
+               {
+                  MFEM_UNROLL(Q)
+                  for (int qz = 0; qz < q; ++qz)
+                  {
+                     u[qz] += DQQ(dx, qy, qz, vd) * Bty(dy, qy);
+                  }
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  DDQ(dx, dy, qz, vd) = u[qz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd, z, vdim)
+      {
+         const int nx = (vd == 0) ? d - 1 : d;
+         const int ny = (vd == 1) ? d - 1 : d;
+         const int nz = (vd == 2) ? d - 1 : d;
+         DeviceTensor<5> Yxyz(Y, nx, ny, nz, vdim, ne);
+         DeviceMatrix Btz = (vd == 2) ? Bot : Bct;
+         MFEM_FOREACH_THREAD(dy, y, ny)
+         {
+            MFEM_FOREACH_THREAD(dx, x, nx)
+            {
+               real_t u[D];
+               MFEM_UNROLL(D)
+               for (int dz = 0; dz < nz; ++dz)
+               {
+                  u[dz] = 0.0;
+               }
+               MFEM_UNROLL(Q)
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  MFEM_UNROLL(D)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += DDQ(dx, dy, qz, vd) * Btz(dz, qz);
+                  }
+               }
+               MFEM_UNROLL(D)
+               for (int dz = 0; dz < nz; ++dz)
+               {
+                  Yxyz(dx, dy, dz, vd, e) += u[dz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+template <FiniteElement::DerivType TestType, int DIM, int TEST_D1D, int Q1D>
+VectorFEDomainLFIntegrator::AssembleKernelType
+VectorFEDomainLFIntegrator::AssembleKernels::Kernel()
+{
+   if constexpr (TestType == FiniteElement::DIV)
+   {
+      if constexpr (DIM == 2)
+      {
+         return HdivDLFAssemble2D<TEST_D1D, Q1D>;
+      }
+      if constexpr (DIM == 3)
+      {
+         return HdivDLFAssemble3D<TEST_D1D, Q1D>;
+      }
+   }
+   if constexpr (TestType == FiniteElement::CURL)
+   {
+      if constexpr (DIM == 3)
+      {
+         return HcurlDLFAssemble3D<TEST_D1D, Q1D>;
+      }
+   }
+   MFEM_ABORT("");
+}
+
 /// \endcond DO_NOT_DOCUMENT
 
 } // namespace mfem

fem/integ/lininteg_domain_vectorfe.cpp

@@ -13,317 +13,76 @@
 #include "../../fem/kernels.hpp"
 #include "../fem.hpp"
 
+#include "lininteg_domain_kernels.hpp"
+
 namespace mfem
 {
 
-template<int T_D1D = 0, int T_Q1D = 0>
-static void HdivDLFAssemble2D(
-   const int ne, const int d, const int q, const int *markers, const real_t *bo,
-   const real_t *bc, const real_t *j, const real_t *weights,
-   const Vector &coeff, real_t *y)
+VectorFEDomainLFIntegrator::Kernels::Kernels()
 {
-   MFEM_VERIFY(T_D1D || d <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
-               "Problem size too large.");
-   MFEM_VERIFY(T_Q1D || q <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
-               "Problem size too large.");
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 1, 1>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 2, 2>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 3, 3>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 4, 4>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 5, 5>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 6, 6>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 7, 7>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 2, 8, 8>();
 
-   static constexpr int vdim = 2;
-   const auto F = coeff.Read();
-   const auto M = Reshape(markers, ne);
-   const auto BO = Reshape(bo, q, d-1);
-   const auto BC = Reshape(bc, q, d);
-   const auto J = Reshape(j, q, q, vdim, vdim, ne);
-   const auto W = Reshape(weights, q, q);
-   const bool cst = coeff.Size() == vdim;
-   const auto C = cst ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,ne);
-   auto Y = Reshape(y, 2*(d-1)*d, ne);
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 1, 1>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 2, 2>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 3, 3>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 4, 4>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 5, 5>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 6, 6>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 7, 7>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::DIV, 3, 8, 8>();
 
-   mfem::forall_3D(ne, q, q, vdim, [=] MFEM_HOST_DEVICE (int e)
-   {
-      if (M(e) == 0) { return; } // ignore
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 1, 1>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 2, 2>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 3, 3>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 4, 4>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 5, 5>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 6, 6>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 7, 7>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 8, 8>();
 
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
-
-      MFEM_SHARED real_t sBot[Q*D];
-      MFEM_SHARED real_t sBct[Q*D];
-      MFEM_SHARED real_t sQQ[vdim*Q*Q];
-      MFEM_SHARED real_t sQD[vdim*Q*D];
-
-      // Bo and Bc into shared memory
-      const DeviceMatrix Bot(sBot, d-1, q);
-      kernels::internal::LoadB<D,Q>(d-1, q, BO, sBot);
-      const DeviceMatrix Bct(sBct, d, q);
-      kernels::internal::LoadB<D,Q>(d, q, BC, sBct);
-
-      const DeviceCube QQ(sQQ, q, q, vdim);
-      const DeviceCube QD(sQD, q, d, vdim);
-
-      MFEM_FOREACH_THREAD(vd,z,vdim)
-      {
-         const real_t cst_val_0 = C(0,0,0,0);
-         const real_t cst_val_1 = C(1,0,0,0);
-         MFEM_FOREACH_THREAD(y,y,q)
-         {
-            MFEM_FOREACH_THREAD(x,x,q)
-            {
-               const real_t J0 = J(x,y,0,vd,e);
-               const real_t J1 = J(x,y,1,vd,e);
-               const real_t C0 = cst ? cst_val_0 : C(0,x,y,e);
-               const real_t C1 = cst ? cst_val_1 : C(1,x,y,e);
-               QQ(x,y,vd) = W(x,y)*(J0*C0 + J1*C1);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,vdim)
-      {
-         const int nx = (vd == 0) ? d : d-1;
-         DeviceMatrix Btx = (vd == 0) ? Bct : Bot;
-         MFEM_FOREACH_THREAD(qy,y,q)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               real_t qd = 0.0;
-               for (int qx = 0; qx < q; ++qx)
-               {
-                  qd += QQ(qx,qy,vd) * Btx(dx,qx);
-               }
-               QD(dx,qy,vd) = qd;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,vdim)
-      {
-         const int nx = (vd == 0) ? d : d-1;
-         const int ny = (vd == 1) ? d : d-1;
-         DeviceMatrix Bty = (vd == 1) ? Bct : Bot;
-         DeviceTensor<4> Yxy(Y, nx, ny, vdim, ne);
-         MFEM_FOREACH_THREAD(dy,y,ny)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               real_t dd = 0.0;
-               for (int qy = 0; qy < q; ++qy)
-               {
-                  dd += QD(dx,qy,vd) * Bty(dy,qy);
-               }
-               Yxy(dx,dy,vd,e) += dd;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 1, 2>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 2, 3>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 3, 4>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 4, 5>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 5, 6>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 6, 7>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 7, 8>();
+   VectorFEDomainLFIntegrator::AddSpecialization<FiniteElement::CURL, 3, 8, 9>();
 }
 
-template<int T_D1D = 0, int T_Q1D = 0>
-static void HdivDLFAssemble3D(
-   const int ne, const int d, const int q, const int *markers, const real_t *bo,
-   const real_t *bc, const real_t *j, const real_t *weights,
-   const Vector &coeff, real_t *y)
+/// \cond DO_NOT_DOCUMENT
+VectorFEDomainLFIntegrator::AssembleKernelType
+VectorFEDomainLFIntegrator::AssembleKernels::Fallback(
+   FiniteElement::DerivType TestType, int DIM, int, int)
 {
-   MFEM_VERIFY(T_D1D || d <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
-               "Problem size too large.");
-   MFEM_VERIFY(T_Q1D || q <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
-               "Problem size too large.");
-
-   static constexpr int vdim = 3;
-   const auto F = coeff.Read();
-   const auto M = Reshape(markers, ne);
-   const auto BO = Reshape(bo, q, d-1);
-   const auto BC = Reshape(bc, q, d);
-   const auto J = Reshape(j, q, q, q, vdim, vdim, ne);
-   const auto W = Reshape(weights, q, q, q);
-   const bool cst = coeff.Size() == vdim;
-   const auto C = cst ? Reshape(F,vdim,1,1,1,1) : Reshape(F,vdim,q,q,q,ne);
-   auto Y = Reshape(y, 2*(d-1)*(d-1)*d, ne);
-
-   mfem::forall_3D(ne, q, q, vdim, [=] MFEM_HOST_DEVICE (int e)
+   if (TestType == FiniteElement::DIV)
    {
-      if (M(e) == 0) { return; } // ignore
-
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
-
-      MFEM_SHARED real_t sBot[Q*D];
-      MFEM_SHARED real_t sBct[Q*D];
-
-      // Bo and Bc into shared memory
-      const DeviceMatrix Bot(sBot, d-1, q);
-      kernels::internal::LoadB<D,Q>(d-1, q, BO, sBot);
-      const DeviceMatrix Bct(sBct, d, q);
-      kernels::internal::LoadB<D,Q>(d, q, BC, sBct);
-
-      MFEM_SHARED real_t sm0[vdim*Q*Q*Q];
-      MFEM_SHARED real_t sm1[vdim*Q*Q*Q];
-      DeviceTensor<4> QQQ(sm1, q, q, q, vdim);
-      DeviceTensor<4> DQQ(sm0, d, q, q, vdim);
-      DeviceTensor<4> DDQ(sm1, d, d, q, vdim);
-
-      MFEM_FOREACH_THREAD(vd,z,vdim)
+      if (DIM == 2)
       {
-         const real_t cst_val_0 = C(0,0,0,0,0);
-         const real_t cst_val_1 = C(1,0,0,0,0);
-         const real_t cst_val_2 = C(2,0,0,0,0);
-         MFEM_FOREACH_THREAD(y,y,q)
-         {
-            MFEM_FOREACH_THREAD(x,x,q)
-            {
-               for (int z = 0; z < q; ++z)
-               {
-                  const real_t J0 = J(x,y,z,0,vd,e);
-                  const real_t J1 = J(x,y,z,1,vd,e);
-                  const real_t J2 = J(x,y,z,2,vd,e);
-                  const real_t C0 = cst ? cst_val_0 : C(0,x,y,z,e);
-                  const real_t C1 = cst ? cst_val_1 : C(1,x,y,z,e);
-                  const real_t C2 = cst ? cst_val_2 : C(2,x,y,z,e);
-                  QQQ(x,y,z,vd) = W(x,y,z)*(J0*C0 + J1*C1 + J2*C2);
-               }
-            }
-         }
+         return HdivDLFAssemble2D<0, 0>;
       }
-      MFEM_SYNC_THREAD;
-      // Apply Bt operator
-      MFEM_FOREACH_THREAD(vd,z,vdim)
+      if (DIM == 3)
       {
-         const int nx = (vd == 0) ? d : d-1;
-         DeviceMatrix Btx = (vd == 0) ? Bct : Bot;
-         MFEM_FOREACH_THREAD(qy,y,q)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               real_t u[Q];
-               MFEM_UNROLL(Q)
-               for (int qz = 0; qz < q; ++qz) { u[qz] = 0.0; }
-               MFEM_UNROLL(Q)
-               for (int qx = 0; qx < q; ++qx)
-               {
-                  MFEM_UNROLL(Q)
-                  for (int qz = 0; qz < q; ++qz)
-                  {
-                     u[qz] += QQQ(qx,qy,qz,vd) * Btx(dx,qx);
-                  }
-               }
-               MFEM_UNROLL(Q)
-               for (int qz = 0; qz < q; ++qz) { DQQ(dx,qy,qz,vd) = u[qz]; }
-            }
-         }
+         return HdivDLFAssemble3D<0, 0>;
       }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,vdim)
-      {
-         const int nx = (vd == 0) ? d : d-1;
-         const int ny = (vd == 1) ? d : d-1;
-         DeviceMatrix Bty = (vd == 1) ? Bct : Bot;
-         MFEM_FOREACH_THREAD(dy,y,ny)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               real_t u[Q];
-               MFEM_UNROLL(Q)
-               for (int qz = 0; qz < q; ++qz) { u[qz] = 0.0; }
-               MFEM_UNROLL(Q)
-               for (int qy = 0; qy < q; ++qy)
-               {
-                  MFEM_UNROLL(Q)
-                  for (int qz = 0; qz < q; ++qz)
-                  {
-                     u[qz] += DQQ(dx,qy,qz,vd) * Bty(dy,qy);
-                  }
-               }
-               MFEM_UNROLL(Q)
-               for (int qz = 0; qz < q; ++qz) { DDQ(dx,dy,qz,vd) = u[qz]; }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,vdim)
-      {
-         const int nx = (vd == 0) ? d : d-1;
-         const int ny = (vd == 1) ? d : d-1;
-         const int nz = (vd == 2) ? d : d-1;
-         DeviceTensor<5> Yxyz(Y, nx, ny, nz, vdim, ne);
-         DeviceMatrix Btz = (vd == 2) ? Bct : Bot;
-         MFEM_FOREACH_THREAD(dy,y,ny)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               real_t u[D];
-               MFEM_UNROLL(D)
-               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
-               MFEM_UNROLL(Q)
-               for (int qz = 0; qz < q; ++qz)
-               {
-                  MFEM_UNROLL(D)
-                  for (int dz = 0; dz < nz; ++dz)
-                  {
-                     u[dz] += DDQ(dx,dy,qz,vd) * Btz(dz,qz);
-                  }
-               }
-               MFEM_UNROLL(D)
-               for (int dz = 0; dz < nz; ++dz) { Yxyz(dx,dy,dz,vd,e) += u[dz]; }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-static void HdivDLFAssemble(const FiniteElementSpace &fes,
-                            const IntegrationRule *ir,
-                            const Array<int> &markers,
-                            const Vector &coeff,
-                            Vector &y)
-{
-   Mesh &mesh = *fes.GetMesh();
-   const int dim = mesh.Dimension();
-   const FiniteElement *el = fes.GetTypicalFE();
-   const auto *vel = dynamic_cast<const VectorTensorFiniteElement *>(el);
-   MFEM_VERIFY(vel != nullptr, "Must be VectorTensorFiniteElement");
-   const MemoryType mt = Device::GetDeviceMemoryType();
-   const DofToQuad &maps_o = vel->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
-   const DofToQuad &maps_c = vel->GetDofToQuad(*ir, DofToQuad::TENSOR);
-   const int d = maps_c.ndof, q = maps_c.nqpt;
-   constexpr int flags = GeometricFactors::JACOBIANS;
-   const GeometricFactors *geom = mesh.GetGeometricFactors(*ir, flags, mt);
-   decltype(&HdivDLFAssemble2D<>) ker =
-      dim == 2 ? HdivDLFAssemble2D<> : HdivDLFAssemble3D<>;
-
-   if (dim==2)
-   {
-      if (d==1 && q==1) { ker=HdivDLFAssemble2D<1,1>; }
-      if (d==2 && q==2) { ker=HdivDLFAssemble2D<2,2>; }
-      if (d==3 && q==3) { ker=HdivDLFAssemble2D<3,3>; }
-      if (d==4 && q==4) { ker=HdivDLFAssemble2D<4,4>; }
-      if (d==5 && q==5) { ker=HdivDLFAssemble2D<5,5>; }
-      if (d==6 && q==6) { ker=HdivDLFAssemble2D<6,6>; }
-      if (d==7 && q==7) { ker=HdivDLFAssemble2D<7,7>; }
-      if (d==8 && q==8) { ker=HdivDLFAssemble2D<8,8>; }
    }
-
-   if (dim==3)
+   else if (TestType == FiniteElement::CURL)
    {
-      if (d==2 && q==2) { ker=HdivDLFAssemble3D<2,2>; }
-      if (d==3 && q==3) { ker=HdivDLFAssemble3D<3,3>; }
-      if (d==4 && q==4) { ker=HdivDLFAssemble3D<4,4>; }
-      if (d==5 && q==5) { ker=HdivDLFAssemble3D<5,5>; }
-      if (d==6 && q==6) { ker=HdivDLFAssemble3D<6,6>; }
-      if (d==7 && q==7) { ker=HdivDLFAssemble3D<7,7>; }
-      if (d==8 && q==8) { ker=HdivDLFAssemble3D<8,8>; }
+      if (DIM == 3)
+      {
+         return HcurlDLFAssemble3D<0, 0>;
+      }
    }
-
-   MFEM_VERIFY(ker, "No kernel ndof " << d << " nqpt " << q);
-
-   const int ne = mesh.GetNE();
-   const int *M = markers.Read();
-   const real_t *Bo = maps_o.B.Read();
-   const real_t *Bc = maps_c.B.Read();
-   const real_t *J = geom->J.Read();
-   const real_t *W = ir->GetWeights().Read();
-   real_t *Y = y.ReadWrite();
-   ker(ne, d, q, M, Bo, Bc, J, W, coeff, Y);
+   MFEM_ABORT("");
 }
+/// \endcond DO_NOT_DOCUMENT
 
 void VectorFEDomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
                                                 const Array<int> &markers,
@@ -337,15 +96,23 @@ void VectorFEDomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
    QuadratureSpace qs(*fes.GetMesh(), *ir);
    CoefficientVector coeff(QF, qs, CoefficientStorage::COMPRESSED);
 
-   const int fe_type = fe.GetDerivType();
-   if (fe_type == FiniteElement::DIV)
-   {
-      HdivDLFAssemble(fes, ir, markers, coeff, b);
-   }
-   else
-   {
-      MFEM_ABORT("Not implemented.");
-   }
+   const FiniteElement::DerivType fe_type =
+      static_cast<FiniteElement::DerivType>(fe.GetDerivType());
+
+   Mesh &mesh = *fes.GetMesh();
+   const int dim = mesh.Dimension();
+   const FiniteElement *el = fes.GetTypicalFE();
+   const auto *vel = dynamic_cast<const VectorTensorFiniteElement *>(el);
+   MFEM_VERIFY(vel != nullptr, "Must be VectorTensorFiniteElement");
+   const MemoryType mt = Device::GetDeviceMemoryType();
+   const DofToQuad &maps_o = vel->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   const DofToQuad &maps_c = vel->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   const int d = maps_c.ndof, q = maps_c.nqpt;
+   constexpr int flags = GeometricFactors::JACOBIANS;
+   const GeometricFactors *geom = mesh.GetGeometricFactors(*ir, flags, mt);
+
+   AssembleKernels::Run(fe_type, dim, d, q, mesh.GetNE(), markers, geom->J,
+                        ir->GetWeights(), maps_o.B, maps_c.B, coeff, b, d, q);
 }
 
 } // namespace mfem

fem/intrules.cpp

@@ -236,6 +236,58 @@ IntegrationRule::ApplyToKnotIntervals(KnotVector const& kv) const
    return kvir;
 }
 
+IntegrationRule IntegrationRule::Reorder(const Array<int> &ordering) const
+{
+   const int np = GetNPoints();
+   MFEM_VERIFY(np == ordering.Size(), "Invalid permutation size");
+   IntegrationRule ir(np);
+   ir.SetOrder(GetOrder());
+
+   for (int i = 0; i < np; i++)
+   {
+      IntegrationPoint &ip_new = ir.IntPoint(i);
+      const IntegrationPoint &ip_old = IntPoint(ordering[i]);
+      ip_new.Set(ip_old.x, ip_old.y, ip_old.z, ip_old.weight);
+   }
+
+   return ir;
+}
+
+IntegrationRule DuffyTrans(const IntegrationRule &ir, int dim)
+{
+   IntegrationRule ir_mapped(ir.GetNPoints());
+   ir_mapped.SetOrder(ir.GetOrder());
+
+   if (dim == 2)
+   {
+      for (int i = 0; i < ir.GetNPoints(); i++)
+      {
+         IntegrationPoint &ip_mapped = ir_mapped.IntPoint(i);
+         ip_mapped.y = ir.IntPoint(i).y * (1 - ir.IntPoint(i).x);
+         ip_mapped.x = ir.IntPoint(i).x;
+         ip_mapped.weight = ir.IntPoint(i).weight;
+      }
+      return ir_mapped;
+   }
+   else if (dim == 3)
+   {
+      for (int i = 0; i < ir.GetNPoints(); i++)
+      {
+         IntegrationPoint &ip_mapped = ir_mapped.IntPoint(i);
+         ip_mapped.z = ir.IntPoint(i).z * (1 - ir.IntPoint(i).x) * (1 - ir.IntPoint(
+                                                                       i).y);
+         ip_mapped.y = ir.IntPoint(i).y * (1 - ir.IntPoint(i).x);
+         ip_mapped.x = ir.IntPoint(i).x;
+         ip_mapped.weight = ir.IntPoint(i).weight;
+      }
+      return ir_mapped;
+   }
+   else
+   {
+      MFEM_ABORT("Duffy transformation not implemented for this dimension!");
+   }
+}
+
 #ifdef MFEM_USE_MPFR
 
 // Class for computing hi-precision (HP) quadrature in 1D
@@ -433,6 +485,142 @@ public:
 #endif // MFEM_USE_MPFR
 
 
+void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
+                                        const real_t beta, IntegrationRule* ir)
+{
+   /* The np-point Gauss-Jacobi quadrature rule is exact for polynomials of
+      degree 2np - 1 with weight function w(x) = (1-x)^alpha * x^beta. The
+      nodes are the zeros of the Jacobi polynomial P_{np}^{alpha,beta} and
+      the weights are
+
+      w_i = C / [(1 - x_i^2) * P'_{np}^{alpha,beta}(x_i)^2]
+      C = 2^{alpha + beta + 1} * Gamma(np + alpha + 1) * Gamma(np + beta + 1)
+            / [Gamma(np + alpha + beta + 1) * Gamma(np + 1)].
+
+      The nodes are computed via nonlinear solve (Newton's method) with an
+      initial guess corresponding to Gatteschi's asymptotic expansions of the
+      Jacobi polynomial roots [1].
+
+      The current initial guess has been tested and performs well for
+      np <= 200 and -1 <= alpha, beta <= 4. For larger np, it may be necessary
+      utilize different initial guesses in the vicinity of x = -1,+1 [2].
+
+      [1] Gautschi, W., & Giordano, C. (2008). Luigi Gatteschi’s work on
+          asymptotics of special functions and their zeros. Numerical Algorithms,
+          49, 11-31.
+      [2] Hale, N., & Townsend, A. (2013). Fast and accurate computation of
+          Gauss--Legendre and Gauss--Jacobi quadrature nodes and weights.
+          SIAM Journal on Scientific Computing, 35(2), A652-A674.
+   */
+   ir->SetSize(np);
+   ir->SetPointIndices();
+   ir->SetOrder(2*np - 1);
+
+   if (alpha <= -1.0 || beta <= -1.0)
+   {
+      MFEM_ABORT("Gauss-Jacobi quadrature only defined for alpha > -1 and beta > -1");
+   }
+   // Jacobi weight function is undefined whenever alpha <= -1 or beta <= -1
+
+   if (alpha > 4.0 || beta > 4.0)
+   {
+      MFEM_ABORT("Current Gauss-Jacobi quadrature implementation only tested for alpha <= 4 and beta <= 4");
+   }
+   // current asymptotic expansions for initial guess may perform poorly for large alpha, beta
+
+   switch (np)
+   {
+      case 1:
+         real_t x = (beta - alpha) / (alpha + beta + 2);
+         real_t w = pow(2, alpha + beta + 1) * tgamma(alpha + 2) * tgamma(
+                       beta + 2) / (tgamma(alpha + beta + 2));
+         w = 0.5 * w / pow(2, alpha + beta);
+         // map weight to  to [0,1], with additional 1/(2^(alpha + beta)) factor coming from mapping
+         // the weight (1-x)^alpha * (1+x)^beta to [0,1] as well.
+         ir->IntPoint(0).Set1w(0.5 * x + 0.5,
+                               4.0 * w / ((1.0 - x*x) * (alpha + beta + 2) * (alpha + beta + 2)));
+         return;
+   }
+
+#ifndef MFEM_USE_MPFR
+
+   const int n = np;
+   // common constants for Jacobi polynomials
+   real_t ab = alpha + beta;
+   real_t a2_minus_b2 = (alpha - beta) * (alpha + beta);
+
+   // roots of P^(alpha,beta)_n in the interval [-1,1]
+   for (int i = 1; i <= n; i++)
+   {
+      // rather than using Chebyshev points for initial guess, use Gatteschi's asymptotic expansion for roots of Jacobi
+      // polynomials
+      real_t n_ab_plus_1 = 2 * n + alpha + beta + 1;
+      real_t v = (2 * i + alpha - 0.5) * M_PI / n_ab_plus_1;
+      real_t theta = v + 1.0 / (n_ab_plus_1*n_ab_plus_1) * ((0.25 - alpha*alpha) *
+                                                            1.0/tan(0.5*v) - (0.25 - beta*beta) * tan(0.5*v));
+      real_t z = cos(theta);
+
+      real_t pp, p1, dz, xi = 0.;
+      bool done = false;
+      while (1)
+      {
+         real_t p2 = 1;
+         p1 = ((alpha-beta) + (alpha + beta + 2) * z) / 2;
+         for (int j = 1; j <= n-1; j++)
+         {
+            real_t p3 = p2;
+            p2 = p1;
+
+            real_t jx2_ab = 2 * j + ab;
+            real_t an = (jx2_ab) * (jx2_ab + 2);
+            real_t bn = a2_minus_b2;
+            real_t cn = 2 * (j + alpha) * (j + beta) * (jx2_ab + 2) / (jx2_ab + 1);
+
+            real_t D = (jx2_ab + 1) / (2 * (j + 1) * (j + ab + 1) * (jx2_ab));
+            p1 = ((an * z + bn) * p2 - cn * p3) * D;
+         }
+         // p1 is Jacobi polynomial
+         pp = n * (alpha - beta - (2 * n + ab) * z) * p1 + 2 * (n + alpha) *
+              (n + beta) * p2;
+         pp = pp / ((2 * n + ab) * (1 - z*z));
+         // derivative of the Jacobi polynomial
+         if (done) { break; }
+
+         dz = p1/pp;
+#ifdef MFEM_USE_SINGLE
+         if (std::abs(dz) < 1e-7)
+#elif defined MFEM_USE_DOUBLE
+         if (std::abs(dz) < std::numeric_limits<real_t>::epsilon())
+            // this seems to cause trouble if we try std::abs(dz) < 1e-16
+#else
+         MFEM_ABORT("Floating point type undefined");
+         // if (std::abs(dz) < 1e-16)
+#endif
+         {
+            done = true;
+            xi = z - dz;
+         }
+         z -= dz;
+      }
+      real_t c0 = exp(lgamma(n + alpha + 1) - lgamma(n + ab + 1)) * exp(lgamma(
+                                                                           n + beta + 1) - lgamma(n + 1));
+      // ratio of gamma functions prone to overflow for large n, so compute logarithms
+      // of Gamma function instead, i.e. Gamma(a)/Gamma(b) = exp(lgamma(a) - lgamma(b))
+      ir->IntPoint(n-i).x = 0.5 * xi + 0.5;
+      ir->IntPoint(n-i).weight = 0.5 * c0 * pow(2.0,
+                                                ab + 1) / ((1.0 - xi*xi)*pp*pp) / pow(2, ab);
+      // map nodes and weights to the interval [0,1]
+   }
+
+#else // MFEM_USE_MPFR is defined
+
+   MFEM_ABORT("MPFR implementation of Gauss-Jacobi quadrature not defined yet");
+
+#endif // MFEM_USE_MPFR
+
+}
+
+
 void QuadratureFunctions1D::GaussLegendre(const int np, IntegrationRule* ir)
 {
    ir->SetSize(np);
@@ -2362,6 +2550,194 @@ IntegrationRule *IntegrationRules::CubeIntegrationRule(int Order)
    return CubeIntRules[Order];
 }
 
+StroudIntegrationRules StroudIntRules;
+
+StroudIntegrationRules::StroudIntegrationRules()
+{
+   const MemoryType h_mt = MemoryType::HOST;
+   SquareStroudIntRules.SetSize(32, h_mt);
+   SquareStroudIntRules = NULL;
+
+   TriangleStroudIntRules.SetSize(32, h_mt);
+   TriangleStroudIntRules = NULL;
+
+   CubeStroudIntRules.SetSize(32, h_mt);
+   CubeStroudIntRules = NULL;
+
+   TetrahedronStroudIntRules.SetSize(32, h_mt);
+   TetrahedronStroudIntRules = NULL;
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   IntRuleLocks.SetSize(Geometry::NUM_GEOMETRIES, h_mt);
+   for (int i = 0; i < Geometry::NUM_GEOMETRIES; i++)
+   {
+      omp_init_lock(&IntRuleLocks[i]);
+   }
+#endif
+}
+
+const IntegrationRule &StroudIntegrationRules::Get(int GeomType, int Order)
+{
+   Array<IntegrationRule *> *ir_array = NULL;
+
+   switch (GeomType)
+   {
+      case Geometry::TRIANGLE:    ir_array = &TriangleStroudIntRules; break;
+      case Geometry::TETRAHEDRON: ir_array = &TetrahedronStroudIntRules; break;
+      case Geometry::INVALID:
+      case Geometry::NUM_GEOMETRIES:
+         MFEM_ABORT("Unknown type of reference element!");
+      default:
+         MFEM_ABORT("Stroud rules only valid for triangular and tetrahedral elements!");
+   }
+
+   if (Order < 0)
+   {
+      Order = 0;
+   }
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_set_lock(&IntRuleLocks[GeomType]);
+#endif
+
+   if (!HaveIntRule(*ir_array, Order))
+   {
+      IntegrationRule *ir = GenerateIntegrationRule(GeomType, Order);
+#ifdef MFEM_DEBUG
+      int RealOrder = Order;
+      while (RealOrder+1 < ir_array->Size() && (*ir_array)[RealOrder+1] == ir)
+      {
+         RealOrder++;
+      }
+      MFEM_VERIFY(RealOrder == ir->GetOrder(), "internal error");
+#else
+      MFEM_CONTRACT_VAR(ir);
+#endif
+   }
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_unset_lock(&IntRuleLocks[GeomType]);
+#endif
+
+   return *(*ir_array)[Order];
+}
+
+void StroudIntegrationRules::DeleteIntRuleArray(
+   Array<IntegrationRule *> &ir_array) const
+{
+   // Many of the intrules have multiple contiguous copies in the ir_array
+   // so we have to be careful to not delete them twice.
+   IntegrationRule *ir = NULL;
+   for (int i = 0; i < ir_array.Size(); i++)
+   {
+      if (ir_array[i] != NULL && ir_array[i] != ir)
+      {
+         ir = ir_array[i];
+         delete ir;
+      }
+   }
+}
+
+StroudIntegrationRules::~StroudIntegrationRules()
+{
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   for (int i = 0; i < Geometry::NUM_GEOMETRIES; i++)
+   {
+      omp_destroy_lock(&IntRuleLocks[i]);
+   }
+#endif
+   DeleteIntRuleArray(SquareStroudIntRules);
+   DeleteIntRuleArray(TriangleStroudIntRules);
+   DeleteIntRuleArray(CubeStroudIntRules);
+   DeleteIntRuleArray(TetrahedronStroudIntRules);
+}
+
+
+IntegrationRule *StroudIntegrationRules::GenerateIntegrationRule(int GeomType,
+                                                                 int Order)
+{
+   switch (GeomType)
+   {
+      case Geometry::TRIANGLE:
+         return TriangleStroudIntegrationRule(Order);
+      case Geometry::TETRAHEDRON:
+         return TetrahedronStroudIntegrationRule(Order);
+      case Geometry::INVALID:
+      case Geometry::NUM_GEOMETRIES:
+         MFEM_ABORT("Unknown type of reference element!");
+      default:
+         MFEM_ABORT("Stroud rules only valid for triangular and tetrahedral elements!");
+   }
+   return NULL;
+}
+
+/* Integration rule in reference triangle according to tensor product Gauss-Jacobi rule.
+  The nodes and weights are used in the original form defined on the reference
+  square to evaluate the component 1D basis functions. Mapping to the reference
+  triangle via IntegrationRule::DuffyTrans() occurs only in evaluation of coefficient
+  vectors, see e.g. MassIntegrator::AssemblePASimplex. */
+IntegrationRule *StroudIntegrationRules::TriangleStroudIntegrationRule(
+   int Order)
+{
+   int RealOrder = GetSegmentRealOrder(Order);
+   // Order is one of {RealOrder-1,RealOrder}
+   // if (!HaveIntRule(SegmentIntRules, RealOrder))
+   // {
+   //    SegmentIntegrationRule(RealOrder);
+   // }
+   IntegrationRule ir_0_0;
+   // Gauss-Jacobi is exact for 2*n-1
+   int n = RealOrder/2 + 1;
+   QuadratureFunctions1D::GaussJacobi(n, 0.0, 0.0, &ir_0_0);
+
+   IntegrationRule ir_1_0;
+   QuadratureFunctions1D::GaussJacobi(n, 1.0, 0.0, &ir_1_0);
+
+   AllocIntRule(TriangleStroudIntRules, RealOrder); // RealOrder >= Order
+   // create rule in unit square
+   TriangleStroudIntRules[RealOrder-1] =
+      TriangleStroudIntRules[RealOrder] =
+         new IntegrationRule(ir_1_0, ir_0_0);
+   // map rule to reference triangle
+   // TriangleStroudIntRules[RealOrder-1]->DuffyTrans(2);
+   *TriangleStroudIntRules[RealOrder-1] =
+      DuffyTrans(*TriangleStroudIntRules[RealOrder-1], 2);
+   return TriangleStroudIntRules[Order];
+}
+
+/* Integration rule in reference tetrahedron according to tensor product Gauss-Jacobi rule.
+  The nodes and weights are used in the original form defined on the reference
+  square to evaluate the component 1D basis functions. Mapping to the reference
+  triangle via IntegrationRule::DuffyTrans() occurs only in evaluation of coefficient
+  vectors, see e.g. MassIntegrator::AssemblePASimplex. */
+IntegrationRule *StroudIntegrationRules::TetrahedronStroudIntegrationRule(
+   int Order)
+{
+   int RealOrder = GetSegmentRealOrder(Order);
+   // Order is one of {RealOrder-1,RealOrder}
+
+   IntegrationRule ir_0_0;
+   int n = RealOrder/2 + 1;
+   QuadratureFunctions1D::GaussJacobi(n, 0.0, 0.0, &ir_0_0);
+
+   IntegrationRule ir_1_0;
+   QuadratureFunctions1D::GaussJacobi(n, 1.0, 0.0, &ir_1_0);
+
+   IntegrationRule ir_2_0;
+   QuadratureFunctions1D::GaussJacobi(n, 2.0, 0.0, &ir_2_0);
+
+   AllocIntRule(TetrahedronStroudIntRules, RealOrder); // RealOrder >= Order
+   // create rule in unit cube
+   TetrahedronStroudIntRules[RealOrder-1] =
+      TetrahedronStroudIntRules[RealOrder] =
+         new IntegrationRule(ir_2_0, ir_1_0, ir_0_0);
+   // map rule to reference tetrahedron
+   // TetrahedronStroudIntRules[RealOrder-1]->DuffyTrans(3);
+   *TetrahedronStroudIntRules[RealOrder-1] =
+      DuffyTrans(*TetrahedronStroudIntRules[RealOrder-1], 3);
+   return TetrahedronStroudIntRules[Order];
+}
+
 IntegrationRule& NURBSMeshRules::GetElementRule(const int elem,
                                                 const int patch, const int *ijk,
                                                 Array<const KnotVector*> const& kv) const

fem/intrules.hpp

@@ -269,6 +269,13 @@ public:
    /// applying this rule on each knot interval.
    IntegrationRule* ApplyToKnotIntervals(KnotVector const& kv) const;
 
+   /** @brief  Returns an integration rule such that the new IntegrationPoints
+    *  are re-ordered based on @a ordering.
+    *
+    *  @details In the new integration rule, ip_new[i] = ip_old[ordering[i]]
+    */
+   IntegrationRule Reorder(const Array<int> &ordering) const;
+
    /// Destroys an IntegrationRule object
    ~IntegrationRule() { }
 };
@@ -378,6 +385,8 @@ public:
        These methods calculate the actual points and weights for the different
        types of quadrature rules. */
    ///@{
+   static void GaussJacobi(const int np, const real_t alpha, const real_t beta,
+                           IntegrationRule* ir);
    static void GaussLegendre(const int np, IntegrationRule* ir);
    static void GaussLobatto(const int np, IntegrationRule *ir);
    static void OpenUniform(const int np, IntegrationRule *ir);
@@ -487,12 +496,71 @@ public:
    ~IntegrationRules();
 };
 
+/// Container class for integration rules
+class StroudIntegrationRules
+{
+private:
+   Array<IntegrationRule *> SquareStroudIntRules;
+   Array<IntegrationRule *> TriangleStroudIntRules;
+   Array<IntegrationRule *> CubeStroudIntRules;
+   Array<IntegrationRule *> TetrahedronStroudIntRules;
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   Array<omp_lock_t> IntRuleLocks;
+#endif
+
+   void AllocIntRule(Array<IntegrationRule *> &ir_array, int Order) const
+   {
+      if (ir_array.Size() <= Order)
+      {
+         ir_array.SetSize(Order + 1, NULL);
+      }
+   }
+   bool HaveIntRule(Array<IntegrationRule *> &ir_array, int Order) const
+   {
+      return (ir_array.Size() > Order && ir_array[Order] != NULL);
+   }
+   int GetSegmentRealOrder(int Order) const
+   {
+      return Order | 1; // valid for all quad_type's
+   }
+   void DeleteIntRuleArray(Array<IntegrationRule *> &ir_array) const;
+
+   /// The following methods allocate new IntegrationRule objects without
+   /// checking if they already exist.  To avoid memory leaks use
+   /// IntegrationRules::Get(int GeomType, int Order) instead.
+   IntegrationRule *GenerateIntegrationRule(int GeomType, int Order);
+   IntegrationRule *TriangleStroudIntegrationRule(int Order);
+   IntegrationRule *TetrahedronStroudIntegrationRule(int Order);
+
+public:
+   /// Sets initial sizes for the integration rule arrays, but rules
+   /// are defined the first time they are requested with the Get method.
+   explicit StroudIntegrationRules();
+
+   /// Returns a Stroud integration rule for given GeomType and Order.
+   const IntegrationRule &Get(int GeomType, int Order);
+
+   /// Destroys an StroudIntegrationRules object
+   ~StroudIntegrationRules();
+};
+
 /// A global object with all integration rules (defined in intrules.cpp)
 extern MFEM_EXPORT IntegrationRules IntRules;
 
 /// A global object with all refined integration rules
 extern MFEM_EXPORT IntegrationRules RefinedIntRules;
 
+/// A global object with all Stroud integration rules (defined in intrules.cpp)
+extern MFEM_EXPORT StroudIntegrationRules StroudIntRules;
+
+/// Duffy Transformation of 2D and 3D tensor product rules of the form
+/// $X(t) = \sum_{i=1}^{d+1} \lambda_i(t) * x_i$, where $x_i$ are the vertices
+/// of the simplex and $\lambda_i = t_i * (1-\lambda_1-...-\lambda_{i-1})$, with
+/// $t$ being the coordinates in the unit square/cube. This function is used only
+/// in the partial assembly of Bernstein elements on simplices and does NOT
+/// modify the quadrature weights.
+IntegrationRule DuffyTrans(const IntegrationRule &ir, int dim);
 }
 
 #endif

fem/lininteg.cpp

@@ -471,6 +471,13 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    }
 }
 
+VectorFEDomainLFIntegrator::VectorFEDomainLFIntegrator(
+   VectorCoefficient &F, const IntegrationRule *ir)
+   : DeltaLFIntegrator(F, ir), QF(F)
+{
+   static Kernels kernels{};
+}
+
 void VectorFEDomainLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {

fem/lininteg.hpp

@@ -369,8 +369,8 @@ private:
    Vector vec;
 
 public:
-   VectorFEDomainLFIntegrator(VectorCoefficient &F)
-      : DeltaLFIntegrator(F), QF(F) { }
+   VectorFEDomainLFIntegrator(VectorCoefficient &F,
+                              const IntegrationRule *ir = nullptr);
 
    void AssembleRHSElementVect(const FiniteElement &el,
                                ElementTransformation &Tr,
@@ -387,6 +387,40 @@ public:
                        Vector &b) override;
 
    using LinearFormIntegrator::AssembleRHSElementVect;
+
+   /// @param ne number of elements
+   /// @param markers array where entry markers[e] == 0 to skip assembly over
+   /// element e element
+   /// @param jac Spatial Jacobians evaluated at all quadrature points
+   /// @param weights 1D quadrature weights
+   /// @param testBO 1D open basis test functions
+   /// @param testBC 1D closed basis test functions
+   /// @param coeff coefficient values evaluated at quadrature points, possibly
+   /// compressed.
+   /// @param d number of 1D closed dofs
+   /// @param q number of 1D quadrature points
+   using AssembleKernelType = void (*)(const int NE, const Array<int> &markers,
+                                       const Vector &jac,
+                                       const Array<real_t> &weights,
+                                       const Array<real_t> &testBO,
+                                       const Array<real_t> &testBC,
+                                       const Vector &coeff, Vector &y,
+                                       const int testd1d, const int q1d);
+
+   /// parameters: test_fetype, ndims, test_d1d, q1d
+   MFEM_REGISTER_KERNELS(AssembleKernels, AssembleKernelType,
+                         (FiniteElement::DerivType, int, int, int));
+
+   struct Kernels
+   {
+      Kernels();
+   };
+
+   template <FiniteElement::DerivType TestType, int DIM, int TEST_D1D, int Q1D>
+   static void AddSpecialization()
+   {
+      AssembleKernels::Specialization<TestType, DIM, TEST_D1D, Q1D>::Add();
+   }
 };
 
 /// $ (Q, \mathrm{curl}(v))_{\Omega} $ for Nedelec Elements

fem/multigrid.cpp

@@ -284,12 +284,12 @@ GeometricMultigrid::GeometricMultigrid(
    ownedProlongations.SetSize(nlevels - 1);
    ownedProlongations = have_ess_bdr;
 
-   if (have_ess_bdr)
+   essentialTrueDofs.SetSize(nlevels);
+   for (int level = 0; level < nlevels; ++level)
    {
-      essentialTrueDofs.SetSize(nlevels);
-      for (int level = 0; level < nlevels; ++level)
+      essentialTrueDofs[level] = new Array<int>;
+      if (have_ess_bdr)
       {
-         essentialTrueDofs[level] = new Array<int>;
          fespaces.GetFESpaceAtLevel(level).GetEssentialTrueDofs(
             ess_bdr, *essentialTrueDofs[level]);
       }

fem/multigrid.hpp

@@ -187,8 +187,7 @@ public:
    /// mesh boundary element attributes that define the essential DOFs.
    ///
    /// If @a ess_bdr is empty, or all its entries are 0, then no essential
-   /// boundary conditions are imposed and the protected array essentialTrueDofs
-   /// remains empty.
+   /// boundary conditions are imposed.
    GeometricMultigrid(const FiniteElementSpaceHierarchy& fespaces_,
                       const Array<int> &ess_bdr);
 

fem/particleset.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "particleset.hpp"
+#include "../general/forall.hpp"
 
 #if defined(MFEM_USE_MPI) && defined(MFEM_USE_GSLIB)
 
@@ -225,6 +226,7 @@ void ParticleSet::AddParticles(const Array<IDType> &new_ids,
       }
    }
    // Add new ids
+   ids.HostReadWrite();
    ids.Append(new_ids);
 
    // Update data
@@ -244,6 +246,102 @@ void ParticleSet::AddParticles(const Array<IDType> &new_ids,
 #if defined(MFEM_USE_MPI) && defined(MFEM_USE_GSLIB)
 
 /// \cond DO_NOT_DOCUMENT
+// Static helper: gather selected particle-vector entries into a compact buffer.
+// nvcc does not allow extended host/device lambdas in non-public members.
+static void GatherParticleVectorDevice(const ParticleVector &pv,
+                                       const Array<int> &send_idxs,
+                                       Vector &send_data,
+                                       int nsend)
+{
+   const int vdim = pv.GetVDim();
+   const int ordering = pv.GetOrdering();
+   const int num_particles = pv.GetNumParticles();
+   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
+   send_data.SetSize(nsend*vdim);
+   real_t *d_send_data =
+      send_data.GetMemory().Write(device_mc, send_data.Size());
+   const real_t *d_src = pv.GetMemory().Read(device_mc, pv.Size());
+   const int *d_send_idxs = send_idxs.GetMemory().Read(device_mc, nsend);
+
+   mfem::forall(nsend, [=] MFEM_HOST_DEVICE (int i)
+   {
+      const int p = d_send_idxs[i];
+      const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
+      const int stride = (ordering == Ordering::byVDIM) ? 1 : num_particles;
+
+      for (int c = 0; c < vdim; c++)
+      {
+         d_send_data[i*vdim + c] = d_src[offset + c*stride];
+      }
+   });
+}
+
+// Static helper: gather selected tag values into a compact buffer.
+// nvcc does not allow extended host/device lambdas in non-public members.
+static void GatherParticleTagsDevice(const Array<int> &tag,
+                                     const Array<int> &send_idxs,
+                                     Array<int> &send_tag,
+                                     int nsend)
+{
+   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
+   send_tag.SetSize(nsend);
+   int *d_send_tag = send_tag.GetMemory().Write(device_mc, nsend);
+   const int *d_tag = tag.GetMemory().Read(device_mc, tag.Size());
+   const int *d_send_idxs = send_idxs.GetMemory().Read(device_mc, nsend);
+
+   mfem::forall(nsend, [=] MFEM_HOST_DEVICE (int i)
+   {
+      d_send_tag[i] = d_tag[d_send_idxs[i]];
+   });
+}
+
+// Static helper: scatter compact particle-vector entries to particle storage.
+// nvcc does not allow extended host/device lambdas in non-public members.
+static void ScatterParticleVectorDevice(ParticleVector &pv,
+                                        const Vector &recv_data,
+                                        const Array<int> &recv_locs,
+                                        int nrecv)
+{
+   const int vdim = pv.GetVDim();
+   const int ordering = pv.GetOrdering();
+   const int num_particles = pv.GetNumParticles();
+   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
+   const real_t *d_recv_data =
+      recv_data.GetMemory().Read(device_mc, recv_data.Size());
+   const int *d_recv_locs = recv_locs.GetMemory().Read(device_mc, nrecv);
+   real_t *d_dst = pv.GetMemory().ReadWrite(device_mc, pv.Size());
+
+   mfem::forall(nrecv, [=] MFEM_HOST_DEVICE (int i)
+   {
+      const int p = d_recv_locs[i];
+      const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
+      const int stride = (ordering == Ordering::byVDIM) ? 1 : num_particles;
+
+      for (int c = 0; c < vdim; c++)
+      {
+         d_dst[offset + c*stride] = d_recv_data[i*vdim + c];
+      }
+   });
+}
+
+// Static helper: scatter compact tag values to particle storage.
+// nvcc does not allow extended host/device lambdas in non-public members.
+static void ScatterParticleTagsDevice(Array<int> &tag,
+                                      const Array<int> &recv_tag,
+                                      const Array<int> &recv_locs,
+                                      int nrecv)
+{
+   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
+   const int *d_recv_tag = recv_tag.GetMemory().Read(device_mc, nrecv);
+   const int *d_recv_locs = recv_locs.GetMemory().Read(device_mc, nrecv);
+   int *d_tag = tag.GetMemory().ReadWrite(device_mc, tag.Size());
+
+   mfem::forall(nrecv, [=] MFEM_HOST_DEVICE (int i)
+   {
+      d_tag[d_recv_locs[i]] = d_recv_tag[i];
+   });
+}
+
 template<size_t NBytes>
 void ParticleSet::TransferParticlesImpl(ParticleSet &pset,
                                         const Array<int> &send_idxs,
@@ -266,37 +364,108 @@ void ParticleSet::TransferParticlesImpl(ParticleSet &pset,
    array_init(parr_t, &gsl_arr, send_idxs.Size());
    pdata_arr = (parr_t*) gsl_arr.ptr;
 
+   int nparticles = pset.GetNParticles();
+   int nsend      = send_idxs.Size();
    gsl_arr.n = send_idxs.Size();
+
+   const int *h_send_idxs_initial = send_idxs.HostRead();
+   const IDType *h_ids = pset.GetIDs().HostRead();
    for (int i = 0; i < send_idxs.Size(); i++)
    {
       parr_t &pdata = pdata_arr[i];
-      pdata.id = pset.GetIDs()[send_idxs[i]];
+      pdata.id = h_ids[h_send_idxs_initial[i]];
+   }
 
-      // Copy particle data directly into pdata
-      size_t counter = 0;
-      for (int f = -1; f < pset.GetNFields(); f++)
+   // Pack coords and fields into the GSLIB send buffer. Device-resident data
+   // is first gathered into a compact device buffer so that only selected
+   // particles are copied back to host. Host-resident data is packed directly.
+   int max_vdim = pset.Coords().GetVDim();
+   for (int f = 0; f < pset.GetNFields(); f++)
+   {
+      int f_vdim = pset.Field(f).GetVDim();
+      if (f_vdim > max_vdim) { max_vdim = f_vdim; }
+   }
+   Vector send_data;
+   Array<int> send_tag;
+   if (Device::IsEnabled())
+   {
+      send_data.SetSize(nsend * max_vdim); // allocate max size over all fields
+      send_tag.SetSize(nsend);
+   }
+
+   size_t counter = 0;
+   for (int f = -1; f < pset.GetNFields(); f++)
+   {
+      const ParticleVector &pv = f == -1 ? pset.Coords() : pset.Field(f);
+      const int vdim = pv.GetVDim();
+      const int ordering = pv.GetOrdering();
+      const int num_particles = pv.GetNumParticles();
+      const bool use_dev = Device::IsEnabled() && pv.UseDevice();
+
+      if (use_dev)
       {
-         ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
-         for (int c = 0; c < pv.GetVDim(); c++)
+         GatherParticleVectorDevice(pv, send_idxs, send_data, nsend);
+
+         const real_t *h_send_data = send_data.HostRead();
+         for (int i = 0; i < nsend; i++)
          {
-            std::memcpy(pdata.data.data() + counter, &pv(send_idxs[i], c),
-                        sizeof(real_t));
-            counter += sizeof(real_t);
+            std::memcpy(pdata_arr[i].data.data() + counter,
+                        h_send_data + i*vdim, vdim * sizeof(real_t));
+         }
+      }
+      else
+      {
+         const real_t *h_src = pv.HostRead();
+         const int *h_send_idxs = send_idxs.HostRead();
+         for (int i = 0; i < nsend; i++)
+         {
+            parr_t &pdata = pdata_arr[i];
+            const int p = h_send_idxs[i];
+            const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
+            const int stride = (ordering == Ordering::byVDIM) ? 1 :
+                               num_particles;
+
+            for (int c = 0; c < vdim; c++)
+            {
+               std::memcpy(pdata.data.data() + counter + c*sizeof(real_t),
+                           h_src + offset + c*stride, sizeof(real_t));
+            }
          }
       }
 
-      // Copy tags
-      for (int t = 0; t < pset.GetNTags(); t++)
-      {
-         Array<int> &tag_arr = pset.Tag(t);
-         std::memcpy(pdata.data.data() + counter, &tag_arr[send_idxs[i]],
-                     sizeof(int));
-         counter += sizeof(int);
-      }
+      counter += vdim*sizeof(real_t);
    }
 
-   int nparticles = pset.GetNParticles();
-   int nsend      = send_idxs.Size();
+   // Pack tags after all real_t data. Each tag uses the same selective
+   // device gather path when its Array is device-resident.
+   for (int t = 0; t < pset.GetNTags(); t++)
+   {
+      const Array<int> &tag = pset.Tag(t);
+      const size_t tag_counter = counter + t*sizeof(int);
+      const bool use_dev = Device::IsEnabled() && tag.UseDevice();
+
+      if (use_dev)
+      {
+         GatherParticleTagsDevice(tag, send_idxs, send_tag, nsend);
+
+         const int *h_send_tag = send_tag.HostRead();
+         for (int i = 0; i < nsend; i++)
+         {
+            std::memcpy(pdata_arr[i].data.data() + tag_counter,
+                        h_send_tag + i, sizeof(int));
+         }
+      }
+      else
+      {
+         const int *h_tag = tag.HostRead();
+         const int *h_send_idxs = send_idxs.HostRead();
+         for (int i = 0; i < nsend; i++)
+         {
+            std::memcpy(pdata_arr[i].data.data() + tag_counter,
+                        h_tag + h_send_idxs[i], sizeof(int));
+         }
+      }
+   }
 
    // Transfer particles
    sarray_transfer_ext(parr_t, &gsl_arr, send_ranks.GetData(),
@@ -304,11 +473,20 @@ void ParticleSet::TransferParticlesImpl(ParticleSet &pset,
 
    // Make sure we have enough space for received particles
    int nrecv = (int) gsl_arr.n;
+
+   Vector recv_data;
+   Array<int> recv_tag;
+   if (Device::IsEnabled())
+   {
+      recv_data.SetSize(nrecv * max_vdim);
+      recv_tag.SetSize(nrecv);
+   }
+
    int ndelete = nsend - nrecv;
    if (ndelete > 0)
    {
       // Remove unneeded particles
-      auto datap = const_cast<int*>(send_idxs.GetData());
+      auto datap = const_cast<int*>(send_idxs.HostRead());
       Array<int> delete_idxs(datap + nrecv, ndelete);
       pset.RemoveParticles(delete_idxs);
    }
@@ -319,47 +497,133 @@ void ParticleSet::TransferParticlesImpl(ParticleSet &pset,
 
    pdata_arr = (parr_t*) gsl_arr.ptr;
 
-   // Add newly-recvd data directly to active state
+   // Make a list of new IDs to add
+   int num_new = nrecv > nsend ? nrecv - nsend : 0;
+   Array<IDType> new_ids(num_new);
+   for (int i = 0; i < num_new; i++)
+   {
+      new_ids[i] = pdata_arr[nsend + i].id;
+   }
+
+   // Add particles in batch
+   Array<int> new_indices;
+   if (num_new > 0)
+   {
+      pset.AddParticles(new_ids, &new_indices);
+   }
+
+   // Map each received packet to the local particle slot it updates.
+   Array<int> recv_locs(nrecv);
+   int *h_recv_locs = recv_locs.HostWrite();
+   const int *h_send_idxs_recv = send_idxs.HostRead();
    for (int i = 0; i < nrecv; i++)
    {
       parr_t &pdata = pdata_arr[i];
-      IDType id = pdata.id;
-
-      int new_loc_idx;
       if (i < nsend) // update existing particle
       {
-         new_loc_idx = send_idxs[i];
-         pset.UpdateID(new_loc_idx, id);
+         h_recv_locs[i] = h_send_idxs_recv[i];
+         pset.UpdateID(h_recv_locs[i], pdata.id);
       }
       else
       {
-         // add new particle
-         Array<int> idx_temp;
-         pset.AddParticles(Array<IDType>({id}), &idx_temp);
-         new_loc_idx = idx_temp[0]; // Get index of newly-added particle
+         h_recv_locs[i] = new_indices[i - nsend];
       }
+   }
 
-      size_t counter = 0;
-      for (int f = -1; f < pset.GetNFields(); f++)
+   // Unpack coords and fields from GSLIB host packets. Device-resident
+   // destinations use a compact host buffer followed by a device scatter.
+   size_t recv_counter = 0;
+   for (int f = -1; f < pset.GetNFields(); f++)
+   {
+      ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
+      const int vdim = pv.GetVDim();
+      const int ordering = pv.GetOrdering();
+      const int num_particles = pv.GetNumParticles();
+      const bool use_dev = Device::IsEnabled() && pv.UseDevice();
+
+      if (use_dev)
       {
-         ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
-         for (int c = 0; c < pv.GetVDim(); c++)
+         recv_data.SetSize(nrecv*vdim);
+         real_t *h_recv_data = recv_data.HostWrite();
+
+         for (int i = 0; i < nrecv; i++)
          {
-            real_t& val = pv(new_loc_idx, c);
-            std::memcpy(&val, pdata.data.data() + counter, sizeof(real_t));
-            counter += sizeof(real_t);
+            std::memcpy(h_recv_data + i*vdim,
+                        pdata_arr[i].data.data() + recv_counter,
+                        vdim*sizeof(real_t));
+         }
+
+         ScatterParticleVectorDevice(pv, recv_data, recv_locs, nrecv);
+      }
+      else
+      {
+         real_t *h_dst = pv.HostReadWrite();
+         const int *h_recv_locs_read = recv_locs.HostRead();
+         for (int i = 0; i < nrecv; i++)
+         {
+            parr_t &pdata = pdata_arr[i];
+            const int p = h_recv_locs_read[i];
+            const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
+            const int stride = (ordering == Ordering::byVDIM) ? 1 :
+                               num_particles;
+
+            for (int c = 0; c < vdim; c++)
+            {
+               std::memcpy(h_dst + offset + c*stride,
+                           pdata.data.data() + recv_counter + c*sizeof(real_t),
+                           sizeof(real_t));
+            }
          }
       }
 
-      for (int t = 0; t < pset.GetNTags(); t++)
+      recv_counter += vdim*sizeof(real_t);
+   }
+
+   // Unpack tags after all real_t data, using the same compact scatter path
+   // for device-resident tag arrays.
+   for (int t = 0; t < pset.GetNTags(); t++)
+   {
+      Array<int> &tag = pset.Tag(t);
+      const size_t tag_counter = recv_counter + t*sizeof(int);
+      const bool use_dev = Device::IsEnabled() && tag.UseDevice();
+
+      if (use_dev)
       {
-         Array<int> &tag_arr = pset.Tag(t);
-         std::memcpy(&tag_arr[new_loc_idx],
-                     pdata.data.data() + counter, sizeof(int));
-         counter += sizeof(int);
+         recv_tag.SetSize(nrecv);
+         int *h_recv_tag = recv_tag.HostWrite();
+
+         for (int i = 0; i < nrecv; i++)
+         {
+            std::memcpy(h_recv_tag + i,
+                        pdata_arr[i].data.data() + tag_counter, sizeof(int));
+         }
+
+         ScatterParticleTagsDevice(tag, recv_tag, recv_locs, nrecv);
+      }
+      else
+      {
+         int *h_tag = tag.HostReadWrite();
+         const int *h_recv_locs_read = recv_locs.HostRead();
+         for (int i = 0; i < nrecv; i++)
+         {
+            std::memcpy(h_tag + h_recv_locs_read[i],
+                        pdata_arr[i].data.data() + tag_counter, sizeof(int));
+         }
       }
    }
    array_free(&gsl_arr);
+
+   // Restore Device validity if needed
+   for (int f = -1; f < pset.GetNFields(); f++)
+   {
+      ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
+      pv.ReadWrite(pv.UseDevice());
+   }
+   for (int t = 0; t < pset.GetNTags(); t++)
+   {
+      Array<int> &tag_arr = pset.Tag(t);
+      if (tag_arr.UseDevice()) { tag_arr.ReadWrite(true); }
+   }
 }
 
 template<size_t NBytes>
@@ -526,11 +790,14 @@ ParticleSet::ParticleSet(int id_stride_, IDType id_counter_, int num_particles,
                          int dim, Ordering::Type coords_ordering, const Array<int> &field_vdims,
                          const Array<Ordering::Type> &field_orderings,
                          const Array<const char*> &field_names_, int num_tags,
-                         const Array<const char*> &tag_names_)
+                         const Array<const char*> &tag_names_,
+                         bool use_device)
    : id_stride(id_stride_),
      id_counter(id_counter_),
      coords(dim, coords_ordering)
 {
+   if (use_device) { coords.UseDevice(true); }
+
    // Initialize fields
    for (int f = 0; f < field_vdims.Size(); f++)
    {
@@ -580,21 +847,22 @@ bool ParticleSet::IsValidParticle(const Particle &p) const
 }
 
 ParticleSet::ParticleSet(int num_particles, int dim,
-                         Ordering::Type coords_ordering)
+                         Ordering::Type coords_ordering,
+                         bool use_device)
    : ParticleSet(1, 0, num_particles, dim, coords_ordering, Array<int>(),
                  Array<Ordering::Type>(), Array<const char*>(), 0,
-                 Array<const char*>())
+                 Array<const char*>(), use_device)
 {
 
 }
 
 ParticleSet::ParticleSet(int num_particles, int dim,
                          const Array<int> &field_vdims, int num_tags,
-                         Ordering::Type all_ordering)
+                         Ordering::Type all_ordering, bool use_device)
    : ParticleSet(1, 0, num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  GetEmptyNameArray(field_vdims.Size()), num_tags,
-                 GetEmptyNameArray(num_tags))
+                 GetEmptyNameArray(num_tags), use_device)
 {
 }
 
@@ -602,11 +870,11 @@ ParticleSet::ParticleSet(int num_particles, int dim,
                          const Array<int> &field_vdims, const Array<const
                          char*> &field_names_, int num_tags,
                          const Array<const char*> &tag_names_,
-                         Ordering::Type all_ordering)
+                         Ordering::Type all_ordering, bool use_device)
    : ParticleSet(1, 0, num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  field_names_, num_tags,
-                 tag_names_)
+                 tag_names_, use_device)
 {
 
 }
@@ -616,9 +884,9 @@ ParticleSet::ParticleSet(int num_particles, int dim,
                          const Array<int> &field_vdims,
                          const Array<Ordering::Type> &field_orderings,
                          const Array<const char*> &field_names_, int num_tags,
-                         const Array<const char*> &tag_names_)
+                         const Array<const char*> &tag_names_, bool use_device)
    : ParticleSet(1, 0, num_particles, dim, coords_ordering, field_vdims,
-                 field_orderings, field_names_, num_tags, tag_names_)
+                 field_orderings, field_names_, num_tags, tag_names_, use_device)
 {
 
 }
@@ -627,21 +895,21 @@ ParticleSet::ParticleSet(int num_particles, int dim,
 
 #ifdef MFEM_USE_MPI
 ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
-                         Ordering::Type coords_ordering)
+                         Ordering::Type coords_ordering, bool use_device)
    : ParticleSet(comm_, rank_num_particles, dim, coords_ordering, Array<int>(),
                  Array<Ordering::Type>(), Array<const char*>(), 0,
-                 Array<const char*>())
+                 Array<const char*>(), use_device)
 {
 
 };
 
 ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                          const Array<int> &field_vdims, int num_tags,
-                         Ordering::Type all_ordering)
+                         Ordering::Type all_ordering, bool use_device)
    : ParticleSet(comm_, rank_num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  GetEmptyNameArray(field_vdims.Size()), num_tags,
-                 GetEmptyNameArray(num_tags))
+                 GetEmptyNameArray(num_tags), use_device)
 {
 
 }
@@ -650,11 +918,11 @@ ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                          const Array<int> &field_vdims, const Array<const
                          char*> &field_names_,
                          int num_tags, const Array<const char*> &tag_names_,
-                         Ordering::Type all_ordering)
+                         Ordering::Type all_ordering, bool use_device)
    : ParticleSet(comm_, rank_num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  field_names_, num_tags,
-                 tag_names_)
+                 tag_names_, use_device)
 {
 
 }
@@ -664,7 +932,7 @@ ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                          const Array<int> &field_vdims,
                          const Array<Ordering::Type> &field_orderings,
                          const Array<const char*> &field_names_, int num_tags,
-                         const Array<const char*> &tag_names_)
+                         const Array<const char*> &tag_names_, bool use_device)
    : ParticleSet(GetSize(comm_), (IDType)GetRank(comm_),
                  rank_num_particles,
                  dim,
@@ -673,7 +941,7 @@ ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                  field_orderings,
                  field_names_,
                  num_tags,
-                 tag_names_)
+                 tag_names_, use_device)
 {
    comm = comm_;
 #ifdef MFEM_USE_GSLIB
@@ -705,6 +973,7 @@ int ParticleSet::AddField(int vdim, Ordering::Type field_ordering,
    }
    fields.emplace_back(std::make_unique<ParticleVector>(vdim, field_ordering,
                                                         GetNParticles()));
+   if (coords.UseDevice()) { fields.back()->UseDevice(true); }
    field_names.emplace_back(field_name_str);
 
    return GetNFields() - 1;
@@ -718,6 +987,7 @@ int ParticleSet::AddTag(const char* tag_name)
       tag_name_str = GetDefaultTagName(tag_names.size());
    }
    tags.emplace_back(std::make_unique<Array<int>>(GetNParticles()));
+   if (coords.UseDevice()) { tags.back()->GetMemory().UseDevice(true); }
    tag_names.emplace_back(tag_name_str);
 
    return GetNTags() - 1;
@@ -782,7 +1052,7 @@ Particle ParticleSet::GetParticle(int i) const
 
    for (int t = 0; t < GetNTags(); t++)
    {
-      p.Tag(t) = Tag(t)[i];
+      p.Tag(t) = Tag(t).HostRead()[i];
    }
 
    return p;
@@ -790,13 +1060,21 @@ Particle ParticleSet::GetParticle(int i) const
 
 bool ParticleSet::IsParticleRefValid() const
 {
-   if (coords.GetOrdering() == Ordering::byNODES)
+   if (coords.GetOrdering() == Ordering::byNODES || coords.UseDevice())
    {
       return false;
    }
    for (int f = 0; f < GetNFields(); f++)
    {
-      if (fields[f]->GetOrdering() == Ordering::byNODES)
+      if (fields[f]->GetOrdering() == Ordering::byNODES ||
+          fields[f]->UseDevice())
+      {
+         return false;
+      }
+   }
+   for (int t = 0; t < GetNTags(); t++)
+   {
+      if (tags[t]->UseDevice())
       {
          return false;
       }
@@ -806,6 +1084,10 @@ bool ParticleSet::IsParticleRefValid() const
 
 Particle ParticleSet::GetParticleRef(int i)
 {
+   MFEM_ASSERT(IsParticleRefValid(),
+               "GetParticleRef is only valid when coordinates and fields are "
+               "ordered byVDIM and particle data is host-resident.");
+
    Particle p = CreateParticle();
 
    Coords().GetValuesRef(i, p.Coords());
@@ -839,7 +1121,7 @@ void ParticleSet::SetParticle(int i, const Particle &p)
 
    for (int t = 0; t < GetNTags(); t++)
    {
-      Tag(t)[i] = p.Tag(t);
+      Tag(t).HostReadWrite()[i] = p.Tag(t);
    }
 }
 
@@ -900,6 +1182,15 @@ void ParticleSet::PrintCSV(const char *fname, const Array<int> &field_idxs,
 #ifdef MFEM_USE_MPI
    int rank = GetRank(comm);
 #endif // MFEM_USE_MPI
+   // make sure we can read tag data on host. fields and coords will be read as
+   // needed in the loop below, so we don't need to pre-read them here.
+   for (int i = 0; i < GetNTags(); i++)
+   {
+      tags[i]->HostRead();
+   }
+   ids.HostRead();
+
+   // Write particle data
    for (int i = 0; i < GetNParticles(); i++)
    {
       ss_data << ids[i];

fem/particleset.hpp

@@ -211,6 +211,12 @@ public:
  *  byVDIM). The unique_ptrs to all the ParticleVectors are stored in the
  *  std::vector \ref fields.
  *
+ *  @par Device Behavior:
+ *  When a ParticleSet is constructed with \p use_device=true, \ref coords and
+ *  all ParticleVector fields are marked to use device memory. Fields added
+ *  later through \ref AddField inherit the current device mode (through
+ *  \ref coords).
+ *
  *  @par Tags:
  *  Tags represent integers associated with each particle. For a given tag,
  *  all particle data are stored in a single Array<int>. The unique_ptrs to all
@@ -369,7 +375,10 @@ protected:
     *  ID of a particle.
     */
    void UpdateID(int local_idx, IDType new_global_id)
-   { ids[local_idx] = new_global_id; }
+   {
+      ids.HostReadWrite();
+      ids[local_idx] = new_global_id;
+   }
 
    /** @brief Create a Particle object with the same spatial dimension,
     *  number of fields and field vdims, and number of tags as this ParticleSet.
@@ -399,12 +408,14 @@ protected:
     *  @param[in] field_names_        Array of field names.
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] tag_names_          Array of tag names.
+    *  @param[in] use_device          Use device memory for particle fields.
     */
    ParticleSet(int id_stride_, IDType id_counter_, int num_particles, int dim,
                Ordering::Type coords_ordering, const Array<int> &field_vdims,
                const Array<Ordering::Type> &field_orderings,
                const Array<const char*> &field_names_, int num_tags,
-               const Array<const char*> &tag_names_);
+               const Array<const char*> &tag_names_,
+               bool use_device);
 
 public:
 
@@ -413,9 +424,12 @@ public:
     *  @param[in] num_particles       Number of particles to initialize.
     *  @param[in] dim                 Particle spatial dimension.
     *  @param[in] coords_ordering     Ordering of coordinates.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(int num_particles, int dim,
-               Ordering::Type coords_ordering=Ordering::byVDIM);
+               Ordering::Type coords_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Construct a serial ParticleSet with specified fields and tags at
     *  construction.
@@ -426,9 +440,12 @@ public:
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(int num_particles, int dim, const Array<int> &field_vdims,
-               int num_tags, Ordering::Type all_ordering=Ordering::byVDIM);
+               int num_tags, Ordering::Type all_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Construct a serial ParticleSet with specified fields and tags at
     *  construction, with names.
@@ -441,11 +458,14 @@ public:
     *  @param[in] tag_names_          Array of tag names.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(int num_particles, int dim, const Array<int> &field_vdims,
                const Array<const char*> &field_names_, int num_tags,
                const Array<const char*> &tag_names_,
-               Ordering::Type all_ordering=Ordering::byVDIM);
+               Ordering::Type all_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Comprehensive serial constructor of ParticleSet.
     *
@@ -457,12 +477,15 @@ public:
     *  @param[in] field_names_        Array of field names.
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] tag_names_          Array of tag names.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(int num_particles, int dim, Ordering::Type coords_ordering,
                const Array<int> &field_vdims,
                const Array<Ordering::Type> &field_orderings,
                const Array<const char*> &field_names_, int num_tags,
-               const Array<const char*> &tag_names_);
+               const Array<const char*> &tag_names_,
+               bool use_device=false);
 
 #ifdef MFEM_USE_MPI
    /** @brief Construct a parallel ParticleSet.
@@ -471,9 +494,12 @@ public:
     *  @param[in] rank_num_particles  Number of particles to initialize.
     *  @param[in] dim                 Particle spatial dimension.
     *  @param[in] coords_ordering     (Optional) Ordering of coordinates.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
-               Ordering::Type coords_ordering=Ordering::byVDIM);
+               Ordering::Type coords_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Construct a parallel ParticleSet with specified fields and tags
     *  at construction.
@@ -485,10 +511,13 @@ public:
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                const Array<int> &field_vdims, int num_tags,
-               Ordering::Type all_ordering=Ordering::byVDIM);
+               Ordering::Type all_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Construct a parallel ParticleSet with specified fields and tags
     *  at construction, with names (for PrintCSV()).
@@ -502,12 +531,15 @@ public:
     *  @param[in] tag_names_          Array of tag names.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                const Array<int> &field_vdims,
                const Array<const char*> &field_names_,
                int num_tags, const Array<const char*> &tag_names_,
-               Ordering::Type all_ordering=Ordering::byVDIM);
+               Ordering::Type all_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Comprehensive parallel constructor of ParticleSet.
     *
@@ -520,12 +552,15 @@ public:
     *  @param[in] field_names_        Array of field names.
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] tag_names_          Array of tag names.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                Ordering::Type coords_ordering, const Array<int> &field_vdims,
                const Array<Ordering::Type> &field_orderings,
                const Array<const char*> &field_names_, int num_tags,
-               const Array<const char*> &tag_names_);
+               const Array<const char*> &tag_names_,
+               bool use_device=false);
 
    /// Get the MPI communicator for this ParticleSet.
    MPI_Comm GetComm() const { return comm; };
@@ -545,6 +580,8 @@ public:
     *  @param[in] field_ordering   (Optional) Ordering::Type of the field.
     *  @param[in] field_name       (Optional) Name of the field.
     *
+    *  @note New fields inherit the current device mode of \ref coords.
+    *
     *  @return Index of the newly-added field.
     */
    int AddField(int vdim, Ordering::Type field_ordering=Ordering::byVDIM,
@@ -637,8 +674,8 @@ public:
 
    /** @brief Determine if GetParticleRef is valid.
     *
-    * If coordinates and all fields are ordered byVDIM, then returns true.
-    * Otherwise, false.
+    * Returns true when coordinates and all fields are ordered byVDIM and
+    * particle data is host-resident. Otherwise, false.
     */
    bool IsParticleRefValid() const;
 

fem/pfespace.cpp

@@ -349,6 +349,7 @@ void ParFiniteElementSpace::GetGroupComm(
       }
    }
 
+   bool have_sign_flips = false;
    if (g_ldof_sign)
    {
       g_ldof_sign->SetSize(GetNDofs());
@@ -428,6 +429,7 @@ void ParFiniteElementSpace::GetGroupComm(
                   if (g_ldof_sign)
                   {
                      (*g_ldof_sign)[dofs[l]] = -1;
+                     have_sign_flips = true;
                   }
                }
                else
@@ -466,6 +468,7 @@ void ParFiniteElementSpace::GetGroupComm(
                   if (g_ldof_sign)
                   {
                      (*g_ldof_sign)[dofs[l]] = -1;
+                     have_sign_flips = true;
                   }
                }
                else
@@ -504,6 +507,7 @@ void ParFiniteElementSpace::GetGroupComm(
                   if (g_ldof_sign)
                   {
                      (*g_ldof_sign)[dofs[l]] = -1;
+                     have_sign_flips = true;
                   }
                }
                else
@@ -527,12 +531,18 @@ void ParFiniteElementSpace::GetGroupComm(
       group_ldof.GetI()[gr+1] = group_ldof_counter;
    }
 
+   if (g_ldof_sign && have_sign_flips == false)
+   {
+      g_ldof_sign->DeleteAll();
+   }
+
    gc.Finalize();
 }
 
 void ParFiniteElementSpace::ApplyLDofSigns(Array<int> &dofs) const
 {
    MFEM_ASSERT(Conforming(), "wrong code path");
+   if (!HaveDofSigns()) { return; }
 
    for (int i = 0; i < dofs.Size(); i++)
    {
@@ -559,6 +569,24 @@ void ParFiniteElementSpace::ApplyLDofSigns(Table &el_dof) const
    ApplyLDofSigns(all_dofs);
 }
 
+void ParFiniteElementSpace::ApplyDofSigns(real_t *h_data) const
+{
+   if (!HaveDofSigns()) { return; }
+
+   const bool byvdim = (ordering == Ordering::byVDIM);
+   for (int i = 0; i < ndofs; i++)
+   {
+      if (ldof_sign[i] < 0)
+      {
+         for (int d = 0; d < vdim; d++)
+         {
+            const int idx = byvdim ? d+vdim*i : i+ndofs*d;
+            h_data[idx] = -h_data[idx];
+         }
+      }
+   }
+}
+
 void ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs,
                                            DofTransformation &doftrans) const
 {
@@ -1193,15 +1221,15 @@ void ParFiniteElementSpace::GetEssentialTrueDofsVar(const Array<int>
    MFEM_VERIFY(IsVariableOrder() && R,
                "GetEssentialTrueDofsVar is only for variable-order spaces");
 
-   true_ess_dofs.SetSize(R->Height(), Device::GetDeviceMemoryType());
+   true_ess_dofs.SetSize(R->Height());
+   true_ess_dofs.HostWrite();
+   true_ess_dofs = 0;
 
    const int ntdofs = tdof2ldof.Size();
    MFEM_VERIFY(vdim * ntdofs == R->NumRows() &&
                vdim * ntdofs == true_ess_dofs.Size(), "");
    MFEM_VERIFY(ldof_ltdof.Size() == ndofs && ess_dofs.Size() == vdim * ndofs, "");
 
-   true_ess_dofs = 0;
-
    const bool bynodes = (ordering == Ordering::byNODES);
    const int vdim_factor = bynodes ? 1 : vdim;
    const int num_true_dofs = R->NumRows() / vdim;

fem/pfespace.hpp

@@ -340,8 +340,20 @@ public:
 
    inline ParMesh *GetParMesh() const { return pmesh; }
 
-   int GetDofSign(int i)
-   { return NURBSext || Nonconforming() ? 1 : ldof_sign[VDofToDof(i)]; }
+   /** @brief Return true if the parallel FE space has DOFs with signs opposite
+       of the DOFs in the respective serial FE space. */
+   bool HaveDofSigns() const { return ldof_sign.Size() != 0; }
+
+   /** @brief Apply the DOF signs to the given host data @a h_data which must be
+       of size GetVSize() if HaveDofSigns() is true. If HaveDofSigns() is false,
+       this method is no-op and returns immediately. */
+   void ApplyDofSigns(real_t *h_data) const;
+
+   /** @brief Return -1 if the given (vector) DOF @a i has a sign opposite of
+       the DOF in the respecive serial FE space. Otherwise, return 1. */
+   int GetDofSign(int i) const
+   { return !HaveDofSigns() ? 1 : ldof_sign[VDofToDof(i)]; }
+
    HYPRE_BigInt *GetDofOffsets()     const { return dof_offsets; }
    HYPRE_BigInt *GetTrueDofOffsets() const { return tdof_offsets; }
    HYPRE_BigInt GlobalVSize() const

fem/pgridfunc.cpp

@@ -80,6 +80,8 @@ ParGridFunction::ParGridFunction(ParMesh *pmesh, std::istream &input)
                                     fes->GetOrdering());
    delete fes;
    fes = pfes;
+
+   pfes->ApplyDofSigns(HostReadWrite());
 }
 
 void ParGridFunction::Update()
@@ -1082,18 +1084,17 @@ real_t ParGridFunction::ComputeDGFaceJumpError(Coefficient *exsol,
 
 void ParGridFunction::Save(std::ostream &os) const
 {
-   real_t *data_  = const_cast<real_t*>(HostRead());
-   for (int i = 0; i < size; i++)
-   {
-      if (pfes->GetDofSign(i) < 0) { data_[i] = -data_[i]; }
-   }
+   // We use const_cast + HostRead (instead of HostReadWrite) because we only
+   // need to change the host data temporarily and this way we do not invalidate
+   // the data if it is on device. If we use HostReadWrite here, later calls to
+   // Read or ReadWrite will need to copy the data from host to device. With the
+   // approach used here, the host-to-device copy is avoided.
+   real_t *h_data = const_cast<real_t*>(HostRead());
+   pfes->ApplyDofSigns(h_data);
 
    GridFunction::Save(os);
 
-   for (int i = 0; i < size; i++)
-   {
-      if (pfes->GetDofSign(i) < 0) { data_[i] = -data_[i]; }
-   }
+   pfes->ApplyDofSigns(h_data);
 }
 
 void ParGridFunction::Save(const char *fname, int precision) const
@@ -1264,7 +1265,13 @@ void ParGridFunction::SaveAsOne(std::ostream &os) const
    int *nfdofs = new int[NRanks];
    int *nrdofs = new int[NRanks];
 
+   // We use const_cast + HostRead (instead of HostReadWrite) because we only
+   // need to change the host data temporarily and this way we do not invalidate
+   // the data if it is on device. If we use HostReadWrite here, later calls to
+   // Read or ReadWrite will need to copy the data from host to device. With the
+   // approach used here, the host-to-device copy is avoided.
    real_t * h_data = const_cast<real_t *>(this->HostRead());
+   pfes->ApplyDofSigns(h_data);  // temporarily flip the dof signs
 
    values[0] = h_data;
    nv[0]     = pfes -> GetVSize();
@@ -1371,6 +1378,8 @@ void ParGridFunction::SaveAsOne(std::ostream &os) const
       MPI_Send(h_data, nv[0], MPITypeMap<real_t>::mpi_type, 0, 460, MyComm);
    }
 
+   pfes->ApplyDofSigns(h_data);  // restore the original h_data
+
    delete [] values;
    delete [] nv;
    delete [] nvdofs;

fem/restriction.cpp

@@ -50,14 +50,21 @@ ElementRestriction::ElementRestriction(const FiniteElementSpace &f,
          const FiniteElement *fe = fes.GetFE(e);
          auto el_t = dynamic_cast<const TensorBasisElement*>(fe);
          auto el_n = dynamic_cast<const NodalFiniteElement*>(fe);
-         if (el_t || el_n) { continue; }
+         auto el_p = dynamic_cast<const H1Pos_TriangleElement*>(fe) ||
+                     dynamic_cast<const H1Pos_TetrahedronElement*>(fe);
+         if (el_t || el_n || el_p) { continue; }
          MFEM_ABORT("Finite element not suitable for lexicographic ordering");
       }
       const FiniteElement *fe = fes.GetTypicalFE();
       auto el_t = dynamic_cast<const TensorBasisElement*>(fe);
       auto el_n = dynamic_cast<const NodalFiniteElement*>(fe);
+      auto el_p_tri = dynamic_cast<const H1Pos_TriangleElement*>(fe);
+      auto el_p_tet = dynamic_cast<const H1Pos_TetrahedronElement*>(fe);
       const Array<int> &fe_dof_map =
-         (el_t) ? el_t->GetDofMap() : el_n->GetLexicographicOrdering();
+         el_n ? el_n->GetLexicographicOrdering() :
+         el_t ? el_t->GetDofMap() :
+         el_p_tri ? el_p_tri->GetDofMap() :
+         el_p_tet->GetDofMap();
       MFEM_VERIFY(fe_dof_map.Size() > 0, "invalid dof map");
       dof_map = fe_dof_map.HostRead();
    }

fem/tmop.cpp

@@ -3758,7 +3758,8 @@ void TMOP_Integrator::SetInitialMeshPos(const GridFunction *x0)
 TMOP_Integrator::~TMOP_Integrator()
 {
    delete lim_func;
-   delete adapt_lim_gf;
+   for (int i = 0; i < adapt_lim_gf.Size(); i++) { delete adapt_lim_gf[i]; }
+   for (int i = 0; i < adapt_lim_gf0.Size(); i++) { delete adapt_lim_gf0[i]; }
    delete surf_fit_gf;
    delete surf_fit_limiter;
    delete surf_fit_grad;
@@ -3797,36 +3798,129 @@ void TMOP_Integrator::EnableLimiting(const GridFunction &n0, Coefficient &w0,
 
 void TMOP_Integrator::EnableAdaptiveLimiting(const GridFunction &z0,
                                              Coefficient &coeff,
-                                             AdaptivityEvaluator &ae)
+                                             AdaptivityEvaluator &ae,
+                                             real_t delta_max)
 {
-   adapt_lim_gf0 = &z0;
-   delete adapt_lim_gf;
-   adapt_lim_gf   = new GridFunction(z0);
-   adapt_lim_coeff = &coeff;
-   adapt_lim_eval = &ae;
-
-   adapt_lim_eval->SetSerialMetaInfo(*z0.FESpace()->GetMesh(),
-                                     *z0.FESpace());
-   adapt_lim_eval->SetInitialField
-   (*adapt_lim_gf->FESpace()->GetMesh()->GetNodes(), *adapt_lim_gf);
+   Array<const GridFunction *> z0_arr(1);
+   Array<Coefficient *> c_arr(1);
+   Array<real_t> d_arr(1);
+   z0_arr[0] = &z0;
+   c_arr[0] = &coeff;
+   d_arr[0] = delta_max;
+   EnableAdaptiveLimiting(z0_arr, c_arr, ae, d_arr);
 }
 
 #ifdef MFEM_USE_MPI
 void TMOP_Integrator::EnableAdaptiveLimiting(const ParGridFunction &z0,
                                              Coefficient &coeff,
-                                             AdaptivityEvaluator &ae)
+                                             AdaptivityEvaluator &ae,
+                                             real_t delta_max)
 {
-   adapt_lim_gf0 = &z0;
-   adapt_lim_pgf0 = &z0;
-   delete adapt_lim_gf;
-   adapt_lim_gf   = new GridFunction(z0);
-   adapt_lim_coeff = &coeff;
-   adapt_lim_eval = &ae;
+   Array<const ParGridFunction *> z0_arr(1);
+   Array<Coefficient *> c_arr(1);
+   Array<real_t> d_arr(1);
+   z0_arr[0] = &z0;
+   c_arr[0] = &coeff;
+   d_arr[0] = delta_max;
+   EnableAdaptiveLimiting(z0_arr, c_arr, ae, d_arr);
+}
+#endif
 
-   adapt_lim_eval->SetParMetaInfo(*z0.ParFESpace()->GetParMesh(),
-                                  *z0.ParFESpace());
-   adapt_lim_eval->SetInitialField
-   (*adapt_lim_gf->FESpace()->GetMesh()->GetNodes(), *adapt_lim_gf);
+void TMOP_Integrator::
+EnableAdaptiveLimiting(const Array<const GridFunction *> &z0,
+                       const Array<Coefficient *> &coeff,
+                       AdaptivityEvaluator &ae, const Array<real_t> &delta_max)
+{
+   MFEM_VERIFY(z0.Size() > 0, "Requires at least one field.");
+   MFEM_VERIFY(z0.Size() == coeff.Size(), "Requires one Coefficient per field.");
+   MFEM_VERIFY(z0.Size() == delta_max.Size(), "Requires one delta_max per field.");
+   for (int i = 0; i < delta_max.Size(); i++)
+   {
+      MFEM_VERIFY(delta_max[i] > 0.0, "Requires delta_max > 0.0.");
+   }
+
+   // Verify compatibility of input fields.
+   const FiniteElementSpace *sfes = z0[0]->FESpace();
+   MFEM_VERIFY(sfes->GetVDim() == 1, "Expects scalar input GridFunctions.");
+   const int ndofs = sfes->GetVSize();
+   Mesh *mesh = sfes->GetMesh();
+   MFEM_VERIFY(mesh->GetNodes(), "EnableAdaptiveLimiting requires mesh Nodes.");
+   for (int i = 0; i < z0.Size(); i++)
+   {
+      MFEM_VERIFY(z0[i], "NULL GridFunction pointer.");
+      const FiniteElementSpace *fes_i = z0[i]->FESpace();
+      MFEM_VERIFY(fes_i->GetVDim() == 1, "Expects scalar input GridFunctions.");
+      MFEM_VERIFY(fes_i->GetVSize() == ndofs,
+                  "All fields must be on the same FE space.");
+      MFEM_VERIFY(fes_i->GetMesh() == mesh,
+                  "All fields must be on the same Mesh.");
+      MFEM_VERIFY(coeff[i], "NULL Coefficient pointer.");
+   }
+
+   // Delete previous adaptive limiting data.
+   for (int i = 0; i < adapt_lim_gf.Size(); i++) { delete adapt_lim_gf[i]; }
+   for (int i = 0; i < adapt_lim_gf0.Size(); i++) { delete adapt_lim_gf0[i]; }
+
+   adapt_lim_coeff.SetSize(coeff.Size());
+   for (int i = 0; i < coeff.Size(); i++) { adapt_lim_coeff[i] = coeff[i]; }
+   adapt_lim_eval = &ae;
+   adapt_lim_delta_max = delta_max;
+   adapt_lim_init_nodes = *mesh->GetNodes();
+
+   // Use one internal vector field (vdim = #fields) so remapping can be done in
+   // one call and incremental remap state (when provided by the evaluator) is
+   // preserved across TMOP iterations.
+   //
+   // Use Ordering::byNODES for the packed vector field so packing / unpacking
+   // can be done with contiguous sub-vector copies (device-friendly).
+   const int nal = z0.Size();
+   const Ordering::Type packed_ord = Ordering::byNODES;
+
+   // Setup the evaluator.
+#ifdef MFEM_USE_MPI
+   if (auto pfes = dynamic_cast<const ParFiniteElementSpace *>(sfes))
+   {
+      auto *pm = pfes->GetParMesh();
+      MFEM_VERIFY(pm, "Invalid ParMesh.");
+      ParFiniteElementSpace vfes(pm, pfes->FEColl(), nal, packed_ord);
+      adapt_lim_eval->SetParMetaInfo(*pm, vfes);
+   }
+   else
+#endif
+   {
+      FiniteElementSpace vfes(mesh, sfes->FEColl(), nal, packed_ord);
+      adapt_lim_eval->SetSerialMetaInfo(*mesh, vfes);
+   }
+
+   // Copy the initial fields; remapped fields are initialized to the same data.
+   adapt_lim_gf0.SetSize(z0.Size());
+   adapt_lim_gf.SetSize(z0.Size());
+   for (int i = 0; i < z0.Size(); i++)
+   {
+      adapt_lim_gf0[i] = new GridFunction(*z0[i]);
+      adapt_lim_gf[i]  = new GridFunction(*z0[i]);
+   }
+
+   // Initialize the evaluator with the packed vector field.
+   Vector init_field_vec;
+   init_field_vec.SetSize(nal * ndofs, *adapt_lim_gf0[0]);
+   init_field_vec.UseDevice(adapt_lim_gf0[0]->UseDevice());
+   for (int c = 0; c < nal; c++)
+   {
+      init_field_vec.SetVector(*adapt_lim_gf0[c], c * ndofs);
+   }
+   adapt_lim_eval->SetInitialField(adapt_lim_init_nodes, init_field_vec);
+}
+
+#ifdef MFEM_USE_MPI
+void TMOP_Integrator::
+EnableAdaptiveLimiting(const Array<const ParGridFunction *> &z0,
+                       const Array<Coefficient *> &coeff,
+                       AdaptivityEvaluator &ae, const Array<real_t> &delta_max)
+{
+   Array<const GridFunction *> z0_base(z0.Size());
+   for (int i = 0; i < z0.Size(); i++) { z0_base[i] = z0[i]; }
+   EnableAdaptiveLimiting(z0_base, coeff, ae, delta_max);
 }
 #endif
 
@@ -4147,26 +4241,61 @@ void TMOP_Integrator::GetSurfaceFittingErrors(const Vector &d_loc,
 
 void TMOP_Integrator::UpdateAfterMeshTopologyChange()
 {
-   if (adapt_lim_gf)
+   if (adapt_lim_gf.Size() > 0)
    {
-      adapt_lim_gf->Update();
-      adapt_lim_eval->SetSerialMetaInfo(*adapt_lim_gf->FESpace()->GetMesh(),
-                                        *adapt_lim_gf->FESpace());
-      adapt_lim_eval->SetInitialField
-      (*adapt_lim_gf->FESpace()->GetMesh()->GetNodes(), *adapt_lim_gf);
+      for (int i = 0; i < adapt_lim_gf0.Size(); i++) { adapt_lim_gf0[i]->Update(); }
+      for (int i = 0; i < adapt_lim_gf.Size(); i++) { adapt_lim_gf[i]->Update(); }
+
+      Mesh *mesh = adapt_lim_gf[0]->FESpace()->GetMesh();
+
+      // Same setup as in EnableAdaptiveLimiting().
+      const int nal = adapt_lim_coeff.Size();
+      const Ordering::Type packed_ord = Ordering::byNODES;
+      FiniteElementSpace vfes(mesh, adapt_lim_gf[0]->FESpace()->FEColl(), nal,
+                              packed_ord);
+      adapt_lim_eval->SetSerialMetaInfo(*mesh, vfes);
+
+      adapt_lim_init_nodes = *mesh->GetNodes();
+      const int ndofs = adapt_lim_gf0[0]->Size();
+      Vector init_field_vec;
+      init_field_vec.SetSize(nal * ndofs, *adapt_lim_gf0[0]);
+      init_field_vec.UseDevice(adapt_lim_gf0[0]->UseDevice());
+      for (int c = 0; c < nal; c++)
+      {
+         init_field_vec.SetVector(*adapt_lim_gf0[c], c * ndofs);
+      }
+      adapt_lim_eval->SetInitialField(adapt_lim_init_nodes, init_field_vec);
    }
 }
 
 #ifdef MFEM_USE_MPI
 void TMOP_Integrator::ParUpdateAfterMeshTopologyChange()
 {
-   if (adapt_lim_gf)
+   if (adapt_lim_gf.Size() > 0)
    {
-      adapt_lim_gf->Update();
-      adapt_lim_eval->SetParMetaInfo(*adapt_lim_pgf0->ParFESpace()->GetParMesh(),
-                                     *adapt_lim_pgf0->ParFESpace());
-      adapt_lim_eval->SetInitialField
-      (*adapt_lim_gf->FESpace()->GetMesh()->GetNodes(), *adapt_lim_gf);
+      for (int i = 0; i < adapt_lim_gf0.Size(); i++) { adapt_lim_gf0[i]->Update(); }
+      for (int i = 0; i < adapt_lim_gf.Size(); i++) { adapt_lim_gf[i]->Update(); }
+
+      // Same setup as in EnableAdaptiveLimiting().
+      auto *pfes = dynamic_cast<ParFiniteElementSpace *>(adapt_lim_gf[0]->FESpace());
+      MFEM_VERIFY(pfes, "internal error");
+      ParMesh *pmesh = pfes->GetParMesh();
+
+      const int nal = adapt_lim_coeff.Size();
+      const Ordering::Type packed_ord = Ordering::byNODES;
+      ParFiniteElementSpace vfes(pmesh, pfes->FEColl(), nal, packed_ord);
+      adapt_lim_eval->SetParMetaInfo(*pmesh, vfes);
+
+      adapt_lim_init_nodes = *pmesh->GetNodes();
+      const int ndofs = adapt_lim_gf0[0]->Size();
+      Vector init_field_vec;
+      init_field_vec.SetSize(nal * ndofs, *adapt_lim_gf0[0]);
+      init_field_vec.UseDevice(adapt_lim_gf0[0]->UseDevice());
+      for (int c = 0; c < nal; c++)
+      {
+         init_field_vec.SetVector(*adapt_lim_gf0[c], c * ndofs);
+      }
+      adapt_lim_eval->SetInitialField(adapt_lim_init_nodes, init_field_vec);
    }
 }
 #endif
@@ -4198,7 +4327,8 @@ real_t TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
    // No adaptive limiting / surface fitting terms if the function is called
    // as part of a FD derivative computation (because we include the exact
    // derivatives of these terms in FD computations).
-   const bool adaptive_limiting = (adapt_lim_gf && fd_call_flag == false);
+   const bool adaptive_limiting = (adapt_lim_gf.Size() > 0 &&
+                                   fd_call_flag == false);
    const bool surface_fit = (surf_fit_marker && fd_call_flag == false);
 
    DSh.SetSize(dof, dim);
@@ -4261,11 +4391,21 @@ real_t TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
    //       the physical coordinates (i.e. changes in 'elfun'), e.g. when the
    //       coefficient is a ConstantCoefficient or a GridFunctionCoefficient.
 
+   const int nal = adapt_lim_coeff.Size();
+   const int nqp = ir.GetNPoints();
    Vector adapt_lim_gf_q, adapt_lim_gf0_q;
    if (adaptive_limiting)
    {
-      adapt_lim_gf->GetValues(el_id, ir, adapt_lim_gf_q);
-      adapt_lim_gf0->GetValues(el_id, ir, adapt_lim_gf0_q);
+      adapt_lim_gf_q.SetSize(nal * nqp);
+      adapt_lim_gf0_q.SetSize(nal * nqp);
+      Vector zc, z0c;
+      for (int c = 0; c < nal; c++)
+      {
+         zc.MakeRef(adapt_lim_gf_q, c * nqp, nqp);
+         z0c.MakeRef(adapt_lim_gf0_q, c * nqp, nqp);
+         adapt_lim_gf[c]->GetValues(el_id, ir, zc);
+         adapt_lim_gf0[c]->GetValues(el_id, ir, z0c);
+      }
    }
 
    for (int i = 0; i < ir.GetNPoints(); i++)
@@ -4297,8 +4437,13 @@ real_t TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
       // Contribution from the adaptive limiting term.
       if (adaptive_limiting)
       {
-         const real_t diff = adapt_lim_gf_q(i) - adapt_lim_gf0_q(i);
-         val += adapt_lim_coeff->Eval(*Tpr, ip) * lim_normal * diff * diff;
+         for (int c = 0; c < nal; c++)
+         {
+            const int idx = c * nqp + i;
+            const real_t diff = (adapt_lim_gf_q(idx) - adapt_lim_gf0_q(idx)) /
+                                adapt_lim_delta_max[c];
+            val += adapt_lim_coeff[c]->Eval(*Tpr, ip) * lim_normal * diff * diff;
+         }
       }
 
       energy += weight * val;
@@ -4591,7 +4736,7 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
 
    // Define ref->physical transformation, when a Coefficient is specified.
    IsoparametricTransformation *Tpr = NULL;
-   if (metric_coeff || lim_coeff || adapt_lim_gf ||
+   if (metric_coeff || lim_coeff || adapt_lim_gf.Size() > 0 ||
        surf_fit_gf || surf_fit_pos || exact_action)
    {
       Tpr = new IsoparametricTransformation;
@@ -4689,7 +4834,7 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
       }
    }
 
-   if (adapt_lim_gf) { AssembleElemVecAdaptLim(el, *Tpr, ir, weights, PMatO); }
+   if (adapt_lim_gf.Size() > 0) { AssembleElemVecAdaptLim(el, *Tpr, ir, weights, PMatO); }
    if (surf_fit_gf || surf_fit_pos) { AssembleElemVecSurfFit(el, *Tpr, PMatO); }
 
    delete Tpr;
@@ -4763,7 +4908,8 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
 
    // Define ref->physical transformation, when a Coefficient is specified.
    IsoparametricTransformation *Tpr = NULL;
-   if (metric_coeff || lim_coeff || adapt_lim_gf || surf_fit_gf || surf_fit_pos)
+   if (metric_coeff || lim_coeff || adapt_lim_gf.Size() > 0 ||
+       surf_fit_gf || surf_fit_pos)
    {
       Tpr = new IsoparametricTransformation;
       Tpr->SetFE(&el);
@@ -4818,7 +4964,7 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
       }
    }
 
-   if (adapt_lim_gf) { AssembleElemGradAdaptLim(el, *Tpr, ir, weights, elmat); }
+   if (adapt_lim_gf.Size() > 0) { AssembleElemGradAdaptLim(el, *Tpr, ir, weights, elmat); }
    if (surf_fit_gf || surf_fit_pos) { AssembleElemGradSurfFit(el, *Tpr, elmat);}
 
    delete Tpr;
@@ -4831,32 +4977,42 @@ void TMOP_Integrator::AssembleElemVecAdaptLim(const FiniteElement &el,
                                               DenseMatrix &mat)
 {
    const int dof = el.GetDof(), dim = el.GetDim(), nqp = weights.Size();
-   Vector shape(dof), adapt_lim_gf_e, adapt_lim_gf_q, adapt_lim_gf0_q(nqp);
+   const int nal = adapt_lim_coeff.Size();
 
+   Vector shape(dof), adapt_lim_gf_e, adapt_lim_gf_q(nqp), adapt_lim_gf0_q(nqp);
    Array<int> dofs;
-   adapt_lim_gf->FESpace()->GetElementDofs(Tpr.ElementNo, dofs);
-   adapt_lim_gf->GetSubVector(dofs, adapt_lim_gf_e);
-   adapt_lim_gf->GetValues(Tpr.ElementNo, ir, adapt_lim_gf_q);
-   adapt_lim_gf0->GetValues(Tpr.ElementNo, ir, adapt_lim_gf0_q);
+   adapt_lim_gf[0]->FESpace()->GetElementDofs(Tpr.ElementNo, dofs);
 
    // Project the gradient of adapt_lim_gf in the same space.
    // The FE coefficients of the gradient go in adapt_lim_gf_grad_e.
-   DenseMatrix adapt_lim_gf_grad_e(dof, dim);
    DenseMatrix grad_phys; // This will be (dof x dim, dof).
    el.ProjectGrad(el, Tpr, grad_phys);
-   Vector grad_ptr(adapt_lim_gf_grad_e.GetData(), dof*dim);
-   grad_phys.Mult(adapt_lim_gf_e, grad_ptr);
 
    Vector adapt_lim_gf_grad_q(dim);
-
-   for (int q = 0; q < nqp; q++)
+   for (int c = 0; c < nal; c++)
    {
-      const IntegrationPoint &ip = ir.IntPoint(q);
-      el.CalcShape(ip, shape);
-      adapt_lim_gf_grad_e.MultTranspose(shape, adapt_lim_gf_grad_q);
-      adapt_lim_gf_grad_q *= 2.0 * (adapt_lim_gf_q(q) - adapt_lim_gf0_q(q));
-      adapt_lim_gf_grad_q *= weights(q) * lim_normal * adapt_lim_coeff->Eval(Tpr, ip);
-      AddMultVWt(shape, adapt_lim_gf_grad_q, mat);
+      const real_t delta2 = adapt_lim_delta_max[c] * adapt_lim_delta_max[c];
+      adapt_lim_gf[c]->GetSubVector(dofs, adapt_lim_gf_e);
+      adapt_lim_gf[c]->GetValues(Tpr.ElementNo, ir, adapt_lim_gf_q);
+      adapt_lim_gf0[c]->GetValues(Tpr.ElementNo, ir, adapt_lim_gf0_q);
+
+      DenseMatrix adapt_lim_gf_grad_e(dof, dim);
+      Vector grad_ptr(adapt_lim_gf_grad_e.GetData(), dof*dim);
+      grad_phys.Mult(adapt_lim_gf_e, grad_ptr);
+
+      for (int q = 0; q < nqp; q++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(q);
+         el.CalcShape(ip, shape);
+
+         adapt_lim_gf_grad_e.MultTranspose(shape, adapt_lim_gf_grad_q);
+         adapt_lim_gf_grad_q *= 2.0 * (adapt_lim_gf_q(q) - adapt_lim_gf0_q(q)) /
+                                delta2;
+         adapt_lim_gf_grad_q *=
+            weights(q) * lim_normal * adapt_lim_coeff[c]->Eval(Tpr, ip);
+
+         AddMultVWt(shape, adapt_lim_gf_grad_q, mat);
+      }
    }
 }
 
@@ -4867,55 +5023,66 @@ void TMOP_Integrator::AssembleElemGradAdaptLim(const FiniteElement &el,
                                                DenseMatrix &mat)
 {
    const int dof = el.GetDof(), dim = el.GetDim(), nqp = weights.Size();
-   Vector shape(dof), adapt_lim_gf_e, adapt_lim_gf_q, adapt_lim_gf0_q(nqp);
+   const int nal = adapt_lim_coeff.Size();
 
+   Vector shape(dof), adapt_lim_gf_e, adapt_lim_gf_q(nqp), adapt_lim_gf0_q(nqp);
    Array<int> dofs;
-   adapt_lim_gf->FESpace()->GetElementDofs(Tpr.ElementNo, dofs);
-   adapt_lim_gf->GetSubVector(dofs, adapt_lim_gf_e);
-   adapt_lim_gf->GetValues(Tpr.ElementNo, ir, adapt_lim_gf_q);
-   adapt_lim_gf0->GetValues(Tpr.ElementNo, ir, adapt_lim_gf0_q);
+   adapt_lim_gf[0]->FESpace()->GetElementDofs(Tpr.ElementNo, dofs);
 
    // Project the gradient of adapt_lim_gf in the same space.
    // The FE coefficients of the gradient go in adapt_lim_gf_grad_e.
-   DenseMatrix adapt_lim_gf_grad_e(dof, dim);
    DenseMatrix grad_phys; // This will be (dof x dim, dof).
    el.ProjectGrad(el, Tpr, grad_phys);
-   Vector grad_ptr(adapt_lim_gf_grad_e.GetData(), dof*dim);
-   grad_phys.Mult(adapt_lim_gf_e, grad_ptr);
-
-   // Project the gradient of each gradient of adapt_lim_gf in the same space.
-   // The FE coefficients of the second derivatives go in adapt_lim_gf_hess_e.
-   DenseMatrix adapt_lim_gf_hess_e(dof*dim, dim);
-   Mult(grad_phys, adapt_lim_gf_grad_e, adapt_lim_gf_hess_e);
-   // Reshape to be more convenient later (no change in the data).
-   adapt_lim_gf_hess_e.SetSize(dof, dim*dim);
-
    Vector adapt_lim_gf_grad_q(dim);
    DenseMatrix adapt_lim_gf_hess_q(dim, dim);
 
-   for (int q = 0; q < nqp; q++)
+   for (int c = 0; c < nal; c++)
    {
-      const IntegrationPoint &ip = ir.IntPoint(q);
-      el.CalcShape(ip, shape);
+      const real_t delta2 = adapt_lim_delta_max[c] * adapt_lim_delta_max[c];
+      adapt_lim_gf[c]->GetSubVector(dofs, adapt_lim_gf_e);
+      adapt_lim_gf[c]->GetValues(Tpr.ElementNo, ir, adapt_lim_gf_q);
+      adapt_lim_gf0[c]->GetValues(Tpr.ElementNo, ir, adapt_lim_gf0_q);
 
-      adapt_lim_gf_grad_e.MultTranspose(shape, adapt_lim_gf_grad_q);
-      Vector gg_ptr(adapt_lim_gf_hess_q.GetData(), dim*dim);
-      adapt_lim_gf_hess_e.MultTranspose(shape, gg_ptr);
+      DenseMatrix adapt_lim_gf_grad_e(dof, dim);
+      Vector grad_ptr(adapt_lim_gf_grad_e.GetData(), dof*dim);
+      grad_phys.Mult(adapt_lim_gf_e, grad_ptr);
 
-      const real_t w = weights(q) * lim_normal * adapt_lim_coeff->Eval(Tpr, ip);
-      for (int i = 0; i < dof * dim; i++)
+      // Project the gradient of each gradient of adapt_lim_gf in the same space.
+      // The FE coefficients of the second derivatives go in adapt_lim_gf_hess_e.
+      DenseMatrix adapt_lim_gf_hess_e(dof*dim, dim);
+      Mult(grad_phys, adapt_lim_gf_grad_e, adapt_lim_gf_hess_e);
+      // Reshape to be more convenient later (no change in the data).
+      adapt_lim_gf_hess_e.SetSize(dof, dim*dim);
+
+      for (int q = 0; q < nqp; q++)
       {
-         const int idof = i % dof, idim = i / dof;
-         for (int j = 0; j <= i; j++)
+         const IntegrationPoint &ip = ir.IntPoint(q);
+         el.CalcShape(ip, shape);
+
+         adapt_lim_gf_grad_e.MultTranspose(shape, adapt_lim_gf_grad_q);
+         Vector gg_ptr(adapt_lim_gf_hess_q.GetData(), dim*dim);
+         adapt_lim_gf_hess_e.MultTranspose(shape, gg_ptr);
+
+         const real_t coeff_q = adapt_lim_coeff[c]->Eval(Tpr, ip);
+         const real_t factor =
+            weights(q) * lim_normal * coeff_q * 2.0 /
+            delta2;
+
+         for (int i = 0; i < dof * dim; i++)
          {
-            const int jdof = j % dof, jdim = j / dof;
-            const real_t entry =
-               w * ( 2.0 * adapt_lim_gf_grad_q(idim) * shape(idof) *
-                     /* */ adapt_lim_gf_grad_q(jdim) * shape(jdof) +
-                     2.0 * (adapt_lim_gf_q(q) - adapt_lim_gf0_q(q)) *
-                     adapt_lim_gf_hess_q(idim, jdim) * shape(idof) * shape(jdof));
-            mat(i, j) += entry;
-            if (i != j) { mat(j, i) += entry; }
+            const int idof = i % dof, idim = i / dof;
+            for (int j = 0; j <= i; j++)
+            {
+               const int jdof = j % dof, jdim = j / dof;
+               const real_t entry =
+                  factor *
+                  (adapt_lim_gf_grad_q(idim) * shape(idof) *
+                   adapt_lim_gf_grad_q(jdim) * shape(jdof) +
+                   (adapt_lim_gf_q(q) - adapt_lim_gf0_q(q)) *
+                   adapt_lim_gf_hess_q(idim, jdim) * shape(idof) * shape(jdof));
+               mat(i, j) += entry;
+               if (i != j) { mat(j, i) += entry; }
+            }
          }
       }
    }
@@ -5188,7 +5355,7 @@ void TMOP_Integrator::AssembleElementVectorFD(const FiniteElement &el,
    fd_call_flag = false;
 
    // Contributions from adaptive limiting, surface fitting (exact derivatives).
-   if (adapt_lim_gf || surf_fit_gf || surf_fit_pos)
+   if (adapt_lim_gf.Size() > 0 || surf_fit_gf || surf_fit_pos)
    {
       const IntegrationRule &ir = ActionIntegrationRule(el);
       const int nqp = ir.GetNPoints();
@@ -5212,7 +5379,7 @@ void TMOP_Integrator::AssembleElementVectorFD(const FiniteElement &el,
       }
 
       PMatO.UseExternalData(elvect.GetData(), dof, dim);
-      if (adapt_lim_gf) { AssembleElemVecAdaptLim(el, Tpr, ir, weights, PMatO); }
+      if (adapt_lim_gf.Size() > 0) { AssembleElemVecAdaptLim(el, Tpr, ir, weights, PMatO); }
       if (surf_fit_gf || surf_fit_pos) { AssembleElemVecSurfFit(el, Tpr, PMatO); }
    }
 }
@@ -5298,7 +5465,7 @@ void TMOP_Integrator::AssembleElementGradFD(const FiniteElement &el,
    fd_call_flag = false;
 
    // Contributions from adaptive limiting.
-   if (adapt_lim_gf || surf_fit_gf || surf_fit_pos)
+   if (adapt_lim_gf.Size() > 0 || surf_fit_gf || surf_fit_pos)
    {
       const IntegrationRule &ir = GradientIntegrationRule(el);
       const int nqp = ir.GetNPoints();
@@ -5321,7 +5488,7 @@ void TMOP_Integrator::AssembleElementGradFD(const FiniteElement &el,
                       ir.IntPoint(q).weight;
       }
 
-      if (adapt_lim_gf) { AssembleElemGradAdaptLim(el, Tpr, ir, weights, elmat); }
+      if (adapt_lim_gf.Size() > 0) { AssembleElemGradAdaptLim(el, Tpr, ir, weights, elmat); }
       if (surf_fit_gf || surf_fit_pos) { AssembleElemGradSurfFit(el, Tpr, elmat); }
    }
 }
@@ -5668,9 +5835,46 @@ UpdateAfterMeshPositionChange(const Vector &d, const FiniteElementSpace &d_fes)
    }
 
    // Update adapt_lim_gf if adaptive limiting is enabled.
-   if (adapt_lim_gf)
+   if (adapt_lim_gf.Size() > 0)
    {
-      adapt_lim_eval->ComputeAtNewPosition(x_loc, *adapt_lim_gf, ordering);
+      // All adapt_lim_gf are remapped as a multi-component vector.
+      const int nal = adapt_lim_coeff.Size();
+      const int ndofs = adapt_lim_gf0[0]->Size();
+      Vector new_field_vec;
+      new_field_vec.SetSize(nal * ndofs, *adapt_lim_gf[0]);
+      new_field_vec.UseDevice(adapt_lim_gf[0]->UseDevice());
+      adapt_lim_eval->ComputeAtNewPosition(x_loc, new_field_vec, ordering);
+      for (int c = 0; c < nal; c++)
+      {
+         const real_t *src = new_field_vec.Read() + c * ndofs;
+         real_t *dst = adapt_lim_gf[c]->Write();
+         internal::device_copy(dst, src, ndofs);
+      }
+
+      if (PA.enabled)
+      {
+         PA.AL_grads_assembled = false;
+
+         // Step 1 of PA.ALFmF0 update: subtract the old ALF.
+         PA.ALFmF0 -= PA.ALF;
+
+         // Refresh PA.ALF from the updated adapt_lim_gf.
+         const ElementDofOrdering ord = ElementDofOrdering::LEXICOGRAPHIC;
+         const FiniteElementSpace *alfes = adapt_lim_gf[0]->FESpace();
+         const Operator *alf_R = alfes->GetElementRestriction(ord);
+
+         const int Esize = alf_R->Height();
+         Vector ALFc;
+         for (int c = 0; c < nal; c++)
+         {
+            MFEM_VERIFY(adapt_lim_gf[c]->Size() == ndofs, "internal error");
+            ALFc.MakeRef(PA.ALF, c * Esize, Esize);
+            alf_R->Mult(*adapt_lim_gf[c], ALFc);
+         }
+
+         // Step 2 of PA.ALFmF0 update: add the new ALF.
+         PA.ALFmF0 += PA.ALF;
+      }
    }
 
    // Update surf_fit_gf (and optionally its gradients) if surface
@@ -5883,7 +6087,7 @@ ComputeUntangleMetricQuantiles(const Vector &d, const FiniteElementSpace &fes)
       dynamic_cast<const ParFiniteElementSpace *>(&fes);
 #endif
 
-   if (wcuo && wcuo->GetBarrierType() ==
+   if (wcuo->GetBarrierType() ==
        TMOP_WorstCaseUntangleOptimizer_Metric::BarrierType::Shifted)
    {
       real_t min_detT = ComputeMinDetT(x_loc, fes);
@@ -5895,7 +6099,7 @@ ComputeUntangleMetricQuantiles(const Vector &d, const FiniteElementSpace &fes)
                        MPITypeMap<real_t>::mpi_type, MPI_MIN, pfes->GetComm());
       }
 #endif
-      if (wcuo) { wcuo->SetMinDetT(min_detT_all); }
+      wcuo->SetMinDetT(min_detT_all);
    }
 
    real_t max_muT = ComputeUntanglerMaxMuBarrier(x_loc, fes);
@@ -5931,6 +6135,48 @@ void TMOPComboIntegrator::EnableLimiting(const GridFunction &n0,
    for (int i = 1; i < tmopi.Size(); i++) { tmopi[i]->DisableLimiting(); }
 }
 
+void TMOPComboIntegrator::EnableAdaptiveLimiting(const GridFunction &z0,
+                                                 Coefficient &coeff,
+                                                 AdaptivityEvaluator &ae,
+                                                 real_t delta_max)
+{
+   MFEM_VERIFY(tmopi.Size() > 0, "No TMOP_Integrators were added.");
+
+   tmopi[0]->EnableAdaptiveLimiting(z0, coeff, ae, delta_max);
+}
+
+void TMOPComboIntegrator::
+EnableAdaptiveLimiting(const Array<const GridFunction *> &z0,
+                       const Array<Coefficient *> &coeff,
+                       AdaptivityEvaluator &ae, const Array<real_t> &delta_max)
+{
+   MFEM_VERIFY(tmopi.Size() > 0, "No TMOP_Integrators were added.");
+
+   tmopi[0]->EnableAdaptiveLimiting(z0, coeff, ae, delta_max);
+}
+
+#ifdef MFEM_USE_MPI
+void TMOPComboIntegrator::EnableAdaptiveLimiting(const ParGridFunction &z0,
+                                                 Coefficient &coeff,
+                                                 AdaptivityEvaluator &ae,
+                                                 real_t delta_max)
+{
+   MFEM_VERIFY(tmopi.Size() > 0, "No TMOP_Integrators were added.");
+
+   tmopi[0]->EnableAdaptiveLimiting(z0, coeff, ae, delta_max);
+}
+
+void TMOPComboIntegrator::
+EnableAdaptiveLimiting(const Array<const ParGridFunction *> &z0,
+                       const Array<Coefficient *> &coeff,
+                       AdaptivityEvaluator &ae, const Array<real_t> &delta_max)
+{
+   MFEM_VERIFY(tmopi.Size() > 0, "No TMOP_Integrators were added.");
+
+   tmopi[0]->EnableAdaptiveLimiting(z0, coeff, ae, delta_max);
+}
+#endif
+
 void TMOPComboIntegrator::SetLimitingNodes(const GridFunction &n0)
 {
    MFEM_VERIFY(tmopi.Size() > 0, "No TMOP_Integrators were added.");

fem/tmop.hpp

@@ -1440,6 +1440,7 @@ public:
    void Eval_d2(const Vector &x, const Vector &x0, real_t dist,
                 DenseMatrix &d2) const override
    {
+      MFEM_CONTRACT_VAR(x0);
       MFEM_ASSERT(x.Size() == x0.Size(), "Bad input.");
 
       d2.Diag(1.0 / (dist * dist), x.Size());
@@ -2037,13 +2038,17 @@ protected:
    real_t lim_normal;
 
    // Adaptive limiting.
-   const GridFunction *adapt_lim_gf0;    // Not owned.
-#ifdef MFEM_USE_MPI
-   const ParGridFunction *adapt_lim_pgf0;
-#endif
-   GridFunction *adapt_lim_gf;           // Owned. Updated by adapt_lim_eval.
-   Coefficient *adapt_lim_coeff;         // Not owned.
-   AdaptivityEvaluator *adapt_lim_eval;  // Not owned.
+   // Adaptive limiting fields. Each field adds a term to the integral:
+   //   int [ c_k (z_k(x) - z_k0(x0))^2 / delta_max_k^2 ] dx
+   // with one Coefficient per field. The fields z_k(x) are remapped from their
+   // initial values z_k0(x0) through a single AdaptivityEvaluator instance.
+   // All GridFunctions must use the same FE space.
+   Array<GridFunction *> adapt_lim_gf0;   // Owned. Initial fields z_k0(x0).
+   Array<GridFunction *> adapt_lim_gf;    // Owned. Remapped fields z_k(x).
+   Vector adapt_lim_init_nodes;           // Owned. Initial mesh nodes (ldofs).
+   Array<Coefficient *> adapt_lim_coeff;  // Not owned, one per field.
+   AdaptivityEvaluator *adapt_lim_eval;   // Not owned. Used for all fields.
+   Array<real_t> adapt_lim_delta_max;     // Per-field delta_max_k (>0).
 
    // Surface fitting.
    const Array<bool> *surf_fit_marker;      // Not owned. Nodes to fit.
@@ -2110,9 +2115,20 @@ protected:
    //     Updated by every call to PANonlinearFormExtension::GetGradient().
    // MC: Q-Vector for the metric Coefficient.
    //     Updated when the mesh nodes change.
+   // ALC:  Q-Vector for spatial weight used for the adaptive limiting term.
+   //       Updated when the mesh nodes change.
+   // ALF: E-Vector constructed using adaptive limiting GF zeta.
+   //      The zeta is remapped when the mesh nodes change.
+   // ALFmF0: E-Vector constructed using adaptive limiting GF zeta.
+   //         It stores difference zeta-zeta0, as all computations use this.
+   // ALFG: Q-Vector for gradient of ALF at quadrature points.
+   //       Updated by every call to PANonlinearFormExtension::GetGradient().
+   // ALFH: Q-Vector for Hessian of ALF at quadrature points.
+   //       Updated by every call to PANonlinearFormExtension::GetGradient().
    //
-   // maps:     Dof2Quad map for fes associated with the nodal coordinates.
-   // maps_lim: Dof2Quad map for fes associated with the limiting dist GridFunc.
+   // maps:       Dof2Quad map for fes associated with the nodal coordinates.
+   // maps_lim:   Dof2Quad map for fes associated with the limiting dist GF.
+   // maps_nodes: like maps, but the quad points are the FE nodes.
    //
    // Jtr_debug_grad
    //     We keep track if Jtr was set by AssembleGradPA() in Jtr_debug_grad: it
@@ -2128,12 +2144,16 @@ protected:
    {
       bool enabled;
       int dim, ne, nq;
+      int nal = 0; // number of adaptive limiting fields
       mutable DenseTensor Jtr;
       mutable bool Jtr_needs_update;
       mutable bool Jtr_debug_grad;
-      mutable Vector E, O, X0, XL, H, C0, LD, H0, MC;
+      mutable Vector E, O, X0, XL, H, C0, LD, H0, MC, ALC,
+              ALF, ALFmF0, ALFG, ALFH, ALD;
+      mutable bool AL_grads_assembled;
       const DofToQuad *maps;
       const DofToQuad *maps_lim = nullptr;
+      const DofToQuad *maps_nodes = nullptr;
       const GeometricFactors *geom;
       const FiniteElementSpace *fes;
       const IntegrationRule *ir;
@@ -2216,16 +2236,25 @@ protected:
       return EnergyIntegrationRule(el);
    }
 
+   //
    // Auxiliary PA methods
+   //
+
+   // PA quadrature data computation - metric term / limiting / adapt limiting.
    void AssembleGradPA_2D(const Vector&) const;
    void AssembleGradPA_3D(const Vector&) const;
    void AssembleGradPA_C0_2D(const Vector&) const;
    void AssembleGradPA_C0_3D(const Vector&) const;
+   void AssembleGradPA_AdaptLim_2D(const Vector&) const;
+   void AssembleGradPA_AdaptLim_3D(const Vector&) const;
 
+   // PA energy computation - metric term / limiting / adaptive limiting.
    void GetLocalStateEnergyPA_2D(const Vector &x, real_t &energy) const;
-   void GetLocalStateEnergyPA_3D(const Vector&, real_t &energy) const;
+   void GetLocalStateEnergyPA_3D(const Vector &x, real_t &energy) const;
    real_t GetLocalStateEnergyPA_C0_2D(const Vector&) const;
    real_t GetLocalStateEnergyPA_C0_3D(const Vector&) const;
+   real_t GetLocalStateEnergyPA_AdaptLim_2D() const;
+   real_t GetLocalStateEnergyPA_AdaptLim_3D() const;
    void GetLocalNormalizationEnergiesPA_2D(const Vector &x,
                                            real_t &met_energy,
                                            real_t &lim_energy) const;
@@ -2233,22 +2262,35 @@ protected:
                                            real_t &met_energy,
                                            real_t &lim_energy) const;
 
+   // PA gradient computation - metric term / limiting / adaptive limiting.
    void AddMultPA_2D(const Vector&, Vector&) const;
    void AddMultPA_3D(const Vector&, Vector&) const;
    void AddMultPA_C0_2D(const Vector&, Vector&) const;
    void AddMultPA_C0_3D(const Vector&, Vector&) const;
+   void AddMultPA_AdaptLim_2D(const Vector&, Vector&) const;
+   void AddMultPA_AdaptLim_3D(const Vector&, Vector&) const;
 
+   // PA Hessian AddMult - metric term / limiting / adaptive limiting.
    void AddMultGradPA_2D(const Vector&, Vector&) const;
    void AddMultGradPA_3D(const Vector&, Vector&) const;
    void AddMultGradPA_C0_2D(const Vector&, Vector&) const;
    void AddMultGradPA_C0_3D(const Vector&, Vector&) const;
+   void AddMultGradPA_AdaptLim_2D(const Vector&, Vector&) const;
+   void AddMultGradPA_AdaptLim_3D(const Vector&, Vector&) const;
 
+   // PA diagonal assemblies - metric term / limiting / adaptive limiting.
    void AssembleDiagonalPA_2D(Vector&) const;
    void AssembleDiagonalPA_3D(Vector&) const;
    void AssembleDiagonalPA_C0_2D(Vector&) const;
    void AssembleDiagonalPA_C0_3D(Vector&) const;
+   void AssembleDiagonalPA_AdaptLim_2D(Vector&) const;
+   void AssembleDiagonalPA_AdaptLim_3D(Vector&) const;
 
+   // Setup of PA data structures related to the limiting term.
    void AssemblePA_Limiting();
+   // Setup of PA data structures related to the adaptive limiting term.
+   void AssemblePA_AdaptLim();
+   // Compute reference->target Jacobians for all quad points.
    void ComputeAllElementTargets(const Vector &xe = Vector()) const;
    // Updates the Q-vectors for the metric_coeff and lim_coeff, based on the
    // new physical positions of the quadrature points.
@@ -2275,7 +2317,6 @@ public:
         integ_order(-1), metric_coeff(NULL), metric_normal(1.0),
         lim_nodes0(NULL), lim_coeff(NULL),
         lim_dist(NULL), lim_func(NULL), lim_normal(1.0),
-        adapt_lim_gf0(NULL), adapt_lim_gf(NULL), adapt_lim_coeff(NULL),
         adapt_lim_eval(NULL),
         surf_fit_marker(NULL), surf_fit_coeff(NULL),
         surf_fit_gf(NULL), surf_fit_eval(NULL),
@@ -2351,21 +2392,34 @@ public:
 
    /** @brief Restriction of the node positions to certain regions.
 
-       Adds the term $ \int c (z(x) - z_0(x_0))^2 $, where z0(x0) is a given
-       function on the starting mesh, and z(x) is its image on the new mesh.
-       Minimizing this term means that a node at x0 is allowed to move to a
-       position x(x0) only if z(x) ~ z0(x0).
+       Adds the term $ \int c (z(x) - z_0(x_0))^2 / delta_max^2 $, where z0(x0)
+       is a given function on the starting mesh, and z(x) is its image on the
+       new mesh. Minimizing this term means that a node at x0 is allowed to
+       move to a position x(x0) only if z(x) ~ z0(x0).
        Such term can be used for tangential mesh relaxation.
 
        @param[in] z0     Function z0 that controls the adaptive limiting.
        @param[in] coeff  Coefficient c for the above integral.
-       @param[in] ae     AdaptivityEvaluator to compute z(x) from z0(x0). */
+       @param[in] ae     AdaptivityEvaluator to compute z(x) from z0(x0).
+       @param[in] delta_max Controls the allowable deviation from z0.
+                            Smaller values activate the term faster. */
    void EnableAdaptiveLimiting(const GridFunction &z0, Coefficient &coeff,
-                               AdaptivityEvaluator &ae);
+                               AdaptivityEvaluator &ae, real_t delta_max = 1.0);
+   /// Multi-field adaptive limiting with per-field delta_max values. All
+   /// GridFunctions must be on the same FiniteElementSpace.
+   void EnableAdaptiveLimiting(const Array<const GridFunction *> &z0,
+                               const Array<Coefficient *> &coeff,
+                               AdaptivityEvaluator &ae,
+                               const Array<real_t> &delta_max);
 #ifdef MFEM_USE_MPI
    /// Parallel support for adaptive limiting.
    void EnableAdaptiveLimiting(const ParGridFunction &z0, Coefficient &coeff,
-                               AdaptivityEvaluator &ae);
+                               AdaptivityEvaluator &ae, real_t delta_max = 1.0);
+   /// Multi-field parallel adaptive limiting with per-field delta_max values.
+   void EnableAdaptiveLimiting(const Array<const ParGridFunction *> &z0,
+                               const Array<Coefficient *> &coeff,
+                               AdaptivityEvaluator &ae,
+                               const Array<real_t> &delta_max);
 #endif
 
    /** @brief Fitting of certain DOFs to the zero level set of a function.
@@ -2588,6 +2642,26 @@ public:
    void EnableLimiting(const GridFunction &n0, Coefficient &w0,
                        TMOP_LimiterFunction *lfunc = NULL);
 
+   /// Adds the adaptive limiting term to the first integrator.
+   void EnableAdaptiveLimiting(const GridFunction &z0, Coefficient &coeff,
+                               AdaptivityEvaluator &ae, real_t delta_max = 1.0);
+   /// Multi-field adaptive limiting with per-field delta_max values.
+   void EnableAdaptiveLimiting(const Array<const GridFunction *> &z0,
+                               const Array<Coefficient *> &coeff,
+                               AdaptivityEvaluator &ae,
+                               const Array<real_t> &delta_max);
+#ifdef MFEM_USE_MPI
+   /// Parallel support for adaptive limiting.
+   void EnableAdaptiveLimiting(const ParGridFunction &z0, Coefficient &coeff,
+                               AdaptivityEvaluator &ae, real_t delta_max = 1.0);
+   /// Multi-field parallel adaptive limiting with per-field delta_max values.
+   void EnableAdaptiveLimiting(const Array<const ParGridFunction *> &z0,
+                               const Array<Coefficient *> &coeff,
+                               AdaptivityEvaluator &ae,
+                               const Array<real_t> &delta_max);
+#endif
+
+
    /// Update the original/reference nodes used for limiting.
    void SetLimitingNodes(const GridFunction &n0);
 

fem/tmop/assemble/diag2_limit.cpp

@@ -11,7 +11,9 @@
 
 #include "../pa.hpp"
 #include "../../tmop.hpp"
+#include "../../kernels.hpp"
 #include "../../../general/forall.hpp"
+#include "../../../linalg/kernels.hpp"
 
 namespace mfem
 {
@@ -64,6 +66,92 @@ void TMOP_AssembleDiagPA_C0_2D(const int NE,
    });
 }
 
+// Diagonal assembly for AdaptLim limiting (2D)
+template <int MD1, int MQ1, int T_D1D = 0, int T_Q1D = 0>
+void TMOP_AssembleDiagPA_AdaptLim_2D(const real_t lim_normal,
+                                     const real_t adapt_lim_delta_max,
+                                     const bool const_coeff,
+                                     const DeviceTensor<3, const real_t> &ALC,
+                                     const int NE,
+                                     const DeviceTensor<5, const real_t> &J,
+                                     const ConstDeviceMatrix &W,
+                                     const real_t *b,
+                                     const DeviceTensor<4, const real_t> &ALF_grad,
+                                     const DeviceTensor<5, const real_t> &ALF_hess,
+                                     const ConstDeviceCube &ALFmF0,
+                                     DeviceTensor<4> &D,
+                                     const int d1d,
+                                     const int q1d)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const real_t normal_inv_delta_sq =
+      2.0 * lim_normal / (adapt_lim_delta_max * adapt_lim_delta_max);
+
+   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      MFEM_SHARED real_t sB[MD1][MQ1];
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+
+      // ALF and ALF0 values at quad points.
+      kernels::internal::s_regs2d_t<MQ1> alf_dof, alf_quad;
+      kernels::internal::LoadDofs2d(e, D1D, ALFmF0, alf_dof);
+      kernels::internal::Eval2d(D1D, Q1D, smem, sB, alf_dof, alf_quad);
+
+      MFEM_SHARED real_t qd[MQ1 * MD1];
+      DeviceTensor<2, real_t> QD(qd, MQ1, MD1);
+
+      for (int v = 0; v < 2; v++)
+      {
+         // Contract in y.
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               QD(qx, dy) = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const real_t By = sB[dy][qy];
+                  const real_t bb = By * By;
+
+                  const real_t *Jtr = &J(0, 0, qx, qy, e);
+                  const real_t detJtr = kernels::Det<2>(Jtr);
+                  const real_t weight = W(qx, qy) * detJtr;
+                  const real_t coeff = const_coeff ? ALC(0,0,0) : ALC(qx, qy, e);
+                  const real_t factor = weight * coeff * normal_inv_delta_sq;
+
+                  const real_t diff = alf_quad(qy, qx);
+                  const real_t grad_v = ALF_grad(v, qx, qy, e);
+                  const real_t hess_vv = ALF_hess(v, v, qx, qy, e);
+
+                  QD(qx, dy) += bb * factor * (grad_v*grad_v + diff * hess_vv);
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Contract in x.
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+            {
+               real_t d = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const real_t Bx = sB[dx][qx];
+                  const real_t bb = Bx * Bx;
+                  d += bb * QD(qx, dy);
+               }
+               D(dx, dy, v, e) += d;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
 MFEM_TMOP_MDQ_REGISTER(TMOPAssembleDiagCoef2D, TMOP_AssembleDiagPA_C0_2D);
 MFEM_TMOP_MDQ_SPECIALIZE(TMOPAssembleDiagCoef2D);
 
@@ -80,4 +168,52 @@ void TMOP_Integrator::AssembleDiagonalPA_C0_2D(Vector &diagonal) const
    TMOPAssembleDiagCoef2D::Run(d, q, NE, B, H0, D, d, q);
 }
 
+MFEM_TMOP_MDQ_REGISTER(TMOPAssembleDiagAdaptLim2D,
+                       TMOP_AssembleDiagPA_AdaptLim_2D);
+MFEM_TMOP_MDQ_SPECIALIZE(TMOPAssembleDiagAdaptLim2D);
+
+void TMOP_Integrator::AssembleDiagonalPA_AdaptLim_2D(Vector &diagonal) const
+{
+   const real_t ln = lim_normal;
+   const int NE = PA.ne, d = PA.maps->ndof, q = PA.maps->nqpt;
+
+   MFEM_VERIFY(d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto J = Reshape(PA.Jtr.Read(), 2, 2, q, q, NE);
+   const auto W = Reshape(PA.ir->GetWeights().Read(), q, q);
+   const auto *B = PA.maps->B.Read();
+   auto D = Reshape(diagonal.ReadWrite(), d, d, 2, NE);
+
+   const int nal = PA.nal;
+   MFEM_VERIFY(nal > 0, "internal error");
+   const real_t *ALD = PA.ALD.HostRead();
+
+   const int ndof_el = d * d;
+   const int nqp_el = q * q;
+   const int ALF_stride = ndof_el * NE;
+   const int ALFG_stride = 2 * nqp_el * NE;
+   const int ALFH_stride = 2 * 2 * nqp_el * NE;
+
+   const bool const_coeff = (PA.ALC.Size() == nal);
+   const int ALC_stride = const_coeff ? 1 : (nqp_el * NE);
+   const real_t *ALC_all = PA.ALC.Read();
+   const real_t *ALFmF0_all = PA.ALFmF0.Read();
+   const real_t *ALFG_all = PA.ALFG.Read();
+   const real_t *ALFH_all = PA.ALFH.Read();
+   for (int c = 0; c < nal; c++)
+   {
+      const real_t delta_max = ALD[c];
+      const auto ALC = const_coeff
+                       ? Reshape(ALC_all + c, 1, 1, 1)
+                       : Reshape(ALC_all + c * ALC_stride, q, q, NE);
+      const auto ALFmF0 = Reshape(ALFmF0_all + c * ALF_stride, d, d, NE);
+      const auto ALF_grad = Reshape(ALFG_all + c * ALFG_stride, 2, q, q, NE);
+      const auto ALF_hess = Reshape(ALFH_all + c * ALFH_stride, 2, 2, q, q, NE);
+
+      TMOPAssembleDiagAdaptLim2D::Run(d, q, ln, delta_max, const_coeff, ALC, NE,
+                                      J, W, B, ALF_grad, ALF_hess, ALFmF0, D, d, q);
+   }
+}
+
 } // namespace mfem

fem/tmop/assemble/diag3_limit.cpp

@@ -13,6 +13,7 @@
 #include "../../tmop.hpp"
 #include "../../kernels.hpp"
 #include "../../../general/forall.hpp"
+#include "../../../linalg/kernels.hpp"
 
 namespace mfem
 {
@@ -128,4 +129,181 @@ void TMOP_Integrator::AssembleDiagonalPA_C0_3D(Vector &diagonal) const
    TMOPAssembleDiagCoef3D::Run(d, q, NE, B, H0, D, d, q);
 }
 
+// Diagonal assembly for AdaptLim limiting (3D)
+template <int MD1, int MQ1, int T_D1D = 0, int T_Q1D = 0>
+void TMOP_AssembleDiagPA_AdaptLim_3D(const real_t lim_normal,
+                                     const real_t adapt_lim_delta_max,
+                                     const bool const_coeff,
+                                     const DeviceTensor<4, const real_t> &ALC,
+                                     const int NE,
+                                     const DeviceTensor<6, const real_t> &J,
+                                     const ConstDeviceCube &W,
+                                     const real_t *b,
+                                     const DeviceTensor<5, const real_t> &ALF_grad,
+                                     const DeviceTensor<6, const real_t> &ALF_hess,
+                                     const DeviceTensor<4, const real_t> &ALFmF0,
+                                     DeviceTensor<5> &D,
+                                     const int d1d,
+                                     const int q1d)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const real_t normal_inv_delta_sq =
+      2.0 * lim_normal / (adapt_lim_delta_max * adapt_lim_delta_max);
+
+   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      MFEM_SHARED real_t sB[MD1][MQ1];
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+
+      // ALF and ALF0 values at quad points.
+      kernels::internal::s_regs3d_t<MQ1> alf_dof, alf_quad;
+      kernels::internal::LoadDofs3d(e, D1D, ALFmF0, alf_dof);
+      kernels::internal::Eval3d(D1D, Q1D, smem, sB, alf_dof, alf_quad);
+
+      kernels::internal::s_regs3d_t<MQ1> r0, r1;
+
+      for (int v = 0; v < 3; ++v)
+      {
+         // Contract in z.
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     const real_t Bz = sB[dz][qz];
+                     const real_t bb = Bz * Bz;
+
+                     const real_t *Jtr = &J(0, 0, qx, qy, qz, e);
+                     const real_t detJtr = kernels::Det<3>(Jtr);
+                     const real_t weight = W(qx, qy, qz) * detJtr;
+                     const real_t coeff =
+                        const_coeff ? ALC(0, 0, 0, 0) : ALC(qx, qy, qz, e);
+                     const real_t factor = weight * coeff * normal_inv_delta_sq;
+
+                     const real_t diff = alf_quad(qz, qy, qx);
+                     const real_t grad_v = ALF_grad(v, qx, qy, qz, e);
+                     const real_t hess_vv = ALF_hess(v, v, qx, qy, qz, e);
+
+                     u += bb * factor * (grad_v * grad_v + diff * hess_vv);
+                  }
+                  r0[dz][qy][qx] = u;
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+
+         // Contract in y.
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  smem[qy][qx] = r0[dz][qy][qx];
+               }
+            }
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     const real_t By = sB[dy][qy];
+                     u += (By * By) * smem[qy][qx];
+                  }
+                  r1[dz][dy][qx] = u;
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+
+         // Contract in x.
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  smem[dy][qx] = r1[dz][dy][qx];
+               }
+            }
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const real_t Bx = sB[dx][qx];
+                     u += (Bx * Bx) * smem[dy][qx];
+                  }
+                  D(dx, dy, dz, v, e) += u;
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   });
+}
+
+MFEM_TMOP_MDQ_REGISTER(TMOPAssembleDiagAdaptLim3D,
+                       TMOP_AssembleDiagPA_AdaptLim_3D);
+MFEM_TMOP_MDQ_SPECIALIZE(TMOPAssembleDiagAdaptLim3D);
+
+void TMOP_Integrator::AssembleDiagonalPA_AdaptLim_3D(Vector &diagonal) const
+{
+   const real_t ln = lim_normal;
+   const int NE = PA.ne, d = PA.maps->ndof, q = PA.maps->nqpt;
+
+   MFEM_VERIFY(d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto J = Reshape(PA.Jtr.Read(), 3, 3, q, q, q, NE);
+   const auto W = Reshape(PA.ir->GetWeights().Read(), q, q, q);
+   const auto *B = PA.maps->B.Read();
+   auto D = Reshape(diagonal.ReadWrite(), d, d, d, 3, NE);
+
+   const int nal = PA.nal;
+   MFEM_VERIFY(nal > 0, "internal error");
+   const real_t *ALD = PA.ALD.HostRead();
+
+   const int ndof_el = d * d * d;
+   const int nqp_el = q * q * q;
+   const int ALF_stride = ndof_el * NE;
+   const int ALFG_stride = 3 * nqp_el * NE;
+   const int ALFH_stride = 3 * 3 * nqp_el * NE;
+
+   const bool const_coeff = (PA.ALC.Size() == nal);
+   const int ALC_stride = const_coeff ? 1 : (nqp_el * NE);
+   const real_t *ALC_all = PA.ALC.Read();
+   const real_t *ALFmF0_all = PA.ALFmF0.Read();
+   const real_t *ALFG_all = PA.ALFG.Read();
+   const real_t *ALFH_all = PA.ALFH.Read();
+   for (int c = 0; c < nal; c++)
+   {
+      const real_t delta_max = ALD[c];
+      const auto ALC = const_coeff
+                       ? Reshape(ALC_all + c, 1, 1, 1, 1)
+                       : Reshape(ALC_all + c * ALC_stride, q, q, q, NE);
+      const auto ALFmF0 = Reshape(ALFmF0_all + c * ALF_stride, d, d, d, NE);
+      const auto ALF_grad = Reshape(ALFG_all + c * ALFG_stride, 3, q, q, q, NE);
+      const auto ALF_hess = Reshape(ALFH_all + c * ALFH_stride, 3, 3, q, q, q, NE);
+
+      TMOPAssembleDiagAdaptLim3D::Run(d, q, ln, delta_max, const_coeff, ALC, NE,
+                                      J, W, B, ALF_grad, ALF_hess, ALFmF0, D, d, q);
+   }
+}
+
 } // namespace mfem

fem/tmop/assemble/grad2_limit.cpp

@@ -113,6 +113,178 @@ void TMOP_AssembleGradPA_C0_2D(const real_t lim_normal,
    });
 }
 
+// Assemble gradient and Hessian of ALF field at quad points for AdaptLim (2D).
+template <int MD1, int MQ1, int T_D1D = 0, int T_Q1D = 0>
+void TMOP_AssembleGradPA_AdaptLim_2D(const int NE,
+                                     const real_t *B_nodes,
+                                     const real_t *G_nodes,
+                                     const real_t *B,
+                                     const DeviceTensor<4, const real_t> &X,
+                                     const ConstDeviceCube &ALF,
+                                     DeviceTensor<4> &ALF_grad,
+                                     DeviceTensor<5> &ALF_hess,
+                                     const int d1d,
+                                     const int q1d)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      // MD1 x MD1 or MQ1 x MQ1 shared memory block.
+      MFEM_SHARED union { real_t d[MD1][MD1]; real_t q[MQ1][MQ1]; } smem;
+      MFEM_SHARED real_t sB_nodes[MD1][MD1], sG_nodes[MD1][MD1];
+      MFEM_SHARED real_t sB_q[MD1][MQ1];
+
+      kernels::internal::s_regs2d_t<MD1> grad_c;
+      kernels::internal::v_regs2d_t<2, MD1> hess_c;
+
+      // Maps nodes - nodes.
+      kernels::internal::LoadMatrix(D1D, D1D, B_nodes, sB_nodes);
+      kernels::internal::LoadMatrix(D1D, D1D, G_nodes, sG_nodes);
+      // Map nodes - quads.
+      kernels::internal::LoadMatrix(D1D, Q1D, B, sB_q);
+
+      // Compute the physical Jacobian at DOF nodes.
+      kernels::internal::vd_regs2d_t<2, 2, MD1> r_X, r_J;
+      kernels::internal::LoadDofs2d(e, D1D, X, r_X);
+      kernels::internal::Grad2d(D1D, D1D, smem.d, sB_nodes, sG_nodes, r_X, r_J);
+
+      // Compute the reference derivatives of ALF at DOF nodes.
+      kernels::internal::s_regs2d_t<MD1> alf_n, dalf_dx_n, dalf_dy_n;
+      kernels::internal::LoadDofs2d(e, D1D, ALF, alf_n);
+      kernels::internal::Contract2d<false, MD1>(D1D, D1D, smem.d,
+                                                sG_nodes, sB_nodes,
+                                                alf_n, dalf_dx_n);
+      kernels::internal::LoadDofs2d(e, D1D, ALF, alf_n);
+      kernels::internal::Contract2d<false, MD1>(D1D, D1D, smem.d,
+                                                sB_nodes, sG_nodes,
+                                                alf_n, dalf_dy_n);
+
+      // Interpolation workspaces.
+      kernels::internal::s_regs2d_t<MQ1> r0, r1;
+
+      // Precompute the inverse of the physical Jacobian.
+      kernels::internal::vd_regs2d_t<2, 2, MD1> Jpr_inv;
+      MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+         {
+            const real_t Jpr[4] =
+            {
+               r_J[0][0][dy][dx], r_J[1][0][dy][dx],
+               r_J[0][1][dy][dx], r_J[1][1][dy][dx]
+            };
+            real_t Jpri[4];
+            kernels::CalcInverse<2>(Jpr, Jpri);
+            Jpr_inv(0, 0, dx, dy) = Jpri[0];
+            Jpr_inv(1, 0, dx, dy) = Jpri[1];
+            Jpr_inv(0, 1, dx, dy) = Jpri[2];
+            Jpr_inv(1, 1, dx, dy) = Jpri[3];
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Compute/interpolate gradient and Hessian, one component at a time.
+      for (int c = 0; c < 2; c++)
+      {
+         kernels::internal::s_regs2d_t<MD1> rgrad_nodes, ddalf_dx_n, ddalf_dy_n;
+
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+            {
+               grad_c[dy][dx] =
+                  Jpr_inv(0, c, dx, dy) * dalf_dx_n[dy][dx] +
+                  Jpr_inv(1, c, dx, dy) * dalf_dy_n[dy][dx];
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Compute ALF_grad with intermediate workspaces
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+            {
+               r0[dy][dx] = grad_c[dy][dx];
+            }
+         }
+         MFEM_SYNC_THREAD;
+         kernels::internal::Eval2d<MQ1>(D1D, Q1D, smem.q, sB_q, r0, r1);
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               ALF_grad(c, qx, qy, e) = r1[qy][qx];
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Compute ddalf_dx_n.
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+            {
+               rgrad_nodes[dy][dx] = grad_c[dy][dx];
+            }
+         }
+         MFEM_SYNC_THREAD;
+         kernels::internal::Contract2d<false, MD1>(D1D, D1D, smem.d,
+                                                   sG_nodes, sB_nodes,
+                                                   rgrad_nodes, ddalf_dx_n);
+         // Compute ddalf_dy_n.
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+            {
+               rgrad_nodes[dy][dx] = grad_c[dy][dx];
+            }
+         }
+         MFEM_SYNC_THREAD;
+         kernels::internal::Contract2d<false, MD1>(D1D, D1D, smem.d,
+                                                   sB_nodes, sG_nodes,
+                                                   rgrad_nodes, ddalf_dy_n);
+         // Compute hess_c with ddalf_[dx, dy]_n.
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+            {
+               const real_t ddalf_dx = ddalf_dx_n[dy][dx];
+               const real_t ddalf_dy = ddalf_dy_n[dy][dx];
+               const real_t ddx = Jpr_inv(0, 0, dy, dx) * ddalf_dx +
+                                  Jpr_inv(1, 0, dy, dx) * ddalf_dy;
+               const real_t ddy = Jpr_inv(0, 1, dy, dx) * ddalf_dx +
+                                  Jpr_inv(1, 1, dy, dx) * ddalf_dy;
+               hess_c[0][dy][dx] = ddx;
+               hess_c[1][dy][dx] = ddy;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         for (int j = 0; j < 2; j++)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+               {
+                  r0[dy][dx] = hess_c[j][dy][dx];
+               }
+            }
+            MFEM_SYNC_THREAD;
+            kernels::internal::Eval2d<MQ1>(D1D, Q1D, smem.q, sB_q, r0, r1);
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  ALF_hess(c, j, qx, qy, e) = r1[qy][qx];
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   });
+}
+
 MFEM_TMOP_MDQ_REGISTER(TMOPAssembleGradCoef2D, TMOP_AssembleGradPA_C0_2D);
 MFEM_TMOP_MDQ_SPECIALIZE(TMOPAssembleGradCoef2D);
 
@@ -142,4 +314,44 @@ void TMOP_Integrator::AssembleGradPA_C0_2D(const Vector &x) const
                                J, W, b, bld, XL, X, H0, exp_lim, d, q);
 }
 
+MFEM_TMOP_MDQ_REGISTER(TMOPAssembleGradAdaptLim2D,
+                       TMOP_AssembleGradPA_AdaptLim_2D);
+MFEM_TMOP_MDQ_SPECIALIZE(TMOPAssembleGradAdaptLim2D);
+
+void TMOP_Integrator::AssembleGradPA_AdaptLim_2D(const Vector &x) const
+{
+   if (PA.AL_grads_assembled) { return; }
+
+   const int NE = PA.ne, d = PA.maps->ndof, q = PA.maps->nqpt;
+   MFEM_VERIFY(d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const int nal = PA.nal;
+   MFEM_VERIFY(nal > 0, "internal error");
+
+   const auto *B_nodes = PA.maps_nodes->B.Read(),
+               *G_nodes = PA.maps_nodes->G.Read();
+   const auto *B = PA.maps->B.Read();
+   const auto X = Reshape(x.Read(), d, d, 2, NE);
+   const int ndof_el = d * d;
+   const int nqp_el = q * q;
+   const int ALF_stride = ndof_el * NE;
+   const int ALFG_stride = 2 * nqp_el * NE;
+   const int ALFH_stride = 2 * 2 * nqp_el * NE;
+
+   const real_t *ALF_all = PA.ALF.Read();
+   real_t *ALFG_all = PA.ALFG.Write();
+   real_t *ALFH_all = PA.ALFH.Write();
+   for (int c = 0; c < nal; c++)
+   {
+      const auto ALF = Reshape(ALF_all + c * ALF_stride, d, d, NE);
+      auto ALF_grad = Reshape(ALFG_all + c * ALFG_stride, 2, q, q, NE);
+      auto ALF_hess = Reshape(ALFH_all + c * ALFH_stride, 2, 2, q, q, NE);
+
+      TMOPAssembleGradAdaptLim2D::Run(d, q, NE, B_nodes, G_nodes, B, X, ALF,
+                                      ALF_grad, ALF_hess, d, q);
+   }
+   PA.AL_grads_assembled = true;
+}
+
 } // namespace mfem

fem/tmop/assemble/grad3_limit.cpp

@@ -164,4 +164,269 @@ void TMOP_Integrator::AssembleGradPA_C0_3D(const Vector &x) const
                                J, W, b, bld, XL, X, H0, exp_lim, d, q);
 }
 
+// Assemble gradient and Hessian of ALF field at quadr points for AdaptLim (3D)
+template <int MD1, int MQ1, int T_D1D = 0, int T_Q1D = 0>
+void TMOP_AssembleGradPA_AdaptLim_3D(const int NE,
+                                     const real_t *B_nodes,
+                                     const real_t *G_nodes,
+                                     const real_t *B,
+                                     const DeviceTensor<5, const real_t> &X,
+                                     const DeviceTensor<4, const real_t> &ALF,
+                                     DeviceTensor<5> &ALF_grad,
+                                     DeviceTensor<6> &ALF_hess,
+                                     const int d1d,
+                                     const int q1d)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      // MD1 x MD1 or MQ1 x MQ1 shared memory block.
+      MFEM_SHARED union { real_t d[MD1][MD1]; real_t q[MQ1][MQ1]; } smem;
+      MFEM_SHARED real_t sB_nodes[MD1][MD1], sG_nodes[MD1][MD1];
+      MFEM_SHARED real_t sB_q[MD1][MQ1];
+
+      kernels::internal::s_regs3d_t<MD1> grad_c;
+      kernels::internal::v_regs3d_t<3, MD1> hess_c;
+
+      // Maps nodes - nodes.
+      kernels::internal::LoadMatrix(D1D, D1D, B_nodes, sB_nodes);
+      kernels::internal::LoadMatrix(D1D, D1D, G_nodes, sG_nodes);
+      // Map nodes - quads.
+      kernels::internal::LoadMatrix(D1D, Q1D, B, sB_q);
+
+      // Compute the physical Jacobian at DOF nodes.
+      kernels::internal::vd_regs3d_t<3, 3, MD1> r_X, r_J;
+      kernels::internal::LoadDofs3d(e, D1D, X, r_X);
+      kernels::internal::Grad3d(D1D, D1D, smem.d, sB_nodes, sG_nodes, r_X, r_J);
+
+      // Compute the reference derivatives of ALF at DOF nodes.
+      kernels::internal::s_regs3d_t<MD1> alf_n;
+      kernels::internal::s_regs3d_t<MD1> dalf_dxi_n, dalf_deta_n, dalf_dzeta_n;
+      kernels::internal::LoadDofs3d(e, D1D, ALF, alf_n);
+      kernels::internal::Contract3d<false, MD1>(D1D, D1D, smem.d,
+                                                sG_nodes, sB_nodes, sB_nodes,
+                                                alf_n, dalf_dxi_n);
+      kernels::internal::LoadDofs3d(e, D1D, ALF, alf_n);
+      kernels::internal::Contract3d<false, MD1>(D1D, D1D, smem.d,
+                                                sB_nodes, sG_nodes, sB_nodes,
+                                                alf_n, dalf_deta_n);
+      kernels::internal::LoadDofs3d(e, D1D, ALF, alf_n);
+      kernels::internal::Contract3d<false, MD1>(D1D, D1D, smem.d,
+                                                sB_nodes, sB_nodes, sG_nodes,
+                                                alf_n, dalf_dzeta_n);
+
+      // Interpolation workspaces.
+      kernels::internal::s_regs3d_t<MQ1> r0, r1;
+
+      // Compute/interpolate gradient and Hessian one vector component at a time.
+      for (int c = 0; c < 3; c++)
+      {
+         kernels::internal::s_regs3d_t<MD1> rgrad_nodes;
+         kernels::internal::s_regs3d_t<MD1> dd_dxi_n, dd_deta_n, dd_dzeta_n;
+
+         // Precompute the inverse of the physical Jacobian.
+         kernels::internal::vd_regs3d_t<3, 3, MD1> Jpr_inv;
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+               {
+                  const real_t Jpr[9] =
+                  {
+                     r_J[0][0][dz][dy][dx], r_J[1][0][dz][dy][dx], r_J[2][0][dz][dy][dx],
+                     r_J[0][1][dz][dy][dx], r_J[1][1][dz][dy][dx], r_J[2][1][dz][dy][dx],
+                     r_J[0][2][dz][dy][dx], r_J[1][2][dz][dy][dx], r_J[2][2][dz][dy][dx]
+                  };
+                  real_t Jpri[9];
+                  kernels::CalcInverse<3>(Jpr, Jpri);
+                  Jpr_inv(0, 0, dx, dy, dz) = Jpri[0];
+                  Jpr_inv(1, 0, dx, dy, dz) = Jpri[1];
+                  Jpr_inv(2, 0, dx, dy, dz) = Jpri[2];
+                  Jpr_inv(0, 1, dx, dy, dz) = Jpri[3];
+                  Jpr_inv(1, 1, dx, dy, dz) = Jpri[4];
+                  Jpr_inv(2, 1, dx, dy, dz) = Jpri[5];
+                  Jpr_inv(0, 2, dx, dy, dz) = Jpri[6];
+                  Jpr_inv(1, 2, dx, dy, dz) = Jpri[7];
+                  Jpr_inv(2, 2, dx, dy, dz) = Jpri[8];
+
+                  grad_c[dz][dy][dx] =
+                     Jpr_inv(0, c, dx, dy, dz) * dalf_dxi_n[dz][dy][dx] +
+                     Jpr_inv(1, c, dx, dy, dz) * dalf_deta_n[dz][dy][dx] +
+                     Jpr_inv(2, c, dx, dy, dz) * dalf_dzeta_n[dz][dy][dx];
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+
+         // Compute ALF_grad with intermediate workspaces.
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+               {
+                  r0[dz][dy][dx] = grad_c[dz][dy][dx];
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+         kernels::internal::Eval3d<MQ1>(D1D, Q1D, smem.q, sB_q, r0, r1);
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  ALF_grad(c, qx, qy, qz, e) = r1[qz][qy][qx];
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         // Compute dd_dxi_n.
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+               {
+                  rgrad_nodes[dz][dy][dx] = grad_c[dz][dy][dx];
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+         kernels::internal::Contract3d<false, MD1>(D1D, D1D, smem.d,
+                                                   sG_nodes, sB_nodes, sB_nodes,
+                                                   rgrad_nodes, dd_dxi_n);
+         // Compute dd_deta_n.
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+               {
+                  rgrad_nodes[dz][dy][dx] = grad_c[dz][dy][dx];
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+         kernels::internal::Contract3d<false, MD1>(D1D, D1D, smem.d,
+                                                   sB_nodes, sG_nodes, sB_nodes,
+                                                   rgrad_nodes, dd_deta_n);
+         // Compute dd_dzeta_n.
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+               {
+                  rgrad_nodes[dz][dy][dx] = grad_c[dz][dy][dx];
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+         kernels::internal::Contract3d<false, MD1>(D1D, D1D, smem.d,
+                                                   sB_nodes, sB_nodes, sG_nodes,
+                                                   rgrad_nodes, dd_dzeta_n);
+
+         // Compute hess_c with dd_[dxi, deta, dzeta]_n.
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+               {
+                  const real_t dd_dxi   = dd_dxi_n[dz][dy][dx];
+                  const real_t dd_deta  = dd_deta_n[dz][dy][dx];
+                  const real_t dd_dzeta = dd_dzeta_n[dz][dy][dx];
+                  const real_t ddx = Jpr_inv(0, 0, dx, dy, dz) * dd_dxi +
+                                     Jpr_inv(1, 0, dx, dy, dz) * dd_deta +
+                                     Jpr_inv(2, 0, dx, dy, dz) * dd_dzeta;
+                  const real_t ddy = Jpr_inv(0, 1, dx, dy, dz) * dd_dxi +
+                                     Jpr_inv(1, 1, dx, dy, dz) * dd_deta +
+                                     Jpr_inv(2, 1, dx, dy, dz) * dd_dzeta;
+                  const real_t ddz = Jpr_inv(0, 2, dx, dy, dz) * dd_dxi +
+                                     Jpr_inv(1, 2, dx, dy, dz) * dd_deta +
+                                     Jpr_inv(2, 2, dx, dy, dz) * dd_dzeta;
+                  hess_c[0][dz][dy][dx] = ddx;
+                  hess_c[1][dz][dy][dx] = ddy;
+                  hess_c[2][dz][dy][dx] = ddz;
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+
+         for (int j = 0; j < 3; j++)
+         {
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
+               {
+                  MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
+                  {
+                     r0[dz][dy][dx] = hess_c[j][dz][dy][dx];
+                  }
+               }
+               MFEM_SYNC_THREAD;
+            }
+            kernels::internal::Eval3d<MQ1>(D1D, Q1D, smem.q, sB_q, r0, r1);
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+                  {
+                     ALF_hess(c, j, qx, qy, qz, e) = r1[qz][qy][qx];
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   });
+}
+
+MFEM_TMOP_MDQ_REGISTER(TMOPAssembleGradAdaptLim3D,
+                       TMOP_AssembleGradPA_AdaptLim_3D);
+MFEM_TMOP_MDQ_SPECIALIZE(TMOPAssembleGradAdaptLim3D);
+
+void TMOP_Integrator::AssembleGradPA_AdaptLim_3D(const Vector &x) const
+{
+   if (PA.AL_grads_assembled) { return; }
+
+   const int NE = PA.ne, d = PA.maps->ndof, q = PA.maps->nqpt;
+   MFEM_VERIFY(d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const int nal = PA.nal;
+   MFEM_VERIFY(nal > 0, "internal error");
+
+   const auto *B_nodes = PA.maps_nodes->B.Read(),
+               *G_nodes = PA.maps_nodes->G.Read();
+   const auto *B = PA.maps->B.Read();
+   const auto X = Reshape(x.Read(), d, d, d, 3, NE);
+   const int ndof_el = d * d * d;
+   const int nqp_el = q * q * q;
+   const int ALF_stride = ndof_el * NE;
+   const int ALFG_stride = 3 * nqp_el * NE;
+   const int ALFH_stride = 3 * 3 * nqp_el * NE;
+
+   const real_t *ALF_all = PA.ALF.Read();
+   real_t *ALFG_all = PA.ALFG.Write();
+   real_t *ALFH_all = PA.ALFH.Write();
+   for (int c = 0; c < nal; c++)
+   {
+      const auto ALF = Reshape(ALF_all + c * ALF_stride, d, d, d, NE);
+      auto ALF_grad = Reshape(ALFG_all + c * ALFG_stride, 3, q, q, q, NE);
+      auto ALF_hess = Reshape(ALFH_all + c * ALFH_stride, 3, 3, q, q, q, NE);
+
+      TMOPAssembleGradAdaptLim3D::Run(d, q, NE, B_nodes, G_nodes, B, X, ALF,
+                                      ALF_grad, ALF_hess, d, q);
+   }
+   PA.AL_grads_assembled = true;
+}
+
 } // namespace mfem

fem/tmop/mult/grad2_limit.cpp

@@ -67,6 +67,96 @@ void TMOP_AddMultGradPA_C0_2D(const int NE,
    });
 }
 
+// Gradient action for AdaptLim limiting (2D)
+template <int MD1, int MQ1, int T_D1D = 0, int T_Q1D = 0>
+void TMOP_AddMultGradPA_AdaptLim_2D(const real_t lim_normal,
+                                    const real_t adapt_lim_delta_max,
+                                    const bool const_coeff,
+                                    const DeviceTensor<3, const real_t> &ALC,
+                                    const int NE,
+                                    const DeviceTensor<5, const real_t> &J,
+                                    const ConstDeviceMatrix &W,
+                                    const real_t *b,
+                                    const DeviceTensor<4, const real_t> &R,
+                                    const DeviceTensor<4, const real_t> &ALF_grad,
+                                    const DeviceTensor<5, const real_t> &ALF_hess,
+                                    const ConstDeviceCube &ALFmF0,
+                                    DeviceTensor<4> &Y,
+                                    const int d1d,
+                                    const int q1d)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const real_t normal_inv_delta_sq =
+      2.0 * lim_normal / (adapt_lim_delta_max * adapt_lim_delta_max);
+
+   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      MFEM_SHARED real_t sB[MD1][MQ1];
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+
+      // ALF and ALF0 values at quad points.
+      kernels::internal::s_regs2d_t<MQ1> alf_dof, alf_quad;
+      kernels::internal::LoadDofs2d(e, D1D, ALFmF0, alf_dof);
+      kernels::internal::Eval2d(D1D, Q1D, smem, sB, alf_dof, alf_quad);
+
+      // Input vector R at quad points.
+      kernels::internal::v_regs2d_t<2,MQ1> r_R_dof, r_R_quad;
+      kernels::internal::LoadDofs2d(e, D1D, R, r_R_dof);
+      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r_R_dof, r_R_quad);
+
+      kernels::internal::v_regs2d_t<2,MQ1> r00, r01;
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            const real_t *Jtr = &J(0, 0, qx, qy, e);
+            const real_t detJtr = kernels::Det<2>(Jtr);
+            const real_t weight = W(qx, qy) * detJtr;
+            const real_t diff = alf_quad(qy, qx);
+
+            // Load precomputed gradient at this quad point.
+            real_t grad_alf[2] =
+            {
+               ALF_grad(0, qx, qy, e),
+               ALF_grad(1, qx, qy, e)
+            };
+
+            // Load precomputed Hessian at this quad point.
+            real_t hess_alf[2][2];
+            for (int i = 0; i < 2; i++)
+            {
+               for (int j = 0; j < 2; j++)
+               {
+                  hess_alf[i][j] = ALF_hess(i, j, qx, qy, e);
+               }
+            }
+
+            // Get input vector at this quad point.
+            const real_t R_q[2] = { r_R_quad(0, qy, qx), r_R_quad(1, qy, qx) };
+
+            // Hessian action:
+            // H = factor * (grad x grad + (gf - gf0) * hess)
+            const real_t coeff = const_coeff ? ALC(0, 0, 0) : ALC(qx, qy, e);
+            const real_t factor = weight * coeff * normal_inv_delta_sq;
+            const real_t grad_dot_R = grad_alf[0] * R_q[0] + grad_alf[1] * R_q[1];
+            real_t hess_R[2];
+            hess_R[0] = hess_alf[0][0] * R_q[0] + hess_alf[0][1] * R_q[1];
+            hess_R[1] = hess_alf[1][0] * R_q[0] + hess_alf[1][1] * R_q[1];
+
+            r00(0, qy, qx) = factor * (grad_alf[0] * grad_dot_R + diff * hess_R[0]);
+            r00(1, qy, qx) = factor * (grad_alf[1] * grad_dot_R + diff * hess_R[1]);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, r00, r01);
+      kernels::internal::WriteDofs2d(e, D1D, r01, Y);
+   });
+}
+
 MFEM_TMOP_MDQ_REGISTER(TMOPMultGradCoefKernels, TMOP_AddMultGradPA_C0_2D);
 MFEM_TMOP_MDQ_SPECIALIZE(TMOPMultGradCoefKernels);
 
@@ -85,4 +175,53 @@ void TMOP_Integrator::AddMultGradPA_C0_2D(const Vector &R, Vector &C) const
    TMOPMultGradCoefKernels::Run(d, q, NE, b, H0, X, Y, d, q);
 }
 
+MFEM_TMOP_MDQ_REGISTER(TMOPMultGradAdaptLim, TMOP_AddMultGradPA_AdaptLim_2D);
+MFEM_TMOP_MDQ_SPECIALIZE(TMOPMultGradAdaptLim);
+
+void TMOP_Integrator::AddMultGradPA_AdaptLim_2D(const Vector &R,
+                                                Vector &C) const
+{
+   const real_t ln = lim_normal;
+   const int NE = PA.ne, d = PA.maps->ndof, q = PA.maps->nqpt;
+
+   MFEM_VERIFY(d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto J = Reshape(PA.Jtr.Read(), 2, 2, q, q, NE);
+   const auto *B = PA.maps->B.Read();
+   const auto W = Reshape(PA.ir->GetWeights().Read(), q, q);
+   const auto RR = Reshape(R.Read(), d, d, 2, NE);
+   auto Y = Reshape(C.ReadWrite(), d, d, 2, NE);
+
+   const int nal = PA.nal;
+   MFEM_VERIFY(nal > 0, "internal error");
+   const real_t *ALD = PA.ALD.HostRead();
+
+   const int ndof_el = d * d;
+   const int nqp_el = q * q;
+   const int ALF_stride = ndof_el * NE;
+   const int ALFG_stride = 2 * nqp_el * NE;
+   const int ALFH_stride = 2 * 2 * nqp_el * NE;
+
+   const bool const_coeff = (PA.ALC.Size() == nal);
+   const int ALC_stride = const_coeff ? 1 : (nqp_el * NE);
+   const real_t *ALC_all = PA.ALC.Read();
+   const real_t *ALFmF0_all = PA.ALFmF0.Read();
+   const real_t *ALFG_all = PA.ALFG.Read();
+   const real_t *ALFH_all = PA.ALFH.Read();
+   for (int c = 0; c < nal; c++)
+   {
+      const real_t delta_max = ALD[c];
+      const auto ALC = const_coeff
+                       ? Reshape(ALC_all + c, 1, 1, 1)
+                       : Reshape(ALC_all + c * ALC_stride, q, q, NE);
+      const auto ALFmF0 = Reshape(ALFmF0_all + c * ALF_stride, d, d, NE);
+      const auto ALF_grad = Reshape(ALFG_all + c * ALFG_stride, 2, q, q, NE);
+      const auto ALF_hess = Reshape(ALFH_all + c * ALFH_stride, 2, 2, q, q, NE);
+
+      TMOPMultGradAdaptLim::Run(d, q, ln, delta_max, const_coeff, ALC, NE, J, W, B,
+                                RR, ALF_grad, ALF_hess, ALFmF0, Y, d, q);
+   }
+}
+
 } // namespace mfem

fem/tmop/mult/grad3_limit.cpp

@@ -98,4 +98,159 @@ void TMOP_Integrator::AddMultGradPA_C0_3D(const Vector &R, Vector &C) const
    TMOPMultGradCoefKernels3D::Run(d, q, NE, b, H0, X, Y, d, q);
 }
 
+// Gradient action for AdaptLim limiting (3D)
+template <int MD1, int MQ1, int T_D1D = 0, int T_Q1D = 0>
+void TMOP_AddMultGradPA_AdaptLim_3D(const real_t lim_normal,
+                                    const real_t adapt_lim_delta_max,
+                                    const bool const_coeff,
+                                    const DeviceTensor<4, const real_t> &ALC,
+                                    const int NE,
+                                    const DeviceTensor<6, const real_t> &J,
+                                    const ConstDeviceCube &W,
+                                    const real_t *b,
+                                    const DeviceTensor<5, const real_t> &R,
+                                    const DeviceTensor<5, const real_t> &ALF_grad,
+                                    const DeviceTensor<6, const real_t> &ALF_hess,
+                                    const DeviceTensor<4, const real_t> &ALFmF0,
+                                    DeviceTensor<5> &Y,
+                                    const int d1d,
+                                    const int q1d)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const real_t normal_inv_delta_sq =
+      2.0 * lim_normal / (adapt_lim_delta_max * adapt_lim_delta_max);
+
+   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      MFEM_SHARED real_t sB[MD1][MQ1];
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+
+      // ALF and ALF0 values at quad points.
+      kernels::internal::s_regs3d_t<MQ1> alf_dof, alf_quad;
+      kernels::internal::LoadDofs3d(e, D1D, ALFmF0, alf_dof);
+      kernels::internal::Eval3d(D1D, Q1D, smem, sB, alf_dof, alf_quad);
+
+      // Input vector R at quad points.
+      kernels::internal::v_regs3d_t<3, MQ1> r_R_dof, r_R_quad;
+      kernels::internal::LoadDofs3d(e, D1D, R, r_R_dof);
+      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r_R_dof, r_R_quad);
+
+      kernels::internal::v_regs3d_t<3, MQ1> r00, r01;
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+            {
+               const real_t *Jtr = &J(0, 0, qx, qy, qz, e);
+               const real_t detJtr = kernels::Det<3>(Jtr);
+               const real_t weight = W(qx, qy, qz) * detJtr;
+               const real_t diff = alf_quad(qz, qy, qx);
+
+               // Load precomputed gradient at this quad point.
+               const real_t grad_alf[3] =
+               {
+                  ALF_grad(0, qx, qy, qz, e),
+                  ALF_grad(1, qx, qy, qz, e),
+                  ALF_grad(2, qx, qy, qz, e)
+               };
+
+               // Get input vector at this quad point.
+               const real_t R_q[3] =
+               {
+                  r_R_quad(0, qz, qy, qx),
+                  r_R_quad(1, qz, qy, qx),
+                  r_R_quad(2, qz, qy, qx)
+               };
+
+               // Hessian action:
+               // H = factor * (grad x grad + (gf - gf0) * hess)
+               const real_t coeff =
+                  const_coeff ? ALC(0, 0, 0, 0) : ALC(qx, qy, qz, e);
+               const real_t factor = weight * coeff * normal_inv_delta_sq;
+               const real_t grad_dot_R = grad_alf[0] * R_q[0] +
+                                         grad_alf[1] * R_q[1] +
+                                         grad_alf[2] * R_q[2];
+               real_t hess_R[3];
+               hess_R[0] =
+                  ALF_hess(0, 0, qx, qy, qz, e) * R_q[0] +
+                  ALF_hess(0, 1, qx, qy, qz, e) * R_q[1] +
+                  ALF_hess(0, 2, qx, qy, qz, e) * R_q[2];
+               hess_R[1] =
+                  ALF_hess(1, 0, qx, qy, qz, e) * R_q[0] +
+                  ALF_hess(1, 1, qx, qy, qz, e) * R_q[1] +
+                  ALF_hess(1, 2, qx, qy, qz, e) * R_q[2];
+               hess_R[2] =
+                  ALF_hess(2, 0, qx, qy, qz, e) * R_q[0] +
+                  ALF_hess(2, 1, qx, qy, qz, e) * R_q[1] +
+                  ALF_hess(2, 2, qx, qy, qz, e) * R_q[2];
+
+               r00(0, qz, qy, qx) = factor * (grad_alf[0] * grad_dot_R +
+                                              diff * hess_R[0]);
+               r00(1, qz, qy, qx) = factor * (grad_alf[1] * grad_dot_R +
+                                              diff * hess_R[1]);
+               r00(2, qz, qy, qx) = factor * (grad_alf[2] * grad_dot_R +
+                                              diff * hess_R[2]);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, r00, r01);
+      kernels::internal::WriteDofs3d(e, D1D, r01, Y);
+   });
+}
+
+MFEM_TMOP_MDQ_REGISTER(TMOPMultGradAdaptLim3D, TMOP_AddMultGradPA_AdaptLim_3D);
+MFEM_TMOP_MDQ_SPECIALIZE(TMOPMultGradAdaptLim3D);
+
+void TMOP_Integrator::AddMultGradPA_AdaptLim_3D(const Vector &R,
+                                                Vector &C) const
+{
+   const real_t ln = lim_normal;
+   const int NE = PA.ne, d = PA.maps->ndof, q = PA.maps->nqpt;
+
+   MFEM_VERIFY(d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto J = Reshape(PA.Jtr.Read(), 3, 3, q, q, q, NE);
+   const auto *B = PA.maps->B.Read();
+   const auto W = Reshape(PA.ir->GetWeights().Read(), q, q, q);
+   const auto RR = Reshape(R.Read(), d, d, d, 3, NE);
+   auto Y = Reshape(C.ReadWrite(), d, d, d, 3, NE);
+
+   const int nal = PA.nal;
+   MFEM_VERIFY(nal > 0, "internal error");
+   const real_t *ALD = PA.ALD.HostRead();
+
+   const int ndof_el = d * d * d;
+   const int nqp_el = q * q * q;
+   const int ALF_stride = ndof_el * NE;
+   const int ALFG_stride = 3 * nqp_el * NE;
+   const int ALFH_stride = 3 * 3 * nqp_el * NE;
+
+   const bool const_coeff = (PA.ALC.Size() == nal);
+   const int ALC_stride = const_coeff ? 1 : (nqp_el * NE);
+   const real_t *ALC_all = PA.ALC.Read();
+   const real_t *ALFmF0_all = PA.ALFmF0.Read();
+   const real_t *ALFG_all = PA.ALFG.Read();
+   const real_t *ALFH_all = PA.ALFH.Read();
+   for (int c = 0; c < nal; c++)
+   {
+      const real_t delta_max = ALD[c];
+      const auto ALC = const_coeff
+                       ? Reshape(ALC_all + c, 1, 1, 1, 1)
+                       : Reshape(ALC_all + c * ALC_stride, q, q, q, NE);
+      const auto ALFmF0 = Reshape(ALFmF0_all + c * ALF_stride, d, d, d, NE);
+      const auto ALF_grad = Reshape(ALFG_all + c * ALFG_stride, 3, q, q, q, NE);
+      const auto ALF_hess = Reshape(ALFH_all + c * ALFH_stride, 3, 3, q, q, q, NE);
+
+      TMOPMultGradAdaptLim3D::Run(d, q, ln, delta_max, const_coeff, ALC, NE, J,
+                                  W, B, RR, ALF_grad, ALF_hess, ALFmF0, Y, d, q);
+   }
+}
+
 } // namespace mfem

fem/tmop/mult/mult2_limit.cpp

@@ -108,6 +108,64 @@ void TMOP_AddMultPA_C0_2D(const real_t lim_normal,
    });
 }
 
+template <int MD1, int MQ1, int T_D1D = 0, int T_Q1D = 0>
+void TMOP_AddMultPA_AdaptLim_2D(const real_t lim_normal,
+                                const real_t adapt_lim_delta_max,
+                                const bool const_coeff,
+                                const DeviceTensor<3, const real_t> &ALC,
+                                const int NE,
+                                const DeviceTensor<5, const real_t> &J,
+                                const ConstDeviceMatrix &W,
+                                const real_t *b,
+                                const DeviceTensor<4, const real_t> &ALF_grad,
+                                const ConstDeviceCube &ALFmF0,
+                                DeviceTensor<4> &Y,
+                                const int d1d,
+                                const int q1d)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const real_t normal_inv_delta_sq =
+      2.0 * lim_normal / (adapt_lim_delta_max * adapt_lim_delta_max);
+
+   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   {
+      MFEM_SHARED real_t smem[MQ1][MQ1];
+      MFEM_SHARED real_t sB[MD1][MQ1];
+
+      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
+
+      // Evaluate ALF and ALF0 at the quad points.
+      kernels::internal::s_regs2d_t<MQ1> alf_dof, alf_quad;
+      kernels::internal::LoadDofs2d(e, D1D, ALFmF0, alf_dof);
+      kernels::internal::Eval2d(D1D, Q1D, smem, sB,
+                                alf_dof, alf_quad);
+
+      kernels::internal::v_regs2d_t<2,MQ1> r00, r01;
+      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+         {
+            const real_t *Jtr = &J(0, 0, qx, qy, e);
+            const real_t detJtr = kernels::Det<2>(Jtr);
+            const real_t weight = W(qx, qy) * detJtr;
+
+            const real_t coeff = const_coeff ? ALC(0, 0, 0) : ALC(qx, qy, e);
+            const real_t factor = weight * coeff * normal_inv_delta_sq *
+                                  alf_quad(qy, qx);
+
+            r00(0, qy, qx) = factor * ALF_grad(0, qx, qy, e);
+            r00(1, qy, qx) = factor * ALF_grad(1, qx, qy, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, r00, r01);
+      kernels::internal::WriteDofs2d(e, D1D, r01, Y);
+   });
+}
+
 MFEM_TMOP_MDQ_REGISTER(TMOPMultCoefKernels, TMOP_AddMultPA_C0_2D);
 MFEM_TMOP_MDQ_SPECIALIZE(TMOPMultCoefKernels);
 
@@ -140,4 +198,48 @@ void TMOP_Integrator::AddMultPA_C0_2D(const Vector &x, Vector &y) const
                             Y, exp_lim, d, q);
 }
 
+MFEM_TMOP_MDQ_REGISTER(TMOPMultAdaptLim, TMOP_AddMultPA_AdaptLim_2D);
+MFEM_TMOP_MDQ_SPECIALIZE(TMOPMultAdaptLim);
+
+void TMOP_Integrator::AddMultPA_AdaptLim_2D([[maybe_unused]] const Vector &x,
+                                            Vector &y) const
+{
+   const real_t ln = lim_normal;
+   const int NE = PA.ne, d = PA.maps->ndof, q = PA.maps->nqpt;
+
+   MFEM_VERIFY(d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   const auto J = Reshape(PA.Jtr.Read(), 2, 2, q, q, NE);
+   const auto *B = PA.maps->B.Read();
+   const auto W = Reshape(PA.ir->GetWeights().Read(), q, q);
+   auto Y = Reshape(y.ReadWrite(), d, d, 2, NE);
+
+   const int nal = PA.nal;
+   MFEM_VERIFY(nal > 0, "internal error");
+   const real_t *ALD = PA.ALD.HostRead();
+
+   const int ndof_el = d * d;
+   const int nqp_el = q * q;
+   const int ALF_stride = ndof_el * NE;
+   const int ALFG_stride = 2 * nqp_el * NE;
+
+   const bool const_coeff = (PA.ALC.Size() == nal);
+   const int ALC_stride = const_coeff ? 1 : (nqp_el * NE);
+   const real_t *ALC_all = PA.ALC.Read();
+   const real_t *ALFmF0_all = PA.ALFmF0.Read();
+   const real_t *ALFG_all = PA.ALFG.Read();
+   for (int c = 0; c < nal; c++)
+   {
+      const real_t delta_max = ALD[c];
+      const auto ALC = const_coeff
+                       ? Reshape(ALC_all + c, 1, 1, 1)
+                       : Reshape(ALC_all + c * ALC_stride, q, q, NE);
+      const auto ALFmF0 = Reshape(ALFmF0_all + c * ALF_stride, d, d, NE);
+      const auto ALF_grad = Reshape(ALFG_all + c * ALFG_stride, 2, q, q, NE);
+      TMOPMultAdaptLim::Run(d, q, ln, delta_max, const_coeff, ALC, NE, J, W,
+                            B, ALF_grad, ALFmF0, Y, d, q);
+   }
+}
+
 } // namespace mfem