Merge branch 'main' into hpcftools/flowSolver-gpu

Merge branch 'master' into hpcftools/flowSolver-gpu
miniapps/navier/incompressible_navier_dfem
2025-06-30 11:17:56 -07:00 · 2025-06-30 11:17:45 -07:00 · 2025-06-30 11:17:11 -07:00 · 2025-06-30 08:43:14 -07:00 · 2025-05-02 09:59:44 -07:00 · 2024-12-04 18:25:05 -08:00
185 changed files with 4197 additions and 7727 deletions
@@ -1,154 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
---
-name: Sanitizer Config
-description: Sets up environment variables for MFEM sanitizer workflow
-
-inputs:
-  DEBUG:
-    description: If true, use intermediate caches to speed up the workflow
-                 by reusing previous builds.
-    default: false
-
-  REPOSITORY:
-    description: Repository to checkout
-    default: mfem/mfem
-
-  BRANCH:
-    description: Branch to checkout
-    default: ubsan
-
-  CLANG_VER:
-    description: CLANG version to use
-    default: 18
-
-  # https://github.com/llvm/llvm-project/releases
-  LLVM_VER:
-    description: LLVM version to use
-    default: 19.1.7
-
-  # https://github.com/hypre-space/hypre/releases
-  HYPRE_VER:
-    description: HYPRE version to use
-    default: 2.19.0
-
-  METIS_VER:
-    description: METIS version to use
-    default: 4.0.3
-
-  CTEST:
-    description: CTest command to use
-    default: ctest -j --test-load $(nproc)
-                      --schedule-random
-                      --stop-on-failure --output-on-failure
-                      --test-dir
-
-  # https://clang.llvm.org/docs/AddressSanitizer.html
-  ASAN_OPTIONS:
-    default: detect_leaks=1,
-             strict_init_order=1,
-             strict_string_checks=1,
-             check_initialization_order=1,
-             detect_stack_use_after_return=1
-  ASAN_CXXFLAGS:
-    default: -fsanitize=address
-             -fsanitize-address-use-after-scope
-  ASAN_LDFLAGS:
-    default: -fsanitize=address
-
-  # https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
-  UBSAN_OPTIONS:
-    default: halt_on_error=1, print_stacktrace=1
-  UBSAN_CXXFLAGS:
-    default: -fsanitize=undefined
-  UBSAN_LDFLAGS:
-    default: -fsanitize=undefined
-
-  # https://clang.llvm.org/docs/MemorySanitizer.html
-  MSAN_OPTIONS:
-    default: "poison_in_dtor=1"
-  MSAN_CXXFLAGS:
-    default: -fsanitize=memory
-             -fsanitize-memory-track-origins
-             -fsanitize-memory-use-after-dtor
-  MSAN_LDFLAGS:
-    default: -fsanitize=memory
-
-  LSAN_DIR:
-    description: LSAN suppression directory
-    default: lsan
-
-  LSAN_FILE:
-    description: LSAN suppression file
-    default: lsan.supp
-
-  NO_FLAGS:
-    description: If true, do not set any CXXFLAGS or LDFLAGS.
-    default: false
-
-runs:
-  using: 'composite'
-  steps:
-    - name: Env (Inputs)
-      run: |
-        echo DEBUG=${{inputs.DEBUG}} >> $GITHUB_ENV
-        echo REPOSITORY=${{inputs.REPOSITORY}} >> $GITHUB_ENV
-        echo BRANCH=${{inputs.BRANCH}} >> $GITHUB_ENV
-        echo CLANG_VER=${{inputs.CLANG_VER}} >> $GITHUB_ENV
-        echo LLVM_VER=${{inputs.LLVM_VER}} >> $GITHUB_ENV
-        echo HYPRE_VER=${{inputs.HYPRE_VER}} >> $GITHUB_ENV
-        echo METIS_VER=${{inputs.METIS_VER}} >> $GITHUB_ENV
-        echo CTEST=${{inputs.CTEST}} >> $GITHUB_ENV
-        echo ASAN_OPTIONS=${{inputs.ASAN_OPTIONS}} >> $GITHUB_ENV
-        echo UBSAN_OPTIONS=${{inputs.UBSAN_OPTIONS}} >> $GITHUB_ENV
-        echo MSAN_OPTIONS=${{inputs.MSAN_OPTIONS}} >> $GITHUB_ENV
-        echo LSAN_DIR=${{inputs.LSAN_DIR}} >> $GITHUB_ENV
-        echo LSAN_FILE=${{inputs.LSAN_FILE}} >> $GITHUB_ENV
-        echo ASAN_CXXFLAGS=${{inputs.ASAN_CXXFLAGS}} >> $GITHUB_ENV
-        echo ASAN_LDFLAGS=${{inputs.ASAN_LDFLAGS}} >> $GITHUB_ENV
-        echo UBSAN_CXXFLAGS=${{inputs.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
-        echo UBSAN_LDFLAGS=${{inputs.UBSAN_LDFLAGS}} >> $GITHUB_ENV
-        echo MSAN_CXXFLAGS=${{inputs.MSAN_CXXFLAGS}} >> $GITHUB_ENV
-        echo MSAN_LDFLAGS=${{inputs.MSAN_LDFLAGS}} >> $GITHUB_ENV
-      shell: bash
-
-    - name: Env (dir)
-      run: |
-        echo LLVM_DIR=${{github.workspace}}/llvm >> $GITHUB_ENV
-        echo HYPRE_DIR=hypre-${{inputs.HYPRE_VER}} >> $GITHUB_ENV
-        echo METIS_DIR=metis-${{inputs.METIS_VER}} >> $GITHUB_ENV
-      shell: bash
-
-    - name: Env (bis)
-      run: |
-        echo CC=clang-${{inputs.CLANG_VER}} >> $GITHUB_ENV
-        echo CXX=clang++-${{inputs.CLANG_VER}} >> $GITHUB_ENV
-        echo LLVM_INC=${{env.LLVM_DIR}}/include/c++/v1 >> $GITHUB_ENV
-        echo LLVM_LIB=${{env.LLVM_DIR}}/lib >> $GITHUB_ENV
-        echo HYPRE_TGZ=v${{inputs.HYPRE_VER}}.tar.gz >> $GITHUB_ENV
-        echo METIS_TGZ=metis-${{inputs.METIS_VER}}.tar.gz >> $GITHUB_ENV
-        LSAN_SUPPRESSIONS="${{github.workspace}}/${{inputs.LSAN_DIR}}/${{inputs.LSAN_FILE}}"
-        echo "LSAN_OPTIONS=suppressions=$LSAN_SUPPRESSIONS" >> $GITHUB_ENV
-      shell: bash
-
-    - name: Env (ter)
-      if: ${{ inputs.NO_FLAGS != 'true' }}
-      run: |
-        echo LLVM_CXXFLAGS=-stdlib=libc++ -I${{env.LLVM_INC}} -Isystem${{env.LLVM_INC}} >> $GITHUB_ENV
-        echo LLVM_LDFLAGS=-L${{env.LLVM_LIB}} -lc++abi -Wl,-rpath,${{env.LLVM_LIB}} >> $GITHUB_ENV
-      shell: bash
-
-    - name: Env (quater)
-      if: ${{ inputs.NO_FLAGS != 'true' }}
-      run: |
-        echo CXXFLAGS=${{env.LLVM_CXXFLAGS}} >> $GITHUB_ENV
-        echo LDFLAGS=${{env.LLVM_LDFLAGS}} >> $GITHUB_ENV
-      shell: bash
@@ -1,91 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
---
-name: 'MFEM Compilation'
-description: 'MFEM Compilation'
-
-inputs:
-  par:
-    description: 'Whether to build for parallel (true/false)'
-    default: false
-  sanitizer:
-    description: 'Sanitizer to use (asan, msan, ubsan)'
-    default: asan
-
-runs:
-  using: 'composite'
-  steps:
-    - uses: ./.github/actions/sanitize/config
-
-    - uses: actions/cache@v4
-      if: ${{env.DEBUG == 'true'}}
-      id: debug
-      with:
-        path: mfem/build
-        key: build-${{inputs.par}}-${{inputs.sanitizer}}
-
-    - uses: ./.github/actions/sanitize/setup
-      if: ${{steps.debug.outputs.cache-hit != 'true'}}
-      with:
-        par: ${{inputs.par}}
-        sanitizer: ${{inputs.sanitizer}}
-
-    - name: Build with ASAN
-      if: inputs.sanitizer == 'asan'
-      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.ASAN_CXXFLAGS}} >> $GITHUB_ENV
-      shell: bash
-
-    - name: Build with MSAN
-      if: inputs.sanitizer == 'msan'
-      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MSAN_CXXFLAGS}} >> $GITHUB_ENV
-      shell: bash
-
-    - name: Build with UBSAN
-      if: inputs.sanitizer == 'ubsan'
-      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
-      shell: bash
-
-    - uses: mfem/github-actions/build-mfem@v2.5
-      if: ${{steps.debug.outputs.cache-hit != 'true'}}
-      env:
-        CXXFLAGS: ${{env.CXXFLAGS}}
-        LDFLAGS: ${{env.LDFLAGS}}
-      with:
-        mpi: ${{inputs.par == 'false' && 'seq' || 'par'}}
-        mfem-dir: mfem
-        os: ${{runner.os}}
-        library-only: true
-        build-system: cmake
-        hypre-dir: ${{env.HYPRE_DIR}}
-        metis-dir: ${{env.METIS_DIR}}
-        config-options: >-
-          -GNinja
-          -DMPICXX=${{env.CXX}}
-          -DCMAKE_CXX_STANDARD=17
-          -DMFEM_USE_MEMALLOC=OFF
-          -DCMAKE_BUILD_TYPE=Release
-          -DCMAKE_VERBOSE_MAKEFILE=ON
-          -DCMAKE_CXX_COMPILER=${{env.CXX}}
-          -DCMAKE_CXX_FLAGS_RELEASE='-g -O1 -fno-omit-frame-pointer'
-
-    - name: Delete object files
-      if: ${{steps.debug.outputs.cache-hit != 'true'}}
-      working-directory: mfem/build
-      run: find . -type f -name '*.o' -delete
-      shell: bash
-
-    - uses: actions/upload-artifact@v4
-      with:
-        name: build-${{inputs.par}}-${{inputs.sanitizer}}
-        path: mfem/build
-        if-no-files-found: error
-        retention-days: 1
-        overwrite: false
@@ -1,33 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
---
-name: 'Install MPI'
-description: 'Installs MPI and set up its environment variables'
-
-runs:
-  using: 'composite'
-  steps:
-    - name: Install
-      run: sudo apt-get install openmpi-bin libopenmpi-dev
-      shell: bash
-
-    - name: Env
-      run: |
-        echo PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe >> $GITHUB_ENV
-        echo MPI_INC=$(mpicxx --showme:compile) >> $GITHUB_ENV
-        echo MPI_LIB=$(mpicxx --showme:link) >> $GITHUB_ENV
-      shell: bash
-
-    - name: Env (bis)
-      run: |
-        echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MPI_INC}} >> $GITHUB_ENV
-        echo LDFLAGS=${{env.LDFLAGS}} ${{env.MPI_LIB}} >> $GITHUB_ENV
-      shell: bash
@@ -1,71 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
---
-name: 'Restore state'
-description: 'Restore state to be able to run checks, tests'
-
-inputs:
-  par:
-    description: 'Whether to build for parallel (true/false)'
-    default: false
-  sanitizer:
-    description: 'Sanitizer to use (asan, msan, ubsan)'
-    default: asan
-  cache-path:
-    description: 'path to what needs to be restored'
-    default: none
-  cache-skip:
-    description: 'Skip cache restoration'
-    default: false
-
-outputs:
-  cache-hit:
-    description: 'Output from a specific step'
-    value: ${{steps.debug.outputs.cache-hit}}
-
-runs:
-  using: 'composite'
-  steps:
-    - uses: ./.github/actions/sanitize/config
-
-    - uses: actions/cache@v4
-      if: ${{env.DEBUG == 'true' && inputs.cache-skip != 'true'}}
-      id: debug
-      with:
-        path: ${{inputs.cache-path}}
-        key: ${{github.job}}-${{inputs.par}}-${{inputs.sanitizer}}
-
-    - uses: ./.github/actions/sanitize/setup
-      if: ${{steps.debug.outputs.cache-hit != 'true'}}
-      with:
-        par: ${{inputs.par}}
-        sanitizer: ${{inputs.sanitizer}}
-
-    - uses: actions/download-artifact@v4
-      with:
-        name: build-${{inputs.par}}-${{inputs.sanitizer}}
-        path: mfem/build
-
-    - name: Ninja Patch
-      working-directory: mfem/build
-      run: |
-        sed -i -e 's/CXX_STATIC_LIBRARY_LINKER__mfem_Release.*/CUSTOM_COMMAND/' build.ninja
-        sed -i -e '/build tests\/unit\/all:/ s/tests\/unit\/[^ ]*unit_tests[^ ]*//g' build.ninja
-        sed -i -e '/^add_test(\[=\[\(unit_tests\|punit_tests\)\]=\]/ s/)/ "--input-file .\/list-test-names-${{matrix.tag}}" "--min-duration 1")/' tests/unit/CTestTestfile.cmake
-      shell: bash
-
-    - name: Copy Data
-      if: ${{steps.debug.outputs.cache-hit != 'true'}}
-      working-directory: mfem/build
-      run: |
-        ninja cmake_object_order_depends_target_unit_tests
-        cp -pR ../tests/unit/data tests/unit
-      shell: bash
@@ -1,64 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
---
-name: 'Setup state'
-description: 'Sets up the state to be able to run build & run'
-
-inputs:
-  par:
-    description: 'Whether to build for parallel (true/false)'
-    default: false
-  sanitizer:
-    description: 'Sanitizer to use (asan, msan, ubsan)'
-    default: asan
-
-runs:
-  using: 'composite'
-  steps:
-    - uses: actions/cache/restore@v4  # Cache for LLVM libcxx
-      with:
-        path: ${{env.LLVM_DIR}}
-        fail-on-cache-miss: true
-        key: build-libcxx-${{env.LLVM_VER}}-${{inputs.sanitizer}}
-
-    - uses: ./.github/actions/sanitize/mpi
-      if: ${{inputs.par == 'true'}}
-
-    - uses: actions/cache/restore@v4  # Cache for Hypre
-      if: ${{inputs.par == 'true'}}
-      with:
-        path: ${{env.HYPRE_DIR}}
-        fail-on-cache-miss: true
-        key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
-
-    - uses: actions/cache/restore@v4  # Cache for Metis
-      if: ${{inputs.par == 'true'}}
-      with:
-        path: ${{env.METIS_DIR}}
-        fail-on-cache-miss: true
-        key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
-
-    - name: Hypre/Metis links
-      if: ${{inputs.par == 'true'}}
-      run: ln -s -f ${{env.HYPRE_DIR}} hypre && ln -s -f ${{env.METIS_DIR}} metis-4.0
-      shell: bash
-
-    - uses: actions/cache/restore@v4  # Cache for LSAN suppression file
-      with:
-        path: ${{env.LSAN_DIR}}
-        fail-on-cache-miss: true
-        key: build-lsan-suppression-file
-
-    - uses: actions/checkout@v4  # Checkout the repository
-      with:
-        path: mfem
-        # ref: ${{env.BRANCH}}
-        # repository: ${{env.REPOSITORY}}
@@ -7,17 +7,18 @@

                               https://mfem.org

+
 This directory contains the GitHub CI scripts for MFEM.

 Note that some of these scripts use the shared MFEM GitHub Actions from the external mfem/github-actions repository:

-  <https://github.com/mfem/github-actions>
+  https://github.com/mfem/github-actions

-For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch in the above from which the action is taken.
+For a particular action, e.g. `mfem/github-actions/build-mfem@v2.1`, the `v2.1` suffix denotes the branch in the above from which the action is taken.

 The current CI workflows are:

-## `repo-check.yml`
+### `repo-check.yml`

 Runs a number of static repository-level sanity checks.

@@ -29,39 +30,19 @@ Runs a number of static repository-level sanity checks.

 - `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.

-## `mfem-analysis.yml` (`build-analysis`)
+### `mfem-analysis.yml` (`build-analysis`)

 Checks if the code builds and satisfies minimal requirements.

 - `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.

-## `builds-and-tests.yml`
+### `builds-and-tests.yml`

 Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.

-Uses the following GitHub Actions from <https://github.com/mfem/github-actions>:
+Uses the following GitHub Actions from https://github.com/mfem/github-actions:

 - `mfem/github-actions/build-hypre`
 - `mfem/github-actions/build-metis`
 - `mfem/github-actions/build-mfem`
 - `mfem/github-actions/upload-coverage`
-
-## Sanitizer Workflow for MFEM Verification
-
-This workflow validates MFEM unit tests, examples, and miniapps using sanitizer tools.
-
- `sanitizers.yml` orchestrates:
-  - Building and caching dependencies: HYPRE, METIS, LSAN suppression file, and LLVM libcxx.
-  - Launching fine-grained jobs for serial (ASAN, MSAN, UBSAN) and parallel (ASAN, UBSAN) sanitizers.
- `sanitize-tests.yml` is a reusable workflow accepting `par` mode (`true` for parallel) and `sanitizer` (ASAN, MSAN, or UBSAN) as inputs. It executes the following jobs:
-  - **Build**: Compiles the MFEM library with specified parallel and sanitizer settings.
-  - **Check**: Runs verification checks.
-  - Parallel jobs to test the following: **Examples**, **Miniapps** and **Unit tests**
-
-The workflow leverages composite actions in `.github/actions/sanitize/`:
-
- `config`: Centralizes settings for the sanitizer workflow.
- `mfem`: Manages the MFEM library build process.
- `mpi`: Installs MPI and applies additional compilation flags.
- `restore`: Restores the testing environment state.
- `setup`: Builds or restores cached dependencies.
@@ -132,14 +132,12 @@ jobs:
            hypre-target: int32
            precision: fp64
            enzyme: true
-            config-opts: MFEM_USE_ENZYME=YES ENZYME_DIR=$(brew --prefix enzyme) LDFLAGS=-L$LLVM_PREFIX/lib/c++
+            config-opts: MFEM_USE_ENZYME=YES ENZYME_DIR=$(brew --prefix enzyme)

    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}

    runs-on: ${{ matrix.os }}

-    continue-on-error: ${{ matrix.enzyme && true || false }}
-
    steps:
      # Fix 'No space left on device' errors for Ubuntu builds.
      - name: Run Actions Cleaner
@@ -170,13 +168,10 @@ jobs:
          env
        shell: bash

-      # For info on Xcode see:
-      #  - https://github.com/actions/runner-images/issues/12541
-      #  - https://github.com/actions/runner-images/blob/releases/macos-15-arm64/20250811/images/macos/macos-15-arm64-Readme.md#xcode
      - name: Xcode version setup (MacOS)
        if: matrix.os == 'macos-latest'
        run: |
-          XCODE_PATH="/Applications/Xcode_16.4.app"
+          XCODE_PATH="/Applications/Xcode_15.3.app"
          echo "> sudo xcode-select -s ${XCODE_PATH}"
          sudo xcode-select -s ${XCODE_PATH}
          echo "> g++ -v"
@@ -294,12 +289,10 @@ jobs:
        run: |
          export HOMEBREW_NO_INSTALL_CLEANUP=1
          brew update
-          brew install enzyme
-          ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required:.*\(llvm[^ ]*\).*/\1/p')
-          LLVM_PREFIX=$(brew --prefix $ENZYME_LLVM)
-          echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV
-          echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
-          echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
+          brew install llvm@19 enzyme
+          echo "LLVM_PREFIX=$(brew --prefix llvm@19)" >> $GITHUB_ENV
+          echo "OMPI_CC=$(brew --prefix llvm@19)/bin/clang" >> $GITHUB_ENV
+          echo "OMPI_CXX=$(brew --prefix llvm@19)/bin/clang++" >> $GITHUB_ENV

      # MFEM build and test
      - name: build
@@ -0,0 +1,69 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+name: "Sanitizer"
+
+permissions:
+  actions: write
+
+on:
+  push:
+    branches:
+      - master
+      - next
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  Serial:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: MFEM Checkout
+        uses: actions/checkout@v4
+        with:
+          path: mfem
+
+      - name: MFEM Build
+        uses: mfem/github-actions/build-mfem@v2.5
+        with:
+          os: ${{ runner.os }}
+          target: opt
+          mpi: seq
+          hypre-dir: unused-hypre-dir
+          metis-dir: unused-metis-dir
+          mfem-dir: mfem
+          build-system: make
+          library-only: false
+          config-options:
+            CXX="clang++-18"
+            CXXFLAGS="-g -O1 -std=c++17
+                      -fsanitize=address
+                      -fno-omit-frame-pointer
+                      -fsanitize-address-use-after-scope"
+
+      - name: MFEM Info
+        working-directory: mfem
+        run: make info
+
+      - name: MFEM Sanitize
+        working-directory: mfem
+        run:
+          ASAN_OPTIONS="detect_leaks=1,
+                        strict_init_order=1,
+                        strict_string_checks=1,
+                        check_initialization_order=1,
+                        detect_stack_use_after_return=1"
+          make test
@@ -1,39 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
---
-name: build-hypre
-on:
-  workflow_call:
-jobs:
-  build-hypre:
-    runs-on: ubuntu-latest
-    name: 2.19.0
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/config
-      - name: Cache
-        id: cache
-        uses: actions/cache@v4
-        with:
-          path: ${{env.HYPRE_DIR}}
-          key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
-      - name: Setup
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: ./.github/actions/sanitize/mpi
-      - name: Build
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.5
-        with:
-          archive: ${{env.HYPRE_TGZ}}
-          dir: ${{env.HYPRE_DIR}}
-          target: int32
-          precision: fp64
-          build-system: make
@@ -1,76 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
---
-name: build-libcxx
-on:
-  workflow_call:
-jobs:
-  build-llvm-libcxx:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        sanitizer: [asan, msan, ubsan]
-        include:
-          - sanitizer: asan
-            llvm_use_sanitizer: "Address"
-          - sanitizer: msan
-            llvm_use_sanitizer: "MemoryWithOrigins"
-          - sanitizer: ubsan
-            llvm_use_sanitizer: "Undefined"
-    name: ${{matrix.sanitizer}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/config
-        with:
-          NO_FLAGS: true
-      - name: Cache
-        id: cache
-        uses: actions/cache@v4
-        with:
-          path: ${{env.LLVM_DIR}}
-          key: build-libcxx-${{env.LLVM_VER}}-${{matrix.sanitizer}}
-      - name: Clone
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        run: >
-          git clone --filter=blob:none --depth=1
-          --branch llvmorg-${{env.LLVM_VER}}
-          --no-checkout https://github.com/llvm/llvm-project.git llvm-project
-      - name: Checkout
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        working-directory: llvm-project
-        run: |
-          git sparse-checkout set --cone
-          git checkout llvmorg-${{env.LLVM_VER}}
-          git sparse-checkout set cmake llvm/cmake runtimes libcxx libcxxabi
-      - name: Mkdir
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        run: mkdir ${{env.LLVM_DIR}}
-      - name: CMake
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        working-directory: ${{env.LLVM_DIR}}
-        run: >
-          VERBOSE=1
-          cmake -GNinja ../llvm-project/runtimes/
-          -DCMAKE_C_COMPILER=${{env.CC}}
-          -DCMAKE_CXX_COMPILER=${{env.CXX}}
-          -DCMAKE_BUILD_TYPE=RelWithDebInfo
-          -DCMAKE_INSTALL_PREFIX=/usr
-          -DLLVM_USE_SANITIZER=${{matrix.llvm_use_sanitizer}}
-          -DLLVM_BUILD_32_BITS=OFF
-          -DLIBCXXABI_USE_LLVM_UNWINDER=OFF
-          -DLLVM_INCLUDE_TESTS=OFF
-          -DLIBCXX_INCLUDE_TESTS=OFF
-          -DLIBCXX_INCLUDE_BENCHMARKS=OFF
-          -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi'
-      - name: Build
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        working-directory: ${{env.LLVM_DIR}}
-        run: cmake --build . -- cxx cxxabi
@@ -1,38 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
---
-name: build-file-lsan
-on:
-  workflow_call:
-jobs:
-  build-file-lsan:
-    runs-on: ubuntu-latest
-    name: lsan.supp
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/config
-      - name: Cache
-        id: cache
-        uses: actions/cache@v4
-        with:
-          path: ${{env.LSAN_DIR}}
-          key: build-lsan-suppression-file
-      - name: Setup
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p ${{env.LSAN_DIR}}
-          cat << EOF > ${{env.LSAN_DIR}}/${{env.LSAN_FILE}}
-          leak:libevent_core-2.1.so
-          leak:ompi_mpi_finalize
-          leak:ompi_mpi_init
-          leak:PMPI_Init
-          leak:strdup
-          EOF
@@ -1,36 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
---
-name: build-metis
-on:
-  workflow_call:
-jobs:
-  build-metis:
-    runs-on: ubuntu-latest
-    name: 4.0.3
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/config
-      - name: Cache
-        id: cache
-        uses: actions/cache@v4
-        with:
-          path: ${{env.METIS_DIR}}
-          key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
-      - name: Setup
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: ./.github/actions/sanitize/mpi
-      - name: Build
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
-        with:
-          archive: ${{env.METIS_TGZ}}
-          dir: ${{env.METIS_DIR}}
@@ -1,197 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
---
-name: Sanitize
-on:
-  workflow_call:
-    inputs:
-      par:
-        description: 'Whether to build for parallel (true/false)'
-        required: false
-        default: false
-        type: boolean
-      sanitizer:
-        description: 'Sanitizer to use (asan, msan, ubsan)'
-        required: true
-        default: asan
-        type: string
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/mfem
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-
-  check:
-    needs: [build]
-    runs-on: ubuntu-latest
-    env:
-      ex: ${{inputs.par && 'ex1p' || 'ex1'}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/examples/${{env.ex}}
-      - name: MFEM Check
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: ninja -v check
-
-  examples:
-    needs: [check]
-    runs-on: ubuntu-latest
-    env:
-      exclude: ${{inputs.par && '-E "_ser"' || ''}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/examples/ex1
-      - name: Build Examples
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: ninja -v examples
-      - name: Test Examples
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: |
-          ${{env.CTEST}} examples ${{env.exclude}} --show-only
-          ${{env.CTEST}} examples ${{env.exclude}}
-
-  miniapps:
-    needs: [check]
-    runs-on: ubuntu-latest
-    env:
-      exclude: ${{inputs.par && '-E "_ser"' || ''}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/miniapps/meshing/minimal-surface
-      - name: Build Miniapps
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: ninja -v miniapps
-      - name: Test Miniapps
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: |
-          ${{env.CTEST}} miniapps ${{env.exclude}} --show-only
-          ${{env.CTEST}} miniapps ${{env.exclude}}
-
-  tests-miniapps:
-    needs: [check]
-    runs-on: ubuntu-latest
-    env:
-      run: ${{inputs.par && '-R "_cpu_np"' || ''}}
-      exclude: ${{inputs.par && '"unit_tests|debug"' || '"^unit_tests$|debug"'}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/tests/unit/sedov_tests_cpu
-      - name: Build Tests Unit Miniapps
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: ninja -v tests/unit/all
-      - name: Run Tests Unit Miniapps
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: |
-          ${{env.CTEST}} tests/unit -E ${{env.exclude}} ${{env.run}} --show-only
-          ${{env.CTEST}} tests/unit -E ${{env.exclude}} ${{env.run}}
-
-  tests-unit-build:
-    needs: [check]
-    runs-on: ubuntu-latest
-    env:
-      unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/tests/unit/${{env.unit_tests}}
-      - name: Build Unit Tests
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: ninja -v ${{env.unit_tests}}
-      - name: Delete object files
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build/tests/unit
-        run: find . -type f -name '*.o' -delete
-      - uses: actions/upload-artifact@v4
-        with:
-          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
-          path: mfem/build/tests/unit/${{env.unit_tests}}
-          if-no-files-found: error
-          retention-days: 1
-          overwrite: false
-
-  tests-unit-run:
-    needs: [tests-unit-build]
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        tag: [0, 1, 2, 3]
-    name: tests-unit-run-${{matrix.tag}}
-    env:
-      unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
-      np: ${{inputs.par && '_np=2' || ''}}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/sanitize/restore
-        id: restore
-        with:
-          par: ${{inputs.par}}
-          sanitizer: ${{inputs.sanitizer}}
-          cache-path: mfem/build/tests/unit/${{env.unit_tests}}
-      - uses: actions/download-artifact@v4
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        with:
-          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
-          path: mfem/build/tests/unit
-      - name: Split Unit Tests
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build/tests/unit
-        run: |
-          chmod 755 ${{env.unit_tests}}
-          ./${{env.unit_tests}} --list-test-names-only | tail -n +2 > list-test-names
-          shuf list-test-names -o list-test-names
-          split --verbose -n l/4 -d -a 1 list-test-names list-test-names-
-      - name: Cat Unit Tests ${{matrix.tag}}
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build/tests/unit
-        run: cat list-test-names-${{matrix.tag}}
-      - name: Run Unit Tests ${{matrix.tag}}
-        if: ${{steps.restore.outputs.cache-hit != 'true'}}
-        working-directory: mfem/build
-        run: |
-          ${{env.CTEST}} tests/unit -R "${{env.unit_tests}}${{env.np}}" --show-only
-          ${{env.CTEST}} tests/unit -R "${{env.unit_tests}}${{env.np}}"
@@ -1,73 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
---
-name: Sanitizers
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches: ["master", "next"]
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{github.workflow}}-${{github.ref}}
-  cancel-in-progress: true
-
-jobs:
-
-  # Build steps for dependencies
-  build-hypre:
-    uses: ./.github/workflows/sanitize-build-hypre.yml
-
-  build-metis:
-    uses: ./.github/workflows/sanitize-build-metis.yml
-
-  build-lsan:
-    uses: ./.github/workflows/sanitize-build-lsan.yml
-
-  build-libcxx:
-    uses: ./.github/workflows/sanitize-build-libcxx.yml
-
-  # Serial sanitizers: asan, msan, ubsan
-  seq-asan:
-    needs: [build-libcxx]
-    uses: ./.github/workflows/sanitize-tests.yml
-    with:
-      sanitizer: asan
-
-  seq-msan:
-    needs: [build-libcxx]
-    uses: ./.github/workflows/sanitize-tests.yml
-    with:
-      sanitizer: msan
-
-  seq-ubsan:
-    needs: [build-libcxx]
-    uses: ./.github/workflows/sanitize-tests.yml
-    with:
-      sanitizer: ubsan
-
-  # Parallel sanitizers: asan, ubsan
-  par-asan:
-    needs: [build-libcxx, build-hypre, build-metis]
-    uses: ./.github/workflows/sanitize-tests.yml
-    with:
-      par: true
-      sanitizer: asan
-  par-ubsan:
-    needs: [build-libcxx, build-hypre, build-metis]
-    uses: ./.github/workflows/sanitize-tests.yml
-    with:
-      par: true
-      sanitizer: ubsan
@@ -19,6 +19,9 @@ CMakeFiles/
 # Clangd server cache
 *.cache*

+# VSCode configuration
+/.vscode/
+
 # Backup files
 *~

@@ -211,7 +214,7 @@ miniapps/electromagnetics/joule
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
-miniapps/electromagnetics/Joule_[0-9]*
+miniapps/electromagnetics/Joule_*

 miniapps/gslib/field-diff
 miniapps/gslib/field-interp
@@ -267,9 +270,9 @@ miniapps/meshing/bounding-box*
 miniapps/meshing/jacobian-determinant*

 miniapps/mtop/parheat
-miniapps/mtop/ParHeat/*
+miniapps/mtop/ParHeat*
 miniapps/mtop/seqheat
-miniapps/mtop/SeqHeat/*
+miniapps/mtop/SeqHeat*

 miniapps/autodiff/paradiff
 miniapps/autodiff/seqadiff
@@ -277,7 +280,7 @@ miniapps/autodiff/seqtest
 miniapps/autodiff/par_example
 miniapps/autodiff/seq_example
 miniapps/autodiff/seq_test
-miniapps/autodiff/Example/*
+miniapps/autodiff/Exampl*

 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
@@ -300,7 +303,6 @@ miniapps/nurbs/nurbs_solenoidal
 miniapps/nurbs/nurbs_printfunc
 miniapps/nurbs/nurbs_patch_ex1
 miniapps/nurbs/nurbs_curveint
-miniapps/nurbs/nurbs_surface
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
 miniapps/nurbs/sol_?.gf
@@ -319,7 +321,6 @@ miniapps/nurbs/nurbs_naca_cmesh
 miniapps/nurbs/naca-cmesh.mesh
 miniapps/nurbs/glvis_naca-cmesh.mesh
 miniapps/nurbs/Naca_cmesh
-miniapps/nurbs/*-Surface.mesh

 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -415,8 +416,8 @@ miniapps/diag-smoothers/mg-abs-l1-jacobi
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
-tests/unit/gpu_unit_tests
-tests/unit/pgpu_unit_tests
+tests/unit/cunit_tests
+tests/unit/pcunit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
@@ -22,7 +22,7 @@ include:
 # the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
-# - Allocate/Release is where Dane resource are allocated/released once for all.
+# - Allocate/Release is where ruby resource are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
@@ -53,7 +53,7 @@ variables:
  AUTOTEST_COMMIT: "YES"

 # Trigger subpipelines:
-dane-build-and-test:
+ruby-build-and-test:
  stage: sub-pipelines
  variables:
    # Explicitly pass down values that we want to be able to set when triggering
@@ -61,10 +61,10 @@ dane-build-and-test:
    AUTOTEST: "${AUTOTEST}"
    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
  trigger:
-    include: .gitlab/dane-build-and-test.yml
+    include: .gitlab/ruby-build-and-test.yml
    strategy: depend

-dane-baseline:
+ruby-baseline:
  stage: sub-pipelines
  variables:
    # Explicitly pass down values that we want to be able to set when triggering
@@ -73,7 +73,7 @@ dane-baseline:
    AUTOTEST: "${AUTOTEST}"
    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
  trigger:
-    include: .gitlab/dane-baseline.yml
+    include: .gitlab/ruby-baseline.yml
    strategy: depend

 lassen-build-and-test:
@@ -24,7 +24,7 @@ and `test type`.

 Machines typically include:

-* Dane: Intel Sapphire Rapids
+* Ruby: 2nd Gen Intel Xeon (Cascade Lake)
 * Lassen: Power9 + Nvidia GPU
 * Corona: AMD GPU

@@ -76,13 +76,13 @@ with a spack spec of MFEM, within the limits permitted by the MFEM spack
 package.

 In any build-and-test sub-pipeline a job basically consists in defining the
-spack spec to use. Adding a job on Dane for example resumes to:
+spack spec to use. Adding a job on ruby for example resumes to:

 ```yaml
 <job_name>:
  variables:
    SPEC: "<spack_spec>"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby
 ```

 The remaining and non trivial work is to make sure this spec is working. To
@@ -24,7 +24,7 @@ variables:
 # TODO: add a clean-up mechanism
  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}

-# On LLNL's Dane, there is only one allocation shared among jobs in order to
+# On LLNL's ruby, there is only one allocation shared among jobs in order to
 # save time and resource. This allocation has to be uniquely named so that we
 # are sure to retrieve it.
  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
@@ -9,17 +9,17 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# GitLab pipelines configurations for the Dane machine at LLNL
+# GitLab pipelines configurations for the Ruby machine at LLNL
 variables:
-  MACHINE_NAME: dane
+  MACHINE_NAME: ruby

-.on_dane:
+.on_ruby:
  tags:
    - shell
-    - dane
+    - ruby
  rules:
-    # Don't run dane jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_DANE == "OFF"'
+    # Don't run ruby jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_RUBY == "OFF"'
      when: never
    # Don't run autotest update if...
    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
@@ -40,17 +40,16 @@ variables:
    - when: on_success

 # Spack helped builds
-# Generic dane build job, extending build script
-.build_and_test_on_dane:
-  extends: [.on_dane]
+# Generic ruby build job, extending build script
+.build_and_test_on_ruby:
+  extends: [.on_ruby]
  stage: build_and_test
  script:
    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
-    - export THREADS=28
+    - export THREADS=16
    - echo ${ALLOC_NAME}
    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
    - echo ${JOBID}
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 60 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -18,7 +18,7 @@
 setup_baseline:
  tags:
    - shell
-    - dane
+    - ruby
  stage: setup
  variables:
    GIT_STRATEGY: none
@@ -16,7 +16,7 @@
 setup:
  tags:
    - shell
-    - dane
+    - ruby
  stage: setup
  variables:
    GIT_STRATEGY: none
@@ -19,8 +19,8 @@ stages:
  - cleanup
  - baseline_publish

-baselinecheck_mfem_intel_dane:
-  extends: [.on_dane]
+baselinecheck_mfem_intel_ruby:
+  extends: [.on_ruby]
  stage: baseline_check
  variables:
    # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
@@ -31,8 +31,8 @@ baselinecheck_mfem_intel_dane:
  script:
    - echo ${BUILD_ROOT}
    - echo ${TPLS_DIR}
-    # Used by the tests in MFEM/tests, dane has 224 threads/node:
-    - export MFEM_TEST_NP=192
+    # Used by the tests in MFEM/tests:
+    - export MFEM_TEST_NP=48
    # The next script uses the following environment variables:
    # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
    # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
@@ -44,7 +44,7 @@ baselinecheck_mfem_intel_dane:
  allow_failure: true

 cleanup:
-  extends: .on_dane
+  extends: .on_ruby
  stage: cleanup
  variables:
    GIT_STRATEGY: none
@@ -53,7 +53,7 @@ cleanup:
    - rm -rf "${BUILD_ROOT}" || true

 report_baseline:
-  extends: [.on_dane]
+  extends: [.on_ruby]
  stage: baseline_report
  script:
    - echo ${MACHINE_NAME}
@@ -113,8 +113,8 @@ report_baseline:
        exit $err
      ) 9> autotest.lock

-baselinepublish_mfem_dane:
-  extends: [.on_dane]
+baselinepublish_mfem_ruby:
+  extends: [.on_ruby]
  stage: baseline_publish
  rules:
    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
@@ -129,5 +129,5 @@ baselinepublish_mfem_dane:

 include:
  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/dane-config.yml
+  - local: .gitlab/configs/ruby-config.yml
  - local: .gitlab/configs/setup-baseline.yml
@@ -19,54 +19,54 @@ stages:
 allocate_resource:
  variables:
    GIT_STRATEGY: none
-  extends: .on_dane
+  extends: .on_ruby
  stage: allocate_resource
  script:
    - echo ${ALLOC_NAME}
    - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME}
  timeout: 6h

-# GitLab jobs for the Dane machine at LLNL
+# GitLab jobs for the Ruby machine at LLNL
 debug_ser_gcc_10:
  variables:
    SPEC: "%gcc@10.3.1 +debug~mpi"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby

 debug_par_gcc_10:
  variables:
    SPEC: "%gcc@10.3.1 +debug+mpi"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby

 opt_ser_gcc_10:
  variables:
    SPEC: "%gcc@10.3.1 ~mpi"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby

 opt_par_gcc_10:
  variables:
    SPEC: "%gcc@10.3.1"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby

 opt_par_gcc_10_sundials:
  variables:
    SPEC: "%gcc@10.3.1 +sundials"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby

 opt_par_gcc_10_petsc:
  variables:
    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby

 opt_par_gcc_10_pumi:
  variables:
    SPEC: "%gcc@10.3.1 +pumi"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby

 # Release
 release_resource:
  variables:
    GIT_STRATEGY: none
-  extends: .on_dane
+  extends: .on_ruby
  stage: release_resource_and_report
  script:
    - echo ${ALLOC_NAME}
@@ -78,17 +78,17 @@ release_resource:
 report_job_success:
  stage: release_resource_and_report
  extends:
-    - .on_dane
+    - .on_ruby
    - .report_job_success

 report_job_failure:
  stage: release_resource_and_report
  extends:
-    - .on_dane
+    - .on_ruby
    - .report_job_failure

 include:
  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/dane-config.yml
+  - local: .gitlab/configs/ruby-config.yml
  - local: .gitlab/configs/setup-build-and-test.yml
  - local: .gitlab/configs/report-build-and-test.yml
@@ -14,7 +14,7 @@
 # locals
 glob_err=${BASELINE_TEST}.err
 base=${BASELINE_TEST}-${SYS_TYPE}
-if [[ "${MACHINE_NAME}" == "dane" ]]; then
+if [[ "${MACHINE_NAME}" == "ruby" ]]; then
  base="${BASELINE_TEST}-${MACHINE_NAME}"
 fi
 base_diff=${base}.diff
@@ -31,7 +31,7 @@ cd tests
 mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}

 # run
-if [[ "${MACHINE_NAME}" == "dane" ]]; then
+if [[ "${MACHINE_NAME}" == "ruby" ]]; then
  salloc --nodes=1 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
@@ -11,7 +11,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# There will be collision between corona and dane baselines.
+# There will be collision between corona and ruby baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
@@ -21,7 +21,7 @@ PATCH_FILE=${ARTIFACT_PATH}.patch
 FULL_FILE=${ARTIFACT_PATH}.out
 DIFF_FILE=${ARTIFACT_PATH}.diff

-# There will be collision between corona and dane baselines.
+# There will be collision between corona and ruby baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 SAVED_NAME=baseline-${SYS_TYPE}.saved
@@ -27,29 +27,11 @@ Discretization improvements
 - In the ParMoonolith integration, added support for variational resampling of
  H1 vector fields.

- Added support for boundary integration to the hyperbolic framework. In this
-  regard, new classes `BdrHyperbolicDirichletIntegrator` and
-  `BoundaryHyperbolicFlowIntegrator` have been introduced for implementation
-  of weak Dirichlet boundary conditions with a general flux or for the linear
-  case respectively.
-
- Added method to compute piecewise linear bounds on high-order functions on
-  tensor-product elements.
-
- Parallel anisotropic refinement of hexahedral meshes is now supported,
-  provided that neighboring hexahedra are not refined in conflicting directions.
-  A new ParMesh method is added to check for such conflicts, before refinement.
-
 Meshing improvements
 --------------------
-
 - Added support for higher order meshes in Mesh::MakeSimplicial and
  ParMesh::MakeSimplicial.

- Added a new miniapp for interpolating a surface grid of points in 3D using a
-  smooth NURBS surface, that can then be sampled at arbitrary resolution while
-  staying close to the original geometry. See miniapps/nurbs/nurbs_surface.
-
 GPU computing
 -------------
 - The function Vector::SetSubVector(const Array<int> &, const real_t) now
@@ -58,15 +40,6 @@ GPU computing
  conditions. A new function Vector::SetSubVectorHost has been added in cases
  where host execution is always needed (e.g. when the DOFs array is small).

- Introduced MFEM_FOREACH_THREAD_DIRECT, which directly maps loop tasks to GPU
-  threads, assigning one task per thread.
-
- Implemented a GPU-accelerated matrix-free AMR derefinement `GridFunction`
-  update operator. This supports mixed geometry meshes and variable order
-  spaces, and is the default derefinement operator constructed by
-  `FiniteElementSpace::Update` and `ParFiniteElementSpace::Update`.
-  The operator requires `FiniteElementSpace::Nonconforming() == true`.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added miniapps to demonstrate an implementation of the absolute-value
@@ -75,43 +48,13 @@ New and updated examples and miniapps
  operators as smoothers.
  These miniapps can be found in `miniapps/diag-smoothers`.

- Added a new miniapp (meshing/mesh-bounding-boxes) that computes the bounding
-  boxes for each element of a given mesh, and the bounds on the determinant of
-  the Jacobian of the transformation.
-
- Added a new miniapp (tools/gridfunction-bounds) to compute piecewise linear
-  bounds on a given high-order grid function.
-
 API changes:
 -----------
 - mfem::internal::tensor and mfem::internal::dual have been moved to
  mfem::future::tensor and mfem::future::dual.
-
 - API addition: in class `Operator`, added virtual functions: `AbsMult`, and
  `AbsMultTranspose`; in class `Vector`, added `Abs` and `Pow`.

-Miscellaneous
-------------
- Added the "gpu", "raja-gpu", and "ceed-gpu" backend aliases/shortcuts which
-  automatically select between CUDA or HIP.
-
- The CUDA-specific names used by some of the unit tests like 'cunit_tests' and
-  'pcunit_tests' were replaced by names using 'gpu' instead of 'c' (short for
-  CUDA) or 'cuda'. These tests automatically run the CUDA/HIP tests based on the
-  MFEM build configuration.
-
- Added the option to enable GPU-aware MPI in MFEM using the environment
-  variable 'MFEM_GPU_AWARE_MPI' set to any value. Setting this environment
-  variable is an alternative to calling 'Device::SetGPUAwareMPI(true)'.
-
- Added parallel Address Sanitizer, serial and parallel Undefined Behavior
-  Sanitizer and serial Memory Sanitizer GitHub actions tests on Ubuntu.
-
- FindPointsGSLIB has a new constructor that accepts the mesh object and
-  internally calls the Setup() method so that the user does not have to.
-  The FreeData() method has also been moved to the destructor so the user does
-  not need to manually free-up the memory if the destructor is called before
-  MPI_Finalize().

 Version 4.8, released on Apr 9, 2025
 ====================================
@@ -598,20 +598,14 @@ set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
    ALGOIM ENZYME)

-# Add all created targets and *_FOUND libraries in the variables TPL_TARGETS and
-# TPL_LIBRARIES, respectively.
-set(TPL_TARGETS)
+# Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
 foreach(TPL IN LISTS MFEM_TPLS)
-  if (${TPL}_FOUND OR TARGET ${TPL})
+  if (${TPL}_FOUND)
    message(STATUS "MFEM: using package ${TPL}")
-    if (TARGET ${TPL})
-      list(APPEND TPL_TARGETS ${TPL})
-    else()
-      list(APPEND TPL_LIBRARIES ${${TPL}_LIBRARIES})
-      list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
-    endif()
+    list(APPEND TPL_LIBRARIES ${${TPL}_LIBRARIES})
+    list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
  endif()
 endforeach(TPL)
 list(REVERSE TPL_LIBRARIES)
@@ -686,10 +680,7 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
-target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
-if (TPL_TARGETS)
-  add_dependencies(mfem ${TPL_TARGETS})
-endif()
+target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
 if (MINGW)
  target_link_libraries(mfem PRIVATE ws2_32)
 endif()
@@ -121,11 +121,6 @@ Parallel build:
   make -j 4
 (For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

-Parallel build with fetching of hypre and METIS:
-   mkdir <mfem-buil-dir> ; cd <mfem-build-dir>
-   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DFETCH_TPLS=YES
-   make -j 4
-
 CUDA build:
   (this build requires CMake 3.17 or newer)
   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
@@ -668,7 +663,6 @@ The specific libraries and their options are:
 - OpenMP (optional), usually part of compiler, used when either MFEM_USE_OPENMP
  or MFEM_USE_LEGACY_OPENMP is set to YES.
  Options: OPENMP_OPT, OPENMP_LIB.
-  Versions: OpenMP >= 3.1 when MFEM_USE_OPENMP=YES.

 - High-resolution POSIX clocks: when using MFEM_TIMER_TYPE = 2, it may be
  necessary to link with a system library (e.g. librt.so).
@@ -848,7 +842,6 @@ The specific libraries and their options are:
 - HIP (optional), used when MFEM_USE_HIP = YES.
  URL: https://rocmdocs.amd.com
  Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
-  Versions: ROCm >= 5.6.1.

 - OCCA (optional), used when MFEM_USE_OCCA = YES.
  URL: https://libocca.org
@@ -1081,9 +1074,6 @@ The following options are CMake specific:
 MFEM_ENABLE_TESTING  - Enable the ctest framework for testing.
 MFEM_ENABLE_EXAMPLES - Build all of the examples by default.
 MFEM_ENABLE_MINIAPPS - Build all of the miniapps by default.
-FETCH_TPLS           - Enable fetching of all supported third-party libraries.
-HYPRE_FETCH          - Enable fetching of hypre.
-METIS_FETCH          - Enable fetching of metis.

 External libraries (CMake):
 ---------------------------
@@ -1145,12 +1135,6 @@ The following built-in CMake packages are also used:
      set the <LIBNAME>_LIBRARIES option directly; the configuration option
      <LIBNAME>_DIR is not supported.

-The MFEM CMake build system also provides fetching (automated building) for the
-packages/libraries listed below. Note that when fetching is enabled, any related
-auto-detection functionality is disabled.
-
- - HYPRE
- - METIS

 Building without GNU make or CMake
 ==================================
@@ -9,18 +9,15 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# Defines the following variables if fetching of TPLs is disabled (default):
+# Defines the following variables:
 #   - HYPRE_FOUND
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
 #   - HYPRE_USING_CUDA (internal)
 #   - HYPRE_USING_HIP (internal)
-# otherwise, the following are defined:
-#   - HYPRE (imported library target)
-#   - HYPRE_VERSION (cache variable)

-if (HYPRE_FOUND OR TARGET HYPRE)
+if (HYPRE_FOUND)
  if (HYPRE_USING_CUDA)
    find_package(CUDAToolkit REQUIRED)
  endif()
@@ -36,55 +33,6 @@ if (HYPRE_FOUND OR TARGET HYPRE)
  endif()
 endif()

-if (HYPRE_FETCH OR FETCH_TPLS)
-  set(HYPRE_FETCH_VERSION 2.33.0)
-  add_library(HYPRE STATIC IMPORTED)
-  # set options and associated dependencies
-  set(CMAKE_OPTIONS)
-  list(APPEND CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
-  if (MFEM_USE_CUDA)
-    list(APPEND CMAKE_OPTIONS -DHYPRE_WITH_CUDA:BOOL=ON)
-    find_package(CUDAToolkit REQUIRED)
-    target_link_libraries(HYPRE INTERFACE CUDA::cusparse CUDA::curand CUDA::cublas)
-  elseif (MFEM_USE_HIP)
-    list(APPEND CMAKE_OPTIONS -DHYPRE_WITH_HIP:BOOL=ON)
-    find_package(rocsparse REQUIRED)
-    find_package(rocrand REQUIRED)
-    target_link_libraries(HYPRE INTERFACE rocsparse rocrand)
-  endif()
-  if (MFEM_USE_SINGLE)
-    list(APPEND CMAKE_OPTIONS -DHYPRE_ENABLE_SINGLE:BOOL=ON)
-  endif()
-  # define external project and create future include directory so it is present
-  # to pass CMake checks at end of MFEM configuration step
-  message(STATUS "Will fetch HYPRE ${HYPRE_FETCH_VERSION} to be built with ${CMAKE_OPTIONS}")
-  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/hypre)
-  include(ExternalProject)
-  ExternalProject_Add(hypre
-    GIT_REPOSITORY https://github.com/hypre-space/hypre.git
-    GIT_TAG v${HYPRE_FETCH_VERSION}
-    GIT_SHALLOW TRUE
-    UPDATE_DISCONNECTED TRUE
-    SOURCE_SUBDIR src
-    PREFIX ${PREFIX}
-    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PREFIX} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${CMAKE_OPTIONS})
-  file(MAKE_DIRECTORY ${PREFIX}/include)
-  # set imported library target properties
-  add_dependencies(HYPRE hypre)
-  set_target_properties(HYPRE PROPERTIES
-    IMPORTED_LOCATION ${PREFIX}/lib/libHYPRE.a
-    INTERFACE_INCLUDE_DIRECTORIES ${PREFIX}/include)
-  # convert HYPRE version to integer
-  string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION ${HYPRE_FETCH_VERSION})
-  list(GET HYPRE_SPLIT_VERSION 0 HYPRE_MAJOR_VERSION)
-  list(GET HYPRE_SPLIT_VERSION 1 HYPRE_MINOR_VERSION)
-  list(GET HYPRE_SPLIT_VERSION 2 HYPRE_PATCH_VERSION)
-  math(EXPR HYPRE_VERSION "10000*${HYPRE_MAJOR_VERSION} + 100*${HYPRE_MINOR_VERSION} + ${HYPRE_PATCH_VERSION}")
-  # set cache variables that would otherwise be set after mfem_find_package call
-  set(HYPRE_VERSION ${HYPRE_VERSION} CACHE STRING "HYPRE version." FORCE)
-  return()
-endif()
-
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
  "Paths to headers required by HYPRE." "Libraries required by HYPRE."
@@ -9,38 +9,10 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# Defines the following variables if fetching of TPLs is disabled (default):
+# Defines the following variables:
 #   - METIS_FOUND
 #   - METIS_LIBRARIES
 #   - METIS_INCLUDE_DIRS
-#   - METIS_VERSION_5
-# otherwise, the following are defined:
-#   - METIS (imported library target)
-#   - METIS_VERSION_5 (cache variable)
-
-if (METIS_FETCH OR FETCH_TPLS)
-  set(METIS_FETCH_VERSION 4.0.3)
-  add_library(METIS STATIC IMPORTED)
-  # define external project
-  message(STATUS "Will fetch METIS ${METIS_FETCH_VERSION} to be built with default options")
-  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/metis)
-  include(ExternalProject)
-  ExternalProject_Add(metis
-    GIT_REPOSITORY https://github.com/mfem/tpls
-    GIT_TAG b60352fbe9675d374b00828055e55be4584c7995 # tag from 1/16/25
-    GIT_SHALLOW TRUE
-    UPDATE_DISCONNECTED TRUE
-    PREFIX ${PREFIX}
-    CONFIGURE_COMMAND tar -xzf ../metis/metis-${METIS_FETCH_VERSION}-mac.tgz --strip=1
-    INSTALL_COMMAND mkdir -p ${PREFIX}/lib && cp libmetis.a ${PREFIX}/lib/)
-  # set imported library target properties
-  add_dependencies(METIS metis)
-  set_target_properties(METIS PROPERTIES
-    IMPORTED_LOCATION ${PREFIX}/lib/libmetis.a)
-  # set cache variables that would otherwise be set after mfem_find_package call
-  set(METIS_VERSION_5 FALSE CACHE BOOL "Is METIS version 5?")
-  return()
-endif()

 include(MfemCmakeUtilities)
 mfem_find_package(METIS METIS METIS_DIR "include;Lib" "metis.h"
@@ -23,14 +23,11 @@
 #include "_config.hpp"
 #endif

-#include <cstdint>
-#include <climits>
-
 namespace mfem
 {

 #if (defined(MFEM_USE_CUDA) && defined(__CUDACC__)) || \
-    (defined(MFEM_USE_HIP) && defined(__HIP__))
+    (defined(MFEM_USE_HIP) && defined(__HIPCC__))
 #define MFEM_HOST_DEVICE __host__ __device__
 #else
 #define MFEM_HOST_DEVICE
@@ -89,12 +89,6 @@ option(MFEM_ENABLE_EXAMPLES "Build all of the examples" OFF)
 option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
 option(MFEM_ENABLE_BENCHMARKS "Build all of the benchmarks" OFF)

-# Allow a user to specify fetching of certain third-party libraries instead of
-# searching for existing installations.
-option(FETCH_TPLS "Enable fetching of all supported third-party libraries" OFF)
-option(HYPRE_FETCH "Enable fetching of hypre" OFF)
-option(METIS_FETCH "Enable fetching of METIS" OFF)
-
 # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the
 # autodetected C++ compiler.
 # set(CXX g++)
@@ -82,8 +82,6 @@ set(SRCS
  fe/fe_ser.cpp
  fe_coll.cpp
  fespace.cpp
-  derefmat_op.cpp
-  pderefmat_op.cpp
  geom.cpp
  gridfunc.cpp
  hybridization.cpp
@@ -249,7 +247,6 @@ set(HDRS
  nonlinearform_ext.hpp
  nonlininteg.hpp
  qfunction.hpp
-  qinterp/det.hpp
  qinterp/eval.hpp
  qinterp/eval_hdiv.hpp
  qinterp/grad.hpp
@@ -515,7 +515,6 @@ struct InvTNewtonSolver<Geometry::SEGMENT, SDim, SType, max_team_x>
            phys_tol += pptr[idx + d * npts] * pptr[idx + d * npts];
         }
         phys_tol = fmax(phys_rtol * phys_rtol, phys_tol * phys_rtol * phys_rtol);
-         hit_bdr[0] = prev_hit_bdr[0] = false;
      }
      // for each iteration
      while (true)
@@ -266,7 +266,11 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)

      // Gather the attributes on the host from all the elements
      const Mesh &mesh = *trial_fes->GetMesh();
-      elem_attributes = &mesh.GetElementAttributes();
+      elem_attributes.SetSize(mesh.GetNE());
+      for (int i = 0; i < mesh.GetNE(); ++i)
+      {
+         elem_attributes[i] = mesh.GetAttribute(i);
+      }
   }

   // Construct face restriction operators only if the bilinear form has
@@ -325,7 +329,45 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
         bdr_face_dYdn.SetSize(bdr_face_restrict_lex->Height());
      }

-      bdr_face_attributes = &trial_fes->GetMesh()->GetBdrFaceAttributes();
+      const Mesh &mesh = *trial_fes->GetMesh();
+      // See LinearFormExtension::Update for explanation of f_to_be logic.
+      std::unordered_map<int,int> f_to_be;
+      for (int i = 0; i < mesh.GetNBE(); ++i)
+      {
+         const int f = mesh.GetBdrElementFaceIndex(i);
+         f_to_be[f] = i;
+      }
+      const int nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary);
+      bdr_attributes.SetSize(nf_bdr);
+      int f_ind = 0;
+      int missing_bdr_elems = 0;
+      for (int f = 0; f < mesh.GetNumFaces(); ++f)
+      {
+         if (!mesh.GetFaceInformation(f).IsOfFaceType(FaceType::Boundary))
+         {
+            continue;
+         }
+         int attribute = 1; // default value
+         if (f_to_be.find(f) != f_to_be.end())
+         {
+            const int be = f_to_be[f];
+            attribute = mesh.GetBdrAttribute(be);
+         }
+         else
+         {
+            // If a boundary face does not correspond to the a boundary element,
+            // we assign it the default attribute of 1. We also generate a
+            // warning at runtime with the number of such missing elements.
+            ++missing_bdr_elems;
+         }
+         bdr_attributes[f_ind] = attribute;
+         ++f_ind;
+      }
+      if (missing_bdr_elems)
+      {
+         MFEM_WARNING("Missing " << missing_bdr_elems << " boundary elements "
+                      "for boundary faces.");
+      }
   }
 }

@@ -387,7 +429,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
         mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
         {
            const int attr = d_attr[e];
-            if (attr <= 0 || d_m[attr - 1] == 0)
+            if (d_m[attr - 1] == 0)
            {
               for (int i = 0; i < nd; ++i)
               {
@@ -408,7 +450,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
         for (int i = 0; i < iSz; ++i)
         {
            assemble_diagonal_with_markers(*integrators[i], elem_markers[i],
-                                           *elem_attributes, localY);
+                                           elem_attributes, localY);
         }
         const ElementRestriction* H1elem_restrict =
            dynamic_cast<const ElementRestriction*>(elem_restrict);
@@ -434,7 +476,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
      for (int i = 0; i < iSz; ++i)
      {
         assemble_diagonal_with_markers(*integrators[i], elem_markers[i],
-                                        *elem_attributes, y);
+                                        elem_attributes, y);
      }
   }

@@ -447,7 +489,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
      for (int i = 0; i < n_bdr_integs; ++i)
      {
         assemble_diagonal_with_markers(*bdr_integs[i], bdr_markers[i],
-                                        *bdr_face_attributes, bdr_face_Y);
+                                        bdr_attributes, bdr_face_Y);
      }
      bdr_face_restrict_lex->AddAbsMultTranspose(bdr_face_Y, y);
   }
@@ -546,7 +588,7 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y,
         for (int i = 0; i < iSz; ++i)
         {
            AddMultWithMarkers(*integrators[i], localX, elem_markers[i],
-                               *elem_attributes, false, localY, useAbs);
+                               elem_attributes, false, localY, useAbs);
         }
         if (H1elem_restrict && useAbs)
         {
@@ -648,8 +690,8 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y,
         }
         for (int i = 0; i < n_bdr_integs; ++i)
         {
-            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i],
-                               *bdr_face_attributes, false, bdr_face_Y);
+            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
+                               false, bdr_face_Y);
         }
         for (int i = 0; i < n_bdr_face_integs; ++i)
         {
@@ -657,14 +699,12 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y,
            {
               AddMultNormalDerivativesWithMarkers(
                  *bdr_face_integs[i], bdr_face_X, bdr_face_dXdn,
-                  bdr_face_markers[i], *bdr_face_attributes, bdr_face_Y,
-                  bdr_face_dYdn);
+                  bdr_face_markers[i], bdr_attributes, bdr_face_Y, bdr_face_dYdn);
            }
            else
            {
-               AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X,
-                                  bdr_face_markers[i], *bdr_face_attributes, false,
-                                  bdr_face_Y);
+               AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
+                                  bdr_attributes, false, bdr_face_Y);
            }
         }
         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
@@ -687,7 +727,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
      localY = 0.0;
      for (int i = 0; i < iSz; ++i)
      {
-         AddMultWithMarkers(*integrators[i], localX, elem_markers[i], *elem_attributes,
+         AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes,
                            true, localY);
      }
      elem_restrict->MultTranspose(localY, y);
@@ -734,14 +774,13 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         bdr_face_Y = 0.0;
         for (int i = 0; i < n_bdr_integs; ++i)
         {
-            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i],
-                               *bdr_face_attributes, true, bdr_face_Y);
+            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
+                               true, bdr_face_Y);
         }
         for (int i = 0; i < n_bdr_face_integs; ++i)
         {
-            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X,
-                               bdr_face_markers[i], *bdr_face_attributes, true,
-                               bdr_face_Y);
+            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
+                               bdr_attributes, true, bdr_face_Y);
         }
         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
@@ -765,7 +804,7 @@ static void AddWithMarkers_(
   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
   {
      const int attr = d_attr[e];
-      if (attr <= 0 || d_m[attr - 1] == 0) { return; }
+      if (d_m[attr - 1] == 0) { return; }
      for (int i = 0; i < nd; ++i)
      {
         d_y(i, e) += d_x(i, e);
@@ -881,8 +920,7 @@ void EABilinearFormExtension::Assemble()
      {
         const int i = idx % sz;
         const int e = idx / sz;
-         const real_t val =
-            d_a[e] > 0 ? (d_m[d_a[e] - 1] ? d_ea_1(i, e) : 0) : 0;
+         const real_t val = d_m[d_a[e] - 1] ? d_ea_1(i, e) : 0.0;
         if (add)
         {
            d_ea_2(i, e) += val;
@@ -915,7 +953,7 @@ void EABilinearFormExtension::Assemble()
            ea_data_tmp.SetSize(ea_data.Size());
            integrators[i]->AssembleEA(*a->FESpace(), ea_data_tmp, false);
            add_with_markers(ea_data_tmp, ea_data, ne, *markers,
-                             *elem_attributes, add);
+                             elem_attributes, add);
         }
      }
   }
@@ -944,7 +982,7 @@ void EABilinearFormExtension::Assemble()
            ea_data_tmp.SetSize(ea_data_bdr.Size());
            bdr_integs[i]->AssembleEABoundary(*a->FESpace(), ea_data_tmp, add);
            add_with_markers(ea_data_tmp, ea_data_bdr, nf_bdr, *markers,
-                             *bdr_face_attributes, add);
+                             bdr_attributes, add);
         }
      }
   }
@@ -993,7 +1031,7 @@ void EABilinearFormExtension::Assemble()
                                                        ea_data_tmp,
                                                        add);
            add_with_markers(ea_data_tmp, ea_data_bdr, nf_bdr, *markers,
-                             *bdr_face_attributes, add);
+                             bdr_attributes, add);
         }
      }
   }
@@ -69,8 +69,7 @@ class PABilinearFormExtension : public BilinearFormExtension
 protected:
   const FiniteElementSpace *trial_fes, *test_fes; // Not owned
   /// Attributes of all mesh elements.
-   const Array<int> *elem_attributes; // Not owned
-   const Array<int> *bdr_face_attributes; // Not owned
+   Array<int> elem_attributes, bdr_attributes;
   mutable Vector tmp_evec; // Work array
   mutable Vector localX, localY;
   mutable Vector int_face_X, int_face_Y;
@@ -23,8 +23,6 @@

 namespace mfem
 {
-class QuadratureSpace;
-class FaceQuadratureSpace;

 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
@@ -814,7 +812,7 @@ protected:
      const FiniteElement & test_fe) const
   {
      return (trial_fe.GetDim() == 1 && test_fe.GetDim() == 1 &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
              test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
   }

@@ -886,7 +884,7 @@ protected:
      const FiniteElement & trial_fe,
      const FiniteElement & test_fe) const
   {
-      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV &&
+      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV  &&
              test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
   }

@@ -921,7 +919,7 @@ protected:
      const FiniteElement & trial_fe,
      const FiniteElement & test_fe) const
   {
-      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV &&
+      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV  &&
              test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
   }

@@ -1602,7 +1600,7 @@ public:
   {
      return (trial_fe.GetCurlDim() == 3 && test_fe.GetRangeDim() == 3 &&
              trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::CURL  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::CURL   &&
              test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
   }

@@ -1637,7 +1635,7 @@ public:
   {
      return (trial_fe.GetDim() == 2 && test_fe.GetDim() == 2 &&
              trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::CURL  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::CURL   &&
              test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
   }

@@ -1671,7 +1669,7 @@ public:
   {
      return (trial_fe.GetDim() == 2 && test_fe.GetDim() == 2 &&
              trial_fe.GetRangeType() == mfem::FiniteElement::SCALAR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD   &&
              test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
   }

@@ -1762,7 +1760,7 @@ public:
      const FiniteElement & test_fe) const
   {
      return (trial_fe.GetRangeType() == mfem::FiniteElement::SCALAR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD   &&
              test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
   }

@@ -1795,7 +1793,7 @@ public:
      const FiniteElement & test_fe) const
   {
      return (trial_fe.GetRangeType() == mfem::FiniteElement::SCALAR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD   &&
              test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR &&
              test_fe.GetDerivType()  == mfem::FiniteElement::DIV   );
   }
@@ -1834,7 +1832,7 @@ public:
      const FiniteElement & test_fe) const
   {
      return (trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::DIV   &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::DIV    &&
              test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR &&
              test_fe.GetDerivType()  == mfem::FiniteElement::GRAD
             );
@@ -1975,7 +1973,7 @@ protected:
      const FiniteElement & test_fe) const override
   {
      return (trial_fe.GetCurlDim() == 3 && test_fe.GetRangeDim() == 3 &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::CURL &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::CURL  &&
              test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
   }

@@ -12,7 +12,6 @@
 // Implementation of Coefficient class

 #include "fem.hpp"
-#include "../general/forall.hpp"

 #include <cmath>
 #include <limits>
@@ -81,49 +80,6 @@ real_t PWConstCoefficient::Eval(ElementTransformation & T,
   return (constants(att-1));
 }

-void PWConstCoefficient::Project(QuadratureFunction &qf)
-{
-   auto &qs = *qf.GetSpace();
-
-   const bool compressed =
-      qs.Offsets(QSpaceOffsetStorage::COMPRESSED).Size() == 1;
-   const int *offsets = qs.Offsets(QSpaceOffsetStorage::COMPRESSED).Read();
-   const int ne = qs.GetNE();
-
-   const int *attributes = [&]()
-   {
-      if (dynamic_cast<QuadratureSpace*>(&qs) != nullptr)
-      {
-         return qs.GetMesh()->GetElementAttributes().Read();
-      }
-      else if (auto *qs_f = dynamic_cast<FaceQuadratureSpace*>(&qs))
-      {
-         MFEM_VERIFY(qs_f->GetFaceType() == FaceType::Boundary,
-                     "Interior faces do not have attributes.");
-         return qs.GetMesh()->GetBdrFaceAttributes().Read();
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported case.");
-      }
-   }();
-
-   const real_t *d_c = constants.Read();
-   real_t *d_qf = qf.Write();
-
-   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
-   {
-      const int a = attributes[e];
-      const real_t elementConstant = d_c[a - 1];
-      const int begin = compressed ? e*offsets[0] : offsets[e];
-      const int end = compressed ? (e+1)*offsets[0] : offsets[e+1];
-      for (int i = begin; i < end; ++i)
-      {
-         d_qf[i] = elementConstant;
-      }
-   });
-}
-
 void PWCoefficient::InitMap(const Array<int> & attr,
                            const Array<Coefficient*> & coefs)
 {
@@ -132,9 +132,6 @@ public:
   /// Evaluate the coefficient.
   real_t Eval(ElementTransformation &T,
               const IntegrationPoint &ip) override;
-
-   /// Fill the QuadratureFunction @a qf with the piecewise constant values.
-   void Project(QuadratureFunction &qf) override;
 };

 /** @brief A piecewise coefficient with the pieces keyed off the element
@@ -912,7 +912,7 @@ ConduitDataCollection::GridFunctionToBlueprintField(mfem::GridFunction *gf,

   if (vdim == 1) // scalar case
   {
-      n_field["values"].set_external(const_cast<real_t *>(gf->HostRead()),
+      n_field["values"].set_external(gf->GetData(),
                                     ndofs);
   }
   else // vector case
@@ -925,18 +925,18 @@ ConduitDataCollection::GridFunctionToBlueprintField(mfem::GridFunction *gf,
      int vdim_stride  = (ordering == Ordering::byNODES ? ndofs : 1);

      index_t offset = 0;
-      index_t stride = sizeof(real_t) * entry_stride;
+      index_t stride = sizeof(double) * entry_stride;

      for (int d = 0;  d < vdim; d++)
      {
         std::ostringstream oss;
         oss << "v" << d;
         std::string comp_name = oss.str();
-         n_field["values"][comp_name].set_external(const_cast<real_t *>(gf->HostRead()),
+         n_field["values"][comp_name].set_external(gf->GetData(),
                                                   ndofs,
                                                   offset,
                                                   stride);
-         offset +=  sizeof(real_t) * vdim_stride;
+         offset +=  sizeof(double) * vdim_stride;
      }
   }

@@ -764,9 +764,9 @@ ParaViewDataCollectionBase::ParaViewDataCollectionBase(
 {
   cycle = 0;
 #ifdef MFEM_USE_ZLIB
-   // If we have zlib, enable compression. Otherwise, compression is disabled in
-   // the DataCollection base class constructor.
-   compression = true;
+   compression = true; // if we have zlib, enable compression
+#else
+   compression = false; // otherwise, disable compression
 #endif
 }

@@ -784,8 +784,13 @@ void ParaViewDataCollectionBase::SetCompressionLevel(int compression_level_)
 {
   MFEM_ASSERT(compression_level_ >= -1 && compression_level_ <= 9,
               "Compression level must be between -1 and 9 (inclusive).");
-   if (compression_level_ != 0) { SetCompression(true);}
   compression_level = compression_level_;
+   compression = compression_level_ != 0;
+}
+
+void ParaViewDataCollectionBase::SetCompression(bool compression_)
+{
+   compression = compression_;
 }

 int ParaViewDataCollectionBase::GetCompressionLevel() const
@@ -1169,14 +1174,7 @@ const char *ParaViewDataCollection::GetDataTypeString() const
 ParaViewHDFDataCollection::ParaViewHDFDataCollection(
   const std::string &collection_name, Mesh *mesh)
   : ParaViewDataCollectionBase(collection_name, mesh)
-{
-   compression = true;
-}
-
-void ParaViewHDFDataCollection::SetCompression(bool compression_)
-{
-   compression = compression_;
-}
+{ }

 void ParaViewHDFDataCollection::EnsureVTKHDF()
 {
@@ -537,6 +537,13 @@ public:
   /// Any nonzero compression level will enable compression.
   void SetCompressionLevel(int compression_level_);

+   /// @brief Enable or disable zlib compression.
+   ///
+   /// If the input is true, use the default zlib compression level (unless the
+   /// compression level has previously been set by calling
+   /// SetCompressionLevel()).
+   void SetCompression(bool compression_) override;
+
   /// @brief Sets whether or not to output the data as high-order elements
   /// (false by default).
   ///
@@ -626,12 +633,6 @@ public:
   ParaViewHDFDataCollection(const std::string& collection_name,
                             Mesh *mesh_ = nullptr);

-   /// @brief Enable or disable compression.
-   ///
-   /// The compression level can be set with SetCompressionLevel()). VTKHDF
-   /// compression does not require MFEM to be compiled with zlib support.
-   void SetCompression(bool compression_) override;
-
   /// Save the collection.
   void Save() override;

@@ -1,266 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "derefmat_op.hpp"
-#include "fes_kernels.hpp"
-
-/// \cond DO_NOT_DOCUMENT
-namespace mfem
-{
-
-namespace internal
-{
-template <Ordering::Type Order, bool Atomic>
-static void DerefMultKernelImpl(const DerefineMatrixOp &op, const Vector &x,
-                                Vector &y)
-{
-   DerefineMatrixOpMultFunctor<Order, Atomic> func;
-   func.xptr = x.Read();
-   y.UseDevice();
-   y = 0.;
-   func.yptr = y.ReadWrite();
-   func.bsptr = op.block_storage.Read();
-   func.boptr = op.block_offsets.Read();
-   func.brptr = op.block_row_idcs_offsets.Read();
-   func.bcptr = op.block_col_idcs_offsets.Read();
-   func.rptr = op.row_idcs.Read();
-   func.cptr = op.col_idcs.Read();
-   func.vdims = op.fespace->GetVDim();
-   func.nblocks = op.block_offsets.Size();
-   func.width = op.Width() / func.vdims;
-   func.height = op.Height() / func.vdims;
-   func.Run(op.max_rows);
-}
-
-} // namespace internal
-
-DerefineMatrixOp::DerefineMatrixOp(FiniteElementSpace &fespace_, int old_ndofs,
-                                   const Table *old_elem_dof,
-                                   const Table *old_elem_fos)
-   : Operator(fespace_.GetVSize(), old_ndofs * fespace_.GetVDim()),
-     fespace(&fespace_)
-{
-   static Kernels kernels;
-   constexpr int max_team_size = 256;
-   /// TODO: Implement DofTransformation support
-
-   MFEM_VERIFY(fespace->Nonconforming(),
-               "Not implemented for conforming meshes.");
-   MFEM_VERIFY(old_ndofs, "Missing previous (finer) space.");
-   MFEM_VERIFY(fespace->GetNDofs() <= old_ndofs,
-               "Previous space is not finer.");
-
-   const CoarseFineTransformations &dtrans =
-      fespace->GetMesh()->ncmesh->GetDerefinementTransforms();
-
-   MFEM_ASSERT(dtrans.embeddings.Size() == old_elem_dof->Size(), "");
-
-   const bool is_dg = fespace->FEColl()->GetContType()
-                      == FiniteElementCollection::DISCONTINUOUS;
-   DenseMatrix localRVO; // for variable-order only
-
-   DenseTensor localR[Geometry::NumGeom];
-   int total_rows = 0;
-   int total_cols = 0;
-   block_offsets.SetSize(dtrans.embeddings.Size());
-   block_offsets.HostWrite();
-   if (fespace->IsVariableOrder())
-   {
-      // TODO: any potential for some compression here?
-      // determine storage size and offsets
-      block_offsets[0] = 0;
-      int total_size = 0;
-      for (int k = 0; k < dtrans.embeddings.Size(); ++k)
-      {
-         const Embedding &emb = dtrans.embeddings[k];
-         const FiniteElement *fe = fespace->GetFE(emb.parent);
-         const int ldof = fe->GetDof();
-         if (k + 1 < dtrans.embeddings.Size())
-         {
-            block_offsets[k + 1] = block_offsets[k] + ldof * ldof;
-         }
-         total_rows += ldof;
-         total_cols += ldof;
-         total_size += ldof * ldof;
-      }
-      block_storage.SetSize(total_size);
-   }
-   else
-   {
-      // compression scheme:
-      // block_offsets is the start of each block, potentially repeated
-      // only need to store localR for used shapes
-      Mesh::GeometryList elem_geoms(*fespace->GetMesh());
-
-      int geom_offsets[Geometry::NumGeom];
-      {
-         int size = 0;
-         for (int i = 0; i < elem_geoms.Size(); ++i)
-         {
-            fespace->GetLocalDerefinementMatrices(elem_geoms[i],
-                                                  localR[elem_geoms[i]]);
-            geom_offsets[elem_geoms[i]] = size;
-            size += localR[elem_geoms[i]].TotalSize();
-         }
-         block_storage.SetSize(size);
-         // copy blocks into block_storage
-         auto bs_ptr = block_storage.HostWrite();
-         for (int i = 0; i < elem_geoms.Size(); ++i)
-         {
-            std::copy(localR[elem_geoms[i]].Data(),
-                      localR[elem_geoms[i]].Data()
-                      + localR[elem_geoms[i]].TotalSize(),
-                      bs_ptr);
-            bs_ptr += localR[elem_geoms[i]].TotalSize();
-         }
-      }
-      for (int k = 0; k < dtrans.embeddings.Size(); ++k)
-      {
-         const Embedding &emb = dtrans.embeddings[k];
-         Geometry::Type geom =
-            fespace->GetMesh()->GetElementBaseGeometry(emb.parent);
-
-         auto size = localR[geom].SizeI() * localR[geom].SizeJ();
-         total_rows += localR[geom].SizeI();
-         total_cols += localR[geom].SizeJ();
-         // set block offsets and sizes
-         block_offsets[k] = geom_offsets[geom] + size * emb.matrix;
-      }
-   }
-   row_idcs.SetSize(total_rows);
-   row_idcs.HostWrite();
-   col_idcs.SetSize(total_cols);
-   col_idcs.HostWrite();
-   block_row_idcs_offsets.SetSize(dtrans.embeddings.Size() + 1);
-   block_row_idcs_offsets.HostWrite();
-   block_col_idcs_offsets.SetSize(dtrans.embeddings.Size() + 1);
-   block_col_idcs_offsets.HostWrite();
-   block_row_idcs_offsets[0] = 0;
-   block_col_idcs_offsets[0] = 0;
-
-   // compute index information
-   Array<int> dofs, old_dofs;
-   max_rows = 1;
-
-   {
-      Array<int> mark(fespace->GetNDofs());
-      mark = 0;
-      auto bs_ptr = block_storage.HostWrite();
-      int ridx = 0;
-      int cidx = 0;
-      int num_marked = 0;
-      for (int k = 0; k < dtrans.embeddings.Size(); k++)
-      {
-         const Embedding &emb = dtrans.embeddings[k];
-         Geometry::Type geom =
-            fespace->GetMesh()->GetElementBaseGeometry(emb.parent);
-
-         if (fespace->IsVariableOrder())
-         {
-            const FiniteElement *fe = fespace->GetFE(emb.parent);
-            const DenseTensor &pmats = dtrans.point_matrices[geom];
-            const int ldof = fe->GetDof();
-
-            IsoparametricTransformation isotr;
-            isotr.SetIdentityTransformation(geom);
-
-            localRVO.SetSize(ldof, ldof);
-            isotr.SetPointMat(pmats(emb.matrix));
-            // Local restriction is size ldofxldof assuming that the parent
-            // and child are of same polynomial order.
-            fe->GetLocalRestriction(isotr, localRVO);
-            // copy block
-            auto size = localRVO.Height() * localRVO.Width();
-            std::copy(localRVO.Data(), localRVO.Data() + size, bs_ptr);
-            bs_ptr += size;
-         }
-         DenseMatrix &lR =
-            fespace->IsVariableOrder() ? localRVO : localR[geom](emb.matrix);
-         block_row_idcs_offsets[k + 1] =
-            block_row_idcs_offsets[k] + lR.Height();
-         block_col_idcs_offsets[k + 1] = block_col_idcs_offsets[k] + lR.Width();
-         max_rows = std::max(lR.Height(), max_rows);
-         // index information
-         fespace->elem_dof->GetRow(emb.parent, dofs);
-         old_elem_dof->GetRow(k, old_dofs);
-         MFEM_VERIFY(old_dofs.Size() == dofs.Size(),
-                     "Parent and child must have same #dofs.");
-         for (int i = 0; i < lR.Height(); ++i, ++ridx)
-         {
-            if (!std::isfinite(lR(i, 0)))
-            {
-               row_idcs[ridx] = INT_MAX;
-               continue;
-            }
-            int r = dofs[i];
-            int m = (r >= 0) ? r : (-1 - r);
-            if (is_dg || !mark[m])
-            {
-               row_idcs[ridx] = r;
-               mark[m] = 1;
-               ++num_marked;
-            }
-            else
-            {
-               row_idcs[ridx] = INT_MAX;
-            }
-         }
-         for (int i = 0; i < lR.Width(); ++i, ++cidx)
-         {
-            col_idcs[cidx] = old_dofs[i];
-         }
-      }
-      if (!is_dg && !fespace->IsVariableOrder())
-      {
-         MFEM_VERIFY(num_marked * fespace->GetVDim() == Height(),
-                     "internal error: not all rows were set.");
-      }
-   }
-   // if not using GPU, set max_rows/max_cols to zero
-   if (Device::Allows(Backend::DEVICE_MASK))
-   {
-      max_rows = std::min(max_rows, max_team_size);
-   }
-   else
-   {
-      max_rows = 1;
-   }
-}
-
-void DerefineMatrixOp::Mult(const Vector &x, Vector &y) const
-{
-   const bool is_dg = fespace->FEColl()->GetContType()
-                      == FiniteElementCollection::DISCONTINUOUS;
-   // DG needs atomic summation
-   MultKernel::Run(fespace->GetOrdering(), is_dg, *this, x, y);
-}
-
-DerefineMatrixOp::Kernels::Kernels()
-{
-   MultKernel::Specialization<Ordering::byNODES, false>::Add();
-   MultKernel::Specialization<Ordering::byVDIM, false>::Add();
-   MultKernel::Specialization<Ordering::byNODES, true>::Add();
-   MultKernel::Specialization<Ordering::byVDIM, true>::Add();
-}
-
-template <Ordering::Type Order, bool Atomic>
-DerefineMatrixOp::MultKernelType DerefineMatrixOp::MultKernel::Kernel()
-{
-   return internal::DerefMultKernelImpl<Order, Atomic>;
-}
-
-DerefineMatrixOp::MultKernelType
-DerefineMatrixOp::MultKernel::Fallback(Ordering::Type, bool)
-{
-   MFEM_ABORT("invalid MultKernel parameters");
-}
-} // namespace mfem
-/// \endcond DO_NOT_DOCUMENT
@@ -1,65 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_DEREFMAT_OP
-#define MFEM_DEREFMAT_OP
-
-#include "fespace.hpp"
-
-#include "kernel_dispatch.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-struct DerefineMatrixOp : public Operator
-{
-   FiniteElementSpace *fespace;
-   /// offsets into block_storage
-   Array<int> block_offsets;
-   /// offsets into row_idcs
-   Array<int> block_row_idcs_offsets;
-   /// offsets into col_idcs
-   Array<int> block_col_idcs_offsets;
-   /// mapping for row dofs, INT_MAX indicates the block row should be ignored.
-   /// negative means the row data should be negated.
-   Array<int> row_idcs;
-   /// mapping for col dofs, negative means the col data should be negated.
-   Array<int> col_idcs;
-   /// dense block matrices which can be reused to construct the full matrix
-   /// operation. These are stored contiguously and blocks have no restrictions
-   /// on shape (can be rectangle and differ from block to block).
-   Vector block_storage;
-   /// maximum height of any block in block_storage for GPU
-   /// parallelization, or 1 for CPU runs.
-   int max_rows;
-
-   using MultKernelType = void (*)(const DerefineMatrixOp &, const Vector &,
-                                   Vector &);
-   /// template args: ordering, atomic
-   MFEM_REGISTER_KERNELS(MultKernel, MultKernelType, (Ordering::Type, bool));
-
-   struct Kernels
-   {
-      Kernels();
-   };
-
-   void Mult(const Vector &x, Vector &y) const;
-
-   DerefineMatrixOp(FiniteElementSpace &fespace_, int old_ndofs,
-                    const Table *old_elem_dof, const Table *old_elem_fos);
-};
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem
-#endif
@@ -231,53 +231,21 @@ public:
      const std::vector<FieldDescriptor> &parameters,
      const ParMesh &mesh);

-   /// MultLevel enum to indicate if the T->L Operators are used in the
-   /// Mult method.
-   enum MultLevel
-   {
-      TVECTOR,
-      LVECTOR
-   };
-
-   /// @brief Set the MultLevel mode for the DifferentiableOperator.
-   /// The default is TVECTOR, which means that the Operator will use
-   /// T->L before Mult and L->T Operators after.
-   void SetMultLevel(MultLevel level)
-   {
-      mult_level = level;
-   }
-
   /// @brief Compute the action of the operator on a given vector.
   ///
-   /// @param solutions_in The solution vector in which to compute the action.
-   /// This has to be a T-dof vector if MultLevel is set to TVECTOR, or L-dof
-   /// Vector if MultLevel is set to LVECTOR.
-   /// @param result_in Result vector of the action of the operator on
-   /// solutions. The result is a T-dof vector or L-dof vector depending on
-   /// the MultLevel.
-   void Mult(const Vector &solutions_in, Vector &result_in) const override
+   /// @param solutions_t The solution vector in which to compute the action.
+   /// This has to be a T-dof vector.
+   /// @param result_t Result vector of the action of the operator on
+   /// solutions_t. The result is a T-dof vector.
+   void Mult(const Vector &solutions_t, Vector &result_t) const override
   {
      MFEM_ASSERT(!action_callbacks.empty(), "no integrators have been set");
-
-      if (mult_level == MultLevel::LVECTOR)
+      prolongation(solutions, solutions_t, solutions_l);
+      for (auto &action : action_callbacks)
      {
-         get_lvectors(solutions, solutions_in, solutions_l);
-         result_in = 0.0;
-         for (auto &action : action_callbacks)
-         {
-            action(solutions_l, parameters_l, result_in);
-         }
-      }
-      else
-      {
-         prolongation(solutions, solutions_in, solutions_l);
-         residual_l = 0.0;
-         for (auto &action : action_callbacks)
-         {
-            action(solutions_l, parameters_l, residual_l);
-         }
-         prolongation_transpose(residual_l, result_in);
+         action(solutions_l, parameters_l, residual_l);
      }
+      prolongation_transpose(residual_l, result_t);
   }

   /// @brief Add a domain integrator to the operator.
@@ -376,8 +344,6 @@ public:
 private:
   const ParMesh &mesh;

-   MultLevel mult_level = TVECTOR;
-
   std::vector<action_t> action_callbacks;
   std::map<size_t,
       std::vector<derivative_action_t>> derivative_action_callbacks;
@@ -387,6 +353,7 @@ private:
       std::vector<assemble_derivative_hypreparmatrix_callback_t>>
       assemble_derivative_hypreparmatrix_callbacks;

+
   std::vector<FieldDescriptor> solutions;
   std::vector<FieldDescriptor> parameters;
   // solutions and parameters
@@ -327,8 +327,8 @@ void print_mpi_sync(const std::string& msg)
   // First gather string lengths
   size_t msg_len = msg.length();
   std::vector<size_t> lengths(nranks);
-   MPI_Gather(&msg_len, 1, MPITypeMap<size_t>::mpi_type,
-              lengths.data(), 1, MPITypeMap<size_t>::mpi_type,
+   MPI_Gather(&msg_len, 1, MPI_INT,
+              lengths.data(), 1, MPI_INT,
              0, MPI_COMM_WORLD);

   if (myrank == 0)
@@ -568,7 +568,7 @@ struct ThreadBlocks
   int z = 1;
 };

-#if defined(MFEM_USE_CUDA_OR_HIP)
+#if (defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
 template <typename func_t>
 __global__ void forall_kernel_shmem(func_t f, int n)
 {
@@ -591,7 +591,7 @@ void forall(func_t f,
   if (Device::Allows(Backend::CUDA_MASK) ||
       Device::Allows(Backend::HIP_MASK))
   {
-#if defined(MFEM_USE_CUDA_OR_HIP)
+#if (defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
      // int gridsize = (N + Z - 1) / Z;
      int num_bytes = num_shmem * sizeof(decltype(shmem));
      dim3 block_size(blocks.x, blocks.y, blocks.z);
@@ -987,7 +987,7 @@ get_restriction_transpose(
   {
      auto RT = [=](const Vector &v_e, Vector &v_l)
      {
-         v_l += v_e;
+         v_l = v_e;
      };
      return std::make_tuple(RT, 1);
   }
@@ -996,7 +996,7 @@ get_restriction_transpose(
      const Operator *R = get_restriction<entity_t>(f, o);
      std::function<void(const Vector&, Vector&)> RT = [=](const Vector &x, Vector &y)
      {
-         R->AddMultTranspose(x, y);
+         R->MultTranspose(x, y);
      };
      return std::make_tuple(RT, R->Height());
   }
@@ -1076,24 +1076,6 @@ void prolongation(const std::vector<FieldDescriptor> fields,
   }
 }

-inline
-void get_lvectors(const std::vector<FieldDescriptor> fields,
-                  const Vector &x,
-                  std::vector<Vector> &fields_l)
-{
-   int data_offset = 0;
-   for (std::size_t i = 0; i < fields.size(); i++)
-   {
-      const int sz = GetVSize(fields[i]);
-      fields_l[i].SetSize(sz);
-
-      const Vector x_i(const_cast<Vector&>(x), data_offset, sz);
-      fields_l[i] = x_i;
-
-      data_offset += sz;
-   }
-}
-
 /// @brief Get a transpose prolongation callback for a field descriptor.
 ///
 /// In the special case of a one field operator, the transpose prolongation
@@ -1726,7 +1708,6 @@ std::array<DofToQuadMap, N> load_dtq_mem(
         const auto B = Reshape(&dtq[i].B[0], nqp_b, dim_b, ndof_b);
         auto mem_Bi = Reshape(reinterpret_cast<real_t *>(mem) + offset, nqp_b, dim_b,
                               ndof_b);
-
         MFEM_FOREACH_THREAD(q, x, nqp_b)
         {
            MFEM_FOREACH_THREAD(d, y, ndof_b)
@@ -2177,7 +2158,7 @@ template <
   std::size_t... Is>
 std::array<DofToQuadMap, N> create_dtq_maps_impl(
   field_operator_ts &fops,
-   std::vector<const DofToQuad*> &dtqs,
+   std::vector<const DofToQuad*> dtqs,
   const std::array<int, N> &field_map,
   std::index_sequence<Is...>)
 {
@@ -2262,7 +2243,7 @@ template <
   std::size_t num_fields>
 std::array<DofToQuadMap, num_fields> create_dtq_maps(
   field_operator_ts &fops,
-   std::vector<const DofToQuad*> &dtqmaps,
+   std::vector<const DofToQuad*> dtqmaps,
   const std::array<int, num_fields> &to_field_map)
 {
   return create_dtq_maps_impl<entity_t>(
@@ -401,6 +401,9 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
   {
      MFEM_ABORT("unknown FiniteElementCollection: " << name);
   }
+   MFEM_VERIFY(!strcmp(fec->Name(), name), "input name: \"" << name
+               << "\" does not match the created collection name: \""
+               << fec->Name() << '"');

   return fec;
 }
@@ -2515,7 +2518,6 @@ RT_FECollection::RT_FECollection(const int p, const int dim,
                                 const int map_type, const bool signs,
                                 const int ob_type)
   : FiniteElementCollection(p + 1)
-   , dim(dim)
   , ob_type(ob_type)
 {
   if (Quadrature1D::CheckOpen(BasisType::GetQuadrature1D(ob_type)) ==
@@ -464,13 +464,6 @@ public:
   RT_Trace_FECollection(const int p, const int dim,
                         const int map_type = FiniteElement::INTEGRAL,
                         const int ob_type = BasisType::GaussLegendre);
-
-   FiniteElementCollection *Clone(int p) const override
-   {
-      const int map_type = (strncmp(rt_name, "RT_Trace", 8) == 0)?
-                           (FiniteElement::INTEGRAL):(FiniteElement::VALUE);
-      return new RT_Trace_FECollection(p, dim, map_type, ob_type);
-   }
 };

 /** Arbitrary order discontinuous finite elements defined on the interface
@@ -482,13 +475,6 @@ public:
   DG_Interface_FECollection(const int p, const int dim,
                             const int map_type = FiniteElement::VALUE,
                             const int ob_type = BasisType::GaussLegendre);
-
-   FiniteElementCollection *Clone(int p) const override
-   {
-      const int map_type = (strncmp(rt_name, "DG_Iface", 8) == 0)?
-                           (FiniteElement::VALUE):(FiniteElement::INTEGRAL);
-      return new DG_Interface_FECollection(p, dim, map_type, ob_type);
-   }
 };

 /// Arbitrary order H(curl)-conforming Nedelec finite elements.
@@ -1,249 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_FES_KERNELS_HPP
-#define MFEM_FES_KERNELS_HPP
-
-#include "../general/forall.hpp"
-
-#include <climits>
-
-namespace mfem
-{
-/// \cond DO_NOT_DOCUMENT
-namespace internal
-{
-
-///
-/// Implements matrix-vector multiply $y = A x$ for a sparse matrix composed of
-/// a sum of smaller dense blocks. There is additional permutation/sign
-/// information associated with each block. The base class only implements
-/// helper routines such as computing block widths, index into x, index into y,
-/// and column in A given sub-block information.
-/// @sa DerefineMatrixOpMultFunctor
-///
-/// @tparam Order vdim ordering for x and y. Note that for Diag = false this is
-/// ignored for x as x has a special interleaved order.
-/// @tparam Base used for the curious recurring template pattern (CRTP) so the
-/// base class can access child class fields without virtual functions
-/// @tparam Diag true if this corresponds to the diagonal block (coarse element
-/// and fine element are on our rank), false otherwise (coarse element is on our
-/// rank, fine element is on a different rank).
-///
-template <Ordering::Type Order, class Base, bool Diag = true>
-struct DerefineMatrixOpFunctorBase;
-
-template <class Base>
-struct DerefineMatrixOpFunctorBase<Ordering::byNODES, Base, true>
-{
-   /// block column indices offsets
-   const int *bcptr;
-   /// column indices
-   const int *cptr;
-
-   int MFEM_HOST_DEVICE BlockWidth(int k) const
-   {
-      return bcptr[k + 1] - bcptr[k];
-   }
-
-   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
-   {
-      col = cptr[bcptr[k] + j];
-      if (col < 0)
-      {
-         col = -1 - col;
-         sign = -sign;
-      }
-   }
-
-   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int) const
-   {
-      return col + vdim * static_cast<const Base *>(this)->width;
-   }
-   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
-   {
-      return row + vdim * static_cast<const Base *>(this)->height;
-   }
-};
-
-template <class Base>
-struct DerefineMatrixOpFunctorBase<Ordering::byVDIM, Base, true>
-{
-   /// block column indices offsets
-   const int *bcptr;
-   /// column indices
-   const int *cptr;
-
-   int MFEM_HOST_DEVICE BlockWidth(int k) const
-   {
-      return bcptr[k + 1] - bcptr[k];
-   }
-
-   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
-   {
-      col = cptr[bcptr[k] + j];
-      if (col < 0)
-      {
-         col = -1 - col;
-         sign = -sign;
-      }
-   }
-
-   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int) const
-   {
-      return vdim + col * static_cast<const Base *>(this)->vdims;
-   }
-   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
-   {
-      return vdim + row * static_cast<const Base *>(this)->vdims;
-   }
-};
-
-template <class Base>
-struct DerefineMatrixOpFunctorBase<Ordering::byNODES, Base, false>
-{
-   /// receive segment offsets
-   const int *segptr;
-   /// receive segment index
-   const int *rsptr;
-   /// off-diagonal block column offsets
-   const int *coptr;
-   /// off-diagonal block widths
-   const int *bwptr;
-
-   int MFEM_HOST_DEVICE BlockWidth(int k) const { return bwptr[k]; }
-
-   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
-   {
-      col = coptr[k] + j;
-   }
-
-   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int k) const
-   {
-      int tmp = rsptr[k];
-      int segwidth = segptr[tmp + 1] - segptr[tmp];
-      return segptr[tmp] * static_cast<const Base *>(this)->vdims + col +
-             vdim * segwidth;
-   }
-   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
-   {
-      return row + vdim * static_cast<const Base *>(this)->height;
-   }
-};
-
-template <class Base>
-struct DerefineMatrixOpFunctorBase<Ordering::byVDIM, Base, false>
-{
-   /// receive segment offsets
-   const int *segptr;
-   /// receive segment index
-   const int *rsptr;
-   /// off-diagonal block column offsets
-   const int *coptr;
-   /// off-diagonal block widths
-   const int *bwptr;
-
-   int MFEM_HOST_DEVICE BlockWidth(int k) const { return bwptr[k]; }
-
-   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
-   {
-      col = coptr[k] + j;
-   }
-
-   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int k) const
-   {
-      int tmp = rsptr[k];
-      int segwidth = segptr[tmp + 1] - segptr[tmp];
-      return segptr[tmp] * static_cast<const Base *>(this)->vdims + col +
-             vdim * segwidth;
-   }
-   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
-   {
-      return vdim + row * static_cast<const Base *>(this)->vdims;
-   }
-};
-
-/// internally used to implement the derefinement operator Mult diagonal
-/// block
-template <Ordering::Type Order, bool Atomic, bool Diag = true>
-struct DerefineMatrixOpMultFunctor
-   : public DerefineMatrixOpFunctorBase<
-     Order, DerefineMatrixOpMultFunctor<Order, Atomic, Diag>, Diag>
-{
-   const real_t *xptr;
-   real_t *yptr;
-   /// block storage
-   const real_t *bsptr;
-   /// block offsets
-   const int *boptr;
-   /// block row index offsets
-   const int *brptr;
-   /// row indices
-   const int *rptr;
-
-   // number of blocks
-   int nblocks;
-   // number of components
-   int vdims;
-   /// overall operator height (for vdim = 1)
-   int height;
-   /// overall operator width (for vdim = 1)
-   int width;
-   void MFEM_HOST_DEVICE operator()(int kidx) const
-   {
-      int k = kidx % nblocks;
-      int vdim = kidx / nblocks;
-
-      int block_height = brptr[k + 1] - brptr[k];
-      int block_width = this->BlockWidth(k);
-      MFEM_FOREACH_THREAD(i, x, block_height)
-      {
-         int row = rptr[brptr[k] + i];
-         int rsign = 1;
-         if (row < 0)
-         {
-            row = -1 - row;
-            rsign = -1;
-         }
-         if (row < INT_MAX)
-         {
-            // row not marked as unused
-            real_t sum = 0;
-            for (int j = 0; j < block_width; ++j)
-            {
-               int col, sign = rsign;
-               this->Col(j, k, col, sign);
-               sum += sign * bsptr[boptr[k] + i + j * block_height] *
-                      xptr[this->IndexX(col, vdim, k)];
-            }
-#if defined(__CUDA_ARCH__) or defined(__HIP_DEVICE_COMPILE__)
-            if (Atomic)
-            {
-               atomicAdd(yptr + this->IndexY(row, vdim), sum);
-            }
-            else
-#endif
-            {
-               yptr[this->IndexY(row, vdim)] += sum;
-            }
-         }
-      }
-   }
-
-   /// N is the max block row size (doesn't have to be a power of 2)
-   void Run(int N) const { forall_2D(nblocks * vdims, N, 1, *this); }
-};
-
-} // namespace internal
-/// \endcond DO_NOT_DOCUMENT
-} // namespace mfem
-
-#endif
@@ -17,9 +17,6 @@
 #include "fem.hpp"
 #include "ceed/interface/util.hpp"

-#include "derefmat_op.hpp"
-
-#include <algorithm>
 #include <cmath>
 #include <cstdarg>

@@ -27,9 +24,9 @@ using namespace std;

 namespace mfem
 {
-template <>
-void Ordering::DofsToVDofs<Ordering::byNODES>(int ndofs, int vdim,
-                                              Array<int> &dofs)
+
+template <> void Ordering::
+DofsToVDofs<Ordering::byNODES>(int ndofs, int vdim, Array<int> &dofs)
 {
   // static method
   int size = dofs.Size();
@@ -43,9 +40,8 @@ void Ordering::DofsToVDofs<Ordering::byNODES>(int ndofs, int vdim,
   }
 }

-template <>
-void Ordering::DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim,
-                                             Array<int> &dofs)
+template <> void Ordering::
+DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim, Array<int> &dofs)
 {
   // static method
   int size = dofs.Size();
@@ -59,6 +55,7 @@ void Ordering::DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim,
   }
 }

+
 FiniteElementSpace::FiniteElementSpace()
   : mesh(NULL), fec(NULL), vdim(0), ordering(Ordering::byNODES),
     ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0),
@@ -4247,11 +4244,7 @@ void FiniteElementSpace::Update(bool want_transform)
         case Mesh::DEREFINE:
         {
            BuildConformingInterpolation();
-#if 0
            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
-#else
-            Th.Reset(new DerefineMatrixOp(*this, old_ndofs, old_elem_dof, old_elem_fos));
-#endif
            if (IsVariableOrder())
            {
               if (cP && cR_hp)
@@ -113,7 +113,7 @@ class QuadratureSpace;
 class QuadratureInterpolator;
 class FaceQuadratureInterpolator;
 class PRefinementTransferOperator;
-struct DerefineMatrixOp;
+

 /** @brief Class FiniteElementSpace - responsible for providing FEM view of the
    mesh, mainly managing the set of degrees of freedom.
@@ -246,7 +246,6 @@ class FiniteElementSpace
   friend class PRefinementTransferOperator;
   friend void Mesh::Swap(Mesh &, bool);
   friend class LORBase;
-   friend struct DerefineMatrixOp;

 protected:
   /// The mesh that FE space lives on (not owned).
@@ -922,9 +921,6 @@ public:
   { return mesh->GetBdrElementType(i); }

   /// Returns ElementTransformation for the @a i-th element.
-   /// @note The returned pointer references an object owned by the associated
-   /// @a Mesh that will be modified by other calls to `GetElementTransformation`.
-   /// As such, this pointer should @b not be deleted by the caller.
   ElementTransformation *GetElementTransformation(int i) const
   { return mesh->GetElementTransformation(i); }

@@ -4334,7 +4334,7 @@ real_t LSZZErrorEstimator(BilinearFormIntegrator &blfi,  // input
         u.GetSubVector(udofs, ul);
         utrans.InvTransformPrimal(ul);
         Transf = ufes->GetElementTransformation(ielem);
-         const auto *dummy = ufes->GetFE(ielem);
+         FiniteElement *dummy = nullptr;
         blfi.ComputeElementFlux(*ufes->GetFE(ielem), *Transf, ul,
                                 *dummy, fl, with_coeff, ir);

@@ -85,9 +85,9 @@ namespace mfem
 {

 FindPointsGSLIB::FindPointsGSLIB()
-   : mesh(nullptr),
-     fec_map_lin(nullptr),
-     fdataD(nullptr), cr(nullptr), gsl_comm(nullptr),
+   : mesh(NULL),
+     fec_map_lin(NULL),
+     fdataD(NULL), cr(NULL), gsl_comm(NULL),
     dim(-1), points_cnt(-1), setupflag(false), default_interp_value(0),
     avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
@@ -97,10 +97,10 @@ FindPointsGSLIB::FindPointsGSLIB()
   gf_rst_map.SetSize(4);
   for (int i = 0; i < mesh_split.Size(); i++)
   {
-      mesh_split[i] = nullptr;
-      ir_split[i] = nullptr;
-      fes_rst_map[i] = nullptr;
-      gf_rst_map[i] = nullptr;
+      mesh_split[i] = NULL;
+      ir_split[i] = NULL;
+      fes_rst_map[i] = NULL;
+      gf_rst_map[i] = NULL;
   }

   gsl_comm = new gslib::comm;
@@ -117,40 +117,27 @@ FindPointsGSLIB::FindPointsGSLIB()
   crystal_init(cr, gsl_comm);
 }

-FindPointsGSLIB::FindPointsGSLIB(Mesh &mesh_in, const double bb_t,
-                                 const double newt_tol, const int npt_max)
-   : FindPointsGSLIB()
-{
-   Setup(mesh_in, bb_t, newt_tol, npt_max);
-}
-
 FindPointsGSLIB::~FindPointsGSLIB()
 {
-   FreeData();
-#ifdef MFEM_USE_MPI
-   if (!Mpi::IsFinalized())  // currently segfaults inside gslib otherwise
-#endif
+   crystal_free(cr);
+   comm_free(gsl_comm);
+   delete gsl_comm;
+   delete cr;
+   for (int i = 0; i < 4; i++)
   {
-      crystal_free(cr);
-      comm_free(gsl_comm);
-      delete gsl_comm;
-      delete cr;
+      if (mesh_split[i]) { delete mesh_split[i]; mesh_split[i] = NULL; }
+      if (ir_split[i]) { delete ir_split[i]; ir_split[i] = NULL; }
+      if (fes_rst_map[i]) { delete fes_rst_map[i]; fes_rst_map[i] = NULL; }
+      if (gf_rst_map[i]) { delete gf_rst_map[i]; gf_rst_map[i] = NULL; }
   }
-   for (int i = 0; i < mesh_split.Size(); i++)
-   {
-      if (mesh_split[i]) { delete mesh_split[i]; mesh_split[i] = nullptr; }
-      if (ir_split[i]) { delete ir_split[i]; ir_split[i] = nullptr; }
-      if (fes_rst_map[i]) { delete fes_rst_map[i]; fes_rst_map[i] = nullptr; }
-      if (gf_rst_map[i]) { delete gf_rst_map[i]; gf_rst_map[i] = nullptr; }
-   }
-   if (fec_map_lin) { delete fec_map_lin; fec_map_lin = nullptr; }
+   if (fec_map_lin) { delete fec_map_lin; fec_map_lin = NULL; }
 }

 #ifdef MFEM_USE_MPI
 FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
-   : mesh(nullptr),
-     fec_map_lin(nullptr),
-     fdataD(nullptr), cr(nullptr), gsl_comm(nullptr),
+   : mesh(NULL),
+     fec_map_lin(NULL),
+     fdataD(NULL), cr(NULL), gsl_comm(NULL),
     dim(-1), points_cnt(-1), setupflag(false), default_interp_value(0),
     avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
@@ -160,10 +147,10 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
   gf_rst_map.SetSize(4);
   for (int i = 0; i < mesh_split.Size(); i++)
   {
-      mesh_split[i] = nullptr;
-      ir_split[i] = nullptr;
-      fes_rst_map[i] = nullptr;
-      gf_rst_map[i] = nullptr;
+      mesh_split[i] = NULL;
+      ir_split[i] = NULL;
+      fes_rst_map[i] = NULL;
+      gf_rst_map[i] = NULL;
   }

   gsl_comm = new gslib::comm;
@@ -171,21 +158,12 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
   comm_init(gsl_comm, comm_);
   crystal_init(cr, gsl_comm);
 }
-
-FindPointsGSLIB::FindPointsGSLIB(ParMesh &mesh_in, const double bb_t,
-                                 const double newt_tol, const int npt_max)
-   : FindPointsGSLIB(mesh_in.GetComm())
-{
-   Setup(mesh_in, bb_t, newt_tol, npt_max);
-}
 #endif

 void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
                            const int npt_max)
 {
   MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
-   MFEM_VERIFY(m.SpaceDimension() == m.Dimension(),
-               "Mesh spatial dimension and reference element dimension must be the same");
   const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();

   // call FreeData if FindPointsGSLIB::Setup has been called already
@@ -193,9 +171,37 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,

   mesh = &m;
   dim  = mesh->Dimension();
-   const unsigned int dof1D = meshOrder+1;
+   unsigned dof1D = meshOrder + 1;

-   SetupSplitMeshesAndIntegrationRules(meshOrder);
+   SetupSplitMeshes();
+   if (dim == 2)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+   }
+   else if (dim == 3)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+
+      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
+      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
+
+      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
+      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
+   }

   GetNodalValues(mesh->GetNodes(), gsl_mesh);

@@ -1122,18 +1128,13 @@ void FindPointsGSLIB::Interpolate(Mesh &m, const Vector &point_pos,
 void FindPointsGSLIB::FreeData()
 {
   if (!setupflag) { return; }
-#ifdef MFEM_USE_MPI
-   if (!Mpi::IsFinalized())  // currently segfaults inside gslib otherwise
-#endif
+   if (dim == 2)
   {
-      if (dim == 2)
-      {
-         findpts_free_2((gslib::findpts_data_2 *)this->fdataD);
-      }
-      else
-      {
-         findpts_free_3((gslib::findpts_data_3 *)this->fdataD);
-      }
+      findpts_free_2((gslib::findpts_data_2 *)this->fdataD);
+   }
+   else
+   {
+      findpts_free_3((gslib::findpts_data_3 *)this->fdataD);
   }
   gsl_code.DeleteAll();
   gsl_proc.DeleteAll();
@@ -1157,8 +1158,8 @@ void FindPointsGSLIB::FreeData()

 void FindPointsGSLIB::SetupSplitMeshes()
 {
-   if (fec_map_lin == nullptr) { fec_map_lin = new H1_FECollection(1, dim); }
-   if (dim == 2)
+   fec_map_lin = new H1_FECollection(1, dim);
+   if (mesh->Dimension() == 2)
   {
      int Nvert = 7;
      int NEsplit = 3;
@@ -1200,7 +1201,7 @@ void FindPointsGSLIB::SetupSplitMeshes()
      mesh_split[1] = new Mesh(Mesh::MakeCartesian2D(1, 1,
                                                     Element::QUADRILATERAL));
   }
-   else if (dim == 3)
+   else if (mesh->Dimension() == 3)
   {
      mesh_split[0] = new Mesh(Mesh::MakeCartesian3D(1, 1, 1,
                                                     Element::HEXAHEDRON));
@@ -1345,6 +1346,41 @@ void FindPointsGSLIB::SetupSplitMeshes()
         }
      }
   }
+
+   NE_split_total = 0;
+   split_element_map.SetSize(0);
+   split_element_index.SetSize(0);
+   int NEsplit = 0;
+   for (int e = 0; e < mesh->GetNE(); e++)
+   {
+      const Geometry::Type gt   = mesh->GetElement(e)->GetGeometryType();
+      if (gt == Geometry::TRIANGLE || gt == Geometry::PRISM)
+      {
+         NEsplit = 3;
+      }
+      else if (gt == Geometry::TETRAHEDRON)
+      {
+         NEsplit = 4;
+      }
+      else if (gt == Geometry::PYRAMID)
+      {
+         NEsplit = 8;
+      }
+      else if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
+      {
+         NEsplit = 1;
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported geometry type.");
+      }
+      NE_split_total += NEsplit;
+      for (int i = 0; i < NEsplit; i++)
+      {
+         split_element_map.Append(e);
+         split_element_index.Append(i);
+      }
+   }
 }

 void FindPointsGSLIB::SetupIntegrationRuleForSplitMesh(Mesh *meshin,
@@ -1395,79 +1431,6 @@ void FindPointsGSLIB::SetupIntegrationRuleForSplitMesh(Mesh *meshin,
   }
 }

-void FindPointsGSLIB::SetupSplitMeshesAndIntegrationRules(const int order)
-{
-   MFEM_VERIFY(mesh, "Setup FindPointsGSLIB with mesh first.");
-   const int dof1D = order+1;
-   const int dim = mesh->Dimension();
-
-   SetupSplitMeshes();
-   if (dim == 2)
-   {
-      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
-      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], order);
-
-      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
-      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], order);
-   }
-   else if (dim == 3)
-   {
-      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
-      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], order);
-
-      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
-      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], order);
-
-      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
-      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], order);
-
-      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
-      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], order);
-   }
-
-   // Setup map for non tensor-product elements
-   NE_split_total = 0;
-   split_element_map.SetSize(0);
-   split_element_index.SetSize(0);
-   int NEsplit = 0;
-   for (int e = 0; e < mesh->GetNE(); e++)
-   {
-      const Geometry::Type gt   = mesh->GetElement(e)->GetGeometryType();
-      if (gt == Geometry::TRIANGLE || gt == Geometry::PRISM)
-      {
-         NEsplit = 3;
-      }
-      else if (gt == Geometry::TETRAHEDRON)
-      {
-         NEsplit = 4;
-      }
-      else if (gt == Geometry::PYRAMID)
-      {
-         NEsplit = 8;
-      }
-      else if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
-      {
-         NEsplit = 1;
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported geometry type.");
-      }
-      NE_split_total += NEsplit;
-      for (int i = 0; i < NEsplit; i++)
-      {
-         split_element_map.Append(e);
-         split_element_index.Append(i);
-      }
-   }
-}
-
 void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
                                     Vector &node_vals)
 {
@@ -2118,19 +2081,6 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
   } // parallel
 }

-Array<unsigned int> FindPointsGSLIB::GetPointsNotFoundIndices() const
-{
-   Array<unsigned int> nf_idxs;
-   for (int i = 0; i < gsl_code.Size(); i++)
-   {
-      if (gsl_code[i] == 2)
-      {
-         nf_idxs.Append(i);
-      }
-   }
-   return nf_idxs;
-}
-
 void FindPointsGSLIB::DistributePointInfoToOwningMPIRanks(
   Array<unsigned int> &recv_elem, Vector &recv_ref,
   Array<unsigned int> &recv_code)
@@ -2436,10 +2386,6 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
 {
   MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
   const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();
-   const int gfOrder = gfmax ? gfmax->FESpace()->GetMaxElementOrder() :
-                       meshOrder;
-   MFEM_VERIFY(meshOrder == gfOrder,
-               "Mesh order must match gfmax order in OversetFindPointsGSLIB.");

   // FreeData if OversetFindPointsGSLIB::Setup has been called already
   if (setupflag) { FreeData(); }
@@ -2449,7 +2395,35 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
   const FiniteElement *fe = mesh->GetNodalFESpace()->GetTypicalFE();
   unsigned dof1D = fe->GetOrder() + 1;

-   SetupSplitMeshesAndIntegrationRules(meshOrder);
+   SetupSplitMeshes();
+   if (dim == 2)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+   }
+   else if (dim == 3)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+
+      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
+      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
+
+      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
+      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
+   }

   GetNodalValues(mesh->GetNodes(), gsl_mesh);

@@ -2506,7 +2480,7 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
 {
   MFEM_VERIFY(setupflag, "Use OversetFindPointsGSLIB::Setup before "
               "finding points.");
-   MFEM_VERIFY(overset, "Please use OversetFindPoints for overlapping grids.");
+   MFEM_VERIFY(overset, "Please setup FindPoints for overlapping grids.");
   points_cnt = point_pos.Size() / dim;
   unsigned int match = 0; // Don't find points in the mesh if point_id=mesh_id

@@ -13,11 +13,7 @@
 #define MFEM_GSLIB

 #include "../config/config.hpp"
-#ifdef MFEM_USE_MPI
-#include "pgridfunc.hpp"
-#else
 #include "gridfunc.hpp"
-#endif

 #ifdef MFEM_USE_GSLIB

@@ -135,10 +131,6 @@ protected:
                                                 IntegrationRule *irule,
                                                 int order);

-   /// Helper function that calls \ref SetupSplitMeshes and
-   /// \ref SetupIntegrationRuleForSplitMesh.
-   virtual void SetupSplitMeshesAndIntegrationRules(const int order);
-
   /// Get GridFunction value at the points expected by GSLIB.
   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals);

@@ -198,23 +190,14 @@ protected:
   void InterpolateOnDevice(const Vector &field_in_evec, Vector &field_out,
                            const int nel, const int ncomp,
                            const int dof1dsol, const int ordering);
-
 public:
   FindPointsGSLIB();
-   FindPointsGSLIB(Mesh &mesh_in, const double bb_t = 0.1,
-                   const double newt_tol = 1.0e-12,
-                   const int npt_max = 256);

 #ifdef MFEM_USE_MPI
   FindPointsGSLIB(MPI_Comm comm_);
-   FindPointsGSLIB(ParMesh &mesh_in, const double bb_t = 0.1,
-                   const double newt_tol = 1.0e-12,
-                   const int npt_max = 256);
 #endif

   virtual ~FindPointsGSLIB();
-   FindPointsGSLIB(const FindPointsGSLIB&) = delete;
-   FindPointsGSLIB& operator=(const FindPointsGSLIB&) = delete;

   /** Initializes the internal mesh in gslib, by sending the positions of the
       Gauss-Lobatto nodes of the input Mesh object \p m.
@@ -229,8 +212,8 @@ public:
       @param[in] npt_max   (Optional) Number of points for simultaneous
                            iteration. This alters performance and
                            memory footprint.*/
-
-   void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12,
+   void Setup(Mesh &m, const double bb_t = 0.1,
+              const double newt_tol = 1.0e-12,
              const int npt_max = 256);
   /** Searches positions given in physical space by \p point_pos.
       These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
@@ -306,12 +289,7 @@ public:

   /** Cleans up memory allocated internally by gslib.
       Note that in parallel, this must be called before MPI_Finalize(), as it
-       calls MPI_Comm_free() for internal gslib communicators. FreeData is
-       also called by the class destructor and there are no memory leaks if the
-       destructor is called before MPI_Finalize(). If the destructor is called
-       after MPI_Finalize(), there will be an error because gslib will try to
-       invoke some MPI functions.
-   */
+       calls MPI_Comm_free() for internal gslib communicators. */
   virtual void FreeData();

   /// Return code for each point searched by FindPoints: inside element (0), on
@@ -334,9 +312,6 @@ public:
   /// point found by FindPoints.
   virtual const Vector &GetGSLIBReferencePosition() const { return gsl_ref; }

-   /// Get array of indices of not-found points.
-   Array<unsigned int> GetPointsNotFoundIndices() const;
-
   /** @name Methods to support a custom interpolation procedure.
       \brief The physical-space point that the user seeks to interpolate at
       could be located inside an element on another mpi rank.
@@ -181,7 +181,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
   // current elements' the number of degrees of freedom
   // does not consider the number of equations
   const int dof1 = el1.GetDof();
-   const int dof2 = (Tr.Elem2No >= 0)?(el2.GetDof()):(0);
+   const int dof2 = el2.GetDof();

 #ifdef MFEM_THREAD_SAFE
   // Local storage for element integration
@@ -219,9 +219,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
   const IntegrationRule *ir = IntRule;
   if (!ir)
   {
-      const int max_el_order = dof2 ? std::max(el1.GetOrder(),
-                                               el2.GetOrder()) : el1.GetOrder();
-      const int order = 2*max_el_order + IntOrderOffset;
+      const int order = 2*std::max(el1.GetOrder(), el2.GetOrder()) + IntOrderOffset;
      ir = &IntRules.Get(Tr.GetGeometryType(), order);
   }
   // loop over integration points
@@ -233,22 +231,18 @@ void HyperbolicFormIntegrator::AssembleFaceVector(

      // Calculate basis functions on both elements at the face
      el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
+      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);

      // Interpolate elfun at the point
      elfun1_mat.MultTranspose(shape1, state1);
-
-      if (dof2)
-      {
-         // Calculate basis functions on both elements at the face
-         el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
-         // Interpolate elfun at the point
-         elfun2_mat.MultTranspose(shape2, state2);
-      }
+      elfun2_mat.MultTranspose(shape2, state2);

      // Get the normal vector and the flux on the face
      if (nor.Size() == 1)  // if 1D, use 1 or -1.
      {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
+         // This assume the 1D integration point is in (0,1). This may not work
+         // if this changes.
+         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
      }
      else
      {
@@ -256,18 +250,14 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
      }
      // Compute F(u+, x) and F(u-, x) with maximum characteristic speed
      // Compute hat(F) using evaluated quantities
-      const real_t speed = (dof2) ? numFlux.Eval(state1, state2, nor, Tr, fluxN):
-                           fluxFunction.ComputeFluxDotN(state1, nor, Tr, fluxN);
+      const real_t speed = numFlux.Eval(state1, state2, nor, Tr, fluxN);

      // Update the global max char speed
      max_char_speed = std::max(speed, max_char_speed);

      // pre-multiply integration weight to flux
      AddMult_a_VWt(-ip.weight*sign, shape1, fluxN, elvect1_mat);
-      if (dof2)
-      {
-         AddMult_a_VWt(+ip.weight*sign, shape2, fluxN, elvect2_mat);
-      }
+      AddMult_a_VWt(+ip.weight*sign, shape2, fluxN, elvect2_mat);
   }
 }

@@ -278,7 +268,7 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
   // current elements' the number of degrees of freedom
   // does not consider the number of equations
   const int dof1 = el1.GetDof();
-   const int dof2 = (Tr.Elem2No >= 0)?(el2.GetDof()):(0);
+   const int dof2 = el2.GetDof();

 #ifdef MFEM_THREAD_SAFE
   // Local storage for element integration
@@ -312,9 +302,7 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
   const IntegrationRule *ir = IntRule;
   if (!ir)
   {
-      const int max_el_order = dof2 ? std::max(el1.GetOrder(),
-                                               el2.GetOrder()) : el1.GetOrder();
-      const int order = 2*max_el_order + IntOrderOffset;
+      const int order = 2*std::max(el1.GetOrder(), el2.GetOrder()) + IntOrderOffset;
      ir = &IntRules.Get(Tr.GetGeometryType(), order);
   }
   // loop over integration points
@@ -324,25 +312,20 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(

      Tr.SetAllIntPoints(&ip); // set face and element int. points

-      // Calculate basis functions of the first element at the face
+      // Calculate basis functions on both elements at the face
      el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
+      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);

      // Interpolate elfun at the point
      elfun1_mat.MultTranspose(shape1, state1);
-
-      if (dof2)
-      {
-         // Calculate basis function of the second element at the face
-         el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
-
-         // Interpolate elfun at the point
-         elfun2_mat.MultTranspose(shape2, state2);
-      }
+      elfun2_mat.MultTranspose(shape2, state2);

      // Get the normal vector and the flux on the face
      if (nor.Size() == 1)  // if 1D, use 1 or -1.
      {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
+         // This assume the 1D integration point is in (0,1). This may not work
+         // if this changes.
+         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
      }
      else
      {
@@ -352,14 +335,7 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
      // Trial side 1

      // Compute hat(J) using evaluated quantities
-      if (dof2)
-      {
-         numFlux.Grad(1, state1, state2, nor, Tr, JDotN);
-      }
-      else
-      {
-         fluxFunction.ComputeFluxJacobianDotN(state1, nor, Tr, JDotN);
-      }
+      numFlux.Grad(1, state1, state2, nor, Tr, JDotN);

      const int ioff = fluxFunction.num_equations * dof1;

@@ -384,325 +360,36 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
            }
         }

-      if (dof2)
-      {
-         // Trial side 2
-
-         // Compute hat(J) using evaluated quantities
-         numFlux.Grad(2, state1, state2, nor, Tr, JDotN);
-
-         const int joff = ioff;
-
-         for (int di = 0; di < fluxFunction.num_equations; di++)
-            for (int dj = 0; dj < fluxFunction.num_equations; dj++)
-            {
-               // pre-multiply integration weight to Jacobian
-               const real_t w = +ip.weight * sign * JDotN(di,dj);
-               for (int j = 0; j < dof2; j++)
-               {
-                  // Test side 1
-                  for (int i = 0; i < dof1; i++)
-                  {
-                     elmat(i+dof1*di, joff+j+dof2*dj) += w * shape1(i) * shape2(j);
-                  }
-
-                  // Test side 2
-                  for (int i = 0; i < dof2; i++)
-                  {
-                     elmat(ioff+i+dof2*di, joff+j+dof2*dj) -= w * shape2(i) * shape2(j);
-                  }
-               }
-            }
-      }
-   }
-}
-
-BdrHyperbolicDirichletIntegrator::BdrHyperbolicDirichletIntegrator(
-   const NumericalFlux &numFlux,
-   VectorCoefficient &bdrState,
-   const int IntOrderOffset,
-   real_t sign)
-   : NonlinearFormIntegrator(),
-     numFlux(numFlux),
-     fluxFunction(numFlux.GetFluxFunction()),
-     u_vcoeff(bdrState),
-     IntOrderOffset(IntOrderOffset),
-     sign(sign),
-     num_equations(fluxFunction.num_equations)
-{
-   MFEM_VERIFY(fluxFunction.num_equations == bdrState.GetVDim(),
-               "Flux function does not match the vector dimension of the coefficient!");
-#ifndef MFEM_THREAD_SAFE
-   state_in.SetSize(num_equations);
-   state_out.SetSize(num_equations);
-   fluxN.SetSize(num_equations);
-   JDotN.SetSize(num_equations);
-   nor.SetSize(fluxFunction.dim);
-#endif
-   ResetMaxCharSpeed();
-}
-
-void BdrHyperbolicDirichletIntegrator::AssembleFaceVector(
-   const FiniteElement &el, const FiniteElement &,
-   FaceElementTransformations &Tr, const Vector &elfun, Vector &elvect)
-{
-   MFEM_ASSERT(Tr.Elem2No < 0, "Not a boundary face!");
-
-   // current elements' the number of degrees of freedom
-   // does not consider the number of equations
-   const int dof = el.GetDof();
-
-#ifdef MFEM_THREAD_SAFE
-   // Local storage for element integration
-
-   // shape function value at an integration point
-   Vector shape(dof);
-   // normal vector (usually not a unit vector)
-   Vector nor(Tr.GetSpaceDim());
-   // state value at an integration point - interior
-   Vector state_in(num_equations);
-   // state value at an integration point - boundary
-   Vector state_out(num_equations);
-   // hat(F)(u,x)
-   Vector fluxN(num_equations);
-#else
-   shape.SetSize(dof);
-#endif
-
-   elvect.SetSize(dof * num_equations);
-   elvect = 0.0;
-
-   const DenseMatrix elfun_mat(elfun.GetData(), dof, num_equations);
-
-   DenseMatrix elvect_mat(elvect.GetData(), dof, num_equations);
-
-   // Obtain integration rule. If integration is rule is given, then use it.
-   // Otherwise, get (2*p + IntOrderOffset) order integration rule
-   const IntegrationRule *ir = IntRule;
-   if (!ir)
-   {
-      const int order = 2*el.GetOrder() + IntOrderOffset;
-      ir = &IntRules.Get(Tr.GetGeometryType(), order);
-   }
-   // loop over integration points
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      Tr.SetAllIntPoints(&ip); // set face and element int. points
-
-      // Calculate basis functions at the face
-      el.CalcShape(Tr.GetElement1IntPoint(), shape);
-
-      // Interpolate elfun at the point
-      elfun_mat.MultTranspose(shape, state_in);
-
-      // Evaluate boundary state at the point
-      u_vcoeff.Eval(state_out, Tr, ip);
-
-      // Get the normal vector and the flux on the face
-      if (nor.Size() == 1)  // if 1D, use 1 or -1.
-      {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
-      }
-      else
-      {
-         CalcOrtho(Tr.Jacobian(), nor);
-      }
-      // Compute F(u+, x) and F(u_b, x) with maximum characteristic speed
-      // Compute hat(F) using evaluated quantities
-      const real_t speed = numFlux.Eval(state_in, state_out, nor, Tr, fluxN);
-
-      // Update the global max char speed
-      max_char_speed = std::max(speed, max_char_speed);
-
-      // pre-multiply integration weight to flux
-      AddMult_a_VWt(-ip.weight*sign, shape, fluxN, elvect_mat);
-   }
-}
-
-void BdrHyperbolicDirichletIntegrator::AssembleFaceGrad(
-   const FiniteElement &el, const FiniteElement &,
-   FaceElementTransformations &Tr, const Vector &elfun, DenseMatrix &elmat)
-{
-   // current elements' the number of degrees of freedom
-   // does not consider the number of equations
-   const int dof = el.GetDof();
-
-#ifdef MFEM_THREAD_SAFE
-   // Local storage for element integration
-
-   // shape function value at an integration point
-   Vector shape(dof);
-   // normal vector (usually not a unit vector)
-   Vector nor(Tr.GetSpaceDim());
-   // state value at an integration point - interior
-   Vector state_in(num_equations);
-   // state value at an integration point - boundary
-   Vector state_out(num_equations);
-   // hat(J)(u,x)
-   DenseMatrix JDotN(num_equations);
-#else
-   shape.SetSize(dof);
-#endif
-
-   elmat.SetSize(dof * num_equations);
-   elmat = 0.0;
-
-   const DenseMatrix elfun_mat(elfun.GetData(), dof, num_equations);
-
-   // Obtain integration rule. If integration is rule is given, then use it.
-   // Otherwise, get (2*p + IntOrderOffset) order integration rule
-   const IntegrationRule *ir = IntRule;
-   if (!ir)
-   {
-      const int order = 2*el.GetOrder() + IntOrderOffset;
-      ir = &IntRules.Get(Tr.GetGeometryType(), order);
-   }
-   // loop over integration points
-   for (int q = 0; q < ir->GetNPoints(); q++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(q);
-
-      Tr.SetAllIntPoints(&ip); // set face and element int. points
-
-      // Calculate basis functions at the face
-      el.CalcShape(Tr.GetElement1IntPoint(), shape);
-
-      // Interpolate elfun at the point
-      elfun_mat.MultTranspose(shape, state_in);
-
-      // Evaluate boundary state at the point
-      u_vcoeff.Eval(state_out, Tr, ip);
-
-      // Get the normal vector and the flux on the face
-      if (nor.Size() == 1)  // if 1D, use 1 or -1.
-      {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
-      }
-      else
-      {
-         CalcOrtho(Tr.Jacobian(), nor);
-      }
+      // Trial side 2

      // Compute hat(J) using evaluated quantities
-      numFlux.Grad(1, state_in, state_out, nor, Tr, JDotN);
+      numFlux.Grad(2, state1, state2, nor, Tr, JDotN);
+
+      const int joff = ioff;

      for (int di = 0; di < fluxFunction.num_equations; di++)
         for (int dj = 0; dj < fluxFunction.num_equations; dj++)
         {
            // pre-multiply integration weight to Jacobian
-            const real_t w = -ip.weight * sign * JDotN(di,dj);
-            for (int j = 0; j < dof; j++)
-               for (int i = 0; i < dof; i++)
+            const real_t w = +ip.weight * sign * JDotN(di,dj);
+            for (int j = 0; j < dof2; j++)
+            {
+               // Test side 1
+               for (int i = 0; i < dof1; i++)
               {
-                  elmat(i+dof*di, j+dof*dj) += w * shape(i) * shape(j);
+                  elmat(i+dof1*di, joff+j+dof2*dj) += w * shape1(i) * shape2(j);
               }
+
+               // Test side 2
+               for (int i = 0; i < dof2; i++)
+               {
+                  elmat(ioff+i+dof2*di, joff+j+dof2*dj) -= w * shape2(i) * shape2(j);
+               }
+            }
         }
   }
 }

-BoundaryHyperbolicFlowIntegrator::BoundaryHyperbolicFlowIntegrator(
-   const FluxFunction &flux, VectorCoefficient &u, real_t alpha_, real_t beta_,
-   const int IntOrderOffset_)
-   : fluxFunction(flux), u_vcoeff(u), alpha(alpha_), beta(beta_),
-     IntOrderOffset(IntOrderOffset_)
-{
-   MFEM_VERIFY(fluxFunction.num_equations == u_vcoeff.GetVDim(),
-               "Flux function does not match the vector dimension of the coefficient!");
-#ifndef MFEM_THREAD_SAFE
-   state.SetSize(fluxFunction.num_equations);
-   nor.SetSize(fluxFunction.dim);
-   fluxN.SetSize(fluxFunction.num_equations);
-#endif
-   ResetMaxCharSpeed();
-}
-
-void BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect(
-   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
-{
-   mfem_error("BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect\n"
-              "  is not implemented as boundary integrator!\n"
-              "  Use LinearForm::AddBdrFaceIntegrator instead of\n"
-              "  LinearForm::AddBoundaryIntegrator.");
-}
-
-void BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect(
-   const FiniteElement &el, FaceElementTransformations &Tr, Vector &elvect)
-{
-   // current elements' the number of degrees of freedom
-   // does not consider the number of equations
-   const int dof = el.GetDof();
-
-#ifdef MFEM_THREAD_SAFE
-   // Local storage for element integration
-
-   // shape function value at an integration point
-   Vector shape(dof);
-   // state value at an integration point
-   Vector state(fluxFunction.num_equations);
-   // normal vector (usually not a unit vector)
-   Vector nor(Tr.GetSpaceDim());
-   // hat(F)(u,x)
-   Vector fluxN(fluxFunction.num_equations);
-#else
-   shape.SetSize(dof);
-#endif
-
-   elvect.SetSize(dof * fluxFunction.num_equations);
-   elvect = 0.0;
-
-   DenseMatrix elvect_mat(elvect.GetData(), dof, fluxFunction.num_equations);
-
-   // Obtain integration rule. If integration is rule is given, then use it.
-   // Otherwise, get (2*p + IntOrderOffset) order integration rule
-   const IntegrationRule *ir = IntRule;
-   if (!ir)
-   {
-      const int order = 2*el.GetOrder() + IntOrderOffset;
-      ir = &IntRules.Get(Tr.GetGeometryType(), order);
-   }
-   // loop over integration points
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      Tr.SetAllIntPoints(&ip); // set face and element int. points
-
-      // Calculate basis functions on both elements at the face
-      el.CalcShape(Tr.GetElement1IntPoint(), shape);
-
-      // Evaluate the coefficient at the point
-      u_vcoeff.Eval(state, Tr, ip);
-
-      // Get the normal vector and the flux on the face
-      if (nor.Size() == 1)  // if 1D, use 1 or -1.
-      {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
-      }
-      else
-      {
-         CalcOrtho(Tr.Jacobian(), nor);
-      }
-      // Compute F(u, x) with maximum characteristic speed
-      const real_t speed = fluxFunction.ComputeFluxDotN(state, nor, Tr, fluxN);
-
-      // Update the global max char speed
-      max_char_speed = std::max(speed, max_char_speed);
-
-      // pre-multiply integration weight to flux
-      const real_t a = 0.5 * alpha  * ip.weight;
-      const real_t b = beta * ip.weight;
-
-      for (int n = 0; n < fluxFunction.num_equations; n++)
-      {
-         fluxN(n) = a * fluxN(n) - b * fabs(fluxN(n));
-      }
-
-      AddMultVWt(shape, fluxN, elvect_mat);
-   }
-}
-
 real_t FluxFunction::ComputeFluxDotN(const Vector &U,
                                     const Vector &normal,
                                     FaceElementTransformations &Tr,
@@ -306,14 +306,12 @@ MFEM_DEPRECATED typedef NumericalFlux RiemannSolver;
 class HyperbolicFormIntegrator : public NonlinearFormIntegrator
 {
 private:
+   // The maximum characteristic speed, updated during element/face vector assembly
+   real_t max_char_speed;
   const NumericalFlux &numFlux;   // Numerical flux that maps F(u±,x) to F̂
   const FluxFunction &fluxFunction;
   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
   const real_t sign;
-
-   // The maximum characteristic speed, updated during element/face vector assembly
-   real_t max_char_speed;
-
 #ifndef MFEM_THREAD_SAFE
   // Local storage for element integration
   Vector shape;              // shape function value at an integration point
@@ -333,9 +331,8 @@ private:

 public:
   const int num_equations;  // the number of equations
-
   /**
-    * @brief Construct a new HyperbolicFormIntegrator object
+    * @brief Construct a new Hyperbolic Form Integrator object
    *
    * @param[in] numFlux numerical flux
    * @param[in] IntOrderOffset integration order offset
@@ -346,14 +343,21 @@ public:
      const int IntOrderOffset = 0,
      const real_t sign = 1.);

-   /// Reset the maximum characteristic speed to zero
-   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
+   /**
+    * @brief Reset the Max Char Speed 0
+    *
+    */
+   void ResetMaxCharSpeed()
+   {
+      max_char_speed = 0.0;
+   }

-   /// Get the maximum characteristic speed
-   real_t GetMaxCharSpeed() const { return max_char_speed; }
+   real_t GetMaxCharSpeed()
+   {
+      return max_char_speed;
+   }

-   /// Get the associated flux function
-   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
+   const FluxFunction &GetFluxFunction() { return fluxFunction; }

   /**
    * @brief Implements (F(u), ∇v) with abstract F computed by
@@ -412,182 +416,6 @@ public:
                         const Vector &elfun, DenseMatrix &elmat) override;
 };

-/**
- * @brief Abstract boundary hyperbolic form integrator, assembling
- * <F̂(u⁻,u_b,x) n, [v]> term for scalar finite elements at the boundary.
- *
- * This form integrator is coupled with a NumericalFlux that implements the
- * numerical flux F̂ at the boundary faces. The flux F is obtained from the
- * FluxFunction assigned to the aforementioned NumericalFlux with the given
- * boundary coefficient for the state u_b.
- *
- * Note the class can be used for imposing conditions on interior interfaces.
- */
-class BdrHyperbolicDirichletIntegrator : public NonlinearFormIntegrator
-{
-private:
-   const NumericalFlux &numFlux;    // Numerical flux that maps F to F̂
-   const FluxFunction &fluxFunction;
-   VectorCoefficient &u_vcoeff;     // Boundary state vector coefficient
-   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
-   const real_t sign;
-
-   // The maximum characteristic speed, updated during element/face vector assembly
-   real_t max_char_speed;
-
-#ifndef MFEM_THREAD_SAFE
-   // Local storage for element integration
-   Vector shape;  // shape function value at an integration point
-   Vector state_in;  // state value at an integration point - interior
-   Vector state_out;  // state value at an integration point - boundary
-   Vector nor;     // normal vector, see mfem::CalcOrtho()
-   Vector fluxN;   // F̂(u⁻,u_b,x) n
-   DenseMatrix JDotN;   // Ĵ(u⁻,u_b,x) n
-#endif
-
-public:
-   const int num_equations;  // the number of equations
-
-   /**
-    * @brief Construct a new BdrHyperbolicDirichletIntegrator object
-    *
-    * @param[in] numFlux numerical flux
-    * @param[in] bdrState boundary state coefficient
-    * @param[in] IntOrderOffset integration order offset
-    * @param[in] sign sign of the convection term
-    */
-   BdrHyperbolicDirichletIntegrator(
-      const NumericalFlux &numFlux,
-      VectorCoefficient &bdrState,
-      const int IntOrderOffset = 0,
-      const real_t sign = 1.);
-
-   /// Reset the maximum characteristic speed to zero
-   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
-
-   /// Get the maximum characteristic speed
-   real_t GetMaxCharSpeed() const { return max_char_speed; }
-
-   /// Get the associated flux function
-   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
-
-   /**
-    * @brief Implements <-F̂(u⁻,u_b,x) n, [v]> with abstract F̂ computed by
-    * NumericalFlux::Eval() of the numerical flux object
-    *
-    * @param[in] el1 finite element of the interior element
-    * @param[in] el2 not used
-    * @param[in] Tr face element transformations
-    * @param[in] elfun local coefficient of basis for the interior element
-    * @param[out] elvect evaluated dual vector <-F̂(u⁻,u_b,x) n, [v]>
-    */
-   void AssembleFaceVector(const FiniteElement &el1,
-                           const FiniteElement &el2,
-                           FaceElementTransformations &Tr,
-                           const Vector &elfun, Vector &elvect) override;
-
-   /**
-    * @brief Implements <-Ĵ(u⁻,u_b,x) n, [v]> with abstract Ĵ computed by
-    * NumericalFlux::Grad() of the numerical flux object
-    *
-    * @param[in] el1 finite element of the interior element
-    * @param[in] el2 not used
-    * @param[in] Tr face element transformations
-    * @param[in] elfun local coefficient of basis for the interior element
-    * @param[out] elmat evaluated Jacobian matrix <-Ĵ(u⁻,u_b,x) n, [v]>
-    */
-   void AssembleFaceGrad(const FiniteElement &el1,
-                         const FiniteElement &el2,
-                         FaceElementTransformations &Tr,
-                         const Vector &elfun, DenseMatrix &elmat) override;
-};
-
-/**
- * @brief Abstract boundary hyperbolic linear form integrator, assembling
- * <ɑ/2 F(u,x) n - β |F(u,x) n|, v> terms for scalar finite elements.
- *
- * This form integrator is coupled with a FluxFunction that evaluates the
- * flux F at the boundary.
- *
- * Note the upwinding is performed component-wise. For general boundary
- * integration with a numerical flux, see BdrHyperbolicDirichletIntegrator.
- */
-class BoundaryHyperbolicFlowIntegrator : public LinearFormIntegrator
-{
-   const FluxFunction &fluxFunction;
-   VectorCoefficient &u_vcoeff;
-   const real_t alpha, beta;
-   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
-
-   // The maximum characteristic speed, updated during face vector assembly
-   real_t max_char_speed;
-
-#ifndef MFEM_THREAD_SAFE
-   // Local storage for element integration
-   Vector shape;   // shape function value at an integration point
-   Vector state;   // state value at an integration point
-   Vector nor;     // normal vector, see mfem::CalcOrtho()
-   Vector fluxN;   // F(u,x) n
-#endif
-
-public:
-   /**
-    * @brief Construct a new BoundaryHyperbolicFlowIntegrator object
-    *
-    * @param[in] flux flux function
-    * @param[in] u vector state coefficient
-    * @param[in] alpha ɑ coefficient (β = ɑ/2)
-    * @param[in] IntOrderOffset integration order offset
-    */
-   BoundaryHyperbolicFlowIntegrator(const FluxFunction &flux, VectorCoefficient &u,
-                                    real_t alpha = -1., int IntOrderOffset = 0)
-      : BoundaryHyperbolicFlowIntegrator(flux, u, alpha, alpha/2., IntOrderOffset) { }
-
-   /**
-    * @brief Construct a new BoundaryHyperbolicFlowIntegrator object
-    *
-    * @param[in] flux flux function
-    * @param[in] u vector state coefficient
-    * @param[in] alpha ɑ coefficient
-    * @param[in] beta β coefficient
-    * @param[in] IntOrderOffset integration order offset
-    */
-   BoundaryHyperbolicFlowIntegrator(const FluxFunction &flux, VectorCoefficient &u,
-                                    real_t alpha, real_t beta, int IntOrderOffset = 0);
-
-   /// Reset the maximum characteristic speed to zero
-   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
-
-   /// Get the maximum characteristic speed
-   real_t GetMaxCharSpeed() const { return max_char_speed; }
-
-   /// Get the associated flux function
-   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
-
-   using LinearFormIntegrator::AssembleRHSElementVect;
-
-   /**
-    * @warning Boundary element integration not implemented, use
-    * AssembleRHSElementVect(const FiniteElement&,
-    * FaceElementTransformations &, Vector &) instead
-    */
-   void AssembleRHSElementVect(const FiniteElement &el,
-                               ElementTransformation &Tr,
-                               Vector &elvect) override;
-
-   /**
-    * @brief Implements <-F(u,x) n, v> with abstract F computed by
-    * FluxFunction::ComputeFluxDotN() of the flux function object
-    *
-    * @param[in] el finite element
-    * @param[in] Tr face element transformations
-    * @param[out] elvect evaluated dual vector <F(u,x) n, v>
-    */
-   void AssembleRHSElementVect(const FiniteElement &el,
-                               FaceElementTransformations &Tr,
-                               Vector &elvect) override;
-};
-

 /**
 * @brief Rusanov flux, also known as local Lax-Friedrichs,
@@ -139,6 +139,8 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
                         Device::GetDeviceMemoryType() : pa_mt;

+   nf = fes.GetNFbyType(type);
+   if (nf==0) { return; }
   // Assumes tensor-product elements
   Mesh *mesh = fes.GetMesh();
   const FiniteElement &el = *fes.GetTypicalTraceElement();
@@ -146,12 +148,6 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
                               IntRule:
                               &GetRule(el.GetGeomType(), el.GetOrder(),
                                        *mesh->GetTypicalElementTransformation());
-
-
-   FaceQuadratureSpace qs(*mesh, *ir, type);
-   nf = qs.GetNumFaces();
-   if (nf==0) { return; }
-
   const int symmDims = 4;
   nq = ir->GetNPoints();
   dim = mesh->Dimension();
@@ -163,6 +159,8 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType());
+
+   FaceQuadratureSpace qs(*mesh, *ir, type);
   CoefficientVector vel(*u, qs, CoefficientStorage::COMPRESSED);

   CoefficientVector r(qs, CoefficientStorage::COMPRESSED);
@@ -1009,7 +1009,6 @@ inline void SmemPADiffusionApply3D(const int NE,
   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_VERIFY(D1D <= Q1D, "THREAD_DIRECT requires D1D <= Q1D");
   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -1039,11 +1038,11 @@ inline void SmemPADiffusionApply3D(const int NE,
      real_t (*QDD0)[MD1][MD1] = (real_t (*)[MD1][MD1]) (sm0+0);
      real_t (*QDD1)[MD1][MD1] = (real_t (*)[MD1][MD1]) (sm0+1);
      real_t (*QDD2)[MD1][MD1] = (real_t (*)[MD1][MD1]) (sm0+2);
-      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
               X[dz][dy][dx] = x(dx,dy,dz,e);
            }
@@ -1051,9 +1050,9 @@ inline void SmemPADiffusionApply3D(const int NE,
      }
      if (MFEM_THREAD_ID(z) == 0)
      {
-         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
               B[qx][dy] = b(qx,dy);
               G[qx][dy] = g(qx,dy);
@@ -1061,11 +1060,11 @@ inline void SmemPADiffusionApply3D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
               real_t u = 0.0, v = 0.0;
               MFEM_UNROLL(MD1)
@@ -1081,11 +1080,11 @@ inline void SmemPADiffusionApply3D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD_DIRECT(qy,y,Q1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
               real_t u = 0.0, v = 0.0, w = 0.0;
               MFEM_UNROLL(MD1)
@@ -1102,11 +1101,11 @@ inline void SmemPADiffusionApply3D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD_DIRECT(qz,z,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
      {
-         MFEM_FOREACH_THREAD_DIRECT(qy,y,Q1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
               real_t u = 0.0, v = 0.0, w = 0.0;
               MFEM_UNROLL(MD1)
@@ -1137,9 +1136,9 @@ inline void SmemPADiffusionApply3D(const int NE,
      MFEM_SYNC_THREAD;
      if (MFEM_THREAD_ID(z) == 0)
      {
-         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
               Bt[dy][qx] = b(qx,dy);
               Gt[dy][qx] = g(qx,dy);
@@ -1147,11 +1146,11 @@ inline void SmemPADiffusionApply3D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD_DIRECT(qz,z,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
      {
-         MFEM_FOREACH_THREAD_DIRECT(qy,y,Q1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
               real_t u = 0.0, v = 0.0, w = 0.0;
               MFEM_UNROLL(MQ1)
@@ -1168,11 +1167,11 @@ inline void SmemPADiffusionApply3D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD_DIRECT(qz,z,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
      {
-         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
               real_t u = 0.0, v = 0.0, w = 0.0;
               MFEM_UNROLL(Q1D)
@@ -1189,11 +1188,11 @@ inline void SmemPADiffusionApply3D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
               real_t u = 0.0, v = 0.0, w = 0.0;
               MFEM_UNROLL(MQ1)
@@ -59,23 +59,26 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)

   QuadratureSpace qs(*mesh, *ir);
   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+
+   const int NE = ne;
+   const int Q1D = quad1D;
+   const int NQ = pow(Q1D, dim);
+   const bool const_c = coeff.Size() == 1;
+   const bool by_val = map_type == FiniteElement::VALUE;
+   const auto W = Reshape(ir->GetWeights().Read(), NQ);
+   const auto J = Reshape(geom->detJ.Read(), NQ, NE);
+   const auto C = const_c ? Reshape(coeff.Read(), 1, 1) :
+                  Reshape(coeff.Read(), NQ,NE);
+   auto v = Reshape(pa_data.Write(), NQ, NE);
+   mfem::forall_2D(NE, NQ, 1, [=] MFEM_HOST_DEVICE (int e)
   {
-      const int NE = ne;
-      const int NQ = nq;
-      const bool const_c = coeff.Size() == 1;
-      const bool by_val = map_type == FiniteElement::VALUE;
-      const auto W = Reshape(ir->GetWeights().Read(), NQ);
-      const auto J = Reshape(geom->detJ.Read(), NQ, NE);
-      const auto C =
-         const_c ? Reshape(coeff.Read(), 1, 1) : Reshape(coeff.Read(), NQ, NE);
-      auto v = Reshape(pa_data.Write(), NQ, NE);
-      mfem::forall(NQ, NE, [=] MFEM_HOST_DEVICE(int q, int e)
+      MFEM_FOREACH_THREAD(i, x, NQ)
      {
-         const real_t detJ = J(q, e);
-         const real_t coeff = const_c ? C(0, 0) : C(q, e);
-         v(q, e) = W(q) * coeff * (by_val ? detJ : 1.0 / detJ);
-      });
-   }
+         const real_t detJ = J(i,e);
+         const real_t coeff = const_c ? C(0,0) : C(i,e);
+         v(i,e) =  W(i) * coeff * (by_val ? detJ : 1.0/detJ);
+      }
+   });
 }

 void MassIntegrator::AssemblePABoundary(const FiniteElementSpace &fes)
@@ -106,22 +109,50 @@ void MassIntegrator::AssemblePABoundary(const FiniteElementSpace &fes)
   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);

   const int NE = ne;
-   const int NQ = nq;
+   const int Q1D = quad1D;
   const bool const_c = coeff.Size() == 1;
   const bool by_val = map_type == FiniteElement::VALUE;
+   if (dim==1)
   {
-      const auto W = Reshape(ir->GetWeights().Read(), NQ);
-      const auto J = Reshape(face_geom->detJ.Read(), NQ, NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1, 1)
-                     : Reshape(coeff.Read(), NQ, NE);
-      auto v = Reshape(pa_data.Write(), NQ, NE);
-      mfem::forall(NQ, NE, [=] MFEM_HOST_DEVICE(int q, int e)
+      const auto W = Reshape(ir->GetWeights().Read(), Q1D);
+      const auto J = Reshape(face_geom->detJ.Read(), Q1D, NE);
+      const auto C = const_c ? Reshape(coeff.Read(), 1, 1) :
+                     Reshape(coeff.Read(), Q1D, NE);
+      auto v = Reshape(pa_data.Write(), Q1D, NE);
+      mfem::forall_2D(NE, Q1D, 1, [=] MFEM_HOST_DEVICE (int e)
      {
-         const real_t detJ = J(q, e);
-         const real_t coeff = const_c ? C(0, 0) : C(q, e);
-         v(q, e) = W(q) * coeff * (by_val ? detJ : 1.0 / detJ);
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            const real_t detJ = J(qx,e);
+            const real_t coeff = const_c ? C(0,0) : C(qx,e);
+            v(qx,e) =  W(qx) * coeff * (by_val ? detJ : 1.0/detJ);
+         }
      });
   }
+   else if (dim==2)
+   {
+      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
+      const auto J = Reshape(face_geom->detJ.Read(), Q1D,Q1D,NE);
+      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
+                     Reshape(coeff.Read(), Q1D,Q1D,NE);
+      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
+      mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               const real_t detJ = J(qx,qy,e);
+               const real_t coeff = const_c ? C(0,0,0) : C(qx,qy,e);
+               v(qx,qy,e) =  W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
+            }
+         }
+      });
+   }
+   else
+   {
+      MFEM_ABORT("Not supported.");
+   }
 }

 void MassIntegrator::AssembleDiagonalPA(Vector &diag)
@@ -346,13 +346,13 @@ private:
      template<typename T>
      T operator() (const blitz::TinyVector<T,3>& x) const
      {
-         const int el_order = el->GetOrder();
-         std::vector<T> u1(el_order+1);
-         std::vector<T> u2(el_order+1);
-         std::vector<T> u3(el_order+1);
-         TmplPoly_1D::CalcBernstein(el_order, x[0], u1.data());
-         TmplPoly_1D::CalcBernstein(el_order, x[1], u2.data());
-         TmplPoly_1D::CalcBernstein(el_order, x[2], u3.data());
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         T u3[el_order+1];
+         TmplPoly_1D::CalcBernstein(el_order, x[0], u1);
+         TmplPoly_1D::CalcBernstein(el_order, x[1], u2);
+         TmplPoly_1D::CalcBernstein(el_order, x[2], u3);

         const Array<int>& dof_map=el->GetDofMap();

@@ -370,17 +370,17 @@ private:
      template<typename T>
      blitz::TinyVector<T,3> grad(const blitz::TinyVector<T,3>& x) const
      {
-         const int el_order = el->GetOrder();
-         std::vector<T> u1(el_order+1);
-         std::vector<T> u2(el_order+1);
-         std::vector<T> u3(el_order+1);
-         std::vector<T> d1(el_order+1);
-         std::vector<T> d2(el_order+1);
-         std::vector<T> d3(el_order+1);
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         T u3[el_order+1];
+         T d1[el_order+1];
+         T d2[el_order+1];
+         T d3[el_order+1];

-         TmplPoly_1D::CalcBernstein(el_order,x[0], u1.data(), d1.data());
-         TmplPoly_1D::CalcBernstein(el_order,x[1], u2.data(), d2.data());
-         TmplPoly_1D::CalcBernstein(el_order,x[2], u3.data(), d3.data());
+         TmplPoly_1D::CalcBernstein(el_order,x[0], u1, d1);
+         TmplPoly_1D::CalcBernstein(el_order,x[1], u2, d2);
+         TmplPoly_1D::CalcBernstein(el_order,x[2], u3, d3);

         blitz::TinyVector<T,3> res(T(0.0),T(0.0),T(0.0));

@@ -415,11 +415,11 @@ private:
      template<typename T>
      T operator() (const blitz::TinyVector<T,2>& x) const
      {
-         const int el_order = el->GetOrder();
-         std::vector<T> u1(el_order+1);
-         std::vector<T> u2(el_order+1);
-         TmplPoly_1D::CalcBernstein(el_order, x[0], u1.data());
-         TmplPoly_1D::CalcBernstein(el_order, x[1], u2.data());
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         TmplPoly_1D::CalcBernstein(el_order, x[0], u1);
+         TmplPoly_1D::CalcBernstein(el_order, x[1], u2);

         const Array<int>& dof_map=el->GetDofMap();

@@ -437,14 +437,14 @@ private:
      template<typename T>
      blitz::TinyVector<T,2> grad(const blitz::TinyVector<T,2>& x) const
      {
-         const int el_order = el->GetOrder();
-         std::vector<T> u1(el_order+1);
-         std::vector<T> u2(el_order+1);
-         std::vector<T> d1(el_order+1);
-         std::vector<T> d2(el_order+1);
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         T d1[el_order+1];
+         T d2[el_order+1];

-         TmplPoly_1D::CalcBernstein(el_order,x[0], u1.data(), d1.data());
-         TmplPoly_1D::CalcBernstein(el_order,x[1], u2.data(), d2.data());
+         TmplPoly_1D::CalcBernstein(el_order,x[0], u1, d1);
+         TmplPoly_1D::CalcBernstein(el_order,x[1], u2, d2);

         blitz::TinyVector<T,2> res(T(0.0),T(0.0));

@@ -51,7 +51,7 @@ void LinearFormExtension::Assemble()
      {
         // scan the attributes to set the markers to 0 or 1
         const int NE = fes.GetNE();
-         const auto attr = attributes->Read();
+         const auto attr = attributes.Read();
         const auto dimk = domain_integs_marker_k->Read();
         auto markers_w = markers.Write();
         mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
@@ -93,14 +93,13 @@ void LinearFormExtension::Assemble()
      else
      {
         // scan the attributes to set the markers to 0 or 1
-         const int NBE = bdr_face_attributes->Size();
-         const auto attr = bdr_face_attributes->Read();
+         const int NBE = bdr_attributes.Size();
+         const auto attr = bdr_attributes.Read();
         const auto attr_markers = boundary_integs_marker_k->Read();
         auto markers_w = bdr_markers.Write();
-         mfem::forall(NBE, [=] MFEM_HOST_DEVICE(int e)
+         mfem::forall(NBE, [=] MFEM_HOST_DEVICE (int e)
         {
-            markers_w[e] =
-               attr[e] > 0 ? (attr_markers[attr[e] - 1] == 1) : false;
+            markers_w[e] = attr_markers[attr[e]-1] == 1;
         });
      }

@@ -126,7 +125,8 @@ void LinearFormExtension::Update()
      //markers.UseDevice(true);

      // Gather the attributes on the host from all the elements
-      attributes = &mesh.GetElementAttributes();
+      attributes.SetSize(NE);
+      for (int i = 0; i < NE; ++i) { attributes[i] = mesh.GetAttribute(i); }

      elem_restrict_lex = fes.GetElementRestriction(ordering);
      MFEM_VERIFY(elem_restrict_lex, "Element restriction not available");
@@ -136,12 +136,34 @@ void LinearFormExtension::Update()

   if (lf->boundary_integs.Size() > 0)
   {
-      bdr_face_attributes = &mesh.GetBdrFaceAttributes();
-
-      const int nf_bdr = bdr_face_attributes->Size();
+      const int nf_bdr = fes.GetNFbyType(FaceType::Boundary);
      bdr_markers.SetSize(nf_bdr);
      // bdr_markers.UseDevice(true);

+      // The face restriction will give us "face E-vectors" on the boundary that
+      // are numbered in the order of the faces of mesh. This numbering will be
+      // different than the numbering of the boundary elements. We compute
+      // mappings so that the array `bdr_attributes[i]` gives the boundary
+      // attribute of the `i`th boundary face in the mesh face order.
+      std::unordered_map<int,int> f_to_be;
+      for (int i = 0; i < mesh.GetNBE(); ++i)
+      {
+         const int f = mesh.GetBdrElementFaceIndex(i);
+         f_to_be[f] = i;
+      }
+      MFEM_VERIFY(size_t(nf_bdr) == f_to_be.size(), "Incompatible sizes");
+      bdr_attributes.SetSize(nf_bdr);
+      int f_ind = 0;
+      for (int f = 0; f < mesh.GetNumFaces(); ++f)
+      {
+         if (f_to_be.find(f) != f_to_be.end())
+         {
+            const int be = f_to_be[f];
+            bdr_attributes[f_ind] = mesh.GetBdrAttribute(be);
+            ++f_ind;
+         }
+      }
+
      bdr_restrict_lex =
         dynamic_cast<const FaceRestriction*>(
            fes.GetFaceRestriction(ordering, FaceType::Boundary,
@@ -25,8 +25,7 @@ class LinearForm;
 class LinearFormExtension
 {
   /// Attributes of all mesh elements.
-   const Array<int> *attributes; // Not owned
-   const Array<int> *bdr_face_attributes; // Not owned
+   Array<int> attributes, bdr_attributes;

   /// Temporary markers for device kernels.
   Array<int> markers, bdr_markers;
@@ -673,7 +673,7 @@ public:
      int myid;
      MPI_Comm_rank(comm, &myid);

-      int seed = (seed_ > 0) ? seed_ + myid : time(nullptr) + myid;
+      int seed = (seed_ > 0) ? seed_ + myid : (int)time(0) + myid;
      SetSeed(seed);
   }
 #else
@@ -1,591 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "pderefmat_op.hpp"
-
-#ifdef MFEM_USE_MPI
-
-#include "fes_kernels.hpp"
-/// \cond DO_NOT_DOCUMENT
-namespace mfem
-{
-namespace internal
-{
-template <Ordering::Type Order, bool Atomic>
-static void ParDerefMultKernelImpl(const ParDerefineMatrixOp &op,
-                                   const Vector &x, Vector &y)
-{
-   // pack sends
-   if (op.xghost_send.Size())
-   {
-      auto src = x.Read();
-      auto idcs = op.send_permutations.Read();
-      auto dst = Device::GetGPUAwareMPI() ? op.xghost_send.Write()
-                 : op.xghost_send.HostWrite();
-      auto vdims = op.fespace->GetVDim();
-      auto sptr = op.send_segment_idcs.Read();
-      auto lptr = op.send_segments.Read();
-      auto old_ndofs = x.Size() / vdims;
-
-      forall(op.send_permutations.Size(), [=] MFEM_HOST_DEVICE(int i)
-      {
-         int seg = sptr[i];
-         int width = lptr[seg + 1] - lptr[seg];
-         auto tdst = dst + i + lptr[seg] * vdims;
-         int sign = 1;
-         int col = idcs[i];
-         if (col < 0)
-         {
-            sign = -1;
-            col = -1 - col;
-         }
-         for (int vdim = 0; vdim < vdims; ++vdim)
-         {
-            tdst[vdim * width] =
-               sign
-               * src[Order == Ordering::byNODES ? (col + vdim * old_ndofs)
-                           : (col * vdims + vdim)];
-         }
-      });
-      // TODO: is this needed so we can send the packed data correctly?
-      // unclear for GPU-aware MPI, definitely required otherwise
-      MFEM_DEVICE_SYNC;
-   }
-   // initialize off-diagonal receive and send
-   op.requests.clear();
-   if (op.xghost_recv.Size())
-   {
-      auto vdims = op.fespace->GetVDim();
-      auto rcv = Device::GetGPUAwareMPI() ? op.xghost_recv.Write()
-                 : op.xghost_recv.HostWrite();
-      for (int i = 0; i < op.recv_ranks.Size(); ++i)
-      {
-         op.requests.emplace_back();
-         MPI_Irecv(rcv + op.recv_segments[i] * vdims,
-                   (op.recv_segments[i + 1] - op.recv_segments[i]) * vdims,
-                   MPITypeMap<real_t>::mpi_type, op.recv_ranks[i],
-                   MessageTag::DEREFINEMENT_MATRIX_CONSTRUCTION_DATA,
-                   op.fespace->GetComm(), &op.requests.back());
-      }
-   }
-   if (op.xghost_send.Size())
-   {
-      auto vdims = op.fespace->GetVDim();
-      // only is a GPU mem ptr if GPU-aware MPI is enabled
-      auto dst = Device::GetGPUAwareMPI() ? op.xghost_send.Write()
-                 : op.xghost_send.HostWrite();
-      for (int i = 0; i < op.send_ranks.Size(); ++i)
-      {
-         op.requests.emplace_back();
-         MPI_Isend(dst + op.send_segments[i] * vdims,
-                   (op.send_segments[i + 1] - op.send_segments[i]) * vdims,
-                   MPITypeMap<real_t>::mpi_type, op.send_ranks[i],
-                   MessageTag::DEREFINEMENT_MATRIX_CONSTRUCTION_DATA,
-                   op.fespace->GetComm(), &op.requests.back());
-      }
-   }
-   {
-      // diagonal
-      DerefineMatrixOpMultFunctor<Order, Atomic, true> func;
-      func.xptr = x.Read();
-      y.UseDevice();
-      y = 0.;
-      func.yptr = y.ReadWrite();
-      func.bsptr = op.block_storage.Read();
-      func.boptr = op.block_offsets.Read();
-      func.brptr = op.block_row_idcs_offsets.Read();
-      func.bcptr = op.block_col_idcs_offsets.Read();
-      func.rptr = op.row_idcs.Read();
-      func.cptr = op.col_idcs.Read();
-      func.vdims = op.fespace->GetVDim();
-      func.nblocks = op.block_offsets.Size();
-      func.width = op.Width() / func.vdims;
-      func.height = op.Height() / func.vdims;
-      func.Run(op.max_rows);
-   }
-   // wait for comm to finish, if any
-   if (op.requests.size())
-   {
-      MPI_Waitall(op.requests.size(), op.requests.data(), MPI_STATUSES_IGNORE);
-      if (op.xghost_recv.Size())
-      {
-         // off-diagonal kernel
-         DerefineMatrixOpMultFunctor<Order, Atomic, false> func;
-         // directly read from host-pinned memory if not using GPU-aware MPI
-         func.xptr = Device::GetGPUAwareMPI() ? op.xghost_recv.Read()
-                     : op.xghost_recv.HostRead();
-         func.yptr = y.ReadWrite();
-         func.bsptr = op.block_storage.Read();
-         func.boptr = op.off_diag_block_offsets.Read();
-         func.brptr = op.block_off_diag_row_idcs_offsets.Read();
-         func.rsptr = op.recv_segment_idcs.Read();
-         func.segptr = op.recv_segments.Read();
-         func.coptr = op.block_off_diag_col_offsets.Read();
-         func.bwptr = op.block_off_diag_widths.Read();
-         func.rptr = op.row_off_diag_idcs.Read();
-         func.vdims = op.fespace->GetVDim();
-         func.nblocks = op.off_diag_block_offsets.Size();
-         func.width = op.xghost_recv.Size() / func.vdims;
-         func.height = op.Height() / func.vdims;
-         func.Run(op.max_rows);
-      }
-   }
-}
-} // namespace internal
-
-template <Ordering::Type Order, bool Atomic>
-ParDerefineMatrixOp::MultKernelType ParDerefineMatrixOp::MultKernel::Kernel()
-{
-   return internal::ParDerefMultKernelImpl<Order, Atomic>;
-}
-
-ParDerefineMatrixOp::MultKernelType
-ParDerefineMatrixOp::MultKernel::Fallback(Ordering::Type, bool)
-{
-   MFEM_ABORT("invalid MultKernel parameters");
-}
-
-ParDerefineMatrixOp::Kernels::Kernels()
-{
-   MultKernel::Specialization<Ordering::byNODES, false>::Add();
-   MultKernel::Specialization<Ordering::byVDIM, false>::Add();
-   MultKernel::Specialization<Ordering::byNODES, true>::Add();
-   MultKernel::Specialization<Ordering::byVDIM, true>::Add();
-}
-
-void ParDerefineMatrixOp::Mult(const Vector &x, Vector &y) const
-{
-   const bool is_dg = fespace->FEColl()->GetContType()
-                      == FiniteElementCollection::DISCONTINUOUS;
-   // DG needs atomic summation
-   MultKernel::Run(fespace->GetOrdering(), is_dg, *this, x, y);
-   // use this to prevent xghost* from being re-purposed for subsequent Mult
-   // calls
-   MFEM_DEVICE_SYNC;
-}
-
-ParDerefineMatrixOp::ParDerefineMatrixOp(ParFiniteElementSpace &fespace_,
-                                         int old_ndofs,
-                                         const Table *old_elem_dof,
-                                         const Table *old_elem_fos)
-   : Operator(fespace_.GetVSize(), old_ndofs * fespace_.GetVDim()),
-     fespace(&fespace_)
-{
-   static Kernels kernels;
-   constexpr int max_team_size = 256;
-
-   const int NRanks = fespace->GetNRanks();
-
-   const int nrk = HYPRE_AssumedPartitionCheck() ? 2 : NRanks;
-
-   MFEM_VERIFY(fespace->Nonconforming(),
-               "Not implemented for conforming meshes.");
-   MFEM_VERIFY(fespace->old_dof_offsets[nrk],
-               "Missing previous (finer) space.");
-
-   const int MyRank = fespace->GetMyRank();
-   ParNCMesh *old_pncmesh = fespace->GetParMesh()->pncmesh;
-   const CoarseFineTransformations &dtrans =
-      old_pncmesh->GetDerefinementTransforms();
-   const Array<int> &old_ranks = old_pncmesh->GetDerefineOldRanks();
-
-   const bool is_dg = fespace->FEColl()->GetContType()
-                      == FiniteElementCollection::DISCONTINUOUS;
-   DenseMatrix localRVO; // for variable-order only
-
-   DenseTensor localR[Geometry::NumGeom];
-   int diag_rows = 0;
-   int off_diag_rows = 0;
-   int diag_cols = 0;
-
-   auto get_ldofs = [&](int k) -> int
-   {
-      const Embedding &emb = dtrans.embeddings[k];
-      if (fespace->IsVariableOrder())
-      {
-         const FiniteElement *fe = fespace->GetFE(emb.parent);
-         return fe->GetDof();
-      }
-      else
-      {
-         Geometry::Type geom =
-         fespace->GetParMesh()->GetElementBaseGeometry(emb.parent);
-         return fespace->FEColl()->FiniteElementForGeometry(geom)->GetDof();
-      }
-   };
-   Array<int> dofs, old_dofs;
-   max_rows = 1;
-   // first pass:
-   // - determine memory block lengths
-   // - identify dofs in x we need to send/receive
-   // don't need to send the indices, fine rank will re-arrange and sign
-   // change x before transmitting the ghost data
-
-   // key: coarse rank to send to
-   // value: old dofs to send (with sign)
-   std::map<int, std::vector<int>> to_send;
-   // key: fine rank
-   // value: indices into dtrans.embeddings
-   std::map<int, std::vector<int>> od_ks;
-   // key: fine rank
-   // value: recv segment length
-   std::map<int, int> od_seg_lens;
-   int send_len = 0;
-   int recv_len = 0;
-   // size of block_storage, if fespace->IsVariableOrder()
-   // otherwise unused
-   int total_size = 0;
-   int num_diagonal_blocks = 0;
-   int num_offdiagonal_blocks = 0;
-   for (int k = 0; k < dtrans.embeddings.Size(); ++k)
-   {
-      const Embedding &emb = dtrans.embeddings[k];
-      int fine_rank = old_ranks[k];
-      int coarse_rank = (emb.parent < 0) ? (-1 - emb.parent)
-                        : old_pncmesh->ElementRank(emb.parent);
-      if (coarse_rank != MyRank && fine_rank == MyRank)
-      {
-         // this rank needs to send data in x to course_rank
-         old_elem_dof->GetRow(k, old_dofs);
-         auto &tmp = to_send[coarse_rank];
-         send_len += old_dofs.Size();
-         for (int i = 0; i < old_dofs.Size(); ++i)
-         {
-            tmp.emplace_back(old_dofs[i]);
-         }
-      }
-      else if (coarse_rank == MyRank && fine_rank != MyRank)
-      {
-         // this rank needs to receive data in x from fine_rank
-         MFEM_ASSERT(emb.parent >= 0, "");
-         auto ldofs = get_ldofs(k);
-         off_diag_rows += ldofs;
-         recv_len += ldofs;
-         od_ks[fine_rank].emplace_back(k);
-         od_seg_lens[fine_rank] += ldofs;
-         ++num_offdiagonal_blocks;
-         if (fespace->IsVariableOrder())
-         {
-            total_size += ldofs * ldofs;
-         }
-      }
-      else if (coarse_rank == MyRank && fine_rank == MyRank)
-      {
-         MFEM_ASSERT(emb.parent >= 0, "");
-         // diagonal
-         ++num_diagonal_blocks;
-         auto ldofs = get_ldofs(k);
-         diag_rows += ldofs;
-         diag_cols += ldofs;
-         if (fespace->IsVariableOrder())
-         {
-            total_size += ldofs * ldofs;
-         }
-      }
-   }
-   send_segments.SetSize(to_send.size() + 1);
-   send_segments.HostWrite();
-   send_ranks.SetSize(to_send.size());
-   send_ranks.HostWrite();
-   {
-      int idx = 0;
-      send_segments[0] = 0;
-      for (auto &tmp : to_send)
-      {
-         send_ranks[idx] = tmp.first;
-         send_segments[idx + 1] = send_segments[idx] + tmp.second.size();
-         ++idx;
-      }
-   }
-   recv_segment_idcs.SetSize(off_diag_rows);
-   recv_segment_idcs.HostWrite();
-   recv_segments.SetSize(od_ks.size() + 1);
-   recv_segments.HostWrite();
-   recv_ranks.SetSize(od_ks.size());
-   recv_ranks.HostWrite();
-
-   // set sizes
-   row_idcs.SetSize(diag_rows);
-   row_idcs.HostWrite();
-   row_off_diag_idcs.SetSize(off_diag_rows);
-   row_off_diag_idcs.HostWrite();
-   col_idcs.SetSize(diag_cols);
-   col_idcs.HostWrite();
-   block_row_idcs_offsets.SetSize(num_diagonal_blocks + 1);
-   block_row_idcs_offsets.HostWrite();
-   block_col_idcs_offsets.SetSize(num_diagonal_blocks + 1);
-   block_col_idcs_offsets.HostWrite();
-   block_off_diag_row_idcs_offsets.SetSize(num_offdiagonal_blocks + 1);
-   block_off_diag_row_idcs_offsets.HostWrite();
-   block_off_diag_col_offsets.SetSize(num_offdiagonal_blocks);
-   block_off_diag_col_offsets.HostWrite();
-   block_off_diag_widths.SetSize(num_offdiagonal_blocks);
-   block_off_diag_widths.HostWrite();
-   pack_col_idcs.SetSize(send_len);
-   // memory manager doesn't appear to have a graceful fallback for
-   // HOST_PINNED if not built with CUDA or HIP
-#if defined(MFEM_USE_CUDA) or defined(MFEM_USE_HIP)
-   xghost_send.SetSize(send_len * fespace->GetVDim(),
-                       Device::GetGPUAwareMPI() ? MemoryType::DEFAULT
-                       : MemoryType::HOST_PINNED);
-   xghost_recv.SetSize(recv_len * fespace->GetVDim(),
-                       Device::GetGPUAwareMPI() ? MemoryType::DEFAULT
-                       : MemoryType::HOST_PINNED);
-#else
-   xghost_send.SetSize(send_len * fespace->GetVDim());
-   xghost_recv.SetSize(recv_len * fespace->GetVDim());
-#endif
-   send_permutations.SetSize(send_len);
-   send_segment_idcs.SetSize(send_len);
-   block_offsets.SetSize(num_diagonal_blocks);
-   block_offsets.HostWrite();
-   off_diag_block_offsets.SetSize(num_offdiagonal_blocks);
-   off_diag_block_offsets.HostWrite();
-   int geom_offsets[Geometry::NumGeom];
-   real_t *bs_ptr;
-
-   if (fespace->IsVariableOrder())
-   {
-      block_storage.SetSize(total_size);
-      bs_ptr = block_storage.HostWrite();
-      // compute block data later
-   }
-   else
-   {
-      // compression scheme:
-      // block_offsets is the start of each block, potentially repeated
-      // only need to store localR for used shapes
-      Mesh::GeometryList elem_geoms(*fespace->GetMesh());
-
-      int size = 0;
-      for (int i = 0; i < elem_geoms.Size(); ++i)
-      {
-         fespace->GetLocalDerefinementMatrices(elem_geoms[i],
-                                               localR[elem_geoms[i]]);
-         geom_offsets[elem_geoms[i]] = size;
-         size += localR[elem_geoms[i]].TotalSize();
-      }
-      block_storage.SetSize(size);
-      bs_ptr = block_storage.HostWrite();
-      // copy blocks into block_storage
-      for (int i = 0; i < elem_geoms.Size(); ++i)
-      {
-         std::copy(localR[elem_geoms[i]].Data(),
-                   localR[elem_geoms[i]].Data()
-                   + localR[elem_geoms[i]].TotalSize(),
-                   bs_ptr);
-         bs_ptr += localR[elem_geoms[i]].TotalSize();
-      }
-   }
-
-   // second pass:
-   // - initialize buffers
-
-   {
-      auto ptr = send_permutations.HostWrite();
-      auto ptr2 = send_segment_idcs.HostWrite();
-      int i = 0;
-      for (auto &v : to_send)
-      {
-         ptr = std::copy(v.second.begin(), v.second.end(), ptr);
-         for (size_t idx = 0; idx < v.second.size(); ++idx)
-         {
-            *ptr2 = i;
-            ++ptr2;
-         }
-         ++i;
-      }
-   }
-
-   block_row_idcs_offsets[0] = 0;
-   block_col_idcs_offsets[0] = 0;
-   block_off_diag_row_idcs_offsets[0] = 0;
-   Array<int> mark(fespace->GetNDofs());
-   mark = 0;
-   {
-      int idx = 0;
-      recv_segments[0] = 0;
-      for (auto &v : od_seg_lens)
-      {
-         recv_ranks[idx] = v.first;
-         recv_segments[idx + 1] = recv_segments[idx] + v.second;
-         ++idx;
-      }
-   }
-   // key: index into dtrans.embeddings
-   // value: off-diagonal block offset, od_ridx, seg id
-   std::unordered_map<int, std::array<int, 3>> ks_map;
-   {
-      int od_ridx = 0;
-      int seg_id = 0;
-      for (auto &v1 : od_ks)
-      {
-         for (auto k : v1.second)
-         {
-            auto &tmp = ks_map[k];
-            tmp[0] = ks_map.size() - 1;
-            tmp[1] = od_ridx;
-            tmp[2] = seg_id;
-            od_ridx += get_ldofs(k);
-         }
-         ++seg_id;
-      }
-   }
-   int diag_idx = 0;
-   int var_offset = 0;
-   int ridx = 0;
-   int cidx = 0;
-   // can't break this up into separate diagonals/off-diagonals loops because
-   // of mark
-   for (int k = 0; k < dtrans.embeddings.Size(); ++k)
-   {
-      const Embedding &emb = dtrans.embeddings[k];
-      if (emb.parent < 0)
-      {
-         continue;
-      }
-      int fine_rank = old_ranks[k];
-      int coarse_rank = (emb.parent < 0) ? (-1 - emb.parent)
-                        : old_pncmesh->ElementRank(emb.parent);
-      if (coarse_rank == MyRank)
-      {
-         // either diagonal or off-diagonal
-         Geometry::Type geom =
-            fespace->GetMesh()->GetElementBaseGeometry(emb.parent);
-         if (fespace->IsVariableOrder())
-         {
-            const FiniteElement *fe = fespace->GetFE(emb.parent);
-            const DenseTensor &pmats = dtrans.point_matrices[geom];
-            const int ldof = fe->GetDof();
-
-            IsoparametricTransformation isotr;
-            isotr.SetIdentityTransformation(geom);
-
-            localRVO.SetSize(ldof, ldof);
-            isotr.SetPointMat(pmats(emb.matrix));
-            // Local restriction is size ldofxldof assuming that the parent
-            // and child are of same polynomial order.
-            fe->GetLocalRestriction(isotr, localRVO);
-            // copy block
-            auto s = localRVO.Height() * localRVO.Width();
-            std::copy(localRVO.Data(), localRVO.Data() + s, bs_ptr);
-            bs_ptr += s;
-         }
-         DenseMatrix &lR =
-            fespace->IsVariableOrder() ? localRVO : localR[geom](emb.matrix);
-         max_rows = std::max(lR.Height(), max_rows);
-         auto size = lR.Height() * lR.Width();
-         fespace->elem_dof->GetRow(emb.parent, dofs);
-         if (fine_rank == MyRank)
-         {
-            // diagonal
-            old_elem_dof->GetRow(k, old_dofs);
-            MFEM_VERIFY(old_dofs.Size() == dofs.Size(),
-                        "Parent and child must have same #dofs.");
-            block_row_idcs_offsets[diag_idx + 1] =
-               block_row_idcs_offsets[diag_idx] + lR.Height();
-            block_col_idcs_offsets[diag_idx + 1] =
-               block_col_idcs_offsets[diag_idx] + lR.Width();
-
-            if (fespace->IsVariableOrder())
-            {
-               block_offsets[diag_idx] = var_offset;
-               var_offset += size;
-            }
-            else
-            {
-               block_offsets[diag_idx] = geom_offsets[geom] + size * emb.matrix;
-            }
-            for (int i = 0; i < lR.Height(); ++i, ++ridx)
-            {
-               if (!std::isfinite(lR(i, 0)))
-               {
-                  row_idcs[ridx] = INT_MAX;
-                  continue;
-               }
-               int r = dofs[i];
-               int m = (r >= 0) ? r : (-1 - r);
-               if (is_dg || !mark[m])
-               {
-                  row_idcs[ridx] = r;
-                  mark[m] = 1;
-               }
-               else
-               {
-                  row_idcs[ridx] = INT_MAX;
-               }
-            }
-            for (int i = 0; i < lR.Width(); ++i, ++cidx)
-            {
-               col_idcs[cidx] = old_dofs[i];
-            }
-            ++diag_idx;
-         }
-         else
-         {
-            // off-diagonal
-            auto &tmp = ks_map.at(k);
-            auto od_idx = tmp[0];
-            auto od_ridx = tmp[1];
-            block_off_diag_row_idcs_offsets[od_idx + 1] =
-               block_off_diag_row_idcs_offsets[od_idx] + lR.Height();
-            block_off_diag_col_offsets[od_idx] = od_ridx;
-            block_off_diag_widths[od_idx] = lR.Width();
-            recv_segment_idcs[od_idx] = tmp[2];
-
-            if (fespace->IsVariableOrder())
-            {
-               off_diag_block_offsets[od_idx] = var_offset;
-               var_offset += size;
-            }
-            else
-            {
-               off_diag_block_offsets[od_idx] =
-                  geom_offsets[geom] + size * emb.matrix;
-            }
-            for (int i = 0; i < lR.Height(); ++i, ++od_ridx)
-            {
-               if (!std::isfinite(lR(i, 0)))
-               {
-                  row_off_diag_idcs[od_ridx] = INT_MAX;
-                  continue;
-               }
-               int r = dofs[i];
-               int m = (r >= 0) ? r : (-1 - r);
-               if (is_dg || !mark[m])
-               {
-                  row_off_diag_idcs[od_ridx] = r;
-                  mark[m] = 1;
-               }
-               else
-               {
-                  row_off_diag_idcs[od_ridx] = INT_MAX;
-               }
-            }
-            ++od_idx;
-         }
-      }
-   }
-   // if not using GPU, set max_rows/max_cols to zero
-   if (Device::Allows(Backend::DEVICE_MASK))
-   {
-      max_rows = std::min(max_rows, max_team_size);
-   }
-   else
-   {
-      max_rows = 1;
-   }
-   requests.reserve(recv_ranks.Size() + send_ranks.Size());
-}
-} // namespace mfem
-/// \endcond DO_NOT_DOCUMENT
-
-#endif
@@ -1,111 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_PDEREFMAT_OP
-#define MFEM_PDEREFMAT_OP
-
-#include "../config/config.hpp"
-
-#ifdef MFEM_USE_MPI
-
-#include "pfespace.hpp"
-
-#include "kernel_dispatch.hpp"
-
-#include <vector>
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-struct ParDerefineMatrixOp : public Operator
-{
-   ParFiniteElementSpace *fespace;
-   /// offsets into block_storage for diagonal
-   Array<int> block_offsets;
-   /// offsets into row_idcs for diagonal
-   Array<int> block_row_idcs_offsets;
-   /// offsets into col_idcs for diagonal
-   Array<int> block_col_idcs_offsets;
-
-   /// offsets into block_storage for off-diagonal
-   Array<int> off_diag_block_offsets;
-   /// offsets into row_idcs for off-diagonal
-   Array<int> block_off_diag_row_idcs_offsets;
-   Array<int> block_off_diag_col_offsets;
-   Array<int> block_off_diag_widths;
-   /// mapping for row dofs, INT_MAX indicates the block row should be ignored.
-   /// negative means the row data should be negated.
-   /// only for diagonal blocks
-   Array<int> row_idcs;
-   /// mapping for col dofs, negative means the col data should be negated.
-   /// only for diagonal blocks
-   Array<int> col_idcs;
-
-   Array<int> pack_col_idcs;
-
-   /// mapping for row dofs, INT_MAX indicates the block row should be ignored.
-   /// negative means the row data should be negated.
-   /// only for off-diagonal blocks
-   Array<int> row_off_diag_idcs;
-   /// dense block matrices which can be reused to construct the full matrix
-   /// operation. These are stored contiguously and blocks have no restrictions
-   /// on shape (can be rectangle and differ from block to block).
-   /// This is only for the diagonal block.
-   Vector block_storage;
-   /// maximum height of any block in block_storage for GPU
-   /// parallelization, or 1 for CPU runs.
-   int max_rows;
-
-   /// quasi Ordering::byNODES, broken into sections by ranks we need to send
-   /// the data to
-   mutable Vector xghost_send;
-   /// quasi Ordering::byNODES, broken into sections by ranks we received
-   /// the data from
-   mutable Vector xghost_recv;
-   /// maps off-diagonal k to segment
-   Array<int> recv_segment_idcs;
-   /// cumulative count of dofs which will be received from other ranks
-   Array<int> recv_segments;
-   /// Source rank of each recv segment
-   Array<int> recv_ranks;
-   /// What send segment each entry in send_permutations corresponds to
-   Array<int> send_segment_idcs;
-   /// cumulative count of dofs which will be sent to other ranks
-   Array<int> send_segments;
-   /// Destination rank of each send segment
-   Array<int> send_ranks;
-   /// how to permute/sign change values from our local x to send to other ranks
-   Array<int> send_permutations;
-   /// internal buffer for MPI requests
-   mutable std::vector<MPI_Request> requests;
-
-   using MultKernelType = void (*)(const ParDerefineMatrixOp &, const Vector &,
-                                   Vector &);
-   /// template args: ordering, atomic
-   MFEM_REGISTER_KERNELS(MultKernel, MultKernelType, (Ordering::Type, bool));
-
-   struct Kernels
-   {
-      Kernels();
-   };
-
-   void Mult(const Vector &x, Vector &y) const;
-
-   ParDerefineMatrixOp(ParFiniteElementSpace &fespace_, int old_ndofs,
-                       const Table *old_elem_dof, const Table *old_elem_fos);
-};
-/// \endcond DO_NOT_DOCUMENT
-} // namespace mfem
-
-#endif
-
-#endif
@@ -22,13 +22,12 @@
 #include "../mesh/mesh_headers.hpp"
 #include "../general/binaryio.hpp"

-#include "pderefmat_op.hpp"
-
 #include <limits>
 #include <list>

 namespace mfem
 {
+
 ParFiniteElementSpace::ParFiniteElementSpace(
   const ParFiniteElementSpace &orig, ParMesh *pmesh,
   const FiniteElementCollection *fec)
@@ -4488,6 +4487,13 @@ ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
   return M;
 }

+
+struct DerefDofMessage
+{
+   std::vector<HYPRE_BigInt> dofs;
+   MPI_Request request;
+};
+
 HypreParMatrix*
 ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
                                                  const Table* old_elem_dof,
@@ -4530,13 +4536,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
      old_pncmesh->GetDerefinementTransforms();
   const Array<int> &old_ranks = old_pncmesh->GetDerefineOldRanks();

-   // key: other rank
-   // value: send or recieve buffer
-   std::map<int, std::vector<HYPRE_BigInt>> to_send;
-   std::map<int, std::vector<HYPRE_BigInt>> to_recv;
-   // key: index into dtrans.embeddings
-   // value: [start, stop]
-   std::unordered_map<int, std::array<size_t, 2>> recv_messages;
+   std::map<int, DerefDofMessage> messages;

   HYPRE_BigInt old_offset = HYPRE_AssumedPartitionCheck()
                             ? old_dof_offsets[0] : old_dof_offsets[MyRank];
@@ -4556,46 +4556,30 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
         old_elem_dof->GetRow(k, dofs);
         DofsToVDofs(dofs, old_ndofs);

-         std::vector<HYPRE_BigInt>& send_buf = to_send[coarse_rank];
-         auto pos = send_buf.size();
-         send_buf.resize(pos + dofs.Size());
+         DerefDofMessage &msg = messages[k];
+         msg.dofs.resize(dofs.Size());
         for (int i = 0; i < dofs.Size(); i++)
         {
-            send_buf[pos + i] = old_offset + dofs[i];
+            msg.dofs[i] = old_offset + dofs[i];
         }
+
+         MPI_Isend(&msg.dofs[0], static_cast<int>(msg.dofs.size()), HYPRE_MPI_BIG_INT,
+                   coarse_rank, 291, MyComm, &msg.request);
      }
      else if (coarse_rank == MyRank && fine_rank != MyRank)
      {
         MFEM_ASSERT(emb.parent >= 0, "");
         Geometry::Type geom = mesh->GetElementBaseGeometry(emb.parent);

-         std::vector<HYPRE_BigInt>& recv_buf = to_recv[fine_rank];
-         auto& msg = recv_messages[k];
-         msg[0] = recv_buf.size();
-         recv_buf.resize(recv_buf.size() + ldof[geom] * vdim);
-         msg[1] = recv_buf.size();
-      }
-   }
+         DerefDofMessage &msg = messages[k];
+         msg.dofs.resize(ldof[geom]*vdim);

-   // assume embedding orders are consistent (i.e. what we expect to receive
-   // first from a given rank is sent first, etc.)
-   std::vector<MPI_Request> requests;
-   requests.reserve(to_send.size() + to_recv.size());
-   // enqueue recvs
-   for (auto &v : to_recv)
-   {
-      requests.emplace_back();
-      MPI_Irecv(v.second.data(), v.second.size(), HYPRE_MPI_BIG_INT, v.first,
-                MessageTag::DEREFINEMENT_MATRIX_CONSTRUCTION_DATA, MyComm,
-                &requests.back());
-   }
-   // enqueue sends
-   for (auto &v : to_send)
-   {
-      requests.emplace_back();
-      MPI_Isend(v.second.data(), v.second.size(), HYPRE_MPI_BIG_INT, v.first,
-                MessageTag::DEREFINEMENT_MATRIX_CONSTRUCTION_DATA, MyComm,
-                &requests.back());
+         MPI_Irecv(&msg.dofs[0], ldof[geom]*vdim, HYPRE_MPI_BIG_INT,
+                   fine_rank, 291, MyComm, &msg.request);
+      }
+      // TODO: coalesce Isends/Irecvs to the same rank. Typically, on uniform
+      // derefinement, there should be just one send to MyRank-1 and one recv
+      // from MyRank+1
   }

   DenseTensor localR[Geometry::NumGeom];
@@ -4653,7 +4637,10 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
   diag->Finalize();

   // wait for all sends/receives to complete
-   MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
+   for (auto it = messages.begin(); it != messages.end(); ++it)
+   {
+      MPI_Wait(&it->second.request, MPI_STATUS_IGNORE);
+   }

   // create the off-diagonal part of the derefinement matrix
   SparseMatrix *offd = new SparseMatrix(ndofs*vdim, 1);
@@ -4674,14 +4661,13 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,

         elem_dof->GetRow(emb.parent, dofs);

-         auto& odofs = to_recv.at(fine_rank);
-         auto &msg = recv_messages[k];
-         MFEM_ASSERT(msg[1] > msg[0], "");
+         DerefDofMessage &msg = messages[k];
+         MFEM_ASSERT(msg.dofs.size(), "");

         for (int vd = 0; vd < vdim; vd++)
         {
            MFEM_ASSERT(ldof[geom], "");
-            HYPRE_BigInt *remote_dofs = odofs.data() + msg[0] + vd * ldof[geom];
+            HYPRE_BigInt* remote_dofs = &msg.dofs[vd*ldof[geom]];

            for (int i = 0; i < lR.Height(); i++)
            {
@@ -4708,6 +4694,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
      }
   }

+   messages.clear();
   offd->Finalize(0);
   offd->SetWidth(static_cast<int>(col_map.size()));

@@ -4959,13 +4946,8 @@ void ParFiniteElementSpace::Update(bool want_transform)

         case Mesh::DEREFINE:
         {
-#if 0
            Th.Reset(ParallelDerefinementMatrix(old_ndofs, old_elem_dof,
                                                old_elem_fos));
-#else
-            Th.Reset(new ParDerefineMatrixOp(*this, old_ndofs, old_elem_dof,
-                                             old_elem_fos));
-#endif
            if (Nonconforming())
            {
               Th.SetOperatorOwner(false);
@@ -5277,7 +5259,7 @@ DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
      gc.GetNeighborLTDofTable(nbr_ltdof);
      const int nb_connections = nbr_ltdof.Size_of_connections();
      shr_ltdof.SetSize(nb_connections);
-      if (nb_connections > 0) { shr_ltdof.CopyFrom(nbr_ltdof.GetJ()); }
+      shr_ltdof.CopyFrom(nbr_ltdof.GetJ());
      shr_buf.SetSize(nb_connections);
      shr_buf.UseDevice(true);
      shr_buf_offsets = nbr_ltdof.GetIMemory();
@@ -5306,7 +5288,7 @@ DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
      gc.GetNeighborLDofTable(nbr_ldof);
      const int nb_connections = nbr_ldof.Size_of_connections();
      ext_ldof.SetSize(nb_connections);
-      if (nb_connections > 0) { ext_ldof.CopyFrom(nbr_ldof.GetJ()); }
+      ext_ldof.CopyFrom(nbr_ldof.GetJ());
      ext_ldof.GetMemory().UseDevice(true);
      ext_buf.SetSize(nb_connections);
      ext_buf.UseDevice(true);
@@ -24,12 +24,9 @@
 namespace mfem
 {

-struct ParDerefineMatrixOp;
-
 /// Abstract parallel finite element space.
 class ParFiniteElementSpace : public FiniteElementSpace
 {
-   friend struct ParDerefineMatrixOp;
 private:
   /// MPI data.
   MPI_Comm MyComm;
@@ -214,16 +214,16 @@ public:
 inline void QuadratureFunction::GetValues(
   int idx, Vector &values)
 {
-   const int s_offset = qspace->Offset(idx);
-   const int sl_size = qspace->Offset(idx + 1) - s_offset;
+   const int s_offset = qspace->offsets[idx];
+   const int sl_size = qspace->offsets[idx+1] - s_offset;
   values.MakeRef(*this, vdim*s_offset, vdim*sl_size);
 }

 inline void QuadratureFunction::GetValues(
   int idx, Vector &values) const
 {
-   const int s_offset = qspace->Offset(idx);
-   const int sl_size = qspace->Offset(idx + 1) - s_offset;
+   const int s_offset = qspace->offsets[idx];
+   const int sl_size = qspace->offsets[idx+1] - s_offset;
   values.SetSize(vdim*sl_size);
   values.HostWrite();
   const real_t *q = HostRead() + vdim*s_offset;
@@ -236,14 +236,14 @@ inline void QuadratureFunction::GetValues(
 inline void QuadratureFunction::GetValues(
   int idx, const int ip_num, Vector &values)
 {
-   const int s_offset = qspace->Offset(idx) * vdim + ip_num * vdim;
+   const int s_offset = qspace->offsets[idx] * vdim + ip_num * vdim;
   values.MakeRef(*this, s_offset, vdim);
 }

 inline void QuadratureFunction::GetValues(
   int idx, const int ip_num, Vector &values) const
 {
-   const int s_offset = qspace->Offset(idx) * vdim + ip_num * vdim;
+   const int s_offset = qspace->offsets[idx] * vdim + ip_num * vdim;
   values.SetSize(vdim);
   values.HostWrite();
   const real_t *q = HostRead() + s_offset;
@@ -256,8 +256,8 @@ inline void QuadratureFunction::GetValues(
 inline void QuadratureFunction::GetValues(
   int idx, DenseMatrix &values)
 {
-   const int s_offset = qspace->Offset(idx);
-   const int sl_size = qspace->Offset(idx + 1) - s_offset;
+   const int s_offset = qspace->offsets[idx];
+   const int sl_size = qspace->offsets[idx+1] - s_offset;
   // Make the values matrix memory an alias of the quadrature function memory
   Memory<real_t> &values_mem = values.GetMemory();
   values_mem.Delete();
@@ -268,8 +268,8 @@ inline void QuadratureFunction::GetValues(
 inline void QuadratureFunction::GetValues(
   int idx, DenseMatrix &values) const
 {
-   const int s_offset = qspace->Offset(idx);
-   const int sl_size = qspace->Offset(idx + 1) - s_offset;
+   const int s_offset = qspace->offsets[idx];
+   const int sl_size = qspace->offsets[idx+1] - s_offset;
   values.SetSize(vdim, sl_size);
   values.HostWrite();
   const real_t *q = HostRead() + vdim*s_offset;
@@ -9,16 +9,278 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-#include "det.hpp"
+#include "../quadinterpolator.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../fem/kernels.hpp"
+#include "../../linalg/kernels.hpp"
+
+using namespace mfem;

 namespace mfem
 {

 namespace internal
 {
+
 namespace quadrature_interpolator
 {

+static void Det1D(const int NE,
+                  const real_t *b,
+                  const real_t *g,
+                  const real_t *x,
+                  real_t *y,
+                  const int d1d,
+                  const int q1d,
+                  Vector *d_buff = nullptr)
+{
+   MFEM_CONTRACT_VAR(b);
+   MFEM_CONTRACT_VAR(d_buff);
+   const auto G = Reshape(g, q1d, d1d);
+   const auto X = Reshape(x, d1d, NE);
+
+   auto Y = Reshape(y, q1d, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int q = 0; q < q1d; q++)
+      {
+         real_t u = 0.0;
+         for (int d = 0; d < d1d; d++)
+         {
+            u += G(q, d) * X(d, e);
+         }
+         Y(q, e) = u;
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void Det2D(const int NE,
+                  const real_t *b,
+                  const real_t *g,
+                  const real_t *x,
+                  real_t *y,
+                  const int d1d = 0,
+                  const int q1d = 0,
+                  Vector *d_buff = nullptr)
+{
+   MFEM_CONTRACT_VAR(d_buff);
+   static constexpr int SDIM = 2;
+   static constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
+   auto Y = Reshape(y, Q1D, Q1D, NE);
+
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      MFEM_SHARED real_t XY[SDIM][NBZ][MD1*MD1];
+      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
+      MFEM_SHARED real_t QQ[2*SDIM][NBZ][MQ1*MQ1];
+
+      kernels::internal::LoadX<MD1,NBZ>(e,D1D,X,XY);
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
+
+      kernels::internal::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
+      kernels::internal::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            real_t J[4];
+            kernels::internal::PullGrad<MQ1,NBZ>(Q1D,qx,qy,QQ,J);
+            Y(qx,qy,e) = kernels::Det<2>(J);
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void Det2DSurface(const int NE,
+                         const real_t *b,
+                         const real_t *g,
+                         const real_t *x,
+                         real_t *y,
+                         const int d1d = 0,
+                         const int q1d = 0,
+                         Vector *d_buff = nullptr)
+{
+   MFEM_CONTRACT_VAR(d_buff);
+
+   static constexpr int SDIM = 3;
+   static constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
+   auto Y = Reshape(y, Q1D, Q1D, NE);
+
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int tidz = MFEM_THREAD_ID(z);
+
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      MFEM_SHARED real_t XYZ[SDIM][NBZ][MD1*MD1];
+      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
+
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
+
+      // Load XYZ components
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            for (int d = 0; d < SDIM; ++d)
+            {
+               XYZ[d][tidz][dx + dy*D1D] = X(dx,dy,d,e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      ConstDeviceMatrix B_mat(BG[0], D1D, Q1D);
+      ConstDeviceMatrix G_mat(BG[1], D1D, Q1D);
+
+      // x contraction
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            for (int d = 0; d < SDIM; ++d)
+            {
+               real_t u = 0.0;
+               real_t v = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t xval = XYZ[d][tidz][dx + dy*D1D];
+                  u += xval * G_mat(dx,qx);
+                  v += xval * B_mat(dx,qx);
+               }
+               DQ[d][tidz][dy + qx*D1D] = u;
+               DQ[3 + d][tidz][dy + qx*D1D] = v;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // y contraction and determinant computation
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            real_t J_[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
+            for (int d = 0; d < SDIM; ++d)
+            {
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  J_[d] += DQ[d][tidz][dy + qx*D1D] * B_mat(dy,qy);
+                  J_[3 + d] += DQ[3 + d][tidz][dy + qx*D1D] * G_mat(dy,qy);
+               }
+            }
+            DeviceTensor<2> J(J_, 3, 2);
+            const real_t E = J(0,0)*J(0,0) + J(1,0)*J(1,0) + J(2,0)*J(2,0);
+            const real_t F = J(0,0)*J(0,1) + J(1,0)*J(1,1) + J(2,0)*J(2,1);
+            const real_t G = J(0,1)*J(0,1) + J(1,1)*J(1,1) + J(2,1)*J(2,1);
+            Y(qx,qy,e) = std::sqrt(E*G - F*F);
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, bool SMEM = true>
+static void Det3D(const int NE,
+                  const real_t *b,
+                  const real_t *g,
+                  const real_t *x,
+                  real_t *y,
+                  const int d1d = 0,
+                  const int q1d = 0,
+                  Vector *d_buff = nullptr) // used only with SMEM = false
+{
+   constexpr int DIM = 3;
+   static constexpr int GRID = SMEM ? 0 : 128;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x, D1D, D1D, D1D, DIM, NE);
+   auto Y = Reshape(y, Q1D, Q1D, Q1D, NE);
+
+   real_t *GM = nullptr;
+   if (!SMEM)
+   {
+      const DeviceDofQuadLimits &limits = DeviceDofQuadLimits::Get();
+      const int max_q1d = T_Q1D ? T_Q1D : limits.MAX_Q1D;
+      const int max_d1d = T_D1D ? T_D1D : limits.MAX_D1D;
+      const int max_qd = std::max(max_q1d, max_d1d);
+      const int mem_size = max_qd * max_qd * max_qd * 9;
+      d_buff->SetSize(2*mem_size*GRID);
+      GM = d_buff->Write();
+   }
+
+   mfem::forall_3D_grid(NE, Q1D, Q1D, Q1D, GRID, [=] MFEM_HOST_DEVICE (int e)
+   {
+      static constexpr int MQ1 = T_Q1D ? T_Q1D :
+                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_Q1D);
+      static constexpr int MD1 = T_D1D ? T_D1D :
+                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_D1D);
+      static constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
+      static constexpr int MSZ = MDQ * MDQ * MDQ * 9;
+
+      const int bid = MFEM_BLOCK_ID(x);
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      MFEM_SHARED real_t SM0[SMEM?MSZ:1];
+      MFEM_SHARED real_t SM1[SMEM?MSZ:1];
+      real_t *lm0 = SMEM ? SM0 : GM + MSZ*bid;
+      real_t *lm1 = SMEM ? SM1 : GM + MSZ*(GRID+bid);
+      real_t (*DDD)[MD1*MD1*MD1] = (real_t (*)[MD1*MD1*MD1]) (lm0);
+      real_t (*DDQ)[MD1*MD1*MQ1] = (real_t (*)[MD1*MD1*MQ1]) (lm1);
+      real_t (*DQQ)[MD1*MQ1*MQ1] = (real_t (*)[MD1*MQ1*MQ1]) (lm0);
+      real_t (*QQQ)[MQ1*MQ1*MQ1] = (real_t (*)[MQ1*MQ1*MQ1]) (lm1);
+
+      kernels::internal::LoadX<MD1>(e,D1D,X,DDD);
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
+
+      kernels::internal::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
+      kernels::internal::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
+      kernels::internal::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               real_t J[9];
+               kernels::internal::PullGrad<MQ1>(Q1D, qx,qy,qz, QQQ, J);
+               Y(qx,qy,qz,e) = kernels::Det<3>(J);
+            }
+         }
+      }
+   });
+}
+
 void InitDetKernels()
 {
   using k = QuadratureInterpolator::DetKernels;
@@ -40,12 +302,27 @@ void InitDetKernels()
 }

 } // namespace quadrature_interpolator
+
 } // namespace internal

 /// @cond Suppress_Doxygen_warnings

-QuadratureInterpolator::DetKernelType
-QuadratureInterpolator::DetKernels::Fallback(
+namespace
+{
+using DetKernel = QuadratureInterpolator::DetKernelType;
+}
+
+template<int DIM, int SDIM, int D1D, int Q1D>
+DetKernel QuadratureInterpolator::DetKernels::Kernel()
+{
+   if (DIM == 1) { return internal::quadrature_interpolator::Det1D; }
+   else if (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D<D1D, Q1D>; }
+   else if (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface<D1D, Q1D>; }
+   else if (DIM == 3) { return internal::quadrature_interpolator::Det3D<D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
+}
+
+DetKernel QuadratureInterpolator::DetKernels::Fallback(
   int DIM, int SDIM, int D1D, int Q1D)
 {
   if (DIM == 1) { return internal::quadrature_interpolator::Det1D; }
@@ -1,304 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_QUADINTERP_DET_HPP
-#define MFEM_QUADINTERP_DET_HPP
-
-#include "../quadinterpolator.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../fem/kernels.hpp"
-#include "../../linalg/kernels.hpp"
-
-namespace mfem
-{
-
-namespace internal
-{
-namespace quadrature_interpolator
-{
-
-inline void Det1D(const int NE,
-                  const real_t *b,
-                  const real_t *g,
-                  const real_t *x,
-                  real_t *y,
-                  const int d1d,
-                  const int q1d,
-                  Vector *d_buff = nullptr)
-{
-   MFEM_CONTRACT_VAR(b);
-   MFEM_CONTRACT_VAR(d_buff);
-   const auto G = Reshape(g, q1d, d1d);
-   const auto X = Reshape(x, d1d, NE);
-
-   auto Y = Reshape(y, q1d, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int q = 0; q < q1d; q++)
-      {
-         real_t u = 0.0;
-         for (int d = 0; d < d1d; d++)
-         {
-            u += G(q, d) * X(d, e);
-         }
-         Y(q, e) = u;
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void Det2D(const int NE,
-                  const real_t *b,
-                  const real_t *g,
-                  const real_t *x,
-                  real_t *y,
-                  const int d1d = 0,
-                  const int q1d = 0,
-                  Vector *d_buff = nullptr)
-{
-   MFEM_CONTRACT_VAR(d_buff);
-   static constexpr int SDIM = 2;
-   static constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
-   auto Y = Reshape(y, Q1D, Q1D, NE);
-
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      MFEM_SHARED real_t XY[SDIM][NBZ][MD1*MD1];
-      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
-      MFEM_SHARED real_t QQ[2*SDIM][NBZ][MQ1*MQ1];
-
-      kernels::internal::LoadX<MD1,NBZ>(e,D1D,X,XY);
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
-
-      kernels::internal::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
-      kernels::internal::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            real_t J[4];
-            kernels::internal::PullGrad<MQ1,NBZ>(Q1D,qx,qy,QQ,J);
-            Y(qx,qy,e) = kernels::Det<2>(J);
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void Det2DSurface(const int NE,
-                         const real_t *b,
-                         const real_t *g,
-                         const real_t *x,
-                         real_t *y,
-                         const int d1d = 0,
-                         const int q1d = 0,
-                         Vector *d_buff = nullptr)
-{
-   MFEM_CONTRACT_VAR(d_buff);
-
-   static constexpr int SDIM = 3;
-   static constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
-   auto Y = Reshape(y, Q1D, Q1D, NE);
-
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int tidz = MFEM_THREAD_ID(z);
-
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      MFEM_SHARED real_t XYZ[SDIM][NBZ][MD1*MD1];
-      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
-
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
-
-      // Load XYZ components
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            for (int d = 0; d < SDIM; ++d)
-            {
-               XYZ[d][tidz][dx + dy*D1D] = X(dx,dy,d,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      ConstDeviceMatrix B_mat(BG[0], D1D, Q1D);
-      ConstDeviceMatrix G_mat(BG[1], D1D, Q1D);
-
-      // x contraction
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            for (int d = 0; d < SDIM; ++d)
-            {
-               real_t u = 0.0;
-               real_t v = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const real_t xval = XYZ[d][tidz][dx + dy*D1D];
-                  u += xval * G_mat(dx,qx);
-                  v += xval * B_mat(dx,qx);
-               }
-               DQ[d][tidz][dy + qx*D1D] = u;
-               DQ[3 + d][tidz][dy + qx*D1D] = v;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      // y contraction and determinant computation
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            real_t J_[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
-            for (int d = 0; d < SDIM; ++d)
-            {
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  J_[d] += DQ[d][tidz][dy + qx*D1D] * B_mat(dy,qy);
-                  J_[3 + d] += DQ[3 + d][tidz][dy + qx*D1D] * G_mat(dy,qy);
-               }
-            }
-            DeviceTensor<2> J(J_, 3, 2);
-            const real_t E = J(0,0)*J(0,0) + J(1,0)*J(1,0) + J(2,0)*J(2,0);
-            const real_t F = J(0,0)*J(0,1) + J(1,0)*J(1,1) + J(2,0)*J(2,1);
-            const real_t G = J(0,1)*J(0,1) + J(1,1)*J(1,1) + J(2,1)*J(2,1);
-            Y(qx,qy,e) = std::sqrt(E*G - F*F);
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, bool SMEM = true>
-inline void Det3D(const int NE,
-                  const real_t *b,
-                  const real_t *g,
-                  const real_t *x,
-                  real_t *y,
-                  const int d1d = 0,
-                  const int q1d = 0,
-                  Vector *d_buff = nullptr) // used only with SMEM = false
-{
-   constexpr int DIM = 3;
-   static constexpr int GRID = SMEM ? 0 : 128;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto X = Reshape(x, D1D, D1D, D1D, DIM, NE);
-   auto Y = Reshape(y, Q1D, Q1D, Q1D, NE);
-
-   real_t *GM = nullptr;
-   if (!SMEM)
-   {
-      const DeviceDofQuadLimits &limits = DeviceDofQuadLimits::Get();
-      const int max_q1d = T_Q1D ? T_Q1D : limits.MAX_Q1D;
-      const int max_d1d = T_D1D ? T_D1D : limits.MAX_D1D;
-      const int max_qd = std::max(max_q1d, max_d1d);
-      const int mem_size = max_qd * max_qd * max_qd * 9;
-      d_buff->SetSize(2*mem_size*GRID);
-      GM = d_buff->Write();
-   }
-
-   mfem::forall_3D_grid(NE, Q1D, Q1D, Q1D, GRID, [=] MFEM_HOST_DEVICE (int e)
-   {
-      static constexpr int MQ1 = T_Q1D ? T_Q1D :
-                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_Q1D);
-      static constexpr int MD1 = T_D1D ? T_D1D :
-                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_D1D);
-      static constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
-      static constexpr int MSZ = MDQ * MDQ * MDQ * 9;
-
-      const int bid = MFEM_BLOCK_ID(x);
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      MFEM_SHARED real_t SM0[SMEM?MSZ:1];
-      MFEM_SHARED real_t SM1[SMEM?MSZ:1];
-      real_t *lm0 = SMEM ? SM0 : GM + MSZ*bid;
-      real_t *lm1 = SMEM ? SM1 : GM + MSZ*(GRID+bid);
-      real_t (*DDD)[MD1*MD1*MD1] = (real_t (*)[MD1*MD1*MD1]) (lm0);
-      real_t (*DDQ)[MD1*MD1*MQ1] = (real_t (*)[MD1*MD1*MQ1]) (lm1);
-      real_t (*DQQ)[MD1*MQ1*MQ1] = (real_t (*)[MD1*MQ1*MQ1]) (lm0);
-      real_t (*QQQ)[MQ1*MQ1*MQ1] = (real_t (*)[MQ1*MQ1*MQ1]) (lm1);
-
-      kernels::internal::LoadX<MD1>(e,D1D,X,DDD);
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
-
-      kernels::internal::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
-      kernels::internal::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
-      kernels::internal::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               real_t J[9];
-               kernels::internal::PullGrad<MQ1>(Q1D, qx,qy,qz, QQQ, J);
-               Y(qx,qy,qz,e) = kernels::Det<3>(J);
-            }
-         }
-      }
-   });
-}
-
-} // namespace quadrature_interpolator
-} // namespace internal
-
-/// @cond Suppress_Doxygen_warnings
-
-template<int DIM, int SDIM, int D1D, int Q1D>
-QuadratureInterpolator::DetKernelType
-QuadratureInterpolator::DetKernels::Kernel()
-{
-   if (DIM == 1) { return internal::quadrature_interpolator::Det1D; }
-   else if (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D<D1D, Q1D>; }
-   else if (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface<D1D, Q1D>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::Det3D<D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
-}
-
-/// @endcond
-
-} // namespace mfem
-
-#endif // MFEM_QUADINTERP_DET_HPP
@@ -22,7 +22,7 @@ QuadratureSpaceBase::QuadratureSpaceBase(Mesh &mesh_, Geometry::Type geom,
 {
   for (int g = 0; g < Geometry::NumGeom; g++)
   {
-      int_rule[g] = nullptr;
+      int_rule[g] = NULL;
   }
   int_rule[geom] = &ir;
 }
@@ -37,29 +37,6 @@ void QuadratureSpaceBase::ConstructIntRules(int dim)
   }
 }

-const Array<int> &QuadratureSpaceBase::Offsets(
-   QSpaceOffsetStorage storage) const
-{
-   if (storage == QSpaceOffsetStorage::COMPRESSED || offsets.Size() > 1)
-   {
-      return offsets;
-   }
-   else
-   {
-      if (full_offset_cache.Size() == 0)
-      {
-         const int nq = size / ne;
-         full_offset_cache.SetSize(ne + 1);
-         int *d_full_offset_cache = full_offset_cache.Write();
-         mfem::forall(ne + 1, [=] MFEM_HOST_DEVICE (int e)
-         {
-            d_full_offset_cache[e] = nq * e;
-         });
-      }
-      return full_offset_cache;
-   }
-}
-
 namespace
 {

@@ -120,31 +97,16 @@ void QuadratureSpaceBase::Integrate(VectorCoefficient &coeff,
 void QuadratureSpace::ConstructOffsets()
 {
   const int num_elem = mesh.GetNE();
-   ne = num_elem;
-
-   if (mesh.GetNumGeometries(mesh.Dimension()) == 1)
+   offsets.SetSize(num_elem + 1);
+   int offset = 0;
+   for (int i = 0; i < num_elem; i++)
   {
-      Array<Geometry::Type> geoms;
-      mesh.GetGeometries(mesh.Dimension(), geoms);
-      offsets.SetSize(1);
-      offsets.HostWrite();
-      offsets[0] = int_rule[geoms[0]]->GetNPoints();
-      size = num_elem * offsets[0];
-   }
-   else
-   {
-      offsets.SetSize(num_elem + 1);
-      int offset = 0;
-      for (int i = 0; i < num_elem; i++)
-      {
-         offsets[i] = offset;
-         const Geometry::Type geom = mesh.GetElementBaseGeometry(i);
-         MFEM_ASSERT(int_rule[geom] != nullptr, "Missing integration rule.");
-         offset += int_rule[geom]->GetNPoints();
-      }
-      offsets[num_elem] = offset;
-      size = offsets.Last();
+      offsets[i] = offset;
+      int geom = mesh.GetElementBaseGeometry(i);
+      MFEM_ASSERT(int_rule[geom] != NULL, "Missing integration rule.");
+      offset += int_rule[geom]->GetNPoints();
   }
+   offsets[num_elem] = size = offset;
 }

 void QuadratureSpace::Construct()
@@ -204,9 +166,9 @@ const Vector &QuadratureSpace::GetGeometricFactorWeights() const

 FaceQuadratureSpace::FaceQuadratureSpace(Mesh &mesh_, int order_,
                                         FaceType face_type_)
-   : QuadratureSpaceBase(mesh_, order_), face_type(face_type_),
-     face_indices(mesh.GetFaceIndices(face_type_)),
-     face_indices_inv(mesh.GetInvFaceIndices(face_type_))
+   : QuadratureSpaceBase(mesh_, order_),
+     face_type(face_type_),
+     num_faces(mesh.GetNFbyType(face_type))
 {
   Construct();
 }
@@ -215,8 +177,7 @@ FaceQuadratureSpace::FaceQuadratureSpace(Mesh &mesh_, const IntegrationRule &ir,
                                         FaceType face_type_)
   : QuadratureSpaceBase(mesh_, mesh_.GetTypicalFaceGeometry(), ir),
     face_type(face_type_),
-     face_indices(mesh.GetFaceIndices(face_type_)),
-     face_indices_inv(mesh.GetInvFaceIndices(face_type_))
+     num_faces(mesh.GetNFbyType(face_type))
 {
   MFEM_VERIFY(mesh.GetNumGeometries(mesh.Dimension() - 1) <= 1,
               "Constructor not valid for mixed meshes");
@@ -225,30 +186,27 @@ FaceQuadratureSpace::FaceQuadratureSpace(Mesh &mesh_, const IntegrationRule &ir,

 void FaceQuadratureSpace::ConstructOffsets()
 {
-   ne = face_indices.Size();
-
-   if (mesh.GetNumGeometries(mesh.Dimension() - 1) == 1)
+   face_indices.SetSize(num_faces);
+   offsets.SetSize(num_faces + 1);
+   int offset = 0;
+   int f_idx = 0;
+   for (int i = 0; i < mesh.GetNumFacesWithGhost(); i++)
   {
-      Array<Geometry::Type> geoms;
-      mesh.GetGeometries(mesh.Dimension() - 1, geoms);
-      offsets.SetSize(1);
-      offsets.HostWrite();
-      offsets[0] = int_rule[geoms[0]]->GetNPoints();
-      size = ne * offsets[0];
-   }
-   else
-   {
-      offsets.SetSize(face_indices.Size() + 1);
-      int offset = 0;
-      for (int i = 0; i < mesh.GetNFbyType(face_type); ++i)
+      const Mesh::FaceInformation face = mesh.GetFaceInformation(i);
+      if (face.IsNonconformingCoarse() || !face.IsOfFaceType(face_type))
      {
-         offsets[i] = offset;
-         Geometry::Type geom = mesh.GetFaceGeometry(face_indices[i]);
-         MFEM_ASSERT(int_rule[geom] != nullptr, "Missing integration rule");
-         offset += int_rule[geom]->GetNPoints();
+         continue;
      }
-      offsets[face_indices.Size()] = size = offset;
+      face_indices[f_idx] = i;
+      face_indices_inv[i] = f_idx;
+      offsets[f_idx] = offset;
+      Geometry::Type geom = mesh.GetFaceGeometry(i);
+      MFEM_ASSERT(int_rule[geom] != NULL, "Missing integration rule");
+      offset += int_rule[geom]->GetNPoints();
+
+      f_idx++;
   }
+   offsets[num_faces] = size = offset;
 }

 void FaceQuadratureSpace::Construct()
@@ -274,16 +232,6 @@ int FaceQuadratureSpace::GetPermutedIndex(int idx, int iq) const
   }
 }

-ElementTransformation *FaceQuadratureSpace::GetTransformation(int idx)
-{
-   ElementTransformation *T = mesh.GetFaceTransformation(face_indices[idx]);
-   if (face_type == FaceType::Boundary)
-   {
-      T->Attribute = mesh.GetBdrFaceAttributes()[idx];
-   }
-   return T;
-}
-
 int FaceQuadratureSpace::GetEntityIndex(const ElementTransformation &T) const
 {
   auto get_face_index = [this](const int idx)
@@ -19,12 +19,6 @@
 namespace mfem
 {

-enum class QSpaceOffsetStorage
-{
-   FULL,
-   COMPRESSED
-};
-
 /// Abstract base class for QuadratureSpace and FaceQuadratureSpace.
 /** This class represents the storage layout for QuadratureFunction%s, that may
    be defined either on mesh elements or mesh faces. */
@@ -36,28 +30,14 @@ protected:
   Mesh &mesh; ///< The underlying mesh.
   int order; ///< The order of integration rule.
   int size; ///< Total number of quadrature points.
-   int ne; ///< Number of entities
   mutable Vector weights; ///< Integration weights.
   mutable long nodes_sequence = 0; ///< Nodes counter for cache invalidation.

-   /// @brief Entity quadrature point offset array.
+   /// @brief Entity quadrature point offset array, of size num_entities + 1.
   ///
-   /// Supports a constant compression scheme for meshes which have a single
-   /// geometry type. When compressed, will have a single value. The true offset
-   /// can be computed as i * offsets[0], where i is the entity index. Otherwise
-   /// has size num_entities + 1.
-   ///
-   /// In the non-compressed case, the quadrature point values for entity i are
-   /// stored in the indices between offsets[i] and offsets[i+1].
+   /// The quadrature point values for entity i are stored in the indices between
+   /// offsets[i] and offsets[i+1].
   Array<int> offsets;
-
-   /// @brief Cached version of the "full" offsets, returned by Offsets() when
-   /// QSpaceOffsetStorage::FULL is provided.
-   ///
-   /// The quadrature point values for entity i are stored in the indices
-   /// between offsets[i] and offsets[i+1].
-   mutable Array<int> full_offset_cache;
-
   /// The quadrature rules used for each geometry type.
   const IntegrationRule *int_rule[Geometry::NumGeom];

@@ -79,30 +59,6 @@ protected:
   void ConstructWeights() const;

 public:
-   /// @brief Gets the offset for a given entity @a idx.
-   ///
-   /// The quadrature point values for entity i are stored in the indices
-   /// between Offset(i) and Offset(i+1)
-   int Offset(int idx) const
-   {
-      return (offsets.Size() == 1) ? (idx * offsets[0]) : offsets[idx];
-   }
-
-   /// @brief Entity quadrature point offset array.
-   ///
-   /// If @a storage is QSpaceOffsetStorage::COMPRESSED, then the returned array
-   /// supports a constant compression scheme for meshes which have a single
-   /// geometry type. When compressed, will have a single value. The true offset
-   /// can be computed as i * offsets[0], where i is the entity index. Otherwise
-   /// has size num_entities + 1.
-   ///
-   /// If @a storage is QSpaceOffsetStorage::FULL, then the array will never be
-   /// compressed.
-   ///
-   /// In the non-compressed case, the quadrature point values for entity i are
-   /// stored in the indices between offsets[i] and offsets[i+1].
-   const Array<int> &Offsets(QSpaceOffsetStorage storage) const;
-
   /// Return the total number of quadrature points.
   int GetSize() const { return size; }

@@ -110,7 +66,7 @@ public:
   int GetOrder() const { return order; }

   /// Return the number of entities.
-   int GetNE() const { return ne; }
+   int GetNE() const { return offsets.Size() - 1; }

   /// Returns the mesh.
   inline Mesh *GetMesh() const { return &mesh; }
@@ -213,12 +169,13 @@ public:
 class FaceQuadratureSpace : public QuadratureSpaceBase
 {
   FaceType face_type; ///< Is the space defined on interior or boundary faces?
+   const int num_faces; ///< Number of faces.

   /// Map from boundary or interior face indices to mesh face indices.
-   const Array<int> &face_indices;
+   Array<int> face_indices;

   /// Inverse of the map @a face_indices.
-   const std::unordered_map<int,int> &face_indices_inv;
+   std::unordered_map<int,int> face_indices_inv;

   const Vector &GetGeometricFactorWeights() const override;
   void ConstructOffsets();
@@ -234,13 +191,14 @@ public:
                       FaceType face_type_);

   /// Returns number of faces in the mesh.
-   inline int GetNumFaces() const { return face_indices.Size(); }
+   inline int GetNumFaces() const { return num_faces; }

   /// Returns the face type (boundary or interior).
   FaceType GetFaceType() const { return face_type; }

   /// Returns the face transformation of face @a idx.
-   ElementTransformation *GetTransformation(int idx) override;
+   ElementTransformation *GetTransformation(int idx) override
+   { return mesh.GetFaceTransformation(face_indices[idx]); }

   /// Returns the geometry type of face @a idx.
   Geometry::Type GetGeometry(int idx) const override
@@ -140,36 +140,6 @@ void add_3D(const scalartype &scalar, const std::vector<type> &u,

 /* Metric definitions */

-// W = ||T||^2 - 2*det(T).
-template <typename type>
-type mu4_ad(const std::vector<type> &T, const std::vector<type> &W)
-{
-   auto fnorm2 = fnorm2_2D(T);
-   auto det = det_2D(T);
-   return fnorm2 - 2*det;
-};
-
-// W = ||T-I||^2.
-template <typename type>
-type mu14_ad(const std::vector<type> &T, const std::vector<type> &W)
-{
-   DenseMatrix Id(2,2); Id = 0.0;
-   Id(0,0) = 1; Id(1,1) = 1;
-
-   std::vector<type> Mat;
-   add_2D(real_t{-1.0}, T, &Id, Mat);
-
-   return fnorm2_2D(Mat);
-};
-
-// W = (det(T)-1)^2.
-template <typename type>
-type mu55_ad(const std::vector<type> &T, const std::vector<type> &W)
-{
-   auto det = det_2D(T);
-   return pow(det-1.0, 2.0);
-};
-
 // W = |T-T'|^2, where T'= |T|*I/sqrt(2).
 template <typename type>
 type mu85_ad(const std::vector<type> &T, const std::vector<type> &W)
@@ -193,63 +163,6 @@ type mu98_ad(const std::vector<type> &T, const std::vector<type> &W)
   return fnorm2_2D(Mat)/det_2D(T);
 };

-template <typename type>
-type make_one_type()
-{
-   return 1.0;
-}
-// add specialization for AD1Type
-template <>
-AD1Type make_one_type<AD1Type>()
-{
-   return AD1Type{1.0, 0.0};
-}
-// add specialization for AD2Type
-template <>
-AD2Type make_one_type<AD2Type>()
-{
-   return AD2Type{AD1Type{1.0, 0.0}, AD1Type{0.0, 0.0}};
-}
-
-using TWCUO = TMOP_WorstCaseUntangleOptimizer_Metric;
-template <typename type>
-type wcuo_ad(type mu,
-             const std::vector<type> &T, const std::vector<type> &W,
-             real_t alpha, real_t min_detT, real_t detT_ep,
-             int exponent, real_t max_muT, real_t muT_ep,
-             TWCUO::BarrierType bt,
-             TWCUO::WorstCaseType wct)
-{
-   type one = make_one_type<type>();
-   type zero = 0.0*one;
-   type denom = one;
-   if (bt == TWCUO::BarrierType::Shifted)
-   {
-      auto val1 = alpha*min_detT-detT_ep < 0.0 ?
-                  (alpha*min_detT-detT_ep)*one :
-                  zero;
-      denom = 2.0*(det_2D(T)-val1);
-   }
-   else if (bt == TWCUO::BarrierType::Pseudo)
-   {
-      auto detT = det_2D(T);
-      denom = detT + sqrt(detT*detT + detT_ep*detT_ep);
-   }
-   mu = mu/denom;
-
-   if (wct == TWCUO::WorstCaseType::PMean)
-   {
-      auto exp = exponent*one;
-      mu = pow(mu, exp);
-   }
-   else if (wct == TWCUO::WorstCaseType::Beta)
-   {
-      auto beta = (max_muT+muT_ep)*one;
-      mu = mu/(beta-mu);
-   }
-   return mu;
-}
-
 // W = 1/(tau^0.5) |T-I|^2.
 template <typename type>
 type mu342_ad(const std::vector<type> &T, const std::vector<type> &W)
@@ -508,7 +421,7 @@ void TMOP_QualityMetric::DefaultAssembleH(const DenseTensor &H,
         {
            for (int cc = 0; cc < dim; cc++)
            {
-               const real_t entry_rr_cc = Hrc(rr, cc);
+               const double entry_rr_cc = Hrc(rr, cc);

               for (int i = 0; i < dof; i++)
               {
@@ -568,30 +481,6 @@ void TMOP_Combo_QualityMetric::EvalPW(const DenseMatrix &Jpt,
   }
 }

-AD1Type TMOP_Combo_QualityMetric::EvalW_AD1(const std::vector<AD1Type> &T,
-                                            const std::vector<AD1Type> &W)
-const
-{
-   AD1Type metric = {0., 0.};
-   for (int i = 0; i < tmop_q_arr.Size(); i++)
-   {
-      metric += wt_arr[i]*tmop_q_arr[i]->EvalW_AD1(T, W);
-   }
-   return metric;
-}
-
-AD2Type TMOP_Combo_QualityMetric::EvalW_AD2(const std::vector<AD2Type> &T,
-                                            const std::vector<AD2Type> &W)
-const
-{
-   AD2Type metric = {{0., 0.},{0., 0.}};
-   for (int i = 0; i < tmop_q_arr.Size(); i++)
-   {
-      metric += wt_arr[i]*tmop_q_arr[i]->EvalW_AD2(T, W);
-   }
-   return metric;
-}
-
 void TMOP_Combo_QualityMetric::AssembleH(const DenseMatrix &Jpt,
                                         const DenseMatrix &DS,
                                         const real_t weight,
@@ -756,64 +645,6 @@ real_t TMOP_WorstCaseUntangleOptimizer_Metric::EvalWBarrier(
   return tmop_metric.EvalW(Jpt)/denominator;
 }

-AD1Type TMOP_WorstCaseUntangleOptimizer_Metric::EvalW_AD1(
-   const std::vector<AD1Type> &T,
-   const std::vector<AD1Type> &W) const
-{
-   return wcuo_ad(tmop_metric.EvalW_AD1(T,W), T, W, alpha, min_detT, detT_ep,
-                  exponent, max_muT, muT_ep, btype, wctype);
-}
-
-AD2Type TMOP_WorstCaseUntangleOptimizer_Metric::EvalW_AD2(
-   const std::vector<AD2Type> &T,
-   const std::vector<AD2Type> &W) const
-{
-   return wcuo_ad(tmop_metric.EvalW_AD2(T,W), T, W, alpha, min_detT, detT_ep,
-                  exponent, max_muT, muT_ep, btype, wctype);
-}
-
-void TMOP_WorstCaseUntangleOptimizer_Metric::EvalP(const DenseMatrix &Jpt,
-                                                   DenseMatrix &P) const
-{
-   auto mu_ad_fn = [this](std::vector<AD1Type> &T, std::vector<AD1Type> &W)
-   {
-      return EvalW_AD1(T,W);
-   };
-   if (tmop_metric.Id() == 4 || tmop_metric.Id() == 14 ||
-       tmop_metric.Id() == 66)
-   {
-      ADGrad(mu_ad_fn, P, Jpt);
-      return;
-   }
-   MFEM_ABORT("EvalW_AD1 not implemented with this metric for "
-              "TMOP_WorstCaseUntangleOptimizer_Metric. "
-              "Please use metric 4/14/66.");
-}
-
-void TMOP_WorstCaseUntangleOptimizer_Metric::AssembleH(
-   const DenseMatrix &Jpt,
-   const DenseMatrix &DS,
-   const real_t weight,
-   DenseMatrix &A) const
-{
-   DenseTensor H(Jpt.Height(), Jpt.Height(), Jpt.TotalSize());
-   H = 0.0;
-   auto mu_ad_fn = [this](std::vector<AD2Type> &T, std::vector<AD2Type> &W)
-   {
-      return EvalW_AD2(T,W);
-   };
-   if (tmop_metric.Id() == 4 || tmop_metric.Id() == 14 ||
-       tmop_metric.Id() == 66)
-   {
-      ADHessian(mu_ad_fn, H, Jpt);
-      this->DefaultAssembleH(H,DS,weight,A);
-      return;
-   }
-   MFEM_ABORT("EvalW_AD1 not implemented with this metric for "
-              "TMOP_WorstCaseUntangleOptimizer_Metric. "
-              "Please use metric 4/14/66.");
-}
-
 real_t TMOP_Metric_001::EvalW(const DenseMatrix &Jpt) const
 {
   ie.SetJacobian(Jpt.GetData());
@@ -1019,25 +850,6 @@ void TMOP_Metric_004::AssembleH(const DenseMatrix &Jpt,
   ie.Assemble_ddI2b(-2.0*weight, A.GetData());
 }

-template <typename type>
-type TMOP_Metric_004::EvalW_AD_impl(const std::vector<type> &T,
-                                    const std::vector<type> &W) const
-{
-   return mu4_ad(T, W);
-}
-
-AD1Type TMOP_Metric_004::EvalW_AD1(const std::vector<AD1Type> &T,
-                                   const std::vector<AD1Type> &W) const
-{
-   return EvalW_AD_impl<AD1Type>(T,W);
-}
-
-AD2Type TMOP_Metric_004::EvalW_AD2(const std::vector<AD2Type> &T,
-                                   const std::vector<AD2Type> &W) const
-{
-   return EvalW_AD_impl<AD2Type>(T,W);
-}
-
 real_t TMOP_Metric_007::EvalW(const DenseMatrix &Jpt) const
 {
   // mu_7 = |J-J^{-t}|^2 = |J|^2 + |J^{-1}|^2 - 4
@@ -1159,25 +971,6 @@ void TMOP_Metric_014::AssembleH(const DenseMatrix &Jpt,
   ie.Assemble_ddI1(weight, A.GetData());
 }

-template <typename type>
-type TMOP_Metric_014::EvalW_AD_impl(const std::vector<type> &T,
-                                    const std::vector<type> &W) const
-{
-   return mu14_ad(T, W);
-}
-
-AD1Type TMOP_Metric_014::EvalW_AD1(const std::vector<AD1Type> &T,
-                                   const std::vector<AD1Type> &W) const
-{
-   return EvalW_AD_impl<AD1Type>(T,W);
-}
-
-AD2Type TMOP_Metric_014::EvalW_AD2(const std::vector<AD2Type> &T,
-                                   const std::vector<AD2Type> &W) const
-{
-   return EvalW_AD_impl<AD2Type>(T,W);
-}
-
 real_t TMOP_Metric_022::EvalW(const DenseMatrix &Jpt) const
 {
   // mu_22 = (0.5*|J|^2 - det(J)) / (det(J) - tau0)
@@ -1307,25 +1100,6 @@ void TMOP_Metric_055::AssembleH(const DenseMatrix &Jpt,
   ie.Assemble_ddI2b(2*weight*(ie.Get_I2b() - 1.0), A.GetData());
 }

-template <typename type>
-type TMOP_Metric_055::EvalW_AD_impl(const std::vector<type> &T,
-                                    const std::vector<type> &W) const
-{
-   return mu55_ad(T, W);
-}
-
-AD1Type TMOP_Metric_055::EvalW_AD1(const std::vector<AD1Type> &T,
-                                   const std::vector<AD1Type> &W) const
-{
-   return EvalW_AD_impl<AD1Type>(T,W);
-}
-
-AD2Type TMOP_Metric_055::EvalW_AD2(const std::vector<AD2Type> &T,
-                                   const std::vector<AD2Type> &W) const
-{
-   return EvalW_AD_impl<AD2Type>(T,W);
-}
-
 real_t TMOP_Metric_056::EvalWMatrixForm(const DenseMatrix &Jpt) const
 {
   // mu_56 = 0.5 (det(J) + 1 / det(J)) - 1.
@@ -4322,7 +4096,7 @@ real_t TMOP_Integrator::GetElementEnergy(const FiniteElement &el,

         const IntegrationPoint &ip_s = ir_s->IntPoint(s);
         Tpr->SetIntPoint(&ip_s);
-         real_t w = surf_fit_coeff->Eval(*Tpr, ip_s) * surf_fit_normal *
+         double w = surf_fit_coeff->Eval(*Tpr, ip_s) * surf_fit_normal *
                    1.0 / surf_fit_dof_count[scalar_dof_id];

         if (surf_fit_gf)
@@ -5348,32 +5122,33 @@ real_t TMOP_Integrator::GetSurfaceFittingWeight()

 void TMOP_Integrator::EnableNormalization(const GridFunction &x)
 {
-   ComputeNormalizationEnergies(x, metric_normal, lim_normal);
+   ComputeNormalizationEnergies(x, metric_normal, lim_normal, surf_fit_normal);
   metric_normal = 1.0 / metric_normal;
   lim_normal = 1.0 / lim_normal;
+   //if (surf_fit_gf) { surf_fit_normal = 1.0 / surf_fit_normal; }
   if (surf_fit_gf || surf_fit_pos) { surf_fit_normal = lim_normal; }
 }

 #ifdef MFEM_USE_MPI
 void TMOP_Integrator::ParEnableNormalization(const ParGridFunction &x)
 {
-   real_t loc[2];
-   ComputeNormalizationEnergies(x, loc[0], loc[1]);
-   real_t rdc[2];
-   MPI_Allreduce(loc, rdc, 2, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+   real_t loc[3];
+   ComputeNormalizationEnergies(x, loc[0], loc[1], loc[2]);
+   real_t rdc[3];
+   MPI_Allreduce(loc, rdc, 3, MPITypeMap<real_t>::mpi_type, MPI_SUM,
                 x.ParFESpace()->GetComm());
   metric_normal = 1.0 / rdc[0];
   lim_normal    = 1.0 / rdc[1];
+   // if (surf_fit_gf) { surf_fit_normal = 1.0 / rdc[2]; }
   if (surf_fit_gf || surf_fit_pos) { surf_fit_normal = lim_normal; }
 }
 #endif

 void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
                                                   real_t &metric_energy,
-                                                   real_t &lim_energy)
+                                                   real_t &lim_energy,
+                                                   real_t &surf_fit_gf_energy)
 {
-   metric_energy = 0.0;
-   lim_energy = 0.0;
   if (PA.enabled)
   {
      MFEM_VERIFY(PA.E.Size() > 0, "Must be called after AssemblePA!");
@@ -5416,6 +5191,9 @@ void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
   Jpr.SetSize(dim);
   Jpt.SetSize(dim);

+   metric_energy = 0.0;
+   lim_energy = 0.0;
+   surf_fit_gf_energy = 0.0;
   for (int i = 0; i < fes->GetNE(); i++)
   {
      const FiniteElement *fe = fes->GetFE(i);
@@ -5447,7 +5225,21 @@ void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
         lim_energy += weight;
      }

-      // TODO: Normalization of the surface fitting term.
+      // Normalization of the surface fitting term.
+      if (surf_fit_gf)
+      {
+         Array<int> dofs;
+         Vector sigma_e;
+         surf_fit_gf->FESpace()->GetElementDofs(i, dofs);
+         surf_fit_gf->GetSubVector(dofs, sigma_e);
+         for (int s = 0; s < dofs.Size(); s++)
+         {
+            if ((*surf_fit_marker)[dofs[s]] == true)
+            {
+               surf_fit_gf_energy += sigma_e(s) * sigma_e(s);
+            }
+         }
+      }
   }

   // Cases when integration is not over the target element, or when the
@@ -14,14 +14,10 @@

 #include "../linalg/invariants.hpp"
 #include "nonlininteg.hpp"
-#include "../linalg/dual.hpp"

 namespace mfem
 {

-using AD1Type = future::dual<real_t, real_t>;
-using AD2Type = future::dual<AD1Type, AD1Type>;
-
 /** @brief Abstract class for local mesh quality metrics in the target-matrix
    optimization paradigm (TMOP) by P. Knupp et al. */
 class TMOP_QualityMetric : public HyperelasticModel
@@ -73,22 +69,6 @@ public:
   virtual void EvalPW(const DenseMatrix &Jpt, DenseMatrix &PW) const
   { PW = 0.0;}

-   /// @brief First-derivative hook for AD-based computations.
-   /// @warning Not for public use. Internal use for AD-based computations.
-   virtual AD1Type EvalW_AD1(const std::vector<AD1Type> &T,
-                             const std::vector<AD1Type> &W) const
-   {
-      MFEM_ABORT("EvalW_AD1 not implemented for this metric");
-   }
-
-   /// @brief Second-derivative hook for AD-based computations.
-   /// @warning Not for public use. Internal use for AD-based computations.
-   virtual AD2Type EvalW_AD2(const std::vector<AD2Type> &T,
-                             const std::vector<AD2Type> &W) const
-   {
-      MFEM_ABORT("EvalW_AD2 not implemented for this metric");
-   }
-
   /** @brief Evaluate the derivative of the 1st Piola-Kirchhoff stress tensor
       and assemble its contribution to the local gradient matrix 'A'.
       @param[in] Jpt     Represents the target->physical transformation
@@ -144,12 +124,6 @@ public:
   void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                  const real_t weight, DenseMatrix &A) const override;

-   AD1Type EvalW_AD1(const std::vector<AD1Type> &T,
-                     const std::vector<AD1Type> &W) const override;
-
-   AD2Type EvalW_AD2(const std::vector<AD2Type> &T,
-                     const std::vector<AD2Type> &W) const override;
-
   /// Computes the averages of all metrics (integral of metric / volume).
   /// Works in parallel when called with a ParGridFunction.
   void ComputeAvgMetrics(const GridFunction &nodes,
@@ -247,16 +221,12 @@ public:

   real_t EvalW(const DenseMatrix &Jpt) const override;

-   AD1Type EvalW_AD1(const std::vector<AD1Type> &T,
-                     const std::vector<AD1Type> &W) const override;
-
-   AD2Type EvalW_AD2(const std::vector<AD2Type> &T,
-                     const std::vector<AD2Type> &W) const override;
-
-   void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const override;
+   void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const override
+   { MFEM_ABORT("Not implemented"); }

   void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
-                  const real_t weight, DenseMatrix &A) const override;
+                  const real_t weight, DenseMatrix &A) const override
+   { MFEM_ABORT("Not implemented"); }

   // Compute mu_hat.
   real_t EvalWBarrier(const DenseMatrix &Jpt) const;
@@ -398,10 +368,6 @@ class TMOP_Metric_004 : public TMOP_QualityMetric
 protected:
   mutable InvariantsEvaluator2D<real_t> ie;

-   template<typename type>
-   type EvalW_AD_impl(const std::vector<type> &T,
-                      const std::vector<type> &W) const;
-
 public:
   // W = |J|^2 - 2*det(J)
   real_t EvalW(const DenseMatrix &Jpt) const override;
@@ -411,12 +377,6 @@ public:
   void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                  const real_t weight, DenseMatrix &A) const override;

-   AD1Type EvalW_AD1(const std::vector<AD1Type> &T,
-                     const std::vector<AD1Type> &W) const override;
-
-   AD2Type EvalW_AD2(const std::vector<AD2Type> &T,
-                     const std::vector<AD2Type> &W) const override;
-
   int Id() const override { return 4; }
 };

@@ -460,10 +420,6 @@ class TMOP_Metric_014 : public TMOP_QualityMetric
 protected:
   mutable InvariantsEvaluator2D<real_t> ie;

-   template <typename type>
-   type EvalW_AD_impl(const std::vector<type> &T,
-                      const std::vector<type> &W) const;
-
 public:
   // W = |J - I|^2.
   real_t EvalWMatrixForm(const DenseMatrix &Jpt) const override;
@@ -475,14 +431,6 @@ public:

   void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                  const real_t weight, DenseMatrix &A) const override;
-
-   AD1Type EvalW_AD1(const std::vector<AD1Type> &T,
-                     const std::vector<AD1Type> &W) const override;
-
-   AD2Type EvalW_AD2(const std::vector<AD2Type> &T,
-                     const std::vector<AD2Type> &W) const override;
-
-   int Id() const override { return 14; }
 };

 /// 2D Shifted barrier form of shape metric (mu_2).
@@ -531,10 +479,6 @@ class TMOP_Metric_055 : public TMOP_QualityMetric
 protected:
   mutable InvariantsEvaluator2D<real_t> ie;

-   template<typename type>
-   type EvalW_AD_impl(const std::vector<type> &T,
-                      const std::vector<type> &W) const;
-
 public:
   // W = (det(J) - 1)^2.
   real_t EvalW(const DenseMatrix &Jpt) const override;
@@ -544,14 +488,6 @@ public:
   void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                  const real_t weight, DenseMatrix &A) const override;

-   AD1Type EvalW_AD1(const std::vector<AD1Type> &T,
-                     const std::vector<AD1Type> &W) const override;
-
-   AD2Type EvalW_AD2(const std::vector<AD2Type> &T,
-                     const std::vector<AD2Type> &W) const override;
-
-   int Id() const override { return 55; }
-
 };

 /// 2D barrier size (V) metric (polyconvex).
@@ -2102,7 +2038,8 @@ protected:
   } PA;

   void ComputeNormalizationEnergies(const GridFunction &x,
-                                     real_t &metric_energy, real_t &lim_energy);
+                                     real_t &metric_energy, real_t &lim_energy,
+                                     real_t &surf_fit_gf_energy);

   void AssembleElementVectorExact(const FiniteElement &el,
                                   ElementTransformation &T,
@@ -111,7 +111,7 @@ public:

   ~InterpolatorFP()
   {
-      if (finder) { finder->FreeData(); }
+      finder->FreeData();
      delete finder;
   }
 };
@@ -326,11 +326,7 @@ public:
       the Size to match this Capacity after this.*/
   template <typename U>
   inline void CopyFrom(const U *src)
-   {
-      if (!begin() || size == 0) { return; }
-      MFEM_ASSERT(begin() && src, "Error in Array::CopyFrom");
-      std::memcpy(begin(), src, MemoryUsage());
-   }
+   { std::memcpy(begin(), src, MemoryUsage()); }

   /// STL-like begin.  Returns pointer to the first element of the array.
   inline T* begin() { return data; }
@@ -14,7 +14,7 @@

 #include "../config/config.hpp"

-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#ifdef MFEM_USE_CUDA
 #include <cusparse.h>
 #include <library_types.h>
 #include <cuda_runtime.h>
@@ -22,7 +22,7 @@
 #endif
 #include "cuda.hpp"

-#if defined(MFEM_USE_HIP) && defined(__HIP__)
+#ifdef MFEM_USE_HIP
 #include <hip/hip_runtime.h>
 #endif
 #include "hip.hpp"
@@ -43,7 +43,7 @@
 #endif
 #endif

-#if !defined(MFEM_USE_CUDA_OR_HIP)
+#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
 #define MFEM_DEVICE
 #define MFEM_HOST
 #define MFEM_LAMBDA
@@ -55,18 +55,17 @@
 #endif

 #if !((defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)) || \
-      (defined(MFEM_USE_HIP) && defined(__HIP_DEVICE_COMPILE__)))
+      (defined(MFEM_USE_HIP)  && defined(__HIP_DEVICE_COMPILE__)))
 #define MFEM_SHARED
 #define MFEM_SYNC_THREAD
 #define MFEM_BLOCK_ID(k) 0
 #define MFEM_THREAD_ID(k) 0
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
-#define MFEM_FOREACH_THREAD_DIRECT(i,k,N) MFEM_FOREACH_THREAD(i,k,N)
 #endif

 // 'double' and 'float' atomicAdd implementation for previous versions of CUDA
-#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 600)
+#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 600
 MFEM_DEVICE inline mfem::real_t atomicAdd(mfem::real_t *add, mfem::real_t val)
 {
   unsigned long long int *ptr = (unsigned long long int *) add;
@@ -94,7 +93,7 @@ template <typename T>
 MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
 {
 #if ((defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)) || \
-     (defined(MFEM_USE_HIP) && defined(__HIP_DEVICE_COMPILE__)))
+     (defined(MFEM_USE_HIP)  && defined(__HIP_DEVICE_COMPILE__)))
   return atomicAdd(&add,val);
 #else
   T old = add;
@@ -347,23 +347,13 @@ void GroupTopology::Swap(GroupTopology &other)
   mfem::Swap(group_mgroup, other.group_mgroup);
 }

-/// \cond DO_NOT_DOCUMENT
 // Initialize the static mpi_type for the specializations of MPITypeMap:
-const MPI_Datatype MPITypeMap<bool>::mpi_type = MFEM_MPI_CXX_BOOL;
-const MPI_Datatype MPITypeMap<char>::mpi_type = MPI_CHAR;
-const MPI_Datatype MPITypeMap<unsigned char>::mpi_type = MPI_UNSIGNED_CHAR;
-const MPI_Datatype MPITypeMap<short>::mpi_type = MPI_SHORT;
-const MPI_Datatype MPITypeMap<unsigned short>::mpi_type = MPI_UNSIGNED_SHORT;
 const MPI_Datatype MPITypeMap<int>::mpi_type = MPI_INT;
-const MPI_Datatype MPITypeMap<unsigned int>::mpi_type = MPI_UNSIGNED;
-const MPI_Datatype MPITypeMap<long>::mpi_type = MPI_LONG;
-const MPI_Datatype MPITypeMap<unsigned long>::mpi_type = MPI_UNSIGNED_LONG;
-const MPI_Datatype MPITypeMap<long long>::mpi_type = MPI_LONG_LONG;
-const MPI_Datatype MPITypeMap<unsigned long long>::mpi_type =
-   MPI_UNSIGNED_LONG_LONG;
 const MPI_Datatype MPITypeMap<float>::mpi_type = MPI_FLOAT;
 const MPI_Datatype MPITypeMap<double>::mpi_type = MPI_DOUBLE;
-/// \endcond DO_NOT_DOCUMENT
+const MPI_Datatype MPITypeMap<int64_t>::mpi_type = MPI_INT64_T;
+const MPI_Datatype MPITypeMap<uint64_t>::mpi_type = MPI_UINT64_T;
+

 GroupCommunicator::GroupCommunicator(const GroupTopology &gt, Mode m)
   : gtopo(gt), mode(m)
@@ -425,24 +425,16 @@ public:
   ~GroupCommunicator();
 };

-/// General MPI message tags used by MFEM
-enum MessageTag
-{
-   DEREFINEMENT_MATRIX_CONSTRUCTION_DATA =
-      291, /// ParFiniteElementSpace ParallelDerefinementMatrix and
-   /// ParDerefineMatrixOp
-};
-
 enum VarMessageTag
 {
-   NEIGHBOR_ELEMENT_RANK_VM, ///< NeighborElementRankMessage
-   NEIGHBOR_ORDER_VM,        ///< NeighborOrderMessage
-   NEIGHBOR_DEREFINEMENT_VM, ///< NeighborDerefinementMessage
-   NEIGHBOR_REFINEMENT_VM,   ///< NeighborRefinementMessage
-   NEIGHBOR_PREFINEMENT_VM,  ///< NeighborPRefinementMessage
-   NEIGHBOR_ROW_VM,          ///< NeighborRowMessage
-   REBALANCE_VM,             ///< RebalanceMessage
-   REBALANCE_DOF_VM,         ///< RebalanceDofMessage
+   NEIGHBOR_ELEMENT_RANK_VM,  ///< NeighborElementRankMessage
+   NEIGHBOR_ORDER_VM,         ///< NeighborOrderMessage
+   NEIGHBOR_DEREFINEMENT_VM,  ///< NeighborDerefinementMessage
+   NEIGHBOR_REFINEMENT_VM,    ///< NeighborRefinementMessage
+   NEIGHBOR_PREFINEMENT_VM,   ///< NeighborPRefinementMessage
+   NEIGHBOR_ROW_VM,           ///< NeighborRowMessage
+   REBALANCE_VM,              ///< RebalanceMessage
+   REBALANCE_DOF_VM           ///< RebalanceDofMessage
 };

 /// \brief Variable-length MPI message containing unspecific binary data.
@@ -607,50 +599,10 @@ protected:
 template <typename Type> struct MPITypeMap;

 // Specializations of MPITypeMap; mpi_type initialized in communication.cpp:
-template<> struct MPITypeMap<bool>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
-template<> struct MPITypeMap<char>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
-template<> struct MPITypeMap<unsigned char>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
-template<> struct MPITypeMap<short>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
-template<> struct MPITypeMap<unsigned short>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
 template<> struct MPITypeMap<int>
 {
   static MFEM_EXPORT const MPI_Datatype mpi_type;
 };
-template<> struct MPITypeMap<unsigned int>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
-template<> struct MPITypeMap<long>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
-template<> struct MPITypeMap<unsigned long>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
-template<> struct MPITypeMap<long long>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
-template<> struct MPITypeMap<unsigned long long>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
 template<> struct MPITypeMap<double>
 {
   static MFEM_EXPORT const MPI_Datatype mpi_type;
@@ -659,6 +611,15 @@ template<> struct MPITypeMap<float>
 {
   static MFEM_EXPORT const MPI_Datatype mpi_type;
 };
+template<> struct MPITypeMap<int64_t>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
+template<> struct MPITypeMap<uint64_t>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
+

 /** Reorder MPI ranks to follow the Z-curve within the physical machine topology
    (provided that functions to query physical node coordinates are available).
@@ -24,7 +24,7 @@ void mfem_cuda_error(cudaError_t err, const char *expr, const char *func,
                     const char *file, int line)
 {
   mfem::err << "\n\nCUDA error: (" << expr << ") failed with error:\n --> "
-             << cudaGetErrorString(err) << " [code: " << (int)err << ']'
+             << cudaGetErrorString(err)
             << "\n ... in function: " << func
             << "\n ... in file: " << file << ':' << line << '\n';
   mfem_error();
@@ -18,7 +18,7 @@
 // CUDA block size used by MFEM.
 #define MFEM_CUDA_BLOCKS 256

-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#ifdef MFEM_USE_CUDA
 #define MFEM_USE_CUDA_OR_HIP
 #define MFEM_DEVICE __device__
 #define MFEM_HOST __host__
@@ -37,23 +37,22 @@
                              __FILE__, __LINE__);                             \
    }                                                                          \
  } while (0)
+#endif // MFEM_USE_CUDA

 // Define the MFEM inner threading macros
-#if defined(__CUDA_ARCH__)
+#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)
 #define MFEM_SHARED __shared__
 #define MFEM_SYNC_THREAD __syncthreads()
 #define MFEM_BLOCK_ID(k) blockIdx.k
 #define MFEM_THREAD_ID(k) threadIdx.k
 #define MFEM_THREAD_SIZE(k) blockDim.k
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=threadIdx.k; i<N; i+=blockDim.k)
-#define MFEM_FOREACH_THREAD_DIRECT(i,k,N) if(const int i=threadIdx.k; i<N)
-#endif // defined(__CUDA_ARCH__)
-#endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#endif

 namespace mfem
 {

-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#ifdef MFEM_USE_CUDA
 // Function used by the macro MFEM_GPU_CHECK.
 void mfem_cuda_error(cudaError_t err, const char *expr, const char *func,
                     const char *file, int line);
@@ -16,7 +16,6 @@
 #include "../fem/ceed/interface/util.hpp"
 #endif
 #ifdef MFEM_USE_MPI
-#include "communication.hpp"
 #include "../linalg/hypre.hpp"
 #endif

@@ -146,11 +145,6 @@ Device::Device()
      Configure(device);
      device_env = true;
   }
-
-   if (GetEnv("MFEM_GPU_AWARE_MPI"))
-   {
-      SetGPUAwareMPI(true);
-   }
 }

 Device::~Device()
@@ -202,29 +196,6 @@ void Device::Configure(const std::string &device, const int device_id)
   {
      bmap[internal::backend_name[i]] = internal::backend_list[i];
   }
-   // auto-detect GPU configurations
-   // assumes only one of HIP or CUDA are available
-#ifdef MFEM_USE_HIP
-   bmap["gpu"] = Backend::HIP;
-#ifdef MFEM_USE_RAJA
-   bmap["raja-gpu"] = Backend::RAJA_HIP;
-#endif
-#ifdef MFEM_USE_CEED
-   bmap["ceed-gpu"] = Backend::CEED_HIP;
-#endif
-   // no OCCA+HIP?
-#elif defined(MFEM_USE_CUDA)
-   bmap["gpu"] = Backend::CUDA;
-#ifdef MFEM_USE_RAJA
-   bmap["raja-gpu"] = Backend::RAJA_CUDA;
-#endif
-#ifdef MFEM_USE_CEED
-   bmap["ceed-gpu"] = Backend::CEED_CUDA;
-#endif
-#ifdef MFEM_USE_OCCA
-   bmap["occa-gpu"] = Backend::OCCA_CUDA;
-#endif
-#endif
   std::string device_option;
   std::string::size_type beg = 0, end;
   while (1)
@@ -342,13 +313,6 @@ void Device::Print(std::ostream &os)
   {
      os << ',' << MemoryTypeName[static_cast<int>(device_mem_type)];
   }
-#ifdef MFEM_USE_MPI
-   if (Allows(Backend::DEVICE_MASK) &&
-       Mpi::IsInitialized() && !Mpi::IsFinalized())
-   {
-      os << "\nUse GPU-aware MPI:    " << (GetGPUAwareMPI() ? "yes" : "no");
-   }
-#endif
   os << std::endl;
 }

@@ -615,7 +579,7 @@ void Device::Setup(const std::string &device_option, const int device_id)
   if (Allows(Backend::DEBUG_DEVICE)) { ngpu = 1; }
 }

-MemoryType Device::QueryMemoryType(const void* ptr)
+MemoryType Device::QueryMemoryType(void *ptr)
 {
   // from HYPRE's hypre_GetPointerLocation
   MemoryType res = MemoryType::HOST;
@@ -198,10 +198,6 @@ public:
         'ceed-hip', 'hip', 'debug',
         'occa-omp', 'raja-omp', 'omp',
         'ceed-cpu', 'occa-cpu', 'raja-cpu', 'cpu'.
-       - The following backend aliases are also available: 'ceed-gpu',
-         'occa-gpu', 'raja-gpu', and 'gpu' where they alias their respective
-         '*-cuda' or '*-hip' backends depending on the MFEM build-time
-         configuration.
       - Multiple backends can be configured at the same time.
       - Only one 'occa-*' backend can be configured at a time.
       - The backend 'occa-cuda' enables the 'cuda' backend unless 'raja-cuda'
@@ -297,9 +293,9 @@ public:
   /// Get the status of GPU-aware MPI flag.
   static bool GetGPUAwareMPI() { return Get().mpi_gpu_aware; }

-   /** Query the device driver for what memory type a given @a ptr is allocated
-    * with. */
-   static MemoryType QueryMemoryType(const void* ptr);
+   /** @brief Query the device driver for what memory type a given @a ptr is
+       allocated with. */
+   static MemoryType QueryMemoryType(void *ptr);

   /** @brief The number of hardware compute units/streaming multiprocessors
       available on a given compute device @a device_id. */
@@ -176,7 +176,7 @@ __device__ void abort_msg(T & msg)
      printf(__VA_ARGS__);     \
      asm("trap;");            \
   }
-#elif defined(__HIP_DEVICE_COMPILE__)
+#elif defined(MFEM_USE_HIP)
 #define MFEM_ABORT_KERNEL(...) \
   {                           \
      printf(__VA_ARGS__);     \
@@ -158,8 +158,8 @@ private:
 #define MFEM_PRAGMA(X) _Pragma(#X)

 // MFEM_UNROLL pragma macro that can be used inside MFEM_FORALL macros.
-#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__) // Clang cuda or nvcc
-#ifdef __NVCC__ // nvcc specifically
+#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)
+#ifdef __NVCC__
 #define MFEM_UNROLL(N) MFEM_PRAGMA(unroll(N))
 #else // Assuming Clang CUDA
 #define MFEM_UNROLL(N) MFEM_PRAGMA(unroll N)
@@ -169,12 +169,12 @@ private:
 #endif

 // MFEM_GPU_FORALL: "parallel for" executed with CUDA or HIP based on the MFEM
-// build-time configuration (MFEM_USE_CUDA or MFEM_USE_HIP), and if compiling
-// with CUDA/HIP language. Otherwise, this macro is a no-op.
-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+// build-time configuration (MFEM_USE_CUDA or MFEM_USE_HIP). If neither CUDA nor
+// HIP is enabled, this macro is a no-op.
+#if defined(MFEM_USE_CUDA)
 #define MFEM_GPU_FORALL(i, N,...) CuWrap1D(N, [=] MFEM_DEVICE      \
                                       (int i) {__VA_ARGS__})
-#elif defined(MFEM_USE_HIP) && defined(__HIP__)
+#elif defined(MFEM_USE_HIP)
 #define MFEM_GPU_FORALL(i, N,...) HipWrap1D(N, [=] MFEM_DEVICE     \
                                        (int i) {__VA_ARGS__})
 #else
@@ -225,52 +225,6 @@ void OmpWrap(const int N, HBODY &&h_body)
 #endif
 }

-template <typename HBODY>
-void OmpWrap2D(const int Nx, const int Ny, HBODY &&h_body)
-{
-#ifdef MFEM_USE_OPENMP
-   // requires OpenMP 3.1
-   #pragma omp parallel for collapse(2)
-   for (int j = 0; j < Ny; j++)
-   {
-      for (int i = 0; i < Nx; i++)
-      {
-         h_body(i, j);
-      }
-   }
-#else
-   MFEM_CONTRACT_VAR(Nx);
-   MFEM_CONTRACT_VAR(Ny);
-   MFEM_CONTRACT_VAR(h_body);
-   MFEM_ABORT("OpenMP requested for MFEM but OpenMP is not enabled!");
-#endif
-}
-
-template <typename HBODY>
-void OmpWrap3D(const int Nx, const int Ny, const int Nz, HBODY &&h_body)
-{
-#ifdef MFEM_USE_OPENMP
-   // requires OpenMP 3.1
-   #pragma omp parallel for collapse(3)
-   for (int k = 0; k < Nz; k++)
-   {
-      for (int j = 0; j < Ny; j++)
-      {
-         for (int i = 0; i < Nx; i++)
-         {
-            h_body(i, j, k);
-         }
-      }
-   }
-#else
-   MFEM_CONTRACT_VAR(Nx);
-   MFEM_CONTRACT_VAR(Ny);
-   MFEM_CONTRACT_VAR(Nz);
-   MFEM_CONTRACT_VAR(h_body);
-   MFEM_ABORT("OpenMP requested for MFEM but OpenMP is not enabled!");
-#endif
-}
-

 /// RAJA Cuda and Hip backends
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA)
@@ -500,42 +454,6 @@ void RajaOmpWrap(const int N, HBODY &&h_body)
   RAJA::forall<RAJA::omp_parallel_for_exec>(RAJA::RangeSegment(0,N), h_body);
 }

-template <typename HBODY>
-void RajaOmpWrap2D(const int Nx, const int Ny, HBODY &&h_body)
-{
-   using omp_launch_policy = RAJA::LaunchPolicy<RAJA::omp_launch_t>;
-   using global_thread_xy = RAJA::LoopPolicy<RAJA::omp_for_exec>;
-   RAJA::RangeSegment xrange(0, Nx);
-   RAJA::RangeSegment yrange(0, Ny);
-   RAJA::launch<omp_launch_policy>(RAJA::ExecPlace::HOST, RAJA::LaunchParams(),
-                                   [=](RAJA::LaunchContext ctx)
-   {
-      // contiguous in x
-      RAJA::expt::loop<global_thread_xy>(ctx, xrange, yrange, [&](int i, int j)
-      {
-         h_body(i, j);
-      });
-   });
-}
-
-template <typename HBODY>
-void RajaOmpWrap3D(const int Nx, const int Ny, const int Nz, HBODY &&h_body)
-{
-   using omp_launch_policy = RAJA::LaunchPolicy<RAJA::omp_launch_t>;
-   using global_thread_xyz = RAJA::LoopPolicy<RAJA::omp_for_exec>;
-   RAJA::RangeSegment xrange(0, Nx);
-   RAJA::RangeSegment yrange(0, Ny);
-   RAJA::RangeSegment zrange(0, Nz);
-   RAJA::launch<omp_launch_policy>(RAJA::ExecPlace::HOST, RAJA::LaunchParams(),
-                                   [=](RAJA::LaunchContext ctx)
-   {
-      // contiguous in x
-      RAJA::expt::loop<global_thread_xyz>(ctx, xrange, yrange, zrange,
-                                          [&](int i, int j, int k)
-      { h_body(i, j, k); });
-   });
-}
-
 #endif


@@ -563,7 +481,7 @@ void RajaSeqWrap(const int N, HBODY &&h_body)


 /// CUDA backend
-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#ifdef MFEM_USE_CUDA

 template <typename BODY> __global__ static
 void CuKernel1D(const int N, BODY body)
@@ -655,11 +573,11 @@ struct CuWrap<3>
   }
 };

-#endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#endif // MFEM_USE_CUDA


 /// HIP backend
-#if defined(MFEM_USE_HIP) && defined(__HIP__)
+#ifdef MFEM_USE_HIP

 template <typename BODY> __global__ static
 void HipKernel1D(const int N, BODY body)
@@ -750,7 +668,7 @@ struct HipWrap<3>
   }
 };

-#endif // defined(MFEM_USE_HIP) && defined(__HIP__)
+#endif // MFEM_USE_HIP


 /// The forall kernel body wrapper
@@ -783,7 +701,7 @@ inline void ForallWrap(const bool use_dev, const int N,
   }
 #endif

-#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
+#ifdef MFEM_USE_CUDA
   // If Backend::CUDA is allowed, use it
   if (Device::Allows(Backend::CUDA))
   {
@@ -791,7 +709,7 @@ inline void ForallWrap(const bool use_dev, const int N,
   }
 #endif

-#if defined(MFEM_USE_HIP) && defined(__HIP__)
+#ifdef MFEM_USE_HIP
   // If Backend::HIP is allowed, use it
   if (Device::Allows(Backend::HIP))
   {
@@ -835,83 +753,6 @@ inline void ForallWrap(const bool use_dev, const int N, lambda &&body,
 template<typename lambda>
 inline void forall(int N, lambda &&body) { ForallWrap<1>(true, N, body); }

-template<typename lambda>
-inline void forall(int Nx, int Ny, lambda &&body)
-{
-   if (Device::Allows(Backend::DEVICE_MASK))
-   {
-      forall(Nx * Ny, [=] MFEM_HOST_DEVICE(int idx)
-      {
-         int j = idx / Nx;
-         int i = idx % Nx;
-         body(i, j);
-      });
-   }
-#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP)
-   else if (Device::Allows(Backend::RAJA_OMP))
-   {
-      return RajaOmpWrap2D(Nx, Ny, body);
-   }
-#endif
-#ifdef MFEM_USE_OPENMP
-   else if (Device::Allows(Backend::OMP))
-   {
-      return OmpWrap2D(Nx, Ny, body);
-   }
-#endif
-   else
-   {
-      for (int j = 0; j < Ny; ++j)
-      {
-         for (int i = 0; i < Nx; ++i)
-         {
-            body(i, j);
-         }
-      }
-   }
-}
-
-template<typename lambda>
-inline void forall(int Nx, int Ny, int Nz, lambda &&body)
-{
-   if (Device::Allows(Backend::DEVICE_MASK))
-   {
-      forall(Nx * Ny * Nz, [=] MFEM_HOST_DEVICE(int idx)
-      {
-         int i = idx % Nx;
-         int j = idx / Nx;
-         int k = j / Ny;
-         j = j % Ny;
-         body(i, j, k);
-      });
-   }
-#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP)
-   else if (Device::Allows(Backend::RAJA_OMP))
-   {
-      return RajaOmpWrap3D(Nx, Ny, Nz, body);
-   }
-#endif
-#ifdef MFEM_USE_OPENMP
-   else if (Device::Allows(Backend::OMP))
-   {
-      return OmpWrap3D(Nx, Ny, Nz, body);
-   }
-#endif
-   else
-   {
-      for (int k = 0; k < Nz; ++k)
-      {
-         for (int j = 0; j < Ny; ++j)
-         {
-            for (int i = 0; i < Nx; ++i)
-            {
-               body(i, j, k);
-            }
-         }
-      }
-   }
-}
-
 template<typename lambda>
 inline void forall_switch(bool use_dev, int N, lambda &&body)
 {
@@ -57,9 +57,9 @@ struct Hashed4
 *  each time this class is invoked.
 *
 *  There are two main methods this class provides. The Get(...) methods always
- *  return an item given the two or four indices. If the item did not previously
+ *  return an item given the two or four indices. If the item didn't previously
 *  exist, the methods creates a new one. The Find(...) methods, on the other
- *  hand, just return NULL or -1 if the item does not exist.
+ *  hand, just return NULL or -1 if the item doesn't exist.
 *
 *  Each new item is automatically assigned a unique ID - the index of the item
 *  inside the BlockArray. The IDs may (but need not) be used as p1, p2, ... of
@@ -95,14 +95,14 @@ public:
       @param[in] init_hash_size The initial size of the hash table. Must be
                                 a power of 2. */
   HashTable(int block_size = 16*1024, int init_hash_size = 32*1024);
-   /// Deep copy
+   /// @brief Deep copy
   HashTable(const HashTable& other);
-   /// Copy assignment not supported
+   /// @brief Copy assignment not supported
   HashTable& operator=(const HashTable&) = delete;
   ~HashTable();

-   /** @brief Item accessor with key (or parents) the pair p1, p2. Default
-       construct an item of type T if no value corresponds to the requested key.
+   /** @brief Item accessor with key (or parents) the pair 'p1', 'p2'. Default
+       construct an item of type T if no value correspond to the requested key.

       @param[in] p1 First part of the key.
       @param[in] p2 Second part of the key.
@@ -111,9 +111,9 @@ public:
       @warning This method should only be called if T inherits from Hashed2. */
   T* Get(int p1, int p2);

-   /** @brief Item accessor with key (or parents) the quadruplet p1, p2, p3, p4.
-       The key p4 is optional. Default construct an item of type T if no value
-       corresponds to the requested key.
+   /** @brief Item accessor with key (or parents) the quadruplet 'p1', 'p2',
+       'p3', 'p4'. The key 'p4' is optional. Default construct an item of type T
+       if no value corresponds to the requested key.

       @param[in] p1 First part of the key.
       @param[in] p2 Second part of the key.
@@ -124,10 +124,10 @@ public:
      @warning This method should only be called if T inherits from Hashed4. */
   T* Get(int p1, int p2, int p3, int p4 = -1 /* p4 optional */);

-   /** @brief Get the "id" of the item whose parents are p1, p2, this "id"
-       corresponding to the index of the item in the underlying BlockArray<T>
-       object. Default construct an item and "id" if no value corresponds to the
-       requested key.
+   /// Get id of item whose parents are p1, p2... Create it if it doesn't exist.
+   /** @brief Get the "id" of an item, this "id" corresponding to the index of the
+       item in the underlying BlockArray<T> object. Default construct an item
+       and id if no value corresponds to the requested key.

       @param[in] p1 First part of the key.
       @param[in] p2 Second part of the key.
@@ -136,9 +136,9 @@ public:
       @warning This method should only be called if T inherits from Hashed2. */
   int GetId(int p1, int p2);

-   /** @brief Get the "id" of an item, this "id" corresponding to the index of
-       the item in the underlying BlockArray<T> object. Default construct an item
-       and "id" if no value corresponds to the requested key.
+   /** @brief Get the "id" of an item, this "id" corresponding to the index of the
+       item in the underlying BlockArray<T> object. Default construct an item
+       and id if no value correspond to the requested key.

       @param[in] p1 First part of the key.
       @param[in] p2 Second part of the key.
@@ -149,8 +149,9 @@ public:
       @warning This method should only be called if T inherits from Hashed4. */
   int GetId(int p1, int p2, int p3, int p4 = -1);

-   /** @brief Item accessor with key (or parents) the pair p1, p2. Return
-       NULL if no value corresponds to the requested key.
+   /// Find item whose parents are p1, p2... Return NULL if it doesn't exist.
+   /** @brief Item accessor with key (or parents) the pair 'p1', 'p2'. Return
+       nullptr if no value correspond to the requested key.

       @param[in] p1 First part of the key.
       @param[in] p2 Second part of the key.
@@ -159,9 +160,9 @@ public:
       @warning This method should only be called if T inherits from Hashed2. */
   T* Find(int p1, int p2);

-   /** @brief Item accessor with key (or parents) the quadruplet p1, p2, p3, p4.
-       The key p4 is optional. Return NULL if no value corresponds to the
-       requested key.
+   /** @brief Item accessor with key (or parents) the quadruplet 'p1', 'p2',
+       'p3', 'p4'. The key 'p4' is optional. Return nullptr if no value
+       correspond to the requested key.

       @param[in] p1 First part of the key.
       @param[in] p2 Second part of the key.
@@ -172,8 +173,8 @@ public:
       @warning This method should only be called if T inherits from Hashed4. */
   T* Find(int p1, int p2, int p3, int p4 = -1);

-   /** @brief Item const accessor with key (or parents) the pair p1, p2.
-       Return NULL if no value corresponds to the requested key.
+   /** @brief Item const accessor with key (or parents) the pair 'p1', 'p2'.
+       Return nullptr if no value correspond to the requested key.

       @param[in] p1 First part of the key.
       @param[in] p2 Second part of the key.
@@ -182,9 +183,9 @@ public:
      @warning This method should only be called if T inherits from Hashed2. */
   const T* Find(int p1, int p2) const;

-   /** @brief Item const accessor with key (or parents) the quadruplet p1, p2,
-       p3, p4. The key p4 is optional. Return NULL if no value corresponds to the
-       requested key.
+   /** @brief Item const accessor with key (or parents) the quadruplet 'p1',
+       'p2', 'p3', 'p4'. The key 'p4' is optional. Return nullptr if no value
+       correspond to the requested key.

       @param[in] p1 First part of the key.
       @param[in] p2 Second part of the key.
@@ -195,12 +196,10 @@ public:
       @warning This method should only be called if T inherits from Hashed4. */
   const T* Find(int p1, int p2, int p3, int p4 = -1) const;

-   /** @brief Find the "id" of an item whose parents are p1, p2. Return -1 if it
-       does not exist.
-
-       This "id" corresponds to the index of the item in the underlying
-       BlockArray<T> object. Default construct an item and "id" if no value
-       corresponds to the requested key.
+   /// Find id of item whose parents are p1, p2... Return -1 if it doesn't exist.
+   /** @brief Find the "id" of an item, this "id" corresponding to the index of
+       the item in the underlying BlockArray<T> object. Default construct an
+       item and id if no value correspond to the requested key.

       @param[in] p1 First part of the key.
       @param[in] p2 Second part of the key.
@@ -210,9 +209,8 @@ public:
   int FindId(int p1, int p2) const;

   /** @brief Find the "id" of an item, this "id" corresponding to the index of
-       the item in the underlying BlockArray<T> object. Return -1 if it does not
-       exist. Default construct an item and "id" if no value corresponds to the
-       requested key.
+       the item in the underlying BlockArray<T> object. Default construct an
+       item and id if no value correspond to the requested key.

       @param[in] p1 First part of the key.
       @param[in] p2 Second part of the key.
@@ -223,16 +221,16 @@ public:
       @warning This method should only be called if T inherits from Hashed4. */
   int FindId(int p1, int p2, int p3, int p4 = -1) const;

-   /// Return the number of elements currently stored in the HashTable.
+   /// @brief Return the number of elements currently stored in the HashTable.
   int Size() const { return Base::Size() - unused.Size(); }

-   /// Return the total number of ids (used and unused) in the HashTable.
+   /// @brief Return the total number of ids (used and unused) in the HashTable.
   int NumIds() const { return Base::Size(); }

-   /// Return the number of free/unused ids in the HashTable.
+   /// @brief Return the number of free/unused ids in the HashTable.
   int NumFreeIds() const { return unused.Size(); }

-   /** @brief Return true if item @a id exists in (is used by) the container.
+   /** @brief Return true if item 'id' exists in (is used by) the container.

       @param[in] id Index of the item in the underlying BlockArray<T>.

@@ -243,13 +241,13 @@ public:

       @param[in] id Index of the item in the underlying BlockArray<T>.

-       @warning Its @a id will be reused by newly added items. */
+       @warning Its id will be reused by newly added items. */
   void Delete(int id);

-   /// Remove all items.
+   /// @brief Remove all items.
   void DeleteAll();

-   /** @brief Allocate an item at @a id. Enlarge the underlying BlockArray if
+   /** @brief Allocate an item at 'id'. Enlarge the underlying BlockArray if
       necessary.

       @param[in] id Index of the item in the underlying BlockArray<T>.
@@ -257,7 +255,7 @@ public:
       @param[in] p2 Second part of the key.

       @warning This is a special purpose method used when loading data from a
-       file. Does nothing if the slot @a id has already been allocated. */
+       file. Does nothing if the slot 'id' has already been allocated. */
   void Alloc(int id, int p1, int p2);

   /** @brief Reinitialize the internal list of unallocated items.
@@ -289,13 +287,13 @@ public:
       @warning This method should only be called if T inherits from Hashed4. */
   void Reparent(int id, int new_p1, int new_p2, int new_p3, int new_p4 = -1);

-   /// Return total size of allocated memory (tables plus items), in bytes.
+   /// @brief Return total size of allocated memory (tables plus items), in bytes.
   std::size_t MemoryUsage() const;

-   /// Write details of the memory usage to the mfem output stream.
+   /// @brief Write details of the memory usage to the mfem output stream.
   void PrintMemoryDetail() const;

-   /// Print a histogram of bin sizes for debugging purposes.
+   /// @brief Print a histogram of bin sizes for debugging purposes.
   void PrintStats() const;

   class iterator : public Base::iterator
@@ -348,7 +346,7 @@ public:

 protected:
   /** The hash table: each bin is a linked list of items. For each non-empty
-       bin, this arrays stores the "id" of the first item in the list, or -1
+       bin, this arrays stores the 'id' of the first item in the list, or -1
       if the bin is empty. */
   int* table;

@@ -386,11 +384,11 @@ protected:
   { return (984120265ul*p1 + 125965121ul*p2 + 495698413ul*p3) & mask; }

   // Delete() and Reparent() use one of these:
-   /// Hash function for items of type T that inherit from Hashed2.
+   /// @brief Hash function for items of type T that inherit from Hashed2.
   inline int Hash(const Hashed2& item) const
   { return Hash(item.p1, item.p2); }

-   /// Hash function for items of type T that inherit from Hashed4.
+   /// @brief Hash function for items of type T that inherit from Hashed4.
   inline int Hash(const Hashed4& item) const
   { return Hash(item.p1, item.p2, item.p3); }

@@ -417,15 +415,15 @@ protected:
       @warning This method should only be called if T inherits from Hashed4. */
   int SearchList(int id, int p1, int p2, int p3) const;

-   /** @brief Insert the item @a id into bin @a idx.
+   /** @brief Insert the item 'id' into bin 'idx'.

       @param[in] idx The bin/bucket index.
       @param[in] id The index of the item in the BlockArray<T>.
       @param[in] item The item to insert at the beginning of the linked list.

-       @warning The method only works with bin @a idx and does not check the
-                overall fill factor of the hash table. If appropriate, use
-                CheckRehash() for that. */
+       @warning The method only works with bin 'idx' and does not check the
+                overall fill factor of the hash table. If appropriate,
+                use CheckRehash() for that. */
   inline void Insert(int idx, int id, T &item);

   /** @brief Unlink an item @a id from the linked list of bin @a idx.
@@ -446,11 +444,11 @@ protected:
       and reinsert all items into the new bins.

       NOTE: Rehashing is computationally expensive (O(N) in the number of items),
-       but since it is only done rarely (when the number of items doubles), the
-       amortized complexity of inserting an item is still O(1). */
+       but since it is only done rarely (when the number of items doubles),
+       the amortized complexity of inserting an item is still O(1). */
   void DoRehash();

-   /** @brief Return the size of the bin @a idx.
+   /** @brief Return the size of the bin "idx".

       @param[in] idx The index of the bin.
       @return The size of the bin. */
@@ -24,7 +24,7 @@ void mfem_hip_error(hipError_t err, const char *expr, const char *func,
                    const char *file, int line)
 {
   mfem::err << "\n\nHIP error: (" << expr << ") failed with error:\n --> "
-             << hipGetErrorString(err) << " [code: " << (int)err << ']'
+             << hipGetErrorString(err)
             << "\n ... in function: " << func
             << "\n ... in file: " << file << ':' << line << '\n';
   mfem_error();
@@ -18,7 +18,7 @@
 // HIP block size used by MFEM.
 #define MFEM_HIP_BLOCKS 256

-#if defined(MFEM_USE_HIP) && defined(__HIP__)
+#ifdef MFEM_USE_HIP
 #define MFEM_USE_CUDA_OR_HIP
 #define MFEM_DEVICE __device__
 #define MFEM_HOST __host__
@@ -37,20 +37,18 @@
                             __FILE__, __LINE__);                              \
    }                                                                          \
  } while (0)
+#endif // MFEM_USE_HIP

 // Define the MFEM inner threading macros
-#if defined(__HIP_DEVICE_COMPILE__)
+#if defined(MFEM_USE_HIP) && defined(__HIP_DEVICE_COMPILE__)
 #define MFEM_SHARED __shared__
 #define MFEM_SYNC_THREAD __syncthreads()
 #define MFEM_BLOCK_ID(k) hipBlockIdx_ ##k
 #define MFEM_THREAD_ID(k) hipThreadIdx_ ##k
 #define MFEM_THREAD_SIZE(k) hipBlockDim_ ##k
 #define MFEM_FOREACH_THREAD(i,k,N) \
-   for(int i=hipThreadIdx_ ##k; i<N; i+=hipBlockDim_ ##k)
-#define MFEM_FOREACH_THREAD_DIRECT(i,k,N) \
-   if(const int i=hipThreadIdx_ ##k; i<N)
-#endif // defined(__HIP_DEVICE_COMPILE__)
-#endif // defined(MFEM_USE_HIP) && defined(__HIP__)
+    for(int i=hipThreadIdx_ ##k; i<N; i+=hipBlockDim_ ##k)
+#endif

 namespace mfem
 {
@@ -513,7 +513,10 @@ public:
   void *HtoD(void *dst, const void *src, size_t bytes) override
   { return HipMemcpyHtoD(dst, src, bytes); }
   void *DtoD(void* dst, const void* src, size_t bytes) override
-   { return HipMemcpyDtoD(dst, src, bytes); }
+   // Unlike cudaMemcpy(DtoD), hipMemcpy(DtoD) causes a host-side synchronization so
+   // instead we use hipMemcpyAsync to get similar behavior.
+   // for more info see: https://github.com/mfem/mfem/pull/2780
+   { return HipMemcpyDtoDAsync(dst, src, bytes); }
   void *DtoH(void *dst, const void *src, size_t bytes) override
   { return HipMemcpyDtoH(dst, src, bytes); }
 };
@@ -655,7 +658,10 @@ public:
      return CuMemcpyDtoD(dst, src, bytes);
 #endif
 #ifdef MFEM_USE_HIP
-      return HipMemcpyDtoD(dst, src, bytes);
+      // Unlike cudaMemcpy(DtoD), hipMemcpy(DtoD) causes a host-side synchronization so
+      // instead we use hipMemcpyAsync to get similar behavior.
+      // for more info see: https://github.com/mfem/mfem/pull/2780
+      return HipMemcpyDtoDAsync(dst, src, bytes);
 #endif
      // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
   }
@@ -1384,11 +1390,8 @@ void MemoryManager::Insert(void *h_ptr, size_t bytes,
   {
      auto &m = res.first->second;
      MFEM_VERIFY(m.bytes >= bytes && m.h_mt == h_mt &&
-                  (m.d_mt == d_mt ||
-                   (d_mt == MemoryType::DEFAULT &&
-                    m.d_mt == GetDualMemoryType(h_mt)) ||
-                   (m.d_mt == MemoryType::DEFAULT &&
-                    d_mt == GetDualMemoryType(m.h_mt))),
+                  (m.d_mt == d_mt || (d_mt == MemoryType::DEFAULT &&
+                                      m.d_mt == GetDualMemoryType(h_mt))),
                  "Address already present with different attributes!");
 #ifdef MFEM_TRACK_MEM_MANAGER
      mfem::out << "[mfem memory manager]: repeated registration of h_ptr: "
@@ -896,7 +896,6 @@ inline HYPRE_MemoryLocation GetHypreMemoryLocation()
 #elif MFEM_HYPRE_VERSION < 23100
   return HYPRE_MEMORY_DEVICE;
 #else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
-   if (!HYPRE_Initialized()) { return HYPRE_MEMORY_HOST; }
   HYPRE_MemoryLocation loc;
   HYPRE_GetMemoryLocation(&loc);
   return loc;
@@ -1058,8 +1057,7 @@ inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
         // register the 'base' if the MemoryManager::Exists():
         MemoryManager::Exists()
 #else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
-         IsDeviceMemory(MemoryManager::GetDeviceMemoryType()) ||
-         (MemoryManager::Exists() && HypreUsingGPU())
+         MemoryManager::Exists() && HypreUsingGPU()
 #endif
      )
      {
@@ -537,7 +537,7 @@ void reduce(int N, T &res, B &&body, const R &reducer, bool use_dev,
      return;
   }

-#if defined(MFEM_USE_CUDA_OR_HIP)
+#if defined(MFEM_USE_HIP) || defined(MFEM_USE_CUDA)
   if (use_dev &&
       mfem::Device::Allows(Backend::CUDA | Backend::HIP | Backend::RAJA_CUDA |
                            Backend::RAJA_HIP))
@@ -1,228 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_SCAN_HPP
-#define MFEM_SCAN_HPP
-
-#include "backends.hpp"
-
-#ifdef MFEM_USE_CUDA
-#include <cub/device/device_scan.cuh>
-#define MFEM_CUB_NAMESPACE cub
-#elif defined(MFEM_USE_HIP)
-#include <hipcub/device/device_scan.hpp>
-#define MFEM_CUB_NAMESPACE hipcub
-#endif
-
-#include <functional>
-#include <numeric>
-#include <cstddef>
-
-namespace mfem
-{
-/// Equivalent to InclusiveScan(use_dev, d_in, d_out, num_items, workspace,
-/// std::plus<>{})
-template <class InputIt, class OutputIt>
-void InclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items)
-{
-   // forward to InclusiveSum for potentially faster kernels
-#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
-   if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
-   {
-      static Array<std::byte> workspace;
-      size_t bytes = workspace.Size();
-      if (bytes)
-      {
-         auto err = MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum(
-                       workspace.Write(), bytes, d_in, d_out, num_items);
-#if defined(MFEM_USE_CUDA)
-         if (err == cudaSuccess)
-         {
-            return;
-         }
-#elif defined(MFEM_USE_HIP)
-         if (err == hipSuccess)
-         {
-            return;
-         }
-#endif
-      }
-      // try allocating a larger buffer
-      bytes = 0;
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum(
-                        nullptr, bytes, d_in, d_out, num_items));
-      workspace.SetSize(bytes);
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum(
-                        workspace.Write(), bytes, d_in, d_out, num_items));
-      return;
-   }
-#endif
-#if 0
-   std::inclusive_scan(d_in, d_in + num_items, d_out);
-#else
-   // work-around to some compilers not fully supporting C++17
-   if (num_items)
-   {
-      *d_out = *d_in;
-      auto prev = d_out;
-      ++d_in;
-      ++d_out;
-      for (size_t i = 1; i < num_items; ++i)
-      {
-         *d_out = (*prev) + (*d_in);
-         prev = d_out;
-         ++d_in;
-         ++d_out;
-      }
-   }
-#endif
-}
-
-/// @brief Performs an inclusive scan of [d_in, d_in+num_items) -> [d_out,
-/// d_out+num_items). This call is potentially asynchronous on the device.
-///
-/// @a d_in input start.
-/// @a d_out output start. Can perform in-place scans with d_out = d_in
-/// @a scan_op binary scan functor. Must be associative. If only weakly
-/// associative (i.e. floating point addition) results are not deterministic. On
-/// device this must also be commutative.
-template <class InputIt, class OutputIt, class ScanOp>
-void InclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
-                   ScanOp scan_op)
-{
-#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
-   if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
-   {
-      static Array<std::byte> workspace;
-      size_t bytes = workspace.Size();
-      if (bytes)
-      {
-         auto err = MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan(
-                       workspace.Write(), bytes, d_in, d_out, scan_op, num_items);
-#if defined(MFEM_USE_CUDA)
-         if (err == cudaSuccess)
-         {
-            return;
-         }
-#elif defined(MFEM_USE_HIP)
-         if (err == hipSuccess)
-         {
-            return;
-         }
-#endif
-      }
-      // try allocating a larger buffer
-      bytes = 0;
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan(
-                        nullptr, bytes, d_in, d_out, scan_op, num_items));
-      workspace.SetSize(bytes);
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan(
-                        workspace.Write(), bytes, d_in, d_out, scan_op, num_items));
-      return;
-   }
-#endif
-#if 0
-   std::inclusive_scan(d_in, d_in + num_items, d_out, scan_op);
-#else
-   // work-around to some compilers not fully supporting C++17
-   if (num_items)
-   {
-      *d_out = *d_in;
-      auto prev = d_out;
-      ++d_in;
-      ++d_out;
-      for (size_t i = 1; i < num_items; ++i)
-      {
-         *d_out = scan_op(*prev, *d_in);
-         prev = d_out;
-         ++d_in;
-         ++d_out;
-      }
-   }
-#endif
-}
-
-/// Performs an exclusive scan of [d_in, d_in+num_items) -> [d_out,
-/// d_out+num_items). This call is potentially asynchronous on the device.
-/// @a d_in input start.
-/// @a d_out output start. Can perform in-place scans with d_out = d_in
-/// @a scan_op binary scan functor. Must be associative. If only weakly
-/// associative (i.e. floating point addition) results are not deterministic. On
-/// device this must also be commutative.
-template <class InputIt, class OutputIt, class T, class ScanOp>
-void ExclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
-                   T init_value, ScanOp scan_op)
-{
-#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
-   if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK))
-   {
-      static Array<std::byte> workspace;
-      size_t bytes = workspace.Size();
-      if (bytes)
-      {
-         auto err = MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan(
-                       workspace.Write(), bytes, d_in, d_out, scan_op, init_value,
-                       num_items);
-#if defined(MFEM_USE_CUDA)
-         if (err == cudaSuccess)
-         {
-            return;
-         }
-#elif defined(MFEM_USE_HIP)
-         if (err == hipSuccess)
-         {
-            return;
-         }
-#endif
-      }
-      // try allocating a larger buffer
-      bytes = 0;
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan(
-                        nullptr, bytes, d_in, d_out, scan_op, init_value, num_items));
-      workspace.SetSize(bytes);
-      MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan(
-                        workspace.Write(), bytes, d_in, d_out, scan_op, init_value,
-                        num_items));
-      return;
-   }
-#endif
-#if 0
-   std::exclusive_scan(d_in, d_in + num_items, d_out, init_value, scan_op);
-#else
-   // work-around to some compilers not fully supporting C++17
-   if (num_items)
-   {
-      for (size_t i = 0; i < num_items; ++i)
-      {
-         auto next = scan_op(init_value, *d_in);
-         *d_out = init_value;
-         init_value = next;
-         ++d_out;
-         ++d_in;
-      }
-   }
-#endif
-}
-
-/// Equivalent to ExclusiveScan(use_dev, d_in, d_out, num_items, init_value,
-/// workspace, std::plus<>{})
-template <class InputIt, class OutputIt, class T>
-void ExclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items,
-                   T init_value)
-{
-   ExclusiveScan(use_dev, d_in, d_out, num_items, init_value, std::plus<> {});
-}
-
-} // namespace mfem
-
-#undef MFEM_CUB_NAMESPACE
-
-#endif
@@ -4405,32 +4405,4 @@ void BatchLUSolve(const DenseTensor &Mlu, const Array<int> &P, Vector &X)
   BatchedLinAlg::LUSolve(Mlu, P, X);
 }

-#ifdef MFEM_USE_LAPACK
-void BandedSolve(int KL, int KU, DenseMatrix &AB, DenseMatrix &B,
-                 Array<int> &ipiv)
-{
-   int LDAB = (2*KL) + KU + 1;
-   int N = AB.NumCols();
-   int NRHS = B.NumCols();
-   int info;
-   ipiv.SetSize(N);
-   MFEM_LAPACK_PREFIX(gbsv_)(&N, &KL, &KU, &NRHS, AB.GetData(), &LDAB,
-                             ipiv.GetData(), B.GetData(), &N, &info);
-   MFEM_ASSERT(info == 0, "BandedSolve failed in LAPACK");
-}
-
-void BandedFactorizedSolve(int KL, int KU, DenseMatrix &AB, DenseMatrix &B,
-                           bool transpose, Array<int> &ipiv)
-{
-   int LDAB = (2*KL) + KU + 1;
-   int N = AB.NumCols();
-   int NRHS = B.NumCols();
-   char trans = transpose ? 'T' : 'N';
-   int info;
-   MFEM_LAPACK_PREFIX(gbtrs_)(&trans, &N, &KL, &KU, &NRHS, AB.GetData(), &LDAB,
-                              ipiv.GetData(), B.GetData(), &N, &info);
-   MFEM_ASSERT(info == 0, "BandedFactorizedSolve failed in LAPACK");
-}
-#endif
-
 } // namespace mfem
@@ -1329,13 +1329,6 @@ void BatchLUFactor(DenseTensor &Mlu, Array<int> &P, const real_t TOL = 0.0);
    dimension m x n. */
 void BatchLUSolve(const DenseTensor &Mlu, const Array<int> &P, Vector &X);

-#ifdef MFEM_USE_LAPACK
-void BandedSolve(int KL, int KU, DenseMatrix &AB, DenseMatrix &B,
-                 Array<int> &ipiv);
-void BandedFactorizedSolve(int KL, int KU, DenseMatrix &AB, DenseMatrix &B,
-                           bool transpose, Array<int> &ipiv);
-#endif
-
 // Inline methods

 inline real_t &DenseMatrix::operator()(int i, int j)
@@ -2574,18 +2574,6 @@ void HypreParMatrix::EliminateBC(const Array<int> &ess_dofs,
 #if defined(HYPRE_USING_GPU)
      if (HypreUsingGPU())
      {
-#if defined(HYPRE_WITH_GPU_AWARE_MPI) || defined(HYPRE_USING_GPU_AWARE_MPI)
-         // hypre_GetGpuAwareMPI() was introduced in v2.31.0, however, its value
-         // is not checked in hypre_ParCSRCommHandleCreate_v2() before v2.33.0,
-         // instead only HYPRE_WITH_GPU_AWARE_MPI is checked.
-#if MFEM_HYPRE_VERSION >= 23300
-         if (hypre_GetGpuAwareMPI())
-#endif
-         {
-            // ensure int_buf_data has been computed before sending it
-            MFEM_STREAM_SYNC;
-         }
-#endif
         // Try to use device-aware MPI for the communication if available
         comm_handle = hypre_ParCSRCommHandleCreate_v2(
                          11, comm_pkg, HYPRE_MEMORY_DEVICE, int_buf_data,
@@ -42,13 +42,6 @@ extern "C" void
 MFEM_LAPACK_PREFIX(getri_)(int *N, real_t *A, int *LDA, int *IPIV, real_t *WORK,
                           int *LWORK, int *INFO);
 extern "C" void
-MFEM_LAPACK_PREFIX(gbsv_)(int *, int *, int *, int *, real_t *, int *, int *,
-                          real_t *, int *, int *);
-extern "C" void
-MFEM_LAPACK_PREFIX(gbtrs_)(char *, int *, int *, int *, int *, real_t *, int *,
-                           int *, real_t *, int *, int *);
-
-extern "C" void
 MFEM_LAPACK_PREFIX(syevr_)(char *JOBZ, char *RANGE, char *UPLO, int *N,
                           real_t *A, int *LDA, real_t *VL, real_t *VU, int *IL,
                           int *IU, real_t *ABSTOL, int *M, real_t *W,
@@ -117,7 +117,7 @@ Vector::Vector(const Vector &v)
   UseDevice(v.UseDevice());
 }

-Vector::Vector(Vector &&v) : Vector()
+Vector::Vector(Vector &&v)
 {
   *this = std::move(v);
 }
@@ -26,12 +26,6 @@
   }\
 }

-#if defined(MFEM_USE_DOUBLE)
-#define MFEM_NETCDF_REAL_T NC_DOUBLE
-#elif defined(MFEM_USE_SINGLE)
-#define MFEM_NETCDF_REAL_T NC_FLOAT
-#endif
-
 namespace mfem
 {

@@ -141,18 +135,18 @@ public:
   /// @brief Writes the mesh to an ExodusII file.
   /// @param fpath The path to the file.
   /// @param flags NC_CLOBBER will overwrite existing file.
-   void PrintExodusII(const std::string &fpath, int flags = NC_CLOBBER);
+   void PrintExodusII(std::string fpath, int flags = NC_CLOBBER);

   /// @brief Static method for writing a mesh to an ExodusII file.
   /// @param mesh The mesh to write to the file.
   /// @param fpath The path to the file.
   /// @param flags NetCDF file flags.
-   static void PrintExodusII(Mesh & mesh, const std::string &fpath,
+   static void PrintExodusII(Mesh & mesh, std::string fpath,
                             int flags = NC_CLOBBER);

 protected:
   /// @brief Closes any open file and creates a NetCDF file using selected flags.
-   void OpenExodusII(const std::string &fpath, int flags);
+   void OpenExodusII(std::string fpath, int flags);

   /// @brief Closes any open file.
   void CloseExodusII();
@@ -173,9 +167,9 @@ protected:
   std::unordered_set<int> GenerateUniqueNodeIDs();

   /// @brief Populates vectors with x, y, z coordinates from mesh.
-   void ExtractVertexCoordinates(std::vector<real_t> &coordx,
-                                 std::vector<real_t> &coordy,
-                                 std::vector<real_t> &coordz);
+   void ExtractVertexCoordinates(std::vector<double> & coordx,
+                                 std::vector<double> & coordy,
+                                 std::vector<double> & coordz);

   /// @brief Writes node connectivity for a particular block.
   /// @param block_id The block to write to the file.
@@ -193,7 +187,7 @@ protected:
   /// @brief Writes the number of elements in the mesh.
   void WriteNumOfElements();

-   /// @brief Writes the floating-point word size (sizeof(real_t)).
+   /// @brief Writes the floating-point word size (4 == float; 8 == double).
   void WriteFloatingPointWordSize();

   /// @brief Writes the API version.
@@ -297,7 +291,7 @@ private:
   std::map<int, std::vector<int>> exodusII_side_ids_for_boundary_id;
 };

-void Mesh::PrintExodusII(const std::string &fpath)
+void Mesh::PrintExodusII(const std::string fpath)
 {
   ExodusIIWriter::PrintExodusII(*this, fpath);
 }
@@ -368,7 +362,7 @@ void ExodusIIWriter::WriteExodusIIMeshInformation()
   WriteNodeSets();
 }

-void ExodusIIWriter::PrintExodusII(const std::string &fpath, int flags)
+void ExodusIIWriter::PrintExodusII(std::string fpath, int flags)
 {
   OpenExodusII(fpath, flags);

@@ -380,7 +374,7 @@ void ExodusIIWriter::PrintExodusII(const std::string &fpath, int flags)
   mfem::out << "Mesh successfully written to Exodus II file" << std::endl;
 }

-void ExodusIIWriter::PrintExodusII(Mesh &mesh, const std::string &fpath,
+void ExodusIIWriter::PrintExodusII(Mesh & mesh, std::string fpath,
                                   int flags)
 {
   ExodusIIWriter writer(mesh);
@@ -388,7 +382,7 @@ void ExodusIIWriter::PrintExodusII(Mesh &mesh, const std::string &fpath,
   writer.PrintExodusII(fpath, flags);
 }

-void ExodusIIWriter::OpenExodusII(const std::string &fpath, int flags)
+void ExodusIIWriter::OpenExodusII(std::string fpath, int flags)
 {
   CloseExodusII();  // Close any open files.

@@ -428,7 +422,7 @@ void ExodusIIWriter::WriteNumOfElements()

 void ExodusIIWriter::WriteFloatingPointWordSize()
 {
-   const int word_size = sizeof(real_t);
+   const int word_size = 8;
   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_FLOATING_POINT_WORD_SIZE_LABEL,
          NC_INT, 1,
          &word_size);
@@ -436,15 +430,13 @@ void ExodusIIWriter::WriteFloatingPointWordSize()

 void ExodusIIWriter::WriteAPIVersion()
 {
-   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_API_VERSION_LABEL, MFEM_NETCDF_REAL_T,
-          1,
+   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_API_VERSION_LABEL, NC_FLOAT, 1,
          &ExodusIILabels::EXODUS_API_VERSION);
 }

 void ExodusIIWriter::WriteDatabaseVersion()
 {
-   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_DATABASE_VERSION_LABEL,
-          MFEM_NETCDF_REAL_T, 1,
+   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_DATABASE_VERSION_LABEL, NC_FLOAT, 1,
          &ExodusIILabels::EXODUS_DATABASE_VERSION);
 }

@@ -615,25 +607,25 @@ void ExodusIIWriter::WriteNodalCoordinates()
   DefineDimension("num_nodes", num_nodes, &num_nodes_id);

   // 3. Extract the nodal coordinates.
-   // NB: writes in format real_t (double or float); ndims = 1 (vector).
+   // NB: assume doubles (could be floats!); ndims = 1 (vector).
   // https://docs.unidata.ucar.edu/netcdf-c/current/group__variables.html#gac7e8662c51f3bb07d1fc6d6c6d9052c8
-   std::vector<real_t> coordx(num_nodes);
-   std::vector<real_t> coordy(num_nodes);
-   std::vector<real_t> coordz(mesh.Dimension() == 3 ? num_nodes : 0);
+   std::vector<double> coordx(num_nodes);
+   std::vector<double> coordy(num_nodes);
+   std::vector<double> coordz(mesh.Dimension() == 3 ? num_nodes : 0);

   ExtractVertexCoordinates(coordx, coordy, coordz);

   // 4. Define and put the nodal coordinates.
-   DefineAndPutVar(ExodusIILabels::EXODUS_COORDX_LABEL, MFEM_NETCDF_REAL_T, 1,
+   DefineAndPutVar(ExodusIILabels::EXODUS_COORDX_LABEL, NC_DOUBLE, 1,
                   &num_nodes_id,
                   coordx.data());
-   DefineAndPutVar(ExodusIILabels::EXODUS_COORDY_LABEL, MFEM_NETCDF_REAL_T, 1,
+   DefineAndPutVar(ExodusIILabels::EXODUS_COORDY_LABEL, NC_DOUBLE, 1,
                   &num_nodes_id,
                   coordy.data());

   if (mesh.Dimension() == 3)
   {
-      DefineAndPutVar(ExodusIILabels::EXODUS_COORDZ_LABEL, MFEM_NETCDF_REAL_T, 1,
+      DefineAndPutVar(ExodusIILabels::EXODUS_COORDZ_LABEL, NC_DOUBLE, 1,
                      &num_nodes_id,
                      coordz.data());
   }
@@ -778,9 +770,9 @@ void ExodusIIWriter::WriteNodeConnectivityForBlock(const int block_id)
 }


-void ExodusIIWriter::ExtractVertexCoordinates(std::vector<real_t> & coordx,
-                                              std::vector<real_t> & coordy,
-                                              std::vector<real_t> & coordz)
+void ExodusIIWriter::ExtractVertexCoordinates(std::vector<double> & coordx,
+                                              std::vector<double> & coordy,
+                                              std::vector<double> & coordz)
 {
   if (mesh.GetNodes()) // Higher-order.
   {
@@ -790,7 +782,7 @@ void ExodusIIWriter::ExtractVertexCoordinates(std::vector<real_t> & coordx,
      sorted_node_ids.assign(unordered_node_ids.begin(), unordered_node_ids.end());
      std::sort(sorted_node_ids.begin(), sorted_node_ids.end());

-      real_t coordinates[3];
+      double coordinates[3];
      for (size_t i = 0; i < sorted_node_ids.size(); i++)
      {
         int node_id = sorted_node_ids[i];
@@ -810,7 +802,7 @@ void ExodusIIWriter::ExtractVertexCoordinates(std::vector<real_t> & coordx,
   {
      for (int ivertex = 0; ivertex < mesh.GetNV(); ivertex++)
      {
-         real_t *coordinates = mesh.GetVertex(ivertex);
+         double * coordinates = mesh.GetVertex(ivertex);

         coordx[ivertex] = coordinates[0];
         coordy[ivertex] = coordinates[1];
@@ -1088,4 +1080,4 @@ void ExodusIIWriter::CheckNodalFESpaceIsSecondOrderH1() const

 #endif

-}
+}
@@ -32,7 +32,6 @@
 #include <cstring>
 #include <ctime>
 #include <functional>
-#include <set>
 #include <numeric>
 #include <unordered_map>
 #include <unordered_set>
@@ -922,103 +921,6 @@ const FaceGeometricFactors* Mesh::GetFaceGeometricFactors(
   return gf;
 }

-const Array<int>& Mesh::GetBdrFaceAttributes() const
-{
-   if (bdr_face_attrs_cache.Size() == 0)
-   {
-      std::unordered_map<int, int> f_to_be;
-      for (int i = 0; i < GetNBE(); ++i)
-      {
-         const int f = GetBdrElementFaceIndex(i);
-         f_to_be[f] = i;
-      }
-      const int nf_bdr = GetNFbyType(FaceType::Boundary);
-      // MFEM_VERIFY(size_t(nf_bdr) == f_to_be.size(), "Incompatible sizes");
-      bdr_face_attrs_cache.SetSize(nf_bdr);
-      int f_ind = 0;
-      const int nf = GetNumFaces();
-      for (int f = 0; f < nf; ++f)
-      {
-         if (!GetFaceInformation(f).IsOfFaceType(FaceType::Boundary))
-         {
-            continue;
-         }
-         int attribute = -1; // default value
-         auto iter = f_to_be.find(f);
-         if (iter != f_to_be.end())
-         {
-            const int be = iter->second;
-            attribute = GetBdrAttribute(be);
-         }
-         else
-         {
-            // If a boundary face does not correspond to the a boundary element,
-            // we assign it the default attribute of -1.
-         }
-         bdr_face_attrs_cache[f_ind] = attribute;
-         ++f_ind;
-      }
-   }
-   return bdr_face_attrs_cache;
-}
-
-const Array<int>& Mesh::GetElementAttributes() const
-{
-   if (elem_attrs_cache.Size() == 0)
-   {
-      // re-compute cache
-      elem_attrs_cache.SetSize(GetNE());
-      elem_attrs_cache.HostWrite();
-      for (int i = 0; i < GetNE(); ++i)
-      {
-         elem_attrs_cache[i] = GetAttribute(i);
-         MFEM_ASSERT(elem_attrs_cache[i] > 0,
-                     "Negative attribute on element " << i);
-      }
-   }
-   return elem_attrs_cache;
-}
-
-void Mesh::ComputeFaceInfo(FaceType ftype) const
-{
-   auto &fidcs = face_indices[static_cast<int>(ftype)];
-   auto &ifidcs = inv_face_indices[static_cast<int>(ftype)];
-   fidcs.SetSize(GetNFbyType(ftype));
-   fidcs.HostWrite();
-   ifidcs.reserve(fidcs.Size());
-   int f_idx = 0;
-   for (int i = 0; i < GetNumFacesWithGhost(); ++i)
-   {
-      const FaceInformation face = GetFaceInformation(i);
-      if (face.IsNonconformingCoarse() || !face.IsOfFaceType(ftype))
-      {
-         continue;
-      }
-      fidcs[f_idx] = i;
-      ifidcs[i] = f_idx;
-      ++f_idx;
-   }
-}
-
-const Array<int> &Mesh::GetFaceIndices(FaceType ftype) const
-{
-   if (face_indices[static_cast<int>(ftype)].Size() == 0)
-   {
-      ComputeFaceInfo(ftype);
-   }
-   return face_indices[static_cast<int>(ftype)];
-}
-
-const std::unordered_map<int, int> &
-Mesh::GetInvFaceIndices(FaceType ftype) const
-{
-   if (inv_face_indices[static_cast<int>(ftype)].empty())
-   {
-      ComputeFaceInfo(ftype);
-   }
-   return inv_face_indices[static_cast<int>(ftype)];
-}
-
 void Mesh::DeleteGeometricFactors()
 {
   for (int i = 0; i < geom_factors.Size(); i++)
@@ -1902,17 +1804,8 @@ void Mesh::Destroy()
   TetMemory.Clear();
 #endif

-   elem_attrs_cache.DeleteAll();
-   bdr_face_attrs_cache.DeleteAll();
   attributes.DeleteAll();
   bdr_attributes.DeleteAll();
-
-   face_indices[0].DeleteAll();
-   face_indices[1].DeleteAll();
-   // force de-allocation so after this mesh has the smallest memory footprint
-   // possible
-   inv_face_indices[0] = std::unordered_map<int, int>();
-   inv_face_indices[1] = std::unordered_map<int, int>();
 }

 void Mesh::ResetLazyData()
@@ -1924,53 +1817,36 @@ void Mesh::ResetLazyData()
   DeleteGeometricFactors();
   nbInteriorFaces = -1;
   nbBoundaryFaces = -1;
-   // set size to 0 so re-computations can potentially avoid a new allocation
-   bdr_face_attrs_cache.SetSize(0);
-   elem_attrs_cache.SetSize(0);
-
-   face_indices[0].SetSize(0);
-   face_indices[1].SetSize(0);
-   inv_face_indices[0].clear();
-   inv_face_indices[1].clear();
 }

-void Mesh::SetAttributes(bool elem_attrs_changed, bool bdr_face_attrs_changed)
+void Mesh::SetAttributes()
 {
-   if (bdr_face_attrs_changed)
+   Array<int> attribs;
+
+   attribs.SetSize(GetNBE());
+   for (int i = 0; i < attribs.Size(); i++)
   {
-      bdr_face_attrs_cache.SetSize(0); // Invalidate the cache
-
-      // Get sorted list of unique boundary element attributes
-      std::set<int> attribs;
-      for (int i = 0; i < GetNBE(); i++)
-      {
-         attribs.emplace(GetBdrAttribute(i));
-      }
-
-      bdr_attributes.SetSize(attribs.size());
-      bdr_attributes.HostWrite();
-      std::copy(attribs.begin(), attribs.end(), bdr_attributes.begin());
-      if (bdr_attributes.Size() > 0 && bdr_attributes[0] <= 0)
-      {
-         MFEM_WARNING("Non-positive attributes on the boundary!");
-      }
+      attribs[i] = GetBdrAttribute(i);
+   }
+   attribs.Sort();
+   attribs.Unique();
+   attribs.Copy(bdr_attributes);
+   if (bdr_attributes.Size() > 0 && bdr_attributes[0] <= 0)
+   {
+      MFEM_WARNING("Non-positive attributes on the boundary!");
   }

-   if (elem_attrs_changed)
+   attribs.SetSize(GetNE());
+   for (int i = 0; i < attribs.Size(); i++)
   {
-      // Re-compute the attributes cache
-      elem_attrs_cache.SetSize(0);
-      GetElementAttributes();
-      // Get sorted list of unique element attributes
-      std::set<int> attribs(elem_attrs_cache.begin(), elem_attrs_cache.end());
-      attributes.SetSize(attribs.size());
-      attributes.HostWrite();
-      std::copy(attribs.begin(), attribs.end(), attributes.begin());
-
-      if (attributes.Size() > 0 && attributes[0] <= 0)
-      {
-         MFEM_WARNING("Non-positive attributes in the domain!");
-      }
+      attribs[i] = GetAttribute(i);
+   }
+   attribs.Sort();
+   attribs.Unique();
+   attribs.Copy(attributes);
+   if (attributes.Size() > 0 && attributes[0] <= 0)
+   {
+      MFEM_WARNING("Non-positive attributes in the domain!");
   }
 }

@@ -3118,7 +2994,6 @@ void Mesh::DoNodeReorder(DSTable *old_v_to_v, Table *old_elem_vert)
   const int num_edge_dofs = old_dofs.Size();

   // Save the original nodes
-   Nodes->HostReadWrite(); // for "(*Nodes)() = "
   const Vector onodes = *Nodes;

   // vertex dofs do not need to be moved
@@ -4578,10 +4453,6 @@ Mesh::Mesh(const Mesh &mesh, bool copy_nodes)
      Nodes = mesh.Nodes;
      own_nodes = 0;
   }
-
-   // copy attribute caches
-   elem_attrs_cache = mesh.elem_attrs_cache;
-   bdr_face_attrs_cache = mesh.bdr_face_attrs_cache;
 }

 Mesh::Mesh(Mesh &&mesh) : Mesh()
@@ -7849,12 +7720,6 @@ void Mesh::GetBdrElementAdjacentElement2(
 void Mesh::SetAttribute(int i, int attr)
 {
  elements[i]->SetAttribute(attr);
-  if (elem_attrs_cache.Size() == GetNE())
-  {
-     // update the existing cache instead of deleting it
-     elem_attrs_cache.HostReadWrite();
-     elem_attrs_cache[i] = attr;
-  }
  if (ncmesh) ncmesh->SetAttribute(i, attr);
 }

@@ -8189,12 +8054,6 @@ void Mesh::GenerateFaces()
      FreeElement(f);
   }

-   // delete caches
-   face_indices[0].SetSize(0);
-   face_indices[1].SetSize(0);
-   inv_face_indices[0].clear();
-   inv_face_indices[1].clear();
-
   // (re)generate the interior faces and the info for them
   faces.SetSize(nfaces);
   faces_info.SetSize(nfaces);
@@ -10990,15 +10849,6 @@ void Mesh::Swap(Mesh& other, bool non_geometry)
      mfem::Swap(nodes_sequence, other.nodes_sequence);
      mfem::Swap(last_operation, other.last_operation);
   }
-
-   // copy attribute caches
-   mfem::Swap(elem_attrs_cache, other.elem_attrs_cache);
-   mfem::Swap(bdr_face_attrs_cache, other.bdr_face_attrs_cache);
-
-   mfem::Swap(face_indices[0], other.face_indices[0]);
-   mfem::Swap(face_indices[1], other.face_indices[1]);
-   inv_face_indices[0].swap(other.inv_face_indices[0]);
-   inv_face_indices[1].swap(other.inv_face_indices[1]);
 }

 void Mesh::GetElementData(const Array<Element*> &elem_array, int geom,
@@ -13406,7 +13256,7 @@ void Mesh::ScaleElements(real_t sf)
   delete [] vn;
 }

-void Mesh::Transform(std::function<void(const Vector &, Vector&)> f)
+void Mesh::Transform(void (*f)(const Vector&, Vector&))
 {
   // TODO: support for different new spaceDim.
   if (Nodes == NULL)
@@ -13419,7 +13269,7 @@ void Mesh::Transform(std::function<void(const Vector &, Vector&)> f)
            vold(j) = vertices[i](j);
         }
         vnew.SetData(vertices[i]());
-         f(vold, vnew);
+         (*f)(vold, vnew);
      }
   }
   else
@@ -107,11 +107,6 @@ protected:
   Array<Element *> boundary;
   Array<Element *> faces;

-   /// internal cache for element attributes
-   mutable Array<int> elem_attrs_cache;
-   /// internal cache for boundary element attributes
-   mutable Array<int> bdr_face_attrs_cache;
-
   /** @brief This structure stores the low level information necessary to
       interpret the configuration of elements on a specific face. This
       information can be accessed using methods like GetFaceElements(),
@@ -278,13 +273,6 @@ protected:

   // used during NC mesh initialization only
   Array<Triple<int, int, int> > tmp_vertex_parents;
-   /// cache for FaceIndices(ftype)
-   mutable Array<int> face_indices[2];
-   /// cache for FaceIndices(ftype)
-   mutable std::unordered_map<int, int> inv_face_indices[2];
-
-   /// compute face_indices[ftype] and inv_face_indices[type]
-   void ComputeFaceInfo(FaceType ftype) const;

 public:
   typedef Geometry::Constants<Geometry::SEGMENT>     seg_t;
@@ -319,11 +307,6 @@ public:
   // (true) is set in mesh_readers.cpp.
   static bool remove_unused_vertices;

-   /// Map from boundary or interior face indices to mesh face indices.
-   const Array<int>& GetFaceIndices(FaceType ftype) const;
-   /// Inverse of the map FaceIndices(ftype)
-   const std::unordered_map<int, int>& GetInvFaceIndices(FaceType ftype) const;
-
 protected:
   Operation last_operation;

@@ -605,10 +588,9 @@ protected:
   void Loader(std::istream &input, int generate_edges = 0,
               std::string parse_tag = "");

-   /** @brief If NURBS mesh, write NURBS format. If NCMesh, write mfem v1.1
-       format. If section_delimiter is empty, write mfem v1.0 format. Otherwise,
-       write mfem v1.2 format with the given section_delimiter at the end.
-
+   /** If NURBS mesh, write NURBS format. If NCMesh, write mfem v1.1 format.
+       If section_delimiter is empty, write mfem v1.0 format. Otherwise, write
+       mfem v1.2 format with the given section_delimiter at the end.
       If @a comments is non-empty, it will be printed after the first line of
       the file, and each line should begin with '#'. */
   void Printer(std::ostream &os = mfem::out,
@@ -1139,14 +1121,13 @@ public:
       Mesh vertices or nodes are set. */
   virtual void Finalize(bool refine = false, bool fix_orientation = false);

-   /// @brief Determine the sets of unique attribute values in domain if @a
-   /// elem_attrs_changed and boundary elements if @a bdr_face_attrs_changed.
+   /// @brief Determine the sets of unique attribute values in domain and
+   /// boundary elements.
   ///
   /// Separately scan the domain and boundary elements to generate unique,
   /// sorted sets of the element attribute values present in the mesh and
   /// store these in the Mesh::attributes and Mesh::bdr_attributes arrays.
-   virtual void SetAttributes(bool elem_attrs_changed = true,
-                              bool bdr_face_attrs_changed = true);
+   virtual void SetAttributes();

   /// Check (and optionally attempt to fix) the orientation of the elements
   /** @param[in] fix_it  If `true`, attempt to fix the orientations of some
@@ -2272,7 +2253,7 @@ public:
   void ScaleSubdomains (real_t sf);
   void ScaleElements (real_t sf);

-   void Transform(std::function<void(const Vector &, Vector&)> f);
+   void Transform(void (*f)(const Vector&, Vector&));
   void Transform(VectorCoefficient &deformation);

   /** @brief This function should be called after the mesh node coordinates
@@ -2285,35 +2266,6 @@ public:
       method does not modify the nodes. */
   void NodesUpdated() { DeleteGeometricFactors(); }

-   /// @brief Returns the attributes for all elements in this mesh. The i'th
-   /// entry of the array is the attribute of the i'th element of the mesh.
-   ///
-   /// The returned array points to an internal object that may be invalidated
-   /// by mesh operations such as refinement or any element attributes are
-   /// modified. Since not all such modifications can be tracked by the Mesh
-   /// class (e.g. if a user calls GetElement() then changes the element
-   /// attribute directly), one needs to account for such changes by calling the
-   /// method SetAttributes().
-   const Array<int>& GetElementAttributes() const;
-
-   /// @brief Returns the attributes for all boundary elements in this mesh.
-   ///
-   /// The face restriction will give "face E-vectors" on the boundary that
-   /// are numbered in the order of the faces of mesh. This numbering will be
-   /// different than the numbering of the boundary elements. We compute
-   /// mappings so that the array `bdr_attributes[i]` gives the boundary
-   /// attribute of the `i`th boundary face in the mesh face order.
-   /// Attributes <= 0 indicate there is no boundary element and should be
-   /// skipped.
-   ///
-   /// The returned array points to an internal object that may be invalidated
-   /// by mesh operations such as refinement or any element attributes are
-   /// modified. Since not all such modifications can be tracked by the Mesh
-   /// class (e.g. if a user calls GetElement() then changes the element
-   /// attribute directly), one needs to account for such changes by calling the
-   /// method SetAttributes().
-   const Array<int>& GetBdrFaceAttributes() const;
-
   /// @}

   /// @anchor mfem_Mesh_gf_nodes
@@ -2530,12 +2482,10 @@ public:
   /// Print the mesh to the given stream using Netgen/Truegrid format.
   virtual void PrintXG(std::ostream &os = mfem::out) const;

-   /** @brief Print the mesh to the given stream using the default MFEM mesh
-       format.
-
-       \see mfem::ofgzstream() for on-the-fly compression of ascii outputs. If
-       @a comments is non-empty, it will be printed after the first line of the
-       file, and each line should begin with '#'. */
+   /// Print the mesh to the given stream using the default MFEM mesh format.
+   /// \see mfem::ofgzstream() for on-the-fly compression of ascii outputs. If
+   /// @a comments is non-empty, it will be printed after the first line of the
+   /// file, and each line should begin with '#'.
   virtual void Print(std::ostream &os = mfem::out,
                      const std::string &comments = "") const
   { Printer(os, "", comments); }
@@ -2587,7 +2537,7 @@ public:

 #ifdef MFEM_USE_NETCDF
   /// @brief Export a mesh to an Exodus II file.
-   void PrintExodusII(const std::string &fpath);
+   void PrintExodusII(const std::string fpath);
 #endif

   /** @brief Prints the mesh with boundary elements given by the boundary of
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
camierjs	f2a424720b	Merge branch 'main' into hpcftools/flowSolver-gpu	2025-06-30 11:17:56 -07:00
camierjs	e5fad8067c	Merge branch 'master' into hpcftools/flowSolver-gpu	2025-06-30 11:17:45 -07:00
camierjs	0b5bb52996	miniapps/navier/incompressible_navier_dfem	2025-06-30 11:17:11 -07:00
camierjs	563bfe9514	Merge branch 'master'	2025-06-30 08:43:14 -07:00
camierjs	9afef578d5	Merge branch 'master' into hpcftools/flowSolver-gpu	2025-05-02 09:59:44 -07:00
camierjs	9ecc414c62	dot reduced tests	2024-12-04 18:25:05 -08:00
camierjs	263b9d32c1	Merge branch 'hpcftools/flowSolver'	2024-12-04 11:59:05 -08:00
Mathias Rainer Schmidt	74e4ad3e2c	- updated flow solver - added comments to Blf and Lf contributions - split setup and step into vel, auxiliary and pressure part	2024-11-25 11:04:30 -08:00
camierjs	79039e0f6f	Switched to PA	2024-11-21 10:52:39 -08:00
Mathias Rainer Schmidt	cf9fcd8dde	Merge remote-tracking branch 'origin/master' into hpcftools/flowSolver	2024-11-14 13:52:27 -08:00
camierjs	c857fde13b	Merge branch 'hpcftools/flowSolver'	2024-11-01 15:59:50 -07:00
Mathias Rainer Schmidt	fa8617ada3	- added partial assembly option	2024-11-01 13:20:29 -07:00
camierjs	c06cbb69d5	Setup and cleanup	2024-10-30 11:37:00 -07:00
Mathias Rainer Schmidt	f7e5db2cea	- added ortho solver to phi field	2024-10-24 16:06:37 -07:00
Mathias Rainer Schmidt	60c11776b6	- added executable	2024-10-21 15:57:23 -07:00
Mathias Rainer Schmidt	6238f8ca76	- update solution	2024-10-16 16:00:08 -07:00
Mathias Rainer Schmidt	6a256db9aa	- added linear solvers to step	2024-10-16 15:56:46 -07:00
Mathias Rainer Schmidt	e0fb9658ca	- added linear form integrators	2024-10-15 17:30:09 -07:00
Mathias Rainer Schmidt	a1089efac3	- added BilinearForms	2024-10-15 13:46:36 -07:00
Mathias Rainer Schmidt	83f7f769dc	- inital flow solver commit	2024-10-15 12:44:25 -07:00