small updates prior to passing the code along for comparison

updating contact and parproblems files
cleanup on finite difference test code. Making it easier for others to use
2024-07-29 16:34:06 -07:00 · 2024-07-15 10:47:54 -07:00 · 2024-07-15 08:08:00 -07:00 · 2024-07-14 20:43:48 -07:00 · 2024-06-17 13:15:59 -07:00 · 2024-06-01 14:03:22 -07:00
190 changed files with 69201 additions and 1883 deletions
@@ -33,6 +33,7 @@ env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
  HYPRE_TOP_DIR: hypre-2.19.0
  METIS_ARCHIVE: metis-4.0.3.tar.gz
+  METIS_ARCHIVE_MAC: metis-4.0.3-mac.tgz
  METIS_TOP_DIR: metis-4.0.3
  MFEM_TOP_DIR: mfem

@@ -52,6 +53,7 @@ jobs:
        mpi: [seq, par]
        build-system: [make, cmake]
        hypre-target: [int32]
+        precision: [fp64]
        exclude:
          - os: ubuntu-latest
            build-system: cmake
@@ -75,6 +77,8 @@ jobs:
          - os: ubuntu-latest
            target: dbg
            config-opts: 'CPPFLAGS+=-Og'
+          - os: macos-latest
+            codecov: NO
          - os: windows-latest
            codecov: NO
          - os: windows-latest
@@ -87,6 +91,7 @@ jobs:
            mpi: par
            build-system: cmake
            hypre-target: int32
+            precision: fp64
            # This option can be set to pass additional configuration options to
            # the MFEM configuration command.
            # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
@@ -96,7 +101,15 @@ jobs:
            mpi: par
            build-system: make
            hypre-target: int64
-    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+            precision: fp64
+          - os: ubuntu-latest
+            target: opt
+            codecov: NO
+            mpi: par
+            build-system: make
+            hypre-target: int32
+            precision: fp32
+    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}

    runs-on: ${{ matrix.os }}

@@ -126,6 +139,17 @@ jobs:
          # Fetch the complete history for codecov to access commits ID
          fetch-depth: 0

+      - name: Xcode version setup (MacOS)
+        if: matrix.os == 'macos-latest'
+        run: |
+          XCODE_PATH="/Applications/Xcode_15.3.app"
+          echo "> sudo xcode-select -s ${XCODE_PATH}"
+          sudo xcode-select -s ${XCODE_PATH}
+          echo "> g++ -v"
+          g++ -v
+          echo "> clang++ -v"
+          clang++ -v
+
      # Only get MPI if defined for the job.
      # TODO: It would be nice to have only one step, e.g. with a dedicated
      # action, but I (@adrienbernede) don't see how at the moment.
@@ -169,25 +193,27 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5

      - name: get hypre
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
          target: ${{ matrix.hypre-target }}
          build-system: make
+          precision: ${{ matrix.precision }}

      - name: get hypre (Windows)
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
          target: ${{ matrix.hypre-target }}
          build-system: cmake
+          precision: ${{ matrix.precision }}

      # Get Metis through cache, or build it.
      # Install will only run on cache miss.
@@ -197,13 +223,13 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: install metis
        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
+        uses: mfem/github-actions/build-metis@v2.5
        with:
-          archive: ${{ env.METIS_ARCHIVE }}
+          archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
          dir: ${{ env.METIS_TOP_DIR }}

      - name: cache vcpkg (Windows)
@@ -228,7 +254,7 @@ jobs:

      # MFEM build and test
      - name: build
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        env:
          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
        with:
@@ -240,6 +266,7 @@ jobs:
          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
          metis-dir: ${{ env.METIS_TOP_DIR }}
          mfem-dir: ${{ env.MFEM_TOP_DIR }}
+          precision: ${{ matrix.precision }}
          config-options: ${{ matrix.config-opts }}
          library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}

@@ -282,7 +309,7 @@ jobs:
      # Code coverage (process and upload reports)
      - name: codecov
        if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.4
+        uses: mfem/github-actions/upload-coverage@v2.5
        with:
          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
          project_dir: ${{ env.MFEM_TOP_DIR }}
@@ -53,11 +53,11 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.5

      - name: Get Hypre
        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
@@ -68,18 +68,18 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: Install Metis
        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
+        uses: mfem/github-actions/build-metis@v2.5
        with:
          archive: ${{ env.METIS_ARCHIVE }}
          dir: ${{ env.METIS_TOP_DIR }}

      # MFEM build and test
      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        with:
          os: ${{ runner.os }}
          target: opt
@@ -44,7 +44,7 @@ jobs:
          path: mfem

      - name: MFEM Build
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        with:
          os: ${{ runner.os }}
          target: opt
@@ -57,6 +57,8 @@ examples/ex2[0-9]
 examples/ex2[0-9]p
 examples/ex3[0-9]
 examples/ex3[0-9]p
+examples/ex4[0-9]
+examples/ex4[0-9]p

 examples/refined.mesh
 examples/displaced.mesh
@@ -232,7 +234,7 @@ miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
 miniapps/meshing/twist-*.mesh
-miniapps/meshing/mesh-explorer.mesh
+miniapps/meshing/mesh-explorer.mesh*
 miniapps/meshing/partitioning.txt
 miniapps/meshing/mesh-explorer-visit*
 miniapps/meshing/mesh-explorer-paraview/
@@ -369,7 +371,7 @@ miniapps/dpg/ParaView
 miniapps/spde/generate_random_field
 miniapps/spde/ParaView

-miniapps/tribol/ContactPatchTest
+miniapps/tribol/contact-patch-test

 # Unit test binary and outputs
 tests/unit/output_meshes
@@ -13,6 +13,9 @@
 # at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
 # LLNL-specific!

+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'

 # The pipeline is divided into stages. Usually, jobs in a given stage wait for
 # the preceding stages to complete before to start. However, we sometimes use
@@ -9,6 +9,10 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
 # We define the following GitLab pipeline variables:
 variables:

@@ -35,9 +35,8 @@ variables:
    - when: on_success

 # Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen
-# to speed-up the allocation.  However this would not be scalable to
-# multiple builds.
+# pre-allocation the same way slurm does.  We use the pci queue on lassen
+# to speed-up the allocation.
 .build_and_test_on_lassen:
  extends: [.on_lassen]
  stage: build_and_test
@@ -45,5 +44,5 @@ variables:
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 45 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
  needs: [setup]
@@ -52,4 +52,4 @@ variables:
    - echo ${JOBID}
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -14,14 +14,14 @@ stages:
  - build_and_test
  - report

-opt_mpi_cuda_xl_16_1_1_12:
+opt_mpi_cuda_gcc:
  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70"
  extends: .build_and_test_on_lassen

-opt_mpi_cuda_hypre_cuda_xl:
+opt_mpi_cuda_hypre_cuda_gcc:
  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
  extends: .build_and_test_on_lassen

 # Jobs report
@@ -32,11 +32,11 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}

 # run
 if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  lalloc 1 -q pci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
  exit 1
@@ -8,77 +8,110 @@
                               https://mfem.org


-Version 4.6.1 (development)
+Version 4.7.1 (development)
 ===========================

+- Added an MFEM example for the eikonal equation. This new solver is based on
+  the proximal Galerkin method introduced by Keith and Surowiec.
+
+
+Version 4.7, released on May 7, 2024
+====================================
+
+- Added support for single precision (with corresponding hypre build). The MFEM
+  floating point type was generalized from `double` to `real_t`. For details see
+  https://github.com/orgs/mfem/discussions/4207.
+
 Meshing improvements
 --------------------
+- Added the capability to partition (big) serial meshes in serial code, see the
+  new classes MeshPartitioner and MeshPart. This capability is also exposed as a
+  menu option in the mesh-explorer miniapp in miniapps/meshing.
+
+- Added named attribute sets and basic supporting methods to the Mesh class as a
+  convenient means of referring to sets of domain or boundary attribute numbers.
+  See the new Example 39/39p and data/compass.mesh.
+
 - Introduced formulas for refinement of patches in NURBS meshes. Refinement by
  arbitrary integer factors is also enabled, e.g. in the mesh-explorer miniapp.
  NURBS coarsening and knot removal are also introduced.

-Discretization improvements
---------------------------
- Introduced support for higher order non conformal Nedelec elements on
-  simplices in ParMesh.
- Introduced support for internal boundary elements in nonconformal adapted
-  meshes.
-
- Added functionality for construction of cut-surface and cut-volume
-  IntegrationRules through a moment-fitting approach. The cut is specified by
-  the zero level set of a Coefficient. See fem/intrules_cut.hpp and Example 38.
-
- Added a new nonlinear integrator, `HyperbolicFormIntegrator`. This implements 
-  both element-wise weak divergence and face-wise numerical flux for a general
-  system of hyperbolic conservation laws. To use this integrator for a specific
-  flux function, users can define a derived class of `FluxFunction`. Currently,
-  advection, Burgers', shallow-water, Euler equations (see, Example 18) are
-  available.
-
-GPU support
----------------------------
- Added support for full assembly on simplices.
- Added functionality for BilinearFormIntegrators to use kernels that work for both
-  tensor and unstructured elements.
- Added partial assembly for linear elasticity. Does not use sum factorization for now.
-
-New and updated examples and miniapps
-------------------------------------
- Added a new block solver in miniapp/solvers for the Darcy problem.
-  The new solver is based on a Bramble-Pasciak preconditioning. User can
-  use and implement their own preconditioner for the mass matrix.
-
- Added miniapp to demonstrate new elasticity integrator and unstructured element GPU support,
-  and a block diagonal preconditioner using low order refinement. Allows comparison with
-  currently existing legacy mode integrator. See miniapps/solvers/lor_elast.
-
- Added a new mortar contact patch test miniapp using the Tribol interface
-  physics library (see https://github.com/LLNL/Tribol).  See miniapps/tribol.
-
-Miscellaneous
-------------
- Added support for single and double precision, with corresponding hypre build.
-  Generalized the floating point type from `double` to `real_t`. For more
-  details see https://github.com/orgs/mfem/discussions/4207.
+- Added support for internal boundary elements in nonconforming meshes.

 - The ReadCubit Genesis mesh importer has been rewritten to improve readability.

+Discretization improvements
+---------------------------
+- Added a new nonlinear integrator, `HyperbolicFormIntegrator` that implements
+  both element-wise weak divergence and face-wise numerical flux for a general
+  system of hyperbolic conservation laws. To use the integrator for a specific
+  flux function, users can define a derived class of `FluxFunction`. Currently,
+  advection, Burgers, shallow-water and Euler equations (see Example 18/18p) are
+  available.
+
+- Added a capability to construct cut-surface and cut-volume IntegrationRules
+  through a moment-fitting approach. The cut is specified by the zero level set
+  of a Coefficient. See fem/intrules_cut.hpp and the new Example 38.
+
+- Introduced support for high-order nonconforming Nedelec elements on simplices.
+
+GPU computing
+-------------
+- Added partial assembly and GPU support for the DG diffusion integrator.
+
+- Efficient GPU-accelerated LOR assembly is now supported on surface meshes.
+
+- Added functionality to automatically configure hypre's compute policy to match
+  MFEM's compute policy when hypre is built with GPU support. Requires version
+  hypre-2.31.0 or later.
+
+- Added support for full assembly on simplices.
+
+- Added partial assembly for linear elasticity (no sum factorization for now).
+
+- Added functionality for BilinearFormIntegrators to use kernels that work for
+  both tensor and unstructured elements.
+
+- The RAJA backend will use `seq_exec` for serial loop execution when RAJA
+  v2023.06.00 and beyond is detected as `loop_exec` is deprecated.
+
+- API change: The macro MFEM_HYPRE_FORALL (from hypre.hpp) which was intended
+  for internal use, has been removed and replaced by the function template
+  mfem::hypre_forall in general/forall.hpp.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new miniapp illustrating elastic contact based on the Tribol library,
+  (https://github.com/LLNL/Tribol). See miniapps/tribol.
+
+- Added a miniapp to demonstrate low order refined (LOR) block preconditioning
+  for linear elasticity on GPUs. See miniapps/solvers/lor_elast.
+
+- Added a new block solver in miniapp/solvers for the Darcy problem. The new
+  solver is based on a Bramble-Pasciak preconditioning. User can use and
+  implement their own preconditioner for the mass matrix.
+
+- Added a small miniapp for printing the shape functions of a KnotVector. See
+  miniapps/nurbs/nurbs_printfunc.cpp.
+
+- Added two new example codes: 38 and 39/39p described above. Substantially
+  updated Example 18/18p.
+
+Miscellaneous
+-------------
 - Updated the Doxygen documentation style, which now requires Doxygen version
  1.9.8 or later. See the doc/ directory.

- Improved thread safety for global variables in the library, for example
-  IntegrationRules IntRules, RefinedIntRules, GeometryRefiner
-  GlobGeometryRefiner, and FiniteElement::dof2quad_array.
+- Improved thread safety for global variables in the library, e.g. for IntRules,
+  RefinedIntRules, GlobGeometryRefiner, and FiniteElement::dof2quad_array.

 - PETSc integration now generally requires PETSc version 3.21 or later, though
  depending on the functionality older versions may still work.

- RAJA backend will use seq_exec for serial loop execution when RAJA
-  v2023.06.00 and beyond is detected as loop_exec is deprecated.
+- Various other simplifications, extensions, and bugfixes in the code.
+
+- Added GSLIB-based gather-scatter operator.

- Adding named attribute sets and basic supporting methods to the Mesh class as
-  a convenient means of referring to sets of domain or boundary attribute
-  numbers. Also adding related serial and parallel examples which illustrate.

 Version 4.6, released on September 27, 2023
 ===========================================
@@ -99,7 +132,6 @@ Meshing improvements
  * The edge to knot map for NURBS meshes can be determined automatically. It is
    no longer needed to specify this in the NURBS mesh.
  * Added curve interpolation method for NURBS.
-  * Added new small miniapp for printing of shape functions of a KnotVector
  * See miniapps/nurbs for example meshes and miniapps.

 Discretization improvements
@@ -146,8 +178,6 @@ Linear and nonlinear solvers

 - Added HIP support to the PETSc and SUNDIALS interfaces.

- Efficient GPU-accelerated LOR assembly now supports surface meshes.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new H(div) solver miniapp demonstrating the use of a matrix-free
@@ -58,7 +58,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.6.1)
+set(${PROJECT_NAME}_VERSION 4.7.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -75,6 +75,8 @@ and miniapps. See https://glvis.org and https://mfem.org/building.

 Quick start with GNU make
 =========================
+See also: https://mfem.org/building
+
 Serial build:
   make serial -j 4

@@ -83,6 +85,7 @@ Parallel build:
   (build METIS 4 in ../metis-4.0 relative to mfem/)
   (build hypre in ../hypre relative to mfem/)
   make parallel -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   make cuda -j 4
@@ -116,6 +119,7 @@ Parallel build:
   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
   make -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   (this build requires CMake 3.8 or newer)
@@ -612,9 +616,13 @@ The specific libraries and their options are:
            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
            HYPRE >= 2.22.1   (HYPRE built with CUDA)
            HYPRE >= 2.23.0   (HYPRE built with HIP)
+            HYPRE >= 2.31.0   (runtime selectable HYPRE execution on CPU/GPU)

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
-  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
+  MFEM_USE_METIS_5 = YES (default is to use METIS 4). For building instructions,
+  see the following:
+    - METIS 4.0.3: https://mfem.org/building/#parallel-mpi-version-of-mfem
+    - METIS 5.1.0: https://mfem.org/building/#parallel-build-using-metis-5
  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
  Options: METIS_OPT, METIS_LIB.
  Versions: METIS 4.0.3 or 5.1.0.
@@ -16,12 +16,22 @@
 #   - MUMPS_VERSION

 include(MfemCmakeUtilities)
+
+# Toggle which precision of MUMPS to use depending on the precision of MFEM.
+if (MFEM_USE_DOUBLE)
+  set(_mumps_header dmumps_c.h)
+  set(_mumps_lib dmumps)
+elseif(MFEM_USE_SINGLE)
+  set(_mumps_header smumps_c.h)
+  set(_mumps_lib smumps)
+endif()
+
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
-  "include" dmumps_c.h "lib" dmumps
+  "include" ${_mumps_header} "lib" ${_mumps_lib}
  "Paths to headers required by MUMPS."
  "Libraries required by MUMPS."
-  ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
-  ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)
+  ADD_COMPONENT mumps_common "include" ${_mumps_header} "lib" mumps_common
+  ADD_COMPONENT pord "include" ${_mumps_header} "lib" pord)

 if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
  try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
@@ -120,6 +120,15 @@ constexpr real_t operator""_r(unsigned long long v)

 // Check dependencies:

+// Define MFEM_MPI_REAL_T to be the appropriate MPI real type
+#ifdef MFEM_USE_MPI
+#ifdef MFEM_USE_SINGLE
+#define MFEM_MPI_REAL_T MPI_FLOAT
+#elif defined MFEM_USE_DOUBLE
+#define MFEM_MPI_REAL_T MPI_DOUBLE
+#endif
+#endif
+
 // Options that require MPI
 #ifndef MFEM_USE_MPI
 #ifdef MFEM_USE_SUPERLU
@@ -170,6 +170,18 @@ MFEM_USE_PARELAG       = NO
 MFEM_USE_TRIBOL        = NO
 MFEM_USE_ENZYME        = NO

+# Process MFEM_PRECISION -> MFEM_USE_SINGLE, MFEM_USE_DOUBLE
+ifneq ($(filter double Double DOUBLE,$(MFEM_PRECISION)),)
+   MFEM_USE_DOUBLE = YES
+   MFEM_USE_SINGLE = NO
+else ifneq ($(filter single Single SINGLE,$(MFEM_PRECISION)),)
+   MFEM_USE_DOUBLE = NO
+   MFEM_USE_SINGLE = YES
+else ifeq ($(MAKECMDGOALS),config)
+   $(error Invalid floating-point precision: \
+     MFEM_PRECISION = $(MFEM_PRECISION))
+endif
+
 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
 ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
@@ -319,13 +331,13 @@ MPI_FORTRAN_LIB = -lmpifort
 # MUMPS library configuration
 MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib \
- -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
+MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib
 ifeq ($(MFEM_USE_SINGLE),YES)
   MUMPS_LIB += -lsmumps
 else
   MUMPS_LIB += -ldmumps
 endif
+MUMPS_LIB += -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)

 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
@@ -376,7 +388,7 @@ GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LINK_LIB_DIR) -L$(GINKGO_LINK_LIB_DIR)\
 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
 AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx
+AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt

 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -110,4 +110,4 @@ config-mk:

 clean:
 	rm -f $(CONFIG_HPP) $(CONFIG_MK) sample-runs-build.log
-	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out
+	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out *.dSYM
@@ -92,4 +92,5 @@ vertices
 -0.70710678 -0.70710678
 0 -1
 0.70710678 -0.70710678
+
 mfem_mesh_end
@@ -48,7 +48,7 @@ PROJECT_NAME           = MFEM
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.6.1
+PROJECT_NUMBER         = v4.7.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -110,9 +110,13 @@ namespace mfem {
 * - <a class="el" href="ex35p_8cpp_source.html">Example 35p</a>: parallel multi-domain damped harmonic oscillators
 * - <a class="el" href="ex36_8cpp_source.html">Example 36</a>: Proximal Galerkin FEM for the obstacle problem
 * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
- * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: Topology optimization
+ * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: topology optimization
 * - <a class="el" href="ex37p_8cpp_source.html">Example 37p</a>: parallel topology optimization
 * - <a class="el" href="ex38_8cpp_source.html">Example 38</a>: cut-surface and cut-volume integration
+ * - <a class="el" href="ex39_8cpp_source.html">Example 39</a>: named mesh attributes
+ * - <a class="el" href="ex39p_8cpp_source.html">Example 39p</a>: parallel named mesh attributes
+ * - <a class="el" href="ex40_8cpp_source.html">Example 40</a>: eikonal equation
+ * - <a class="el" href="ex40p_8cpp_source.html">Example 40p</a>: parallel eikonal equation
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -214,6 +218,8 @@ namespace mfem {
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
+ * - <a class="el" href="contact-patch-test_8cpp_source.html">Contact</a>: mortar contact patch test for elasticity
+ * - <a class="el" href="multidomain_8cpp_source.html">Multidomain miniapp</a>: Multidomain and Submesh demonstration miniapp
 * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
 * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
 * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
@@ -46,7 +46,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
            DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
        })
        // Update the color scheme when the tab is made visible again.
-        // It is possible that the appearance was changed in another tab 
+        // It is possible that the appearance was changed in another tab
        // while this tab was in the background.
        document.addEventListener("visibilitychange", visibilityState => {
            if (document.visibilityState === 'visible') {
@@ -97,7 +97,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
     * @returns `true` for dark-mode, `false` for light-mode user preference
     */
    static get userPreference() {
-        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) || 
+        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) ||
        (DoxygenAwesomeDarkModeToggle.systemPreference && !localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey))
    }

@@ -45,6 +45,7 @@ list(APPEND ALL_EXE_SRCS
  ex37.cpp
  ex38.cpp
  ex39.cpp
+  ex40.cpp
  )

 if (MFEM_USE_MPI)
@@ -87,6 +88,7 @@ if (MFEM_USE_MPI)
    ex36p.cpp
    ex37p.cpp
    ex39p.cpp
+    ex40p.cpp
  )
 endif()

@@ -146,10 +148,10 @@ if (MFEM_ENABLE_TESTING)
  # Add CUDA/HIP tests.
  set(DEVICE_EXAMPLES
      # serial examples with device support:
-      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
+      ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
      # parallel examples with device support:
-      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p
-      ex34p ex35p)
+      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p ex22p ex24p ex25p
+      ex26p ex34p ex35p)
  set(MFEM_TEST_DEVICE)
  if (MFEM_USE_CUDA)
    set(MFEM_TEST_DEVICE "cuda")
@@ -159,6 +161,11 @@ if (MFEM_ENABLE_TESTING)
  if (MFEM_TEST_DEVICE)
    foreach(TEST_NAME ${DEVICE_EXAMPLES})
      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+      if (${TEST_NAME} MATCHES "ex14p")
+        list(APPEND THIS_TEST_OPTIONS "-rs" "2" "-rp" "0" "-pa")
+      elseif (${TEST_NAME} MATCHES "ex14")
+        list(APPEND THIS_TEST_OPTIONS "-r" "2" "-pa")
+      endif()
      if (NOT (${TEST_NAME} MATCHES ".*p$"))
        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
          COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
@@ -646,10 +646,7 @@ real_t HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 real_t HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   real_t loc_energy = 0.5*M.InnerProduct(v, v);
-   real_t energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPITypeMap<real_t>::mpi_type,
-                 MPI_SUM, fespace.GetComm());
+   real_t energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -18,6 +18,12 @@
 //               ex14 -m ../data/amr-quad.mesh -r 3
 //               ex14 -m ../data/amr-hex.mesh
 //               ex14 -m ../data/fichera-amr.mesh
+//               ex14 -pa -r 1 -o 3
+//               ex14 -pa -r 1 -o 3 -m ../data/fichera.mesh
+//
+// Device sample runs:
+//               ex14 -pa -r 2 -d cuda -o 3
+//               ex14 -pa -r 2 -d cuda -o 3 -m ../data/fichera.mesh
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -46,7 +52,9 @@ int main(int argc, char *argv[])
   real_t sigma = -1.0;
   real_t kappa = -1.0;
   real_t eta = 0.0;
+   bool pa = false;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -62,9 +70,13 @@ int main(int argc, char *argv[])
                  "One of the three DG penalty parameters, should be positive."
                  " Negative values are replaced with (order+1)^2.");
   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -77,117 +89,129 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
   //    NURBS meshes are projected to second order meshes.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   Mesh mesh(mesh_file);
+   const int dim = mesh.Dimension();

-   // 3. Refine the mesh to increase the resolution. In this example we do
+   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. By default, or if ref_levels < 0,
   //    we choose it to be the largest number that gives a final mesh with no
   //    more than 50,000 elements.
   {
      if (ref_levels < 0)
      {
-         ref_levels = (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+         ref_levels = (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
      }
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh->UniformRefinement();
+         mesh.UniformRefinement();
      }
   }
-   if (mesh->NURBSext)
+   if (mesh.NURBSext)
   {
-      mesh->SetCurvature(max(order, 1));
+      mesh.SetCurvature(max(order, 1));
   }

-   // 4. Define a finite element space on the mesh. Here we use discontinuous
+   // 5. Define a finite element space on the mesh. Here we use discontinuous
   //    finite elements of the specified order >= 0.
-   FiniteElementCollection *fec = new DG_FECollection(order, dim);
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of unknowns: " << fespace->GetVSize() << endl;
+   const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
+   DG_FECollection fec(order, dim, bt);
+   FiniteElementSpace fespace(&mesh, &fec);
+   cout << "Number of unknowns: " << fespace.GetVSize() << endl;

-   // 5. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system.
-   LinearForm *b = new LinearForm(fespace);
+   LinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   ConstantCoefficient zero(0.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->AddBdrFaceIntegrator(
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.AddBdrFaceIntegrator(
      new DGDirichletLFIntegrator(zero, one, sigma, kappa));
-   b->Assemble();
+   b.Assemble();

-   // 6. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
-   GridFunction x(fespace);
+   GridFunction x(&fespace);
   x = 0.0;

-   // 7. Set up the bilinear form a(.,.) on the finite element space
+   // 8. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    domain integrator and the interior and boundary DG face integrators.
   //    Note that boundary conditions are imposed weakly in the form, so there
   //    is no need for dof elimination. After assembly and finalizing we
   //    extract the corresponding sparse matrix A.
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   BilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      MFEM_VERIFY(!pa, "BR2 not yet compatible with partial assembly.");
+      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
-   a->Assemble();
-   a->Finalize();
-   const SparseMatrix &A = a->SpMat();
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.Assemble();
+   a.Finalize();

-#ifndef MFEM_USE_SUITESPARSE
-   // 8. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   // 9. Define a simple symmetric Gauss-Seidel preconditioner and use it to
   //    solve the system Ax=b with PCG in the symmetric case, and GMRES in the
-   //    non-symmetric one.
-   GSSmoother M(A);
-   if (sigma == -1.0)
+   //    non-symmetric one. (Note that tolerances are squared: 1e-12 corresponds
+   //    to a relative tolerance of 1e-6).
+   //
+   //    If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   if (pa)
   {
-      PCG(A, M, *b, x, 1, 500, 1e-12, 0.0);
+      MFEM_VERIFY(sigma == -1.0,
+                  "The case of PA with sigma != -1 is not yet supported.");
+      CG(a, b, x, 1, 500, 1e-12, 0.0);
   }
   else
   {
-      GMRES(A, M, *b, x, 1, 500, 10, 1e-12, 0.0);
-   }
+      const SparseMatrix &A = a.SpMat();
+#ifndef MFEM_USE_SUITESPARSE
+      GSSmoother M(A);
+      if (sigma == -1.0)
+      {
+         PCG(A, M, b, x, 1, 500, 1e-12, 0.0);
+      }
+      else
+      {
+         GMRES(A, M, b, x, 1, 500, 10, 1e-12, 0.0);
+      }
 #else
-   // 8. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(A);
-   umf_solver.Mult(*b, x);
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(A);
+      umf_solver.Mult(b, x);
 #endif
+   }

-   // 9. Save the refined mesh and the solution. This output can be viewed later
-   //    using GLVis: "glvis -m refined.mesh -g sol.gf".
+   // 10. Save the refined mesh and the solution. This output can be viewed
+   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
   ofstream mesh_ofs("refined.mesh");
   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
+   mesh.Print(mesh_ofs);
   ofstream sol_ofs("sol.gf");
   sol_ofs.precision(8);
   x.Save(sol_ofs);

-   // 10. Send the solution by socket to a GLVis server.
+   // 11. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
+      sol_sock << "solution\n" << mesh << x << flush;
   }

-   // 11. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete mesh;
-
   return 0;
 }
@@ -17,6 +17,12 @@
 //               mpirun -np 4 ex14p -m ../data/inline-segment.mesh -rs 5
 //               mpirun -np 4 ex14p -m ../data/amr-quad.mesh -rs 3
 //               mpirun -np 4 ex14p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -o 3
+//               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -m ../data/fichera.mesh -o 3
+//
+// Device sample runs:
+//               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -o 3
+//               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -m ../data/fichera.mesh -o 3
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -38,11 +44,14 @@ using namespace mfem;

 class CustomSolverMonitor : public IterativeSolverMonitor
 {
+private:
+   const ParMesh &pmesh;
+   ParGridFunction &pgf;
 public:
-   CustomSolverMonitor(const ParMesh *m,
-                       ParGridFunction *f) :
-      pmesh(m),
-      pgf(f) {}
+   CustomSolverMonitor(const ParMesh &pmesh_,
+                       ParGridFunction &pgf_) :
+      pmesh(pmesh_),
+      pgf(pgf_) {}

   void MonitorSolution(int i, real_t norm, const Vector &x, bool final)
   {
@@ -50,30 +59,24 @@ public:
      int  visport   = 19916;
      int  num_procs, myid;

-      MPI_Comm_size(pmesh->GetComm(),&num_procs);
-      MPI_Comm_rank(pmesh->GetComm(),&myid);
+      MPI_Comm_size(pmesh.GetComm(), &num_procs);
+      MPI_Comm_rank(pmesh.GetComm(), &myid);

-      pgf->SetFromTrueDofs(x);
+      pgf.SetFromTrueDofs(x);

      socketstream sol_sock(vishost, visport);
      sol_sock << "parallel " << num_procs << " " << myid << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << *pgf
+      sol_sock << "solution\n" << pmesh << pgf
               << "window_title 'Iteration no " << i << "'"
               << "keys rRjlc\n" << flush;
   }
-
-private:
-   const ParMesh *pmesh;
-   ParGridFunction *pgf;
 };

 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI and HYPRE.
   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
   Hypre::Init();

   // 2. Parse command-line options.
@@ -84,7 +87,9 @@ int main(int argc, char *argv[])
   real_t sigma = -1.0;
   real_t kappa = -1.0;
   real_t eta = 0.0;
+   bool pa = false;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -103,13 +108,17 @@ int main(int argc, char *argv[])
                  "One of the three DG penalty parameters, should be positive."
                  " Negative values are replaced with (order+1)^2.");
   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
-      if (myid == 0)
+      if (Mpi::Root())
      {
         args.PrintUsage(cout);
      }
@@ -119,16 +128,19 @@ int main(int argc, char *argv[])
   {
      kappa = (order+1)*(order+1);
   }
-   if (myid == 0)
+   if (Mpi::Root())
   {
      args.PrintOptions(cout);
   }

+   Device device(device_config);
+   if (Mpi::Root()) { device.Print(); }
+
   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral and hexahedral meshes
   //    with the same code. NURBS meshes are projected to second order meshes.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   Mesh mesh(mesh_file);
+   int dim = mesh.Dimension();

   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ser_ref_levels' of uniform refinement. By default,
@@ -137,53 +149,54 @@ int main(int argc, char *argv[])
   {
      if (ser_ref_levels < 0)
      {
-         ser_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+         ser_ref_levels = (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
      }
      for (int l = 0; l < ser_ref_levels; l++)
      {
-         mesh->UniformRefinement();
+         mesh.UniformRefinement();
      }
   }
-   if (mesh->NURBSext)
+   if (mesh.NURBSext)
   {
-      mesh->SetCurvature(max(order, 1));
+      mesh.SetCurvature(max(order, 1));
   }

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
   {
      for (int l = 0; l < par_ref_levels; l++)
      {
-         pmesh->UniformRefinement();
+         pmesh.UniformRefinement();
      }
   }

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use discontinuous finite elements of the specified order >= 0.
-   FiniteElementCollection *fec = new DG_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
-   if (myid == 0)
+   const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
+   DG_FECollection fec(order, dim, bt);
+   ParFiniteElementSpace fespace(&pmesh, &fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   if (Mpi::Root())
   {
      cout << "Number of unknowns: " << size << endl;
   }

   // 7. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system.
-   ParLinearForm *b = new ParLinearForm(fespace);
+   ParLinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   ConstantCoefficient zero(0.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->AddBdrFaceIntegrator(
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.AddBdrFaceIntegrator(
      new DGDirichletLFIntegrator(zero, one, sigma, kappa));
-   b->Assemble();
+   b.Assemble();

   // 8. Define the solution vector x as a parallel finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
-   ParGridFunction x(fespace);
+   ParGridFunction x(&fespace);
   x = 0.0;

   // 9. Set up the bilinear form a(.,.) on the finite element space
@@ -192,42 +205,51 @@ int main(int argc, char *argv[])
   //    Note that boundary conditions are imposed weakly in the form, so there
   //    is no need for dof elimination. After serial and parallel assembly we
   //    extract the corresponding parallel matrix A.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      MFEM_VERIFY(!pa, "BR2 not yet compatible with partial assembly.");
+      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
-   a->Assemble();
-   a->Finalize();
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.Assemble();
+   a.Finalize();

   // 10. Define the parallel (hypre) matrix and vectors representing a(.,.),
   //     b(.) and the finite element approximation.
-   HypreParMatrix *A = a->ParallelAssemble();
-   HypreParVector *B = b->ParallelAssemble();
-   HypreParVector *X = x.ParallelProject();
+   OperatorHandle A;

-   delete a;
-   delete b;
-
-   // 11. Depending on the symmetry of A, define and apply a parallel PCG or
-   //     GMRES solver for AX=B using the BoomerAMG preconditioner from hypre.
-   HypreSolver *amg = new HypreBoomerAMG(*A);
-   if (sigma == -1.0)
+   std::unique_ptr<HypreBoomerAMG> amg;
+   if (pa)
   {
-      HyprePCG pcg(*A);
-      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(500);
-      pcg.SetPrintLevel(2);
-      pcg.SetPreconditioner(*amg);
-      pcg.Mult(*B, *X);
+      A.Reset(&a, false);
   }
   else
   {
-      CustomSolverMonitor monitor(pmesh, &x);
+      A.SetType(Operator::Hypre_ParCSR);
+      a.ParallelAssemble(A);
+      amg.reset(new HypreBoomerAMG(*A.As<HypreParMatrix>()));
+   }
+
+   // 11. Depending on the symmetry of A, define and apply a parallel PCG or
+   //     GMRES solver for AX=B using the BoomerAMG preconditioner from hypre.
+   if (sigma == -1.0)
+   {
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(500);
+      cg.SetPrintLevel(1);
+      cg.SetOperator(*A);
+      if (amg) { cg.SetPreconditioner(*amg); }
+      cg.Mult(b, x);
+   }
+   else
+   {
+      CustomSolverMonitor monitor(pmesh, x);
      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetAbsTol(0.0);
      gmres.SetRelTol(1e-12);
@@ -235,51 +257,37 @@ int main(int argc, char *argv[])
      gmres.SetKDim(10);
      gmres.SetPrintLevel(1);
      gmres.SetOperator(*A);
-      gmres.SetPreconditioner(*amg);
+      if (amg) { gmres.SetPreconditioner(*amg); }
      gmres.SetMonitor(monitor);
-      gmres.Mult(*B, *X);
+      gmres.Mult(b, x);
   }
-   delete amg;

-   // 12. Extract the parallel grid function corresponding to the finite element
-   //     approximation X. This is the local solution on each processor.
-   x = *X;
-
-   // 13. Save the refined mesh and the solution in parallel. This output can
+   // 12. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      mesh_name << "mesh." << setfill('0') << setw(6) << Mpi::WorldRank();
+      sol_name << "sol." << setfill('0') << setw(6) << Mpi::WorldRank();

      ofstream mesh_ofs(mesh_name.str().c_str());
      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
+      pmesh.Print(mesh_ofs);

      ofstream sol_ofs(sol_name.str().c_str());
      sol_ofs.precision(8);
      x.Save(sol_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock << "parallel " << Mpi::WorldSize() << " " << Mpi::WorldRank() << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
+      sol_sock << "solution\n" << pmesh << x << flush;
   }

-   // 15. Free the used memory.
-   delete X;
-   delete B;
-   delete A;
-
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
   return 0;
 }
@@ -39,8 +39,8 @@ private:
   // Base Nonlinear Form
   std::unique_ptr<NonlinearForm> nonlinearForm;
   // element-wise inverse mass matrix
-   std::vector<DenseMatrix> invmass; // local scalar inverse mass.
-   std::vector<DenseMatrix> weakdiv; // local weakdivergence. Trial space is ByDim.
+   std::vector<DenseMatrix> invmass; // local scalar inverse mass
+   std::vector<DenseMatrix> weakdiv; // local weak divergence (trial space ByDim)
   // global maximum characteristic speed. Updated by form integrators
   mutable real_t max_char_speed;
   // auxiliary variable used in Mult
@@ -169,9 +169,9 @@ void DGHyperbolicConservationLaws::Mult(const Vector &x, Vector &y) const
 {
   // 0. Reset wavespeed computation before operator application.
   formIntegrator->ResetMaxCharSpeed();
-   // 1. Apply Nonlinear form to obtain an axiliary result
+   // 1. Apply Nonlinear form to obtain an auxiliary result
   //         z = - <F̂(u_h,n), [[v]]>_e
-   //    If weak-divergencee is not preassembled, we also have weak-divergence
+   //    If weak-divergence is not preassembled, we also have weak-divergence
   //         z = - <F̂(u_h,n), [[v]]>_e + (F(u_h), ∇v)
   nonlinearForm->Mult(x, z);
   if (!weakdiv.empty()) // if weak divergence is pre-assembled
@@ -0,0 +1,374 @@
+//                                MFEM Example 40
+//
+// Compile with: make ex40
+//
+// Sample runs: ex40 -step 10 -gr 2.0
+//              ex40 -step 10 -gr 2.0 -o 3 -r 1
+//              ex40 -step 10 -gr 2.0 -r 4 -m ../data/l-shape.mesh
+//              ex40 -step 10 -gr 2.0 -r 2 -m ../data/fichera.mesh
+//
+// Description: This example code demonstrates how to use MFEM to solve the
+//              eikonal equation,
+//
+//                      |∇𝑢| = 1 in Ω,  𝑢 = g on ∂Ω.
+//
+//              The solution of this problem coincides with the unique optimum of
+//              the nonlinear program
+//
+//                   maximize ∫_Ω 𝑢 d𝑥 subject to |∇𝑢| ≤ 1,   𝑢 = g on Ω,      (⋆)
+//
+//              which is the foundation for method implemented below.
+//
+//              Following the proximal Galerkin methodology [1] (see also Example
+//              36), we construct a Legendre function for the unit ball
+//              𝐵₁ := {𝑥 ∈ Rⁿ | |𝑥| < 1}. Our choice is the Hellinger entropy,
+//
+//                    h(𝑥) = −( 1 − |𝑥|² )^{1/2},
+//
+//              although other choices are possible, each leading to a slightly
+//              different algorithm. We then adaptively regularize the optimization
+//              problem (⋆) with the Bregman divergence of the Hellinger entropy,
+//
+//                 maximize  ∫_Ω 𝑢 d𝑥 - αₖ⁻¹ Dₕ(∇𝑢,∇𝑢ₖ₋₁)  subject to  𝑢 = g on Ω.
+//
+//              This results in a sequence of functions ( 𝜓ₖ , 𝑢ₖ ),
+//
+//                      𝑢ₖ → 𝑢,    𝜓ₖ/|𝜓ₖ| → ∇𝑢    as k → \infty,
+//
+//              defined by the nonlinear saddle-point problems
+//
+//               Find 𝜓ₖ ∈ H(div,Ω) and 𝑢ₖ ∈ L²(Ω) such that
+//               ( Zₖ(𝜓ₖ) , τ ) + ( 𝑢ₖ , ∇⋅τ ) = ⟨ g , τ⋅n ⟩         ∀ τ ∈ H(div,Ω)
+//               ( ∇⋅𝜓ₖ , v )                 = ( ∇⋅𝜓ₖ₋₁ - 1 , v )   ∀ v ∈ L²(Ω)
+//
+//              where Zₖ(𝜓) := ∇h⁻¹(αₖ 𝜓) = 𝜓 / ( αₖ⁻² + |𝜓|² )^{1/2} and step size
+//              αₖ > 0. These saddle-point problems are solved using a damped Newton's
+//              method. This example assumes that g = 0 and allows the step size to
+//              grow geometrically, αₖ = α₀rᵏ, where r ≥ 1 is the growth rate.
+//
+//              [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//                  preserving finite element method for pointwise bound constraints.
+//                  arXiv:2307.12444 [math.NA]
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ZCoefficient : public VectorCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   ZCoefficient(int vdim, GridFunction &psi_, real_t alpha_ = 1.0)
+      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+class DZCoefficient : public MatrixCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   DZCoefficient(int height, GridFunction &psi_, real_t alpha_ = 1.0)
+      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int max_it = 5;
+   int ref_levels = 3;
+   real_t alpha = 1.0;
+   real_t growth_rate = 1.0;
+   real_t newton_scaling = 0.9;
+   real_t tichonov = 1e-1;
+   real_t tol = 1e-4;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Initial size alpha");
+   args.AddOption(&growth_rate, "-gr", "--growth-rate",
+                  "Growth rate of the step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+
+   MFEM_ASSERT(mesh.bdr_attributes.Size(),
+               "This example does not currently support meshes"
+               " without boundary attributes."
+              )
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   RT_FECollection RTfec(order, dim);
+   FiniteElementSpace RTfes(&mesh, &RTfec);
+
+   L2_FECollection L2fec(order, dim);
+   FiniteElementSpace L2fes(&mesh, &L2fec);
+
+   cout << "Number of H(div) dofs: "
+        << RTfes.GetTrueVSize() << endl;
+   cout << "Number of L² dofs: "
+        << L2fes.GetTrueVSize() << endl;
+
+   // 5. Define the offsets for the block matrices
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = RTfes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   offsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   // 6. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   GridFunction u_gf, delta_psi_gf;
+   delta_psi_gf.MakeRef(&RTfes,x,offsets[0]);
+   u_gf.MakeRef(&L2fes,x,offsets[1]);
+
+   GridFunction psi_old_gf(&RTfes);
+   GridFunction psi_gf(&RTfes);
+   GridFunction u_old_gf(&L2fes);
+
+   // 7. Define initial guesses for the solution variables.
+   delta_psi_gf = 0.0;
+   psi_gf = 0.0;
+   u_gf = 0.0;
+   psi_old_gf = psi_gf;
+   u_old_gf = u_gf;
+
+   // 8. Prepare for glvis output.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 9. Coefficients to be used later.
+   ConstantCoefficient neg_one(-1.0);
+   ConstantCoefficient zero(0.0);
+   ConstantCoefficient tichonov_cf(tichonov);
+   ConstantCoefficient neg_tichonov_cf(-1.0*tichonov);
+   ZCoefficient Z(sdim, psi_gf, alpha);
+   DZCoefficient DZ(sdim, psi_gf, alpha);
+   ScalarVectorProductCoefficient neg_Z(-1.0, Z);
+   DivergenceGridFunctionCoefficient div_psi_cf(&psi_gf);
+   DivergenceGridFunctionCoefficient div_psi_old_cf(&psi_old_gf);
+   SumCoefficient psi_old_minus_psi(div_psi_old_cf, div_psi_cf, 1.0, -1.0);
+
+   // 10. Assemble constant matrices/vectors to avoid reassembly in the loop.
+   LinearForm b0, b1;
+   b0.MakeRef(&RTfes,rhs.GetBlock(0),0);
+   b1.MakeRef(&L2fes,rhs.GetBlock(1),0);
+
+   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_Z));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+
+   BilinearForm a00(&RTfes);
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(DZ));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(tichonov_cf));
+
+   MixedBilinearForm a10(&RTfes,&L2fes);
+   a10.AddDomainIntegrator(new VectorFEDivergenceIntegrator());
+   a10.Assemble();
+   a10.Finalize();
+   SparseMatrix &A10 = a10.SpMat();
+   SparseMatrix *A01 = Transpose(A10);
+
+   BilinearForm a11(&L2fes);
+   a11.AddDomainIntegrator(new MassIntegrator(neg_tichonov_cf));
+   a11.Assemble();
+   a11.Finalize();
+   SparseMatrix &A11 = a11.SpMat();
+
+   // 11. Iterate.
+   int k;
+   int total_iterations = 0;
+   real_t increment_u = 0.1;
+   GridFunction u_tmp(&L2fes);
+   for (k = 0; k < max_it; k++)
+   {
+      u_tmp = u_old_gf;
+      Z.SetAlpha(alpha);
+      DZ.SetAlpha(alpha);
+
+      mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+
+      int j;
+      for ( j = 0; j < 5; j++)
+      {
+         total_iterations++;
+
+         b0.Assemble();
+         b1.Assemble();
+
+         a00.Assemble(false);
+         a00.Finalize(false);
+         SparseMatrix &A00 = a00.SpMat();
+
+         // Construct Schur-complement preconditioner
+         Vector A00_diag(a00.Height());
+         A00.GetDiag(A00_diag);
+         A00_diag.Reciprocal();
+         SparseMatrix *S = Mult_AtDA(*A01, A00_diag);
+
+         BlockDiagonalPreconditioner prec(offsets);
+         prec.SetDiagonalBlock(0,new DSmoother(A00));
+#ifndef MFEM_USE_SUITESPARSE
+         prec.SetDiagonalBlock(1,new GSSmoother(*S));
+#else
+         prec.SetDiagonalBlock(1,new UMFPackSolver(*S));
+#endif
+         prec.owns_blocks = 1;
+
+         BlockOperator A(offsets);
+         A.SetBlock(0,0,&A00);
+         A.SetBlock(1,0,&A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,&A11);
+
+         GMRES(A,prec,rhs,x,0,2000,500,1e-12,0.0);
+         delete S;
+
+         u_tmp -= u_gf;
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         // Damped Newton update
+         psi_gf.Add(newton_scaling, delta_psi_gf);
+         a00.Update();
+
+         if (visualization)
+         {
+            sol_sock << "solution\n" << mesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+         }
+
+         mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      mfem::out << "Number of Newton iterations = " << j+1 << endl;
+      mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      alpha *= max(growth_rate, 1_r);
+
+   }
+
+   mfem::out << "\n Outer iterations: " << k+1
+             << "\n Total iterations: " << total_iterations
+             << "\n Total dofs:       " << RTfes.GetTrueVSize() + L2fes.GetTrueVSize()
+             << endl;
+
+   delete A01;
+   return 0;
+}
+
+void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
+                        const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(vdim);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   V = psi_vals;
+   V *= phi;
+}
+
+void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                         const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(height);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   K = 0.0;
+   for (int i = 0; i < height; i++)
+   {
+      K(i,i) = phi;
+      for (int j = 0; j < height; j++)
+      {
+         K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
+      }
+   }
+}
@@ -0,0 +1,436 @@
+//                       MFEM Example 40 - Parallel Version
+//
+// Compile with: make ex40p
+//
+// Sample runs: mpirun -np 4 ex40p -step 10 -gr 2.0
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -o 3 -r 1
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -r 4 -m ../data/l-shape.mesh
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -r 2 -m ../data/fichera.mesh
+//
+// Description: This example code demonstrates how to use MFEM to solve the
+//              eikonal equation,
+//
+//                      |∇𝑢| = 1 in Ω,  𝑢 = g on ∂Ω.
+//
+//              The solution of this problem coincides with the unique optimum of
+//              the nonlinear program
+//
+//                   maximize ∫_Ω 𝑢 d𝑥 subject to |∇𝑢| ≤ 1,   𝑢 = g on Ω,      (⋆)
+//
+//              which is the foundation for method implemented below.
+//
+//              Following the proximal Galerkin methodology [1] (see also Example
+//              36), we construct a Legendre function for the unit ball
+//              𝐵₁ := {𝑥 ∈ Rⁿ | |𝑥| < 1}. Our choice is the Hellinger entropy,
+//
+//                    h(𝑥) = −( 1 − |𝑥|² )^{1/2},
+//
+//              although other choices are possible, each leading to a slightly
+//              different algorithm. We then adaptively regularize the optimization
+//              problem (⋆) with the Bregman divergence of the Hellinger entropy,
+//
+//                 maximize  ∫_Ω 𝑢 d𝑥 - αₖ⁻¹ Dₕ(∇𝑢,∇𝑢ₖ₋₁)  subject to  𝑢 = g on Ω.
+//
+//              This results in a sequence of functions ( 𝜓ₖ , 𝑢ₖ ),
+//
+//                      𝑢ₖ → 𝑢,    𝜓ₖ/|𝜓ₖ| → ∇𝑢    as k → \infty,
+//
+//              defined by the nonlinear saddle-point problems
+//
+//               Find 𝜓ₖ ∈ H(div,Ω) and 𝑢ₖ ∈ L²(Ω) such that
+//               ( Zₖ(𝜓ₖ) , τ ) + ( 𝑢ₖ , ∇⋅τ ) = ⟨ g , τ⋅n ⟩         ∀ τ ∈ H(div,Ω)
+//               ( ∇⋅𝜓ₖ , v )                 = ( ∇⋅𝜓ₖ₋₁ - 1 , v )   ∀ v ∈ L²(Ω)
+//
+//              where Zₖ(𝜓) := ∇h⁻¹(αₖ 𝜓) = 𝜓 / ( αₖ⁻² + |𝜓|² )^{1/2} and step size
+//              αₖ > 0. These saddle-point problems are solved using a damped Newton's
+//              method. This example assumes that g = 0 and allows the step size to
+//              grow geometrically, αₖ = α₀rᵏ, where r ≥ 1 is the growth rate.
+//
+//              [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//                  preserving finite element method for pointwise bound constraints.
+//                  arXiv:2307.12444 [math.NA]
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ZCoefficient : public VectorCoefficient
+{
+protected:
+   ParGridFunction *psi;
+   real_t alpha;
+
+public:
+   ZCoefficient(int vdim, ParGridFunction &psi_, real_t alpha_ = 1.0)
+      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+class DZCoefficient : public MatrixCoefficient
+{
+protected:
+   ParGridFunction *psi;
+   real_t alpha;
+
+public:
+   DZCoefficient(int height, ParGridFunction &psi_, real_t alpha_ = 1.0)
+      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int max_it = 5;
+   int ref_levels = 3;
+   real_t alpha = 1.0;
+   real_t growth_rate = 1.0;
+   real_t newton_scaling = 0.9;
+   real_t tichonov = 1e-1;
+   real_t tol = 1e-4;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Initial size alpha");
+   args.AddOption(&growth_rate, "-gr", "--growth-rate",
+                  "Growth rate of the step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2. Read the mesh from the mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+
+   MFEM_ASSERT(mesh.bdr_attributes.Size(),
+               "This example does not currently support meshes"
+               " without boundary attributes."
+              )
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   RT_FECollection RTfec(order, dim);
+   ParFiniteElementSpace RTfes(&pmesh, &RTfec);
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2fes(&pmesh, &L2fec);
+
+   int num_dofs_RT = RTfes.GlobalTrueVSize();
+   int num_dofs_L2 = L2fes.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of H(div) dofs: "
+           << num_dofs_RT << endl;
+      cout << "Number of L² dofs: "
+           << num_dofs_L2 << endl;
+   }
+
+   // 5. Define the offsets for the block matrices
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = RTfes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   offsets.PartialSum();
+
+   Array<int> toffsets(3);
+   toffsets[0] = 0;
+   toffsets[1] = RTfes.GetTrueVSize();
+   toffsets[2] = L2fes.GetTrueVSize();
+   toffsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   BlockVector tx(toffsets), trhs(toffsets);
+   tx = 0.0; trhs = 0.0;
+
+   // 6. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   ParGridFunction u_gf, delta_psi_gf;
+   delta_psi_gf.MakeRef(&RTfes,x,offsets[0]);
+   u_gf.MakeRef(&L2fes,x,offsets[1]);
+
+   ParGridFunction psi_old_gf(&RTfes);
+   ParGridFunction psi_gf(&RTfes);
+   ParGridFunction u_old_gf(&L2fes);
+
+   // 7. Define initial guesses for the solution variables.
+   delta_psi_gf = 0.0;
+   psi_gf = 0.0;
+   u_gf = 0.0;
+   psi_old_gf = psi_gf;
+   u_old_gf = u_gf;
+
+   // 8. Prepare for glvis output.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 9. Coefficients to be used later.
+   ConstantCoefficient neg_one(-1.0);
+   ConstantCoefficient zero(0.0);
+   ConstantCoefficient tichonov_cf(tichonov);
+   ConstantCoefficient neg_tichonov_cf(-1.0*tichonov);
+   ZCoefficient Z(sdim, psi_gf, alpha);
+   DZCoefficient DZ(sdim, psi_gf, alpha);
+   ScalarVectorProductCoefficient neg_Z(-1.0, Z);
+   DivergenceGridFunctionCoefficient div_psi_cf(&psi_gf);
+   DivergenceGridFunctionCoefficient div_psi_old_cf(&psi_old_gf);
+   SumCoefficient psi_old_minus_psi(div_psi_old_cf, div_psi_cf, 1.0, -1.0);
+
+   // 10. Assemble constant matrices/vectors to avoid reassembly in the loop.
+   ParLinearForm b0, b1;
+   b0.MakeRef(&RTfes,rhs.GetBlock(0),0);
+   b1.MakeRef(&L2fes,rhs.GetBlock(1),0);
+
+   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_Z));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+
+   ParBilinearForm a00(&RTfes);
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(DZ));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(tichonov_cf));
+
+   ParMixedBilinearForm a10(&RTfes,&L2fes);
+   a10.AddDomainIntegrator(new VectorFEDivergenceIntegrator());
+   a10.Assemble();
+   a10.Finalize();
+   HypreParMatrix *A10 = a10.ParallelAssemble();
+
+   HypreParMatrix *A01 = A10->Transpose();
+
+   ParBilinearForm a11(&L2fes);
+   a11.AddDomainIntegrator(new MassIntegrator(neg_tichonov_cf));
+   a11.Assemble();
+   a11.Finalize();
+   HypreParMatrix *A11 = a11.ParallelAssemble();
+
+   // 11. Iterate.
+   int k;
+   int total_iterations = 0;
+   real_t increment_u = 0.1;
+   ParGridFunction u_tmp(&L2fes);
+   for (k = 0; k < max_it; k++)
+   {
+      u_tmp = u_old_gf;
+      Z.SetAlpha(alpha);
+      DZ.SetAlpha(alpha);
+
+      if (myid == 0)
+      {
+         mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+      }
+
+      int j;
+      for ( j = 0; j < 5; j++)
+      {
+         total_iterations++;
+
+         b0.Assemble();
+         b0.ParallelAssemble(trhs.GetBlock(0));
+
+         b1.Assemble();
+         b1.ParallelAssemble(trhs.GetBlock(1));
+
+         a00.Assemble(false);
+         a00.Finalize(false);
+         HypreParMatrix *A00 = a00.ParallelAssemble();
+
+         // Construct Schur-complement preconditioner
+         HypreParVector A00_diag(MPI_COMM_WORLD, A00->GetGlobalNumRows(),
+                                 A00->GetRowStarts());
+         A00->GetDiag(A00_diag);
+         HypreParMatrix S_tmp(*A01);
+         S_tmp.InvScaleRows(A00_diag);
+         HypreParMatrix *S = ParMult(A10, &S_tmp, true);
+
+         BlockDiagonalPreconditioner prec(toffsets);
+         HypreBoomerAMG P00(*A00);
+         P00.SetPrintLevel(0);
+         HypreBoomerAMG P11(*S);
+         P11.SetPrintLevel(0);
+         prec.SetDiagonalBlock(0,&P00);
+         prec.SetDiagonalBlock(1,&P11);
+
+         BlockOperator A(toffsets);
+         A.SetBlock(0,0,A00);
+         A.SetBlock(1,0,A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,A11);
+
+         GMRESSolver gmres(MPI_COMM_WORLD);
+         gmres.SetPrintLevel(-1);
+         gmres.SetRelTol(1e-8);
+         gmres.SetMaxIter(2000);
+         gmres.SetKDim(500);
+         gmres.SetOperator(A);
+         gmres.SetPreconditioner(prec);
+         gmres.Mult(trhs,tx);
+         delete S;
+         delete A00;
+
+         delta_psi_gf.SetFromTrueDofs(tx.GetBlock(0));
+         u_gf.SetFromTrueDofs(tx.GetBlock(1));
+
+         u_tmp -= u_gf;
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         // Damped Newton update
+         psi_gf.Add(newton_scaling, delta_psi_gf);
+         a00.Update();
+
+         if (visualization)
+         {
+            sol_sock << "parallel " << num_procs << " " << myid << "\n";
+            sol_sock << "solution\n" << pmesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+         }
+
+         if (myid == 0)
+         {
+            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+         }
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      if (myid == 0)
+      {
+         mfem::out << "Number of Newton iterations = " << j+1 << endl;
+         mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+      }
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      alpha *= max(growth_rate, 1_r);
+
+   }
+
+   // 12. Print stats.
+   if (myid == 0)
+   {
+      mfem::out << "\n Outer iterations: " << k+1
+                << "\n Total iterations: " << total_iterations
+                << "\n Total dofs:       " << RTfes.GetTrueVSize() + L2fes.GetTrueVSize()
+                << endl;
+   }
+
+   // 13. Free the used memory.
+   delete A01;
+   delete A10;
+   delete A11;
+   return 0;
+}
+
+void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
+                        const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(vdim);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   V = psi_vals;
+   V *= phi;
+}
+
+void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                         const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(height);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   K = 0.0;
+   for (int i = 0; i < height; i++)
+   {
+      K(i,i) = phi;
+      for (int j = 0; j < height; j++)
+      {
+         K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
+      }
+   }
+}
@@ -23,14 +23,14 @@ MFEM_LIB_FILE = mfem_is_not_built

 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
   ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-   ex31 ex33 ex34 ex36 ex37 ex38 ex39
+   ex31 ex33 ex34 ex36 ex37 ex38 ex39 ex40
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
   ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-   ex37p ex39p
-SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
-PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-   ex24p ex25p ex26p ex34p ex35p
+   ex37p ex39p ex40p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
+PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
+   ex22p ex24p ex25p ex26p ex34p ex35p

 ifeq ($(MFEM_USE_LAPACK),YES)
   SEQ_EXAMPLES += ex38
@@ -138,6 +138,14 @@ ex10-test-seq: ex10
 	@$(call mfem-test,$<,, Serial example,-tf 5)
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-tf 5)
+ex14-test-seq-cuda: ex14
+	@$(call mfem-test,$<,, Serial CUDA example,-r 2 -pa -d cuda)
+ex14p-test-par-cuda: ex14p
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-rs 2 -rp 0 -pa -d cuda)
+ex14-test-seq-hip: ex14
+	@$(call mfem-test,$<,, Serial HIP example,-r 2 -pa -d hip)
+ex14p-test-par-hip: ex14p
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-rs 2 -rp 0 -pa -d hip)
 ex15-test-seq: ex15
 	@$(call mfem-test,$<,, Serial example,-e 1)
 ex15p-test-par: ex15p
@@ -1,3 +1,14 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
 #include <algorithm>
 #include <assert.h>
 #include <cstdlib>
@@ -709,10 +709,7 @@ real_t HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 real_t HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   real_t loc_energy = 0.5*M.InnerProduct(v, v);
-   real_t energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
-                 fespace.GetComm());
+   real_t energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -856,10 +856,7 @@ double HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 double HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   double loc_energy = 0.5*M.InnerProduct(v, v);
-   double energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPI_DOUBLE, MPI_SUM,
-                 fespace.GetComm());
+   double energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -18,6 +18,7 @@ set(SRCS
  integ/bilininteg_convection_pa.cpp
  integ/bilininteg_convection_ea.cpp
  integ/bilininteg_curlcurl_pa.cpp
+  integ/bilininteg_dgdiffusion_pa.cpp
  integ/bilininteg_dgtrace_pa.cpp
  integ/bilininteg_dgtrace_ea.cpp
  integ/bilininteg_diffusion_mf.cpp
@@ -117,6 +118,7 @@ set(SRCS
  quadinterpolator.cpp
  quadinterpolator_face.cpp
  restriction.cpp
+  normal_deriv_restriction.cpp
  staticcond.cpp
  tmop.cpp
  tmop/tmop_pa.cpp
@@ -228,6 +230,7 @@ set(HDRS
  quadinterpolator.hpp
  quadinterpolator_face.hpp
  restriction.hpp
+  normal_deriv_restriction.hpp
  fespacehierarchy.hpp
  staticcond.hpp
  tbilinearform.hpp
@@ -340,9 +340,9 @@ public:
       $ M^{-1} $ (currently returns NULL) */
   virtual MatrixInverse *Inverse() const;

-   /** @brief  Finalizes the matrix initialization if the ::AssemblyLevel is
+   /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
       AssemblyLevel::LEGACY.
-       THe matrix that gets finalized is different if you are using static
+       The matrix that gets finalized is different if you are using static
       condensation or hybridization.*/
   virtual void Finalize(int skip_zeros = 1);

@@ -643,7 +643,7 @@ public:
   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
                       DiagonalPolicy dpolicy = DIAG_ONE);

-   /** @brief  Eliminate the given @a vdofs, storing the eliminated part
+   /** @brief Eliminate the given @a vdofs, storing the eliminated part
       internally in $ M_e $.

       This method works in conjunction with EliminateVDofsInRHS() and allows
@@ -826,7 +826,7 @@ public:
       $ M^{-1} $ (currently unimplemented and returns NULL)*/
   virtual MatrixInverse *Inverse() const;

-   /** @brief  Finalizes the matrix initialization if the ::AssemblyLevel is
+   /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
       AssemblyLevel::LEGACY.*/
   virtual void Finalize(int skip_zeros = 1);

@@ -282,6 +282,22 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
      int_face_X.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
      int_face_Y.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
      int_face_Y.UseDevice(true); // ensure 'int_face_Y = 0.0' is done on device
+
+      bool needs_normal_derivs = false;
+      auto &integs = *a->GetFBFI();
+      for (int i = 0; i < integs.Size(); ++i)
+      {
+         if (integs[i]->RequiresFaceNormalDerivatives())
+         {
+            needs_normal_derivs = true;
+            break;
+         }
+      }
+      if (needs_normal_derivs)
+      {
+         int_face_dXdn.SetSize(int_face_restrict_lex->Height());
+         int_face_dYdn.SetSize(int_face_restrict_lex->Height());
+      }
   }

   const bool has_bdr_integs = (a->GetBFBFI()->Size() > 0 ||
@@ -296,6 +312,22 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
      bdr_face_Y.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
      bdr_face_Y.UseDevice(true); // ensure 'faceBoundY = 0.0' is done on device

+      bool needs_normal_derivs = false;
+      auto &integs = *a->GetBFBFI();
+      for (int i = 0; i < integs.Size(); ++i)
+      {
+         if (integs[i]->RequiresFaceNormalDerivatives())
+         {
+            needs_normal_derivs = true;
+            break;
+         }
+      }
+      if (needs_normal_derivs)
+      {
+         bdr_face_dXdn.SetSize(bdr_face_restrict_lex->Height());
+         bdr_face_dYdn.SetSize(bdr_face_restrict_lex->Height());
+      }
+
      const Mesh &mesh = *trial_fes->GetMesh();
      // See LinearFormExtension::Update for explanation of f_to_be logic.
      std::unordered_map<int,int> f_to_be;
@@ -542,8 +574,8 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         localY = 0.0;
         for (int i = 0; i < iSz; ++i)
         {
-            AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes,
-                               false, localY);
+            AddMultWithMarkers(*integrators[i], localX, elem_markers[i],
+                               elem_attributes, false, localY);
         }
         elem_restrict->MultTranspose(localY, y);
      }
@@ -557,15 +589,57 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
   const int iFISz = intFaceIntegrators.Size();
   if (int_face_restrict_lex && iFISz>0)
   {
-      int_face_restrict_lex->Mult(x, int_face_X);
-      if (int_face_X.Size()>0)
+      // When assembling interior face integrators for DG spaces, we need to
+      // exchange the face-neighbor information. This happens inside member
+      // functions of the 'int_face_restrict_lex'. To avoid repeated calls to
+      // ParGridFunction::ExchangeFaceNbrData, if we have a parallel space
+      // with interior face integrators, we create a ParGridFunction that
+      // will be used to cache the face-neighbor data. x_dg should be passed
+      // to any restriction operator that may need to use face-neighbor data.
+      const Vector *x_dg = &x;
+#ifdef MFEM_USE_MPI
+      ParGridFunction x_pgf;
+      if (auto *pfes = dynamic_cast<ParFiniteElementSpace*>(a->FESpace()))
+      {
+         x_pgf.MakeRef(pfes, const_cast<Vector&>(x), 0);
+         x_dg = &x_pgf;
+      }
+#endif
+
+      int_face_restrict_lex->Mult(*x_dg, int_face_X);
+      if (int_face_dXdn.Size() > 0)
+      {
+         int_face_restrict_lex->NormalDerivativeMult(*x_dg, int_face_dXdn);
+      }
+      if (int_face_X.Size() > 0)
      {
         int_face_Y = 0.0;
+
+         // if normal derivatives are needed by at least one integrator...
+         if (int_face_dYdn.Size() > 0)
+         {
+            int_face_dYdn = 0.0;
+         }
+
         for (int i = 0; i < iFISz; ++i)
         {
-            intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
+            if (intFaceIntegrators[i]->RequiresFaceNormalDerivatives())
+            {
+               intFaceIntegrators[i]->AddMultPAFaceNormalDerivatives(
+                  int_face_X, int_face_dXdn,
+                  int_face_Y, int_face_dYdn);
+            }
+            else
+            {
+               intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
+            }
         }
         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         if (int_face_dYdn.Size() > 0)
+         {
+            int_face_restrict_lex->NormalDerivativeAddMultTranspose(
+               int_face_dYdn, y);
+         }
      }
   }

@@ -579,9 +653,19 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
      Array<Array<int>*> &bdr_markers = *a->GetBBFI_Marker();
      Array<Array<int>*> &bdr_face_markers = *a->GetBFBFI_Marker();
      bdr_face_restrict_lex->Mult(x, bdr_face_X);
-      if (bdr_face_X.Size()>0)
+      if (bdr_face_dXdn.Size() > 0)
+      {
+         bdr_face_restrict_lex->NormalDerivativeMult(x, bdr_face_dXdn);
+      }
+      if (bdr_face_X.Size() > 0)
      {
         bdr_face_Y = 0.0;
+
+         // if normal derivatives are needed by at least one integrator...
+         if (bdr_face_dYdn.Size() > 0)
+         {
+            bdr_face_dYdn = 0.0;
+         }
         for (int i = 0; i < n_bdr_integs; ++i)
         {
            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
@@ -589,10 +673,23 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         }
         for (int i = 0; i < n_bdr_face_integs; ++i)
         {
-            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
-                               bdr_attributes, false, bdr_face_Y);
+            if (bdr_face_integs[i]->RequiresFaceNormalDerivatives())
+            {
+               AddMultNormalDerivativesWithMarkers(
+                  *bdr_face_integs[i], bdr_face_X, bdr_face_dXdn,
+                  bdr_face_markers[i], bdr_attributes, bdr_face_Y, bdr_face_dYdn);
+            }
+            else
+            {
+               AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
+                                  bdr_attributes, false, bdr_face_Y);
+            }
         }
         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         if (bdr_face_dYdn.Size() > 0)
+         {
+            bdr_face_restrict_lex->NormalDerivativeAddMultTranspose(bdr_face_dYdn, y);
+         }
      }
   }
 }
@@ -693,6 +790,37 @@ static void AddWithMarkers_(
   });
 }

+void PABilinearFormExtension::AddMultNormalDerivativesWithMarkers(
+   const BilinearFormIntegrator &integ,
+   const Vector &x,
+   const Vector &dxdn,
+   const Array<int> *markers,
+   const Array<int> &attributes,
+   Vector &y,
+   Vector &dydn) const
+{
+   if (markers)
+   {
+      tmp_evec.SetSize(y.Size() + dydn.Size());
+      tmp_evec = 0.0;
+      Vector tmp_y(tmp_evec, 0, y.Size());
+      Vector tmp_dydn(tmp_evec, y.Size(), dydn.Size());
+
+      integ.AddMultPAFaceNormalDerivatives(x, dxdn, tmp_y, tmp_dydn);
+
+      const int ne = attributes.Size();
+      const int nd_1 = x.Size() / ne;
+      const int nd_2 = dxdn.Size() / ne;
+
+      AddWithMarkers_(ne, nd_1, tmp_y, *markers, attributes, y);
+      AddWithMarkers_(ne, nd_2, tmp_dydn, *markers, attributes, dydn);
+   }
+   else
+   {
+      integ.AddMultPAFaceNormalDerivatives(x, dxdn, y, dydn);
+   }
+}
+
 void PABilinearFormExtension::AddMultWithMarkers(
   const BilinearFormIntegrator &integ,
   const Vector &x,
@@ -74,6 +74,8 @@ protected:
   mutable Vector localX, localY;
   mutable Vector int_face_X, int_face_Y;
   mutable Vector bdr_face_X, bdr_face_Y;
+   mutable Vector int_face_dXdn, int_face_dYdn;
+   mutable Vector bdr_face_dXdn, bdr_face_dYdn;
   const Operator *elem_restrict; // Not owned
   const FaceRestriction *int_face_restrict_lex; // Not owned
   const FaceRestriction *bdr_face_restrict_lex; // Not owned
@@ -113,6 +115,23 @@ protected:
                           const Array<int> &attributes,
                           const bool transpose,
                           Vector &y) const;
+
+   /// @brief Performs the same function as AddMultWithMarkers, but takes as
+   /// input and output face normal derivatives.
+   ///
+   /// This is required when the integrator requires face normal derivatives,
+   /// for example, DGDiffusionIntegrator.
+   ///
+   /// This is called when the integrator's member function
+   /// BilinearFormIntegrator::RequiresFaceNormalDerivatives() returns true.
+   void AddMultNormalDerivativesWithMarkers(
+      const BilinearFormIntegrator &integ,
+      const Vector &x,
+      const Vector &dxdn,
+      const Array<int> *markers,
+      const Array<int> &attributes,
+      Vector &y,
+      Vector &dydn) const;
 };

 /// Data and methods for element-assembled bilinear forms
@@ -189,6 +189,12 @@ void BilinearFormIntegrator::AssembleTraceFaceMatrix (int elem,
              " Integrator class.");
 }

+void BilinearFormIntegrator::AddMultPAFaceNormalDerivatives(
+   const Vector &x, const Vector &dxdn, Vector &y, Vector &dydn) const
+{
+   MFEM_ABORT("Not implemented.");
+}
+
 void BilinearFormIntegrator::AssembleElementVector(
   const FiniteElement &el, ElementTransformation &Tr, const Vector &elfun,
   Vector &elvect)
@@ -3423,7 +3429,7 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   const FiniteElement &el1, const FiniteElement &el2,
   FaceElementTransformations &Trans, DenseMatrix &elmat)
 {
-   int dim, ndof1, ndof2, ndofs;
+   int ndof1, ndof2, ndofs;
   bool kappa_is_nonzero = (kappa != 0.);
   real_t w, wq = 0.0;

@@ -3466,17 +3472,9 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
   {
-      // a simple choice for the integration order; is this OK?
-      int order;
-      if (ndof2)
-      {
-         order = 2*max(el1.GetOrder(), el2.GetOrder());
-      }
-      else
-      {
-         order = 2*el1.GetOrder();
-      }
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      const int order = (ndof2) ? max(el1.GetOrder(),
+                                      el2.GetOrder()) : el1.GetOrder();
+      ir = &GetRule(order, Trans);
   }

   // assemble: < {(Q \nabla u).n},[v] >      --> elmat
@@ -3654,6 +3652,13 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   }
 }

+const IntegrationRule &DGDiffusionIntegrator::GetRule(
+   int order, FaceElementTransformations &T)
+{
+   // order is typically the maximum of the order of the left and right elements
+   // neighboring the given face.
+   return IntRules.Get(T.GetGeometryType(), 2*order);
+}

 // static method
 void DGElasticityIntegrator::AssembleBlock(
@@ -266,6 +266,39 @@ public:
                                    Vector &flux, Vector *d_energy = NULL)
   { return 0.0; }

+   /** @brief For bilinear forms on element faces, specifies if the normal
+              derivatives are needed on the faces or just the face restriction.
+
+       @details if RequiresFaceNormalDerivatives() == true, then
+                AddMultPAFaceNormalDerivatives(...) should be invoked in place
+                of AddMultPA(...) and L2NormalDerivativeFaceRestriction should
+                be used to compute the normal derivatives. This is used for some
+                DG integrators, for example DGDiffusionIntegrator.
+
+       @returns whether normal derivatives appear in the bilinear form.
+   */
+   virtual bool RequiresFaceNormalDerivatives() const { return false; }
+
+   /// Method for partially assembled action.
+   /** @brief For bilinear forms on element faces that depend on the normal
+              derivative on the faces, computes the action of integrator to the
+              face values @a x and reference-normal derivatives @a dxdn and adds
+              the result to @a y and @a dydn.
+
+      @details This method can be called only after the method AssemblePA() has
+               been called.
+
+      @param[in]     x E-vector of face values (provided by
+                       FaceRestriction::Mult)
+      @param[in]     dxdn E-vector of face reference-normal derivatives
+                          (provided by FaceRestriction::NormalDerivativeMult)
+      @param[in,out] y E-vector of face values to add action to.
+      @param[in,out] dydn E-vector of face reference-normal derivative values to
+                          add action to.
+   */
+   virtual void AddMultPAFaceNormalDerivatives(const Vector &x, const Vector &dxdn,
+                                               Vector &y, Vector &dydn) const;
+
   virtual ~BilinearFormIntegrator() { }
 };

@@ -1708,7 +1741,7 @@ public:
   { vector_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \cdot  \nabla \cdot u, \nabla \cdot v)$ in 2D
+/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \cdot \nabla u, \nabla \cdot v)$ in 2D
    or 3D and where $\hat{V}$ is a vector coefficient, $u$ is in $H^1$ and $v$ is in $H(div)$. */
 class MixedGradDivIntegrator : public MixedScalarVectorIntegrator
 {
@@ -1747,7 +1780,7 @@ public:
   { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \nabla \cdot u, \nabla \cdot v)$ in 2D
+/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \nabla \cdot u, \nabla v)$ in 2D
    or 3D and where $\hat{V}$ is a vector coefficient, $u$ is in $H(div)$ and $v$ is in $H^1$. */
 class MixedDivGradIntegrator : public MixedScalarVectorIntegrator
 {
@@ -1787,7 +1820,7 @@ public:
   { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} u, \nabla \cdot v)$ in 2D or 3D
+/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} u, \nabla v)$ in 2D or 3D
    and where $\hat{V}$ is a vector coefficient, $u$ is in $H^1$ or $L_2$ and $v$ is in $H^1$. */
 class MixedScalarWeakDivergenceIntegrator : public MixedScalarVectorIntegrator
 {
@@ -3229,6 +3262,13 @@ protected:
   Vector shape1, shape2, dshape1dn, dshape2dn, nor, nh, ni;
   DenseMatrix jmat, dshape1, dshape2, mq, adjJ;

+
+   // PA extension
+   Vector pa_data; // (Q, h, dot(n,J)|el0, dot(n,J)|el1)
+   const DofToQuad *maps; ///< Not owned
+   int dim, nf, nq, dofs1D, quad1D;
+   IntegrationRules irs{0, Quadrature1D::GaussLobatto};
+
 public:
   DGDiffusionIntegrator(const real_t s, const real_t k)
      : Q(NULL), MQ(NULL), sigma(s), kappa(k) { }
@@ -3237,10 +3277,26 @@ public:
   DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k)
      : Q(NULL), MQ(&q), sigma(s), kappa(k) { }
   using BilinearFormIntegrator::AssembleFaceMatrix;
-   virtual void AssembleFaceMatrix(const FiniteElement &el1,
-                                   const FiniteElement &el2,
-                                   FaceElementTransformations &Trans,
-                                   DenseMatrix &elmat);
+   void AssembleFaceMatrix(const FiniteElement &el1,
+                           const FiniteElement &el2,
+                           FaceElementTransformations &Trans,
+                           DenseMatrix &elmat) override;
+
+   bool RequiresFaceNormalDerivatives() const override { return true; }
+
+   using BilinearFormIntegrator::AssemblePA;
+
+   void AssemblePAInteriorFaces(const FiniteElementSpace &fes) override;
+
+   void AssemblePABoundaryFaces(const FiniteElementSpace &fes) override;
+
+   void AddMultPAFaceNormalDerivatives(const Vector &x, const Vector &dxdn,
+                                       Vector &y, Vector &dydn) const override;
+
+   const IntegrationRule &GetRule(int order, FaceElementTransformations &T);
+
+private:
+   void SetupPA(const FiniteElementSpace &fes, FaceType type);
 };

 /** Integrator for the "BR2" diffusion stabilization term
@@ -807,6 +807,7 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)

   QuadratureSpaceBase &qspace = *qf.GetSpace();
   const int ne = qspace.GetNE();
+   qf.HostWrite();
   DenseMatrix values;
   DenseSymmetricMatrix matrix;
   for (int iel = 0; iel < ne; ++iel)
@@ -818,7 +819,7 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
      {
         const IntegrationPoint &ip = ir[iq];
         T.SetIntPoint(&ip);
-         matrix.UseExternalData(&values(0, iq), vdim);
+         matrix.UseExternalData(&values(0, iq), height);
         Eval(matrix, T, ip);
      }
   }
@@ -828,13 +829,12 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
 void SymmetricMatrixCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
-   mat.SetSize(height);
-   Eval(mat, T, ip);
+   Eval(mat_aux, T, ip);
   for (int j = 0; j < width; ++j)
   {
      for (int i = 0; i < height; ++ i)
      {
-         K(i, j) = mat(i, j);
+         K(i, j) = mat_aux(i, j);
      }
   }
 }
@@ -924,6 +924,75 @@ void MatrixArrayCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
   }
 }

+MatrixArrayVectorCoefficient::MatrixArrayVectorCoefficient (int dim)
+   : MatrixCoefficient (dim)
+{
+   Coeff.SetSize(height);
+   ownCoeff.SetSize(height);
+   for (int i = 0; i < height; i++)
+   {
+      Coeff[i] = NULL;
+      ownCoeff[i] = true;
+   }
+}
+
+void MatrixArrayVectorCoefficient::SetTime(real_t t)
+{
+   for (int i=0; i < height; i++)
+   {
+      if (Coeff[i]) { Coeff[i]->SetTime(t); }
+   }
+   this->MatrixCoefficient::SetTime(t);
+}
+
+void MatrixArrayVectorCoefficient::Set(int i, VectorCoefficient * c, bool own)
+{
+   MFEM_ASSERT(i < height && i >= 0, "Row "
+               << i << " does not exist. " <<
+               "Matrix height = " << height << ".");
+   if (ownCoeff[i]) { delete Coeff[i]; }
+   Coeff[i] = c;
+   ownCoeff[i] = own;
+}
+
+MatrixArrayVectorCoefficient::~MatrixArrayVectorCoefficient ()
+{
+   for (int i=0; i < height; i++)
+   {
+      if (ownCoeff[i]) { delete Coeff[i]; }
+   }
+}
+
+void MatrixArrayVectorCoefficient::Eval(int i, Vector &V,
+                                        ElementTransformation &T,
+                                        const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(i < height && i >= 0, "Row "
+               << i << " does not exist. " <<
+               "Matrix height = " << height << ".");
+   if (Coeff[i])
+   {
+      Coeff[i] -> Eval(V, T, ip);
+   }
+   else
+   {
+      V = 0.0;
+   }
+}
+
+void MatrixArrayVectorCoefficient::Eval(DenseMatrix &K,
+                                        ElementTransformation &T,
+                                        const IntegrationPoint &ip)
+{
+   K.SetSize(height, width);
+   Vector V(width);
+   for (int i = 0; i < height; i++)
+   {
+      this->Eval(i, V, T, ip);
+      K.SetRow(i, V);
+   }
+}
+
 void MatrixRestrictedCoefficient::SetTime(real_t t)
 {
   if (c) { c->SetTime(t); }
@@ -1041,6 +1110,27 @@ real_t DeterminantCoefficient::Eval(ElementTransformation &T,
   return ma.Det();
 }

+TraceCoefficient::TraceCoefficient(MatrixCoefficient &A)
+   : a(&A), ma(A.GetHeight(), A.GetWidth())
+{
+   MFEM_ASSERT(A.GetHeight() == A.GetWidth(),
+               "TraceCoefficient:  "
+               "Argument must be a square matrix.");
+}
+
+void TraceCoefficient::SetTime(real_t t)
+{
+   if (a) { a->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
+real_t TraceCoefficient::Eval(ElementTransformation &T,
+                              const IntegrationPoint &ip)
+{
+   a->Eval(ma, T, ip);
+   return ma.Trace();
+}
+
 VectorSumCoefficient::VectorSumCoefficient(int dim)
   : VectorCoefficient(dim),
     ACoef(NULL), BCoef(NULL),
@@ -1326,6 +1416,30 @@ void InverseMatrixCoefficient::Eval(DenseMatrix &M,
   M.Invert();
 }

+ExponentialMatrixCoefficient::ExponentialMatrixCoefficient(MatrixCoefficient &A)
+   : MatrixCoefficient(A.GetHeight(), A.GetWidth()), a(&A)
+{
+   MFEM_ASSERT(A.GetHeight() == A.GetWidth() && A.GetHeight() == 2,
+               "ExponentialMatrixCoefficient:  "
+               << "Argument must be a square 2x2 matrix."
+               << "  Height = " << A.GetHeight()
+               << ", Width = " << A.GetWidth());
+}
+
+void ExponentialMatrixCoefficient::SetTime(real_t t)
+{
+   if (a) { a->SetTime(t); }
+   this->MatrixCoefficient::SetTime(t);
+}
+
+void ExponentialMatrixCoefficient::Eval(DenseMatrix &M,
+                                        ElementTransformation &T,
+                                        const IntegrationPoint &ip)
+{
+   a->Eval(M, T, ip);
+   M.Exponential();
+}
+
 OuterProductCoefficient::OuterProductCoefficient(VectorCoefficient &A,
                                                 VectorCoefficient &B)
   : MatrixCoefficient(A.GetVDim(), B.GetVDim()), a(&A), b(&B),
@@ -1334,6 +1334,46 @@ public:
   virtual ~MatrixArrayCoefficient();
 };

+/** @brief Matrix coefficient defined row-wise by an array of vector
+    coefficients. Rows that are not set will evaluate to zero. The
+    matrix coefficient is stored as an array indexing the rows of
+    the matrix. */
+class MatrixArrayVectorCoefficient : public MatrixCoefficient
+{
+private:
+   Array<VectorCoefficient *> Coeff;
+   Array<bool> ownCoeff;
+
+public:
+   /** @brief Construct a coefficient matrix of dimensions @a dim * @a dim. The
+       actual coefficients still need to be added with Set(). */
+   explicit MatrixArrayVectorCoefficient (int dim);
+
+   /// Set the time for internally stored coefficients
+   void SetTime(real_t t) override;
+
+   /// Get the vector coefficient located at the i-th row of the matrix
+   VectorCoefficient* GetCoeff (int i) { return Coeff[i]; }
+
+   /** @brief Set the coefficient located at the i-th row of the matrix.
+       By this will take ownership of the Coefficient passed in, but this
+       can be overridden with the @a own parameter. */
+   void Set(int i, VectorCoefficient * c, bool own=true);
+
+   using MatrixCoefficient::Eval;
+
+   /// Evaluate coefficient located at the i-th row of the matrix using integration
+   /// point @a ip.
+   void Eval(int i, Vector &V, ElementTransformation &T,
+             const IntegrationPoint &ip);
+
+   /// Evaluate the matrix coefficient @a ip.
+   void Eval(DenseMatrix &K, ElementTransformation &T,
+             const IntegrationPoint &ip) override;
+
+   virtual ~MatrixArrayVectorCoefficient();
+};
+

 /** @brief Derived matrix coefficient that has the value of the parent matrix
    coefficient where it is active and is zero otherwise. */
@@ -1426,12 +1466,13 @@ public:
 class SymmetricMatrixCoefficient : public MatrixCoefficient
 {
 protected:
+
   /// Internal matrix used when evaluating this coefficient as a DenseMatrix.
-   DenseSymmetricMatrix mat;
+   mutable DenseSymmetricMatrix mat_aux;
 public:
   /// Construct a dim x dim matrix coefficient.
   explicit SymmetricMatrixCoefficient(int dimension)
-      : MatrixCoefficient(dimension, true) { }
+      : MatrixCoefficient(dimension, true), mat_aux(height) { }

   /// Get the size of the matrix.
   int GetSize() const { return height; }
@@ -1464,8 +1505,9 @@ public:
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip);

-   /// Return a reference to the constant matrix.
-   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
+   /// @deprecated Return a reference to the internal matrix used when evaluating this coefficient as a DenseMatrix.
+   MFEM_DEPRECATED const DenseSymmetricMatrix& GetMatrix() { return mat_aux; }

   virtual ~SymmetricMatrixCoefficient() { }
 };
@@ -1485,6 +1527,10 @@ public:
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseSymmetricMatrix &M, ElementTransformation &T,
                     const IntegrationPoint &ip) { M = mat; }
+
+   /// Return a reference to the constant matrix.
+   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
 };


@@ -1761,6 +1807,31 @@ public:
                       const IntegrationPoint &ip);
 };

+/// Scalar coefficient defined as the trace of a matrix coefficient
+class TraceCoefficient : public Coefficient
+{
+private:
+   MatrixCoefficient * a;
+
+   mutable DenseMatrix ma;
+
+public:
+   /// Construct with the matrix.
+   TraceCoefficient(MatrixCoefficient &A);
+
+   /// Set the time for internally stored coefficients
+   void SetTime(real_t t);
+
+   /// Reset the matrix coefficient
+   void SetACoef(MatrixCoefficient &A) { a = &A; }
+   /// Return the matrix coefficient
+   MatrixCoefficient * GetACoef() const { return a; }
+
+   /// Evaluate the trace coefficient at @a ip.
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
 /// Vector coefficient defined as the linear combination of two vectors
 class VectorSumCoefficient : public VectorCoefficient
 {
@@ -2112,7 +2183,7 @@ public:
                     const IntegrationPoint &ip);
 };

-/// Matrix coefficient defined as the transpose a matrix coefficient
+/// Matrix coefficient defined as the transpose of a matrix coefficient
 class TransposeMatrixCoefficient : public MatrixCoefficient
 {
 private:
@@ -2135,7 +2206,7 @@ public:
                     const IntegrationPoint &ip);
 };

-/// Matrix coefficient defined as the inverse a matrix coefficient.
+/// Matrix coefficient defined as the inverse of a matrix coefficient.
 class InverseMatrixCoefficient : public MatrixCoefficient
 {
 private:
@@ -2158,6 +2229,29 @@ public:
                     const IntegrationPoint &ip);
 };

+/// Matrix coefficient defined as the exponential of a matrix coefficient.
+class ExponentialMatrixCoefficient : public MatrixCoefficient
+{
+private:
+   MatrixCoefficient * a;
+
+public:
+   /// Construct the matrix coefficient.  Result is $ \exp(A) $.
+   ExponentialMatrixCoefficient(MatrixCoefficient &A);
+
+   /// Set the time for internally stored coefficients
+   void SetTime(real_t t);
+
+   /// Reset the matrix coefficient
+   void SetACoef(MatrixCoefficient &A) { a = &A; }
+   /// Return the matrix coefficient
+   MatrixCoefficient * GetACoef() const { return a; }
+
+   /// Evaluate the matrix coefficient at @a ip.
+   virtual void Eval(DenseMatrix &M, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+};
+
 /// Matrix coefficient defined as the outer product of two vector coefficients.
 class OuterProductCoefficient : public MatrixCoefficient
 {
@@ -1243,25 +1243,16 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
         HypreParMatrix * Ah;
         A_i.Get(Ah);
         hypre_ParCSRMatrix *Aih = *Ah;
-#if !defined(HYPRE_USING_GPU)
-         ess_tdof_list.HostRead();
-         for (int k = 0; k < n; k++)
-         {
-            const int j = ess_tdof_list[k];
-            Aih->diag->data[Aih->diag->i[j]] = 0.0;
-         }
-#else
         Ah->HypreReadWrite();
         const int *d_ess_tdof_list =
-            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
-         const int *d_diag_i = Aih->diag->i;
+            ess_tdof_list.GetMemory().Read(GetHypreMemoryClass(), n);
+         HYPRE_Int *d_diag_i = Aih->diag->i;
         real_t *d_diag_data = Aih->diag->data;
-         MFEM_GPU_FORALL(k, n,
+         mfem::hypre_forall(n, [=] MFEM_HOST_DEVICE (int k)
         {
            const int j = d_ess_tdof_list[k];
            d_diag_data[d_diag_i[j]] = 0.0;
         });
-#endif
      }
      else
      {
@@ -1,8 +1,18 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
 #include "convergence.hpp"

 using namespace std;

-
 namespace mfem
 {

@@ -101,7 +101,7 @@ void DGMassInverse::SetRelTol(const real_t rel_tol_) { rel_tol = rel_tol_; }

 void DGMassInverse::SetAbsTol(const real_t abs_tol_) { abs_tol = abs_tol_; }

-void DGMassInverse::SetMaxIter(const real_t max_iter_) { max_iter = max_iter_; }
+void DGMassInverse::SetMaxIter(const int max_iter_) { max_iter = max_iter_; }

 void DGMassInverse::Update()
 {
@@ -137,7 +137,7 @@ void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const

   const real_t RELTOL = rel_tol;
   const real_t ABSTOL = abs_tol;
-   const real_t MAXIT = max_iter;
+   const int MAXIT = max_iter;
   const bool IT_MODE = iterative_mode;
   const bool CHANGE_BASIS = (d2q != nullptr);

@@ -96,7 +96,7 @@ public:
   /// Set the absolute tolerance.
   void SetAbsTol(const real_t abs_tol_);
   /// Set the maximum number of iterations.
-   void SetMaxIter(const real_t max_iter_);
+   void SetMaxIter(const int max_iter_);
   /// Recompute operator and preconditioner (when coefficient or mesh changes).
   void Update();

@@ -13,6 +13,7 @@
 #define MFEM_FACE_MAP_UTILS_HPP

 #include "../../general/array.hpp"
+#include "../../general/backends.hpp"
 #include <utility> // std::pair
 #include <vector>

@@ -51,6 +52,189 @@ void FillFaceMap(const int n_face_dofs_per_component,
 void GetTensorFaceMap(const int dim, const int order, const int face_id,
                      Array<int> &face_map);

+/// @brief Given a face DOF index in native (counter-clockwise) ordering, return
+/// the corresponding DOF index in lexicographic ordering (for a quadrilateral
+/// element).
+MFEM_HOST_DEVICE
+inline int ToLexOrdering2D(const int face_id, const int size1d, const int i)
+{
+   if (face_id==2 || face_id==3)
+   {
+      return size1d-1-i;
+   }
+   else
+   {
+      return i;
+   }
+}
+
+/// @brief Given a face DOF index on a shared face, ordered lexicographically
+/// relative to element 1, return the corresponding face DOF index ordered
+/// lexicographically relative to element 2.
+MFEM_HOST_DEVICE
+inline int PermuteFace2D(const int face_id1, const int face_id2,
+                         const int orientation, const int size1d,
+                         const int index)
+{
+   int new_index;
+   // Convert from element 1 lex ordering to native ordering
+   if (face_id1 == 2 || face_id1 == 3)
+   {
+      new_index = size1d-1-index;
+   }
+   else
+   {
+      new_index = index;
+   }
+   // Permute based on face orientations
+   if (orientation == 1)
+   {
+      new_index = size1d-1-new_index;
+   }
+   // Covert to element 2 lex ordering
+   return ToLexOrdering2D(face_id2, size1d, new_index);
+}
+
+/// @brief Given a face DOF index in native (counter-clockwise) ordering, return
+/// the corresponding DOF index in lexicographic ordering (for a hexahedral
+/// element).
+MFEM_HOST_DEVICE
+inline int ToLexOrdering3D(const int face_id, const int size1d, const int i,
+                           const int j)
+{
+   if (face_id==2 || face_id==1 || face_id==5)
+   {
+      return i + j*size1d;
+   }
+   else if (face_id==3 || face_id==4)
+   {
+      return (size1d-1-i) + j*size1d;
+   }
+   else // face_id==0
+   {
+      return i + (size1d-1-j)*size1d;
+   }
+}
+
+/// @brief Given the index of a face DOF in lexicographic ordering relative
+/// element 1, permute the index so that it is lexicographically ordered
+/// relative to element 2.
+///
+/// The given face corresponds to local face index @a face_id1 relative to
+/// element 1, and @a face_id2 (with @a orientation) relative to element 2.
+MFEM_HOST_DEVICE
+inline int PermuteFace3D(const int face_id1, const int face_id2,
+                         const int orientation,
+                         const int size1d, const int index)
+{
+   int i=0, j=0, new_i=0, new_j=0;
+   i = index%size1d;
+   j = index/size1d;
+   // Convert from lex ordering
+   if (face_id1==3 || face_id1==4)
+   {
+      i = size1d-1-i;
+   }
+   else if (face_id1==0)
+   {
+      j = size1d-1-j;
+   }
+   // Permute based on face orientations
+   switch (orientation)
+   {
+      case 0:
+         new_i = i;
+         new_j = j;
+         break;
+      case 1:
+         new_i = j;
+         new_j = i;
+         break;
+      case 2:
+         new_i = j;
+         new_j = (size1d-1-i);
+         break;
+      case 3:
+         new_i = (size1d-1-i);
+         new_j = j;
+         break;
+      case 4:
+         new_i = (size1d-1-i);
+         new_j = (size1d-1-j);
+         break;
+      case 5:
+         new_i = (size1d-1-j);
+         new_j = (size1d-1-i);
+         break;
+      case 6:
+         new_i = (size1d-1-j);
+         new_j = i;
+         break;
+      case 7:
+         new_i = i;
+         new_j = (size1d-1-j);
+         break;
+   }
+   return ToLexOrdering3D(face_id2, size1d, new_i, new_j);
+}
+
+/// @brief Given a face DOF (or quadrature) index ordered lexicographically
+/// relative to element 1, return the associated (i, j) coordinates.
+///
+/// The returned coordinates will be relative to element 1 or element 2
+/// according to the value of side (side == 0 corresponds element 1).
+MFEM_HOST_DEVICE
+inline void FaceIdxToVolIdx2D(const int qi, const int nq, const int face_id0,
+                              const int face_id1, const int side, int &i, int &j)
+{
+   // Note: in 2D, a consistently ordered mesh will always have the element 2
+   // face reversed relative to element 1, so orientation is determined entirely
+   // by side. (In 3D, separate orientation information is needed).
+   const int orientation = side;
+
+   const int face_id = (side == 0) ? face_id0 : face_id1;
+   const int edge_idx = (side == 0) ? qi : PermuteFace2D(face_id0, face_id1,
+                                                         orientation, nq, qi);
+
+   const int level = (face_id == 0 || face_id == 3) ? 0 : (nq-1);
+   const bool x_axis = (face_id == 0 || face_id == 2);
+
+   i = x_axis ? edge_idx : level;
+   j = x_axis ? level : edge_idx;
+}
+
+/// @brief Given a face DOF (or quadrature) index ordered lexicographically
+/// relative to element 1, return the associated (i, j, k) coordinates.
+///
+/// The returned coordinates will be relative to element 1 or element 2
+/// according to the value of side (side == 0 corresponds element 1).
+MFEM_HOST_DEVICE
+inline void FaceIdxToVolIdx3D(const int index, const int size1d,
+                              const int face_id0, const int face_id1,
+                              const int side, const int orientation,
+                              int& i, int& j, int& k)
+{
+   MFEM_VERIFY_KERNEL(face_id1 >= 0 || side == 0,
+                      "Accessing second side but face_id1 is not valid.");
+
+   const int face_id = (side == 0) ? face_id0 : face_id1;
+   const int fidx = (side == 0) ? index
+                    : PermuteFace3D(face_id0, face_id1, orientation, size1d, index);
+
+   const bool xy_plane = (face_id == 0 || face_id == 5);
+   const bool yz_plane = (face_id == 2 || face_id == 4);
+
+   const int level = (face_id == 0 || face_id == 1 || face_id == 4)
+                     ? 0 : (size1d-1);
+
+   const int _i = fidx % size1d;
+   const int _j = fidx / size1d;
+
+   k = xy_plane ? level : _j;
+   j = yz_plane ? _i : xy_plane ? _j : level;
+   i = yz_plane ? level : _i;
+}
+
 } // namespace internal

 } // namespace mfem
@@ -316,7 +316,7 @@ public:
   int GetDim() const { return dim; }

   /** @brief Returns the vector dimension for vector-valued finite elements,
-       which is also the dimension of the interpolation operatrion. */
+       which is also the dimension of the interpolation operation. */
   int GetRangeDim() const { return vdim; }

   /// Returns the dimension of the curl for vector-valued finite elements.
@@ -1321,9 +1321,9 @@ void GridFunction::ProjectVectorFieldOn(GridFunction &vec_field, int comp)
   }
 }

-void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
-                                                      GridFunction &der,
-                                                      Array<int> &zones_per_dof)
+void GridFunction::AccumulateAndCountDerivativeValues(
+   int comp, int der_comp, GridFunction &der,
+   Array<int> &zones_per_dof) const
 {
   FiniteElementSpace * der_fes = der.FESpace();
   ElementTransformation * transf;
@@ -1374,7 +1374,8 @@ void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
   }
 }

-void GridFunction::GetDerivative(int comp, int der_comp, GridFunction &der)
+void GridFunction::GetDerivative(int comp, int der_comp,
+                                 GridFunction &der) const
 {
   Array<int> overlap;
   AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
@@ -2061,41 +2062,37 @@ void GridFunction::AccumulateAndCountBdrValues(
   Coefficient *coeff[], VectorCoefficient *vcoeff, const Array<int> &attr,
   Array<int> &values_counter)
 {
-   int i, j, fdof, d, ind, vdim;
-   real_t val;
-   const FiniteElement *fe;
-   ElementTransformation *transf;
   Array<int> vdofs;
   Vector vc;

   values_counter.SetSize(Size());
   values_counter = 0;

-   vdim = fes->GetVDim();
-
+   const int vdim = fes->GetVDim();
   HostReadWrite();

-   for (i = 0; i < fes->GetNBE(); i++)
+   for (int i = 0; i < fes->GetNBE(); i++)
   {
      if (attr[fes->GetBdrAttribute(i) - 1] == 0) { continue; }

-      fe = fes->GetBE(i);
-      fdof = fe->GetDof();
-      transf = fes->GetBdrElementTransformation(i);
+      const FiniteElement *fe = fes->GetBE(i);
+      const int fdof = fe->GetDof();
+      ElementTransformation *transf = fes->GetBdrElementTransformation(i);
      const IntegrationRule &ir = fe->GetNodes();
      fes->GetBdrElementVDofs(i, vdofs);

-      for (j = 0; j < fdof; j++)
+      for (int j = 0; j < fdof; j++)
      {
         const IntegrationPoint &ip = ir.IntPoint(j);
         transf->SetIntPoint(&ip);
         if (vcoeff) { vcoeff->Eval(vc, *transf, ip); }
-         for (d = 0; d < vdim; d++)
+         for (int d = 0; d < vdim; d++)
         {
            if (!vcoeff && !coeff[d]) { continue; }

-            val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
-            if ( (ind = vdofs[fdof*d+j]) < 0 )
+            real_t val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
+            int ind = vdofs[fdof*d+j];
+            if ( ind < 0 )
            {
               val = -val, ind = -1-ind;
            }
@@ -2117,10 +2114,11 @@ void GridFunction::AccumulateAndCountBdrValues(
   // iff A_ij != 0. It is sufficient to resolve just the first level of
   // dependency, since A is a projection matrix: A^n = A due to cR.cP = I.
   // Cases like these arise in 3D when boundary edges are constrained by
-   // (depend on) internal faces/elements. We use the virtual method
-   // GetBoundaryClosure from NCMesh to resolve the dependencies.
-
-   if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3)
+   // (depend on) internal faces/elements, or for internal boundaries in 2 or
+   // 3D. We use the virtual method GetBoundaryClosure from NCMesh to resolve
+   // the dependencies.
+   if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 ||
+                                fes->GetMesh()->Dimension() == 3))
   {
      Vector vals;
      Mesh *mesh = fes->GetMesh();
@@ -2128,26 +2126,19 @@ void GridFunction::AccumulateAndCountBdrValues(
      Array<int> bdr_edges, bdr_vertices, bdr_faces;
      ncmesh->GetBoundaryClosure(attr, bdr_vertices, bdr_edges, bdr_faces);

-      for (i = 0; i < bdr_edges.Size(); i++)
+      auto mark_dofs = [&](ElementTransformation &transf, const FiniteElement &fe)
      {
-         int edge = bdr_edges[i];
-         fes->GetEdgeVDofs(edge, vdofs);
-         if (vdofs.Size() == 0) { continue; }
-
-         transf = mesh->GetEdgeTransformation(edge);
-         transf->Attribute = -1; // TODO: set the boundary attribute
-         fe = fes->GetEdgeElement(edge);
         if (!vcoeff)
         {
-            vals.SetSize(fe->GetDof());
-            for (d = 0; d < vdim; d++)
+            vals.SetSize(fe.GetDof());
+            for (int d = 0; d < vdim; d++)
            {
               if (!coeff[d]) { continue; }

-               fe->Project(*coeff[d], *transf, vals);
+               fe.Project(*coeff[d], transf, vals);
               for (int k = 0; k < vals.Size(); k++)
               {
-                  ind = vdofs[d*vals.Size()+k];
+                  const int ind = vdofs[d*vals.Size()+k];
                  if (++values_counter[ind] == 1)
                  {
                     (*this)(ind) = vals(k);
@@ -2161,11 +2152,11 @@ void GridFunction::AccumulateAndCountBdrValues(
         }
         else // vcoeff != NULL
         {
-            vals.SetSize(vdim*fe->GetDof());
-            fe->Project(*vcoeff, *transf, vals);
+            vals.SetSize(vdim*fe.GetDof());
+            fe.Project(*vcoeff, transf, vals);
            for (int k = 0; k < vals.Size(); k++)
            {
-               ind = vdofs[k];
+               const int ind = vdofs[k];
               if (++values_counter[ind] == 1)
               {
                  (*this)(ind) = vals(k);
@@ -2176,6 +2167,26 @@ void GridFunction::AccumulateAndCountBdrValues(
               }
            }
         }
+      };
+
+      for (auto edge : bdr_edges)
+      {
+         fes->GetEdgeVDofs(edge, vdofs);
+         if (vdofs.Size() == 0) { continue; }
+
+         ElementTransformation *transf = mesh->GetEdgeTransformation(edge);
+         const FiniteElement *fe = fes->GetEdgeElement(edge);
+         mark_dofs(*transf, *fe);
+      }
+
+      for (auto face : bdr_faces)
+      {
+         fes->GetFaceVDofs(face, vdofs);
+         if (vdofs.Size() == 0) { continue; }
+
+         ElementTransformation *transf = mesh->GetFaceTransformation(face);
+         const FiniteElement *fe = fes->GetFaceElement(face);
+         mark_dofs(*transf, *fe);
      }
   }
 }
@@ -2228,26 +2239,37 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
      accumulate_dofs(dofs, lvec, *this, values_counter);
   }

-   if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3)
+   if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 ||
+                                fes->GetMesh()->Dimension() == 3))
   {
      Mesh *mesh = fes->GetMesh();
      NCMesh *ncmesh = mesh->ncmesh;
      Array<int> bdr_edges, bdr_vertices, bdr_faces;
      ncmesh->GetBoundaryClosure(bdr_attr, bdr_vertices, bdr_edges, bdr_faces);

-      for (int i = 0; i < bdr_edges.Size(); i++)
+      for (auto edge : bdr_edges)
      {
-         int edge = bdr_edges[i];
         fes->GetEdgeDofs(edge, dofs);
         if (dofs.Size() == 0) { continue; }

         T = mesh->GetEdgeTransformation(edge);
-         T->Attribute = -1; // TODO: set the boundary attribute
         fe = fes->GetEdgeElement(edge);
         lvec.SetSize(fe->GetDof());
         fe->Project(vcoeff, *T, lvec);
         accumulate_dofs(dofs, lvec, *this, values_counter);
      }
+
+      for (auto face : bdr_faces)
+      {
+         fes->GetFaceDofs(face, dofs);
+         if (dofs.Size() == 0) { continue; }
+
+         T = mesh->GetFaceTransformation(face);
+         fe = fes->GetFaceElement(face);
+         lvec.SetSize(fe->GetDof());
+         fe->Project(vcoeff, *T, lvec);
+         accumulate_dofs(dofs, lvec, *this, values_counter);
+      }
   }
 }

@@ -321,7 +321,7 @@ public:
       @param[out] der       The resulting derivative (scalar function). The
                             FiniteElementSpace of this function must be set
                             before the call. */
-   void GetDerivative(int comp, int der_comp, GridFunction &der);
+   void GetDerivative(int comp, int der_comp, GridFunction &der) const;

   real_t GetDivergence(ElementTransformation &tr) const;

@@ -443,7 +443,7 @@ protected:
       GetDerivative() method; see its documentation. */
   void AccumulateAndCountDerivativeValues(int comp, int der_comp,
                                           GridFunction &der,
-                                           Array<int> &zones_per_dof);
+                                           Array<int> &zones_per_dof) const;

   void AccumulateAndCountBdrValues(Coefficient *coeff[],
                                    VectorCoefficient *vcoeff,
@@ -1352,6 +1352,85 @@ void OversetFindPointsGSLIB::Interpolate(const Vector &point_pos,
   Interpolate(field_in, field_out);
 }

+GSOPGSLIB::GSOPGSLIB(Array<long long> &ids)
+{
+   gsl_comm = new gslib::comm;
+   cr       = new gslib::crystal;
+#ifdef MFEM_USE_MPI
+   int initialized;
+   MPI_Initialized(&initialized);
+   if (!initialized) { MPI_Init(NULL, NULL); }
+   MPI_Comm comm = MPI_COMM_WORLD;
+   comm_init(gsl_comm, comm);
+#else
+   comm_init(gsl_comm, 0);
+#endif
+   crystal_init(cr, gsl_comm);
+   UpdateIdentifiers(ids);
+}
+
+#ifdef MFEM_USE_MPI
+GSOPGSLIB::GSOPGSLIB(MPI_Comm comm_, Array<long long> &ids)
+   : cr(NULL), gsl_comm(NULL)
+{
+   gsl_comm = new gslib::comm;
+   cr      = new gslib::crystal;
+   comm_init(gsl_comm, comm_);
+   crystal_init(cr, gsl_comm);
+   UpdateIdentifiers(ids);
+}
+#endif
+
+GSOPGSLIB::~GSOPGSLIB()
+{
+   crystal_free(cr);
+   gslib_gs_free(gsl_data);
+   comm_free(gsl_comm);
+   delete gsl_comm;
+   delete cr;
+}
+
+void GSOPGSLIB::UpdateIdentifiers(const Array<long long> &ids)
+{
+   long long minval = ids.Min();
+#ifdef MFEM_USE_MPI
+   MPI_Allreduce(MPI_IN_PLACE, &minval, 1, MPI_LONG_LONG_INT,
+                 MPI_MIN, gsl_comm->c);
+#endif
+   MFEM_VERIFY(minval >= 0, "Unique identifier cannot be negative.");
+   if (gsl_data != NULL) { gslib_gs_free(gsl_data); }
+   num_ids = ids.Size();
+   gsl_data = gslib_gs_setup(ids.GetData(),
+                             ids.Size(),
+                             gsl_comm, 0,
+                             gslib::gs_crystal_router, 0);
+}
+
+void GSOPGSLIB::GS(Vector &senddata, GSOp op)
+{
+   MFEM_VERIFY(senddata.Size() == num_ids,
+               "Incompatible setup and GOP operation.");
+   if (op == GSOp::ADD)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_add,0,gsl_data,0);
+   }
+   else if (op == GSOp::MUL)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_mul,0,gsl_data,0);
+   }
+   else if (op == GSOp::MAX)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_max,0,gsl_data,0);
+   }
+   else if (op == GSOp::MIN)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_min,0,gsl_data,0);
+   }
+   else
+   {
+      MFEM_ABORT("Invalid GSOp operation.");
+   }
+}

 } // namespace mfem

@@ -23,13 +23,16 @@ struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
 struct crystal;
+struct gs_data;
 }

 namespace mfem
 {

 /** \brief FindPointsGSLIB can robustly evaluate a GridFunction on an arbitrary
- *  collection of points. There are three key functions in FindPointsGSLIB:
+ *  collection of points.
+ *
+ *  There are three key functions in FindPointsGSLIB:
 *
 *  1. Setup - constructs the internal data structures of gslib.
 *
@@ -226,6 +229,7 @@ public:

 /** \brief OversetFindPointsGSLIB enables use of findpts for arbitrary number of
    overlapping grids.
+
    The parameters in this class are the same as FindPointsGSLIB with the
    difference of additional inputs required to account for more than 1 mesh. */
 class OversetFindPointsGSLIB : public FindPointsGSLIB
@@ -290,6 +294,63 @@ public:
   using FindPointsGSLIB::Interpolate;
 };

+/** \brief  Class for gather-scatter (gs) operations on Vectors based on
+    corresponding global identifiers.
+
+    This functionality is useful for gs-ops on DOF values across processor
+    boundary, where the global identifier would be the corresponding true DOF
+    index. Operations currently supported are min, max, sum, and multiplication.
+    Note: identifier 0 does not participate in the gather-scatter operation and
+    a given identifier can be included multiple times on a given rank.
+    For example, consider a vector, v:
+    - v = [0.3, 0.4, 0.25, 0.7] on rank1,
+    - v = [0.6, 0.1] on rank 2,
+    - v = [-0.2, 0.3, 0.7, 0.] on rank 3.
+
+    Consider a corresponding Array<int>, a:
+    - a = [1, 2, 3, 1] on rank 1,
+    - a = [3, 2] on rank 2,
+    - a = [1, 2, 0, 3] on rank 3.
+
+    A gather-scatter "minimum" operation, done as follows:
+    GSOPGSLIB gs = GSOPGSLIB(MPI_COMM_WORLD, a);
+    gs.GS(v, GSOp::MIN);
+    would return into v:
+    - v = [-0.2, 0.1, 0., -0.2] on rank 1,
+    - v = [0., 0.1] on rank 2,
+    - v = [-0.2, 0.1, 0.7, 0.] on rank 3,
+    where the values have been compared across all processors based on the
+    integer identifier. */
+class GSOPGSLIB
+{
+protected:
+   struct gslib::crystal *cr;               // gslib's internal data
+   struct gslib::comm *gsl_comm;            // gslib's internal data
+   struct gslib::gs_data *gsl_data = NULL;
+   int num_ids;
+
+public:
+   GSOPGSLIB(Array<long long> &ids);
+
+#ifdef MFEM_USE_MPI
+   GSOPGSLIB(MPI_Comm comm_, Array<long long> &ids);
+#endif
+
+   virtual ~GSOPGSLIB();
+
+   /// Supported operation types. See class description.
+   enum GSOp {ADD, MUL, MIN, MAX};
+
+   /// Update the identifiers used for the gather-scatter operator.
+   /// Same @a ids get grouped together and id == 0 does not participate.
+   /// See class description.
+   void UpdateIdentifiers(const Array<long long> &ids);
+
+   /// Gather-Scatter operation on senddata. Must match length of unique
+   /// identifiers used in the constructor. See class description.
+   void GS(Vector &senddata, GSOp op);
+};
+
 } // namespace mfem

 #endif // MFEM_USE_GSLIB
@@ -18,7 +18,6 @@
 namespace mfem
 {

-
 void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
                                                     ElementTransformation &Tr,
                                                     const Vector &elfun,
@@ -29,7 +28,7 @@ void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
   const int dof = el.GetDof();

 #ifdef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration

   // shape function value at an integration point
   Vector shape(dof);
@@ -62,7 +61,7 @@ void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
      ir = &IntRules.Get(Tr.GetGeometryType(), order);
   }

-   // loop over interation points
+   // loop over integration points
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
@@ -92,7 +91,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
   const int dof2 = el2.GetDof();

 #ifdef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration

   // shape function value at an integration point - first elem
   Vector shape1(dof1);
@@ -122,7 +121,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
   DenseMatrix elvect2_mat(elvect.GetData() + dof1 * num_equations, dof2,
                           num_equations);

-   // obtain integration rule. If integration is rule is given, then use it.
+   // Obtain integration rule. If integration is rule is given, then use it.
   // Otherwise, get (2*p + IntOrderOffset) order integration rule
   const IntegrationRule *ir = IntRule;
   if (!ir)
@@ -149,7 +148,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
      if (nor.Size() == 1)  // if 1D, use 1 or -1.
      {
         // This assume the 1D integration point is in (0,1). This may not work
-         // if this chages.
+         // if this changes.
         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
      }
      else
@@ -18,43 +18,36 @@
 namespace mfem
 {

-//                     MFEM Hyperbolic Conservation Laws
+// This file contains general hyperbolic conservation element/face form
+// integrators. HyperbolicFormIntegrator and RiemannSolver are defined.
 //
-// Description:
+// HyperbolicFormIntegrator is a NonlinearFormIntegrator that implements
+// element weak divergence and interface flux
 //
-//    This file contains general hyperbolic conservation element/face form
-//    integrators.
+//     ∫_T F(u):∇v,   -∫_e F̂(u)⋅[[v]]
 //
-//    HyperbolicFormIntegrator and RiemannSolver are defined.
-//    HyperbolicFormIntegrator is a NonlinearFormIntegrator that implements
-//    element weak divergence and interface flux
+// Here, T is an element, e is an edge, and [[⋅]] is jump. This form integrator
+// is coupled with RiemannSolver that implements the numerical flux F̂. For
+// RiemannSolver, the Rusanov flux, also known as local Lax-Friedrichs flux, is
+// provided.
 //
-//       ∫_T F(u):∇v,   -∫_e F̂(u)⋅[[v]]
+// To implement a specific hyperbolic conservation laws, users can create
+// derived classes from FluxFunction with overloaded ComputeFlux. One can
+// optionally overload ComputeFluxDotN to avoid creating dense matrix when
+// computing normal flux. Several example equations are also defined including:
+// advection, Burgers', shallow water, and Euler equations. Users can control
+// the quadrature rule by either providing the integration rule, or integration
+// order offset. Integration will use 2*p + IntOrderOffset order quadrature
+// rule.
 //
-//    Here, T is an element, e is an edge, and [[⋅]] is jump. This form
-//    integrator is coupled with RiemannSolver that implements the numerical
-//    flux F̂. For RiemannSolver, the Rusanov flux, also known as local
-//    Lax-Friedrichs flux, is provided.
-//
-//    To implement a specific hyperbolic conservation laws, users can create
-//    derived classes from FluxFunction with overloaded ComputeFlux. One can
-//    optionally overload ComputeFluxDotN to avoid creating dense matrix when
-//    computing normal flux. Several example equations are also defined
-//    including: advection, Burgers', shallow water, and Euler equations. Users
-//    can control the quadrature rule by either providing the integration rule,
-//    or integration order offset. Integration will use 2*p + IntOrderOffset
-//    order quadrature rule.
-//
-//    At each call of HyperbolicFormIntegrator::AssembleElementVector
-//    HyperbolicFormIntegrator::AssembleFaceVector, the maximum characteristic
-//    speed will be updated. This will not be reinitialized automatically.
-//    To reinitialize, use HyperbolicFormIntegrator::ResetMaxCharSpeed. See,
-//    ex18.hpp.
-//
-//    Note: To avoid communication overhead, we update the maximum
-//    characteristic speed within each process. Use a proper MPI routine to
-//    gather the information.
+// At each call of HyperbolicFormIntegrator::AssembleElementVector
+// HyperbolicFormIntegrator::AssembleFaceVector, the maximum characteristic
+// speed will be updated. This will not be reinitialized automatically. To
+// reinitialize, use HyperbolicFormIntegrator::ResetMaxCharSpeed. See, ex18.hpp.
 //
+// Note: To avoid communication overhead, we update the maximum characteristic
+// speed within each MPI process only. Use the appropriate MPI routine to gather
+// the information.

 /**
 * @brief Abstract class for hyperbolic flux for a system of hyperbolic
@@ -88,7 +81,7 @@ public:
   virtual real_t ComputeFlux(const Vector &state, ElementTransformation &Tr,
                              DenseMatrix &flux) const = 0;
   /**
-    * @brief Compute normal flux. Optionally overloadded in the
+    * @brief Compute normal flux. Optionally overloaded in the
    * derived class to avoid creating full dense matrix for flux.
    *
    * @param[in] state state at the current integration point
@@ -168,13 +161,13 @@ protected:
 class HyperbolicFormIntegrator : public NonlinearFormIntegrator
 {
 private:
-   // The maximum characterstic speed, updated during element/face vector assembly
+   // The maximum characteristic speed, updated during element/face vector assembly
   real_t max_char_speed;
   const RiemannSolver &rsolver;   // Numerical flux that maps F(u±,x) to hat(F)
   const FluxFunction &fluxFunction;
   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
 #ifndef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration
   Vector shape;              // shape function value at an integration point
   Vector state;              // state value at an integration point
   DenseMatrix flux;          // flux value at an integration point
@@ -563,7 +563,7 @@ void DiffusionIntegrator::AssemblePatchMatrix_fullQuadrature(
   cdofs.SetSize(maxw[0], maxw[1], maxw[2]);

   // Compute sparsity of the sparse matrix
-   smati = new int[ndof+1];
+   smati = Memory<int>(ndof+1);
   smati[0] = 0;

   for (int dof_j=0; dof_j<ndof; ++dof_j)
@@ -586,8 +586,8 @@ void DiffusionIntegrator::AssemblePatchMatrix_fullQuadrature(
      nnz += ndd;
   }

-   smatj = new int[nnz];
-   smata = new real_t[nnz];
+   smatj = Memory<int>(nnz);
+   smata = Memory<real_t>(nnz);

   for (int i=0; i<nnz; ++i)
   {
@@ -973,7 +973,7 @@ void DiffusionIntegrator::AssemblePatchMatrix_reducedQuadrature(
   cdofs.SetSize(maxw[0], maxw[1], maxw[2]);

   // Compute sparsity of the sparse matrix
-   smati = new int[ndof+1];
+   smati = Memory<int>(ndof+1);
   smati[0] = 0;

   for (int dof_j=0; dof_j<ndof; ++dof_j)
@@ -996,8 +996,8 @@ void DiffusionIntegrator::AssemblePatchMatrix_reducedQuadrature(
      nnz += ndd;
   }

-   smatj = new int[nnz];
-   smata = new real_t[nnz];
+   smatj = Memory<int>(nnz);
+   smata = Memory<real_t>(nnz);

   for (int i=0; i<nnz; ++i)
   {
@@ -157,7 +157,7 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
   static constexpr int aSize = aUpper-aLower;
   static constexpr bool isComponent = (i_block >= 0);

-   //Assuming all elements are the same
+   // Assuming all elements are the same
   const auto &ir = QVec.GetIntRule(0);
   const QuadratureInterpolator *E_To_Q_Map = fespace.GetQuadratureInterpolator(
                                                 ir);
@@ -180,7 +180,7 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
         auto invJ = inv(make_tensor<d, d>(
         [&](int i, int j) { return J(p, i, j, e); }));
         tensor<real_t, aSize, d> gradx;
-         //load grad(x) into gradx
+         // load grad(x) into gradx
         if (isComponent)
         {
            for (int i = 0; i < d; i++)
@@ -198,11 +198,11 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
               }
            }
         }
-         //compute divergence
+         // compute divergence
         real_t div = 0.;
         for (int i = aLower; i < aUpper; i++)
         {
-            //take size of gradx into account
+            // take size of gradx into account
            const int iIndex = isComponent ? 0 : i;
            div += gradx(iIndex,i);
         }
@@ -211,11 +211,11 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
         {
            for (int q = qLower; q < qUpper; q++)
            {
-               //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-               //this contraction could be made slightly cheaper using Voigt
-               //notation, but repeated entries are summed for simplicity.
+               // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+               // this contraction could be made slightly cheaper using Voigt
+               // notation, but repeated entries are summed for simplicity.
               real_t contraction = 0.;
-               //not sure how to combine cases
+               // not sure how to combine cases
               if (isComponent)
               {
                  for (int a = 0; a < d; a++)
@@ -276,7 +276,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
                                   const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
-   //Assuming all elements are the same
+   // Assuming all elements are the same
   const auto &ir = QVec.GetIntRule(0);
   static constexpr int d = dim;
   const int numPoints = ir.GetNPoints();
@@ -299,9 +299,9 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
            {
               for (int q = 0; q < d; q++)
               {
-                  //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-                  //this contraction could be made slightly cheaper using Voigt
-                  //notation, but repeated entries are summed for simplicity.
+                  // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                  // this contraction could be made slightly cheaper using Voigt
+                  // notation, but repeated entries are summed for simplicity.
                  real_t contraction = 0.;
                  for (int a = 0; a < d; a++)
                  {
@@ -321,7 +321,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
      }
   });

-   //Reduce quadrature function to an E-Vector
+   // Reduce quadrature function to an E-Vector
   const auto QRead = Reshape(QVec.Read(), numPoints, d, d, d, numEls);
   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
@@ -348,7 +348,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
   });
 }

-//Templated implementation of ElasticityAssembleEA.
+// Templated implementation of ElasticityAssembleEA.
 template<int dim>
 void ElasticityAssembleEA_(const int i_block,
                           const int j_block,
@@ -360,7 +360,7 @@ void ElasticityAssembleEA_(const int i_block,
                           const DofToQuad &maps,
                           Vector &emat)
 {
-   //Assuming all elements are the same
+   // Assuming all elements are the same
   static constexpr int d = dim;
   const int numPoints = ir.GetNPoints();
   const int numEls = lambda.Size()/numPoints;
@@ -386,7 +386,7 @@ void ElasticityAssembleEA_(const int i_block,
               {
                  for (int m = 0; m < d; m++)
                  {
-                     //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                     // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
                     real_t contraction = 0.;
                     for (int a = 0; a < d; a++)
                     {
@@ -101,7 +101,7 @@ void MomentFittingIntRules::InitVolume(int order, Coefficient& levelset,
         }
      }

-      // assamble the matrix
+      // assemble the matrix
      DenseMatrix Mat(nBasisVolume, ir.GetNPoints());
      for (int ip = 0; ip < ir.GetNPoints(); ip++)
      {
@@ -118,7 +118,7 @@ void MomentFittingIntRules::InitVolume(int order, Coefficient& levelset,
         Mat.SetCol(ip, shape);
      }

-      // compute the svd for the matrix
+      // compute the SVD for the matrix
      VolumeSVD = new DenseMatrixSVD(Mat, 'A', 'A');
      VolumeSVD->Eval(Mat);
   }
@@ -1239,7 +1239,7 @@ void MomentFittingIntRules::OrthoBasis2D(const IntegrationPoint& ip,

   shape.SetSize(nBasis, 2);

-   // evaluate basis inthe point
+   // evaluate basis in the point
   DenseMatrix preshape(nBasis, 2);
   DivFreeBasis2D(ip, shape);

@@ -1597,6 +1597,6 @@ void MomentFittingIntRules::GetSurfaceWeights(ElementTransformation& Tr,
   }
 }

-#endif //MFEM_USE_LAPACK
+#endif // MFEM_USE_LAPACK

 }
@@ -291,7 +291,7 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   const auto ltdof_ldof = HypreRead(R->GetMemoryJ());

   // Go from E-vector format directly to T-vector format
-   MFEM_HYPRE_FORALL(i, ntdofs,
+   mfem::hypre_forall(ntdofs, [=] MFEM_HOST_DEVICE (int i)
   {
      const int j = d_offsets[ltdof_ldof[i]];
      for (int c = 0; c < sdim; ++c)
@@ -269,13 +269,13 @@ void BatchedLOR_H1::Assemble3D()
               real_t vx[8], vy[8], vz[8];
               LORVertexCoordinates3D<ORDER>(X, iel_ho, kx, ky, kz, vx, vy, vz);

-               //MFEM_UNROLL(2)
+               // MFEM_UNROLL(2)
               for (int iqz=0; iqz<2; ++iqz)
               {
-                  //MFEM_UNROLL(2)
+                  // MFEM_UNROLL(2)
                  for (int iqy=0; iqy<2; ++iqy)
                  {
-                     //MFEM_UNROLL(2)
+                     // MFEM_UNROLL(2)
                     for (int iqx=0; iqx<2; ++iqx)
                     {
                        const real_t x = iqx;
@@ -307,21 +307,21 @@ void BatchedLOR_H1::Assemble3D()
                  }
               }

-               //MFEM_UNROLL(2)
+               // MFEM_UNROLL(2)
               for (int iqx=0; iqx<2; ++iqx)
               {
-                  //MFEM_UNROLL(2)
+                  // MFEM_UNROLL(2)
                  for (int jz=0; jz<2; ++jz)
                  {
                     // Note loop starts at iz=jz here, taking advantage of
                     // symmetries.
-                     //MFEM_UNROLL(2)
+                     // MFEM_UNROLL(2)
                     for (int iz=jz; iz<2; ++iz)
                     {
-                        //MFEM_UNROLL(2)
+                        // MFEM_UNROLL(2)
                        for (int iqy=0; iqy<2; ++iqy)
                        {
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int iqz=0; iqz<2; ++iqz)
                           {
                              const real_t mq = const_mq ? MQ(0,0,0,0) : MQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
@@ -356,10 +356,10 @@ void BatchedLOR_H1::Assemble3D()
                              real_t wdetJ = Q(6,iqz,iqy,iqx);
                              mass_A(iqy,iz,jz,iqx) += mq*wdetJ*biz*bjz;
                           }
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int jy=0; jy<2; ++jy)
                           {
-                              //MFEM_UNROLL(2)
+                              // MFEM_UNROLL(2)
                              for (int iy=0; iy<2; ++iy)
                              {
                                 const real_t biy = (iy == iqy) ? 1.0 : 0.0;
@@ -382,16 +382,16 @@ void BatchedLOR_H1::Assemble3D()
                              }
                           }
                        }
-                        //MFEM_UNROLL(2)
+                        // MFEM_UNROLL(2)
                        for (int jy=0; jy<2; ++jy)
                        {
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int jx=0; jx<2; ++jx)
                           {
-                              //MFEM_UNROLL(2)
+                              // MFEM_UNROLL(2)
                              for (int iy=0; iy<2; ++iy)
                              {
-                                 //MFEM_UNROLL(2)
+                                 // MFEM_UNROLL(2)
                                 for (int ix=0; ix<2; ++ix)
                                 {
                                    const real_t bix = (ix == iqx) ? 1.0 : 0.0;
@@ -431,7 +431,7 @@ void BatchedLOR_H1::Assemble3D()
               // Assemble the local matrix into the macro-element sparse matrix
               // in a format similar to coordinate format. The (I,J) arrays
               // are implicit (not stored explicitly).
-               //MFEM_UNROLL(8)
+               // MFEM_UNROLL(8)
               for (int ii_loc=0; ii_loc<nv; ++ii_loc)
               {
                  const int ix = ii_loc%2;
@@ -0,0 +1,828 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "normal_deriv_restriction.hpp"
+#include "fespace.hpp"
+#include "pgridfunc.hpp"
+#include "fe/face_map_utils.hpp"
+#include "../general/forall.hpp"
+
+namespace mfem
+{
+
+/// Compute the face index to volume index map "face_to_vol" in 2D
+static void NormalDerivativeSetupFaceIndexMap2D(
+   int nf, int d, const Array<int>& face_to_elem, Array<int>& face_to_vol)
+{
+   const auto f2e = Reshape(face_to_elem.HostRead(), 2, 2, nf);
+   auto f2v = Reshape(face_to_vol.HostWrite(), d, 2, nf);
+
+   for (int f = 0; f < nf; ++f)
+   {
+      const int fid0 = f2e(0, 1, f);
+      const int fid1 = f2e(1, 1, f);
+      for (int side = 0; side < 2; ++side)
+      {
+         const int el = f2e(side, 0, f);
+
+         if (el < 0)
+         {
+            for (int p = 0; p < d; ++p)
+            {
+               f2v(p, side, f) = -1;
+            }
+         }
+         else
+         {
+            for (int p = 0; p < d; ++p)
+            {
+               int i, j;
+               internal::FaceIdxToVolIdx2D(p, d, fid0, fid1, side, i, j);
+
+               f2v(p, side, f) = i + d * j;
+            }
+         }
+      }
+   }
+}
+
+/// Compute the face index to volume index map "face_to_vol" in 3D
+static void NormalDerivativeSetupFaceIndexMap3D(
+   int nf, int d, const Array<int>& face_to_elem, Array<int>& face_to_vol)
+{
+   const auto f2e = Reshape(face_to_elem.HostRead(), 2, 3, nf);
+   auto f2v = Reshape(face_to_vol.HostWrite(), d*d, 2, nf);
+
+   for (int f = 0; f < nf; ++f)
+   {
+      const int fid0 = f2e(0, 1, f);
+      const int fid1 = f2e(1, 1, f);
+      for (int side = 0; side < 2; ++side)
+      {
+         const int el = f2e(side, 0, f);
+         const int orientation = f2e(side, 2, f);
+
+         if (el < 0)
+         {
+            for (int p = 0; p < d*d; ++p)
+            {
+               f2v(p, side, f) = -1;
+            }
+         }
+         else
+         {
+            for (int p = 0; p < d*d; ++p)
+            {
+               int i, j, k; // 3D lexicographic index of quad point p
+               internal::FaceIdxToVolIdx3D(p, d, fid0, fid1, side, orientation, i, j, k);
+
+               f2v(p, side, f) = i + d * (j + d * k);
+            }
+         }
+      }
+   }
+}
+
+L2NormalDerivativeFaceRestriction::L2NormalDerivativeFaceRestriction(
+   const FiniteElementSpace &fes_,
+   const ElementDofOrdering f_ordering,
+   const FaceType face_type_)
+   : fes(fes_),
+     face_type(face_type_),
+     dim(fes.GetMesh()->Dimension()),
+     nf(fes.GetNFbyType(face_type)),
+     ne(fes.GetNE())
+{
+   MFEM_VERIFY(f_ordering == ElementDofOrdering::LEXICOGRAPHIC,
+               "Non-lexicographic ordering not currently supported in "
+               "L2NormalDerivativeFaceRestriction.");
+
+   Mesh &mesh = *fes.GetMesh();
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const int d = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR).ndof;
+
+   if (dim == 2)
+   {
+      // (el0, el1, fid0, fid1)
+      face_to_elem.SetSize(nf * 4);
+      face_to_vol.SetSize(2 * nf * d);
+   }
+   else if (dim == 3)
+   {
+      // (el0, el1, fid0, fid1, or0, or1)
+      face_to_elem.SetSize(nf * 6);
+      face_to_vol.SetSize(2 * nf * d * d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension.");
+   }
+   auto f2e = Reshape(face_to_elem.HostWrite(), 2, (dim == 2) ? 2 : 3, nf);
+
+   // Populate the face_to_elem array. The elem_indicator will be used to count
+   // the number of elements that are adjacent to faces of the given type.
+   Array<int> elem_indicator(ne);
+   elem_indicator = 0;
+
+   int f_ind = 0;
+   for (int f = 0; f < fes.GetNF(); ++f)
+   {
+      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
+
+      if (face.IsOfFaceType(face_type))
+      {
+         f2e(0, 0, f_ind) = face.element[0].index;
+         f2e(0, 1, f_ind) = face.element[0].local_face_id;
+         if (dim == 3)
+         {
+            f2e(0, 2, f_ind) = face.element[0].orientation;
+         }
+
+         elem_indicator[face.element[0].index] = 1;
+
+         if (face_type == FaceType::Interior)
+         {
+            const int el_idx_1 = face.element[1].index;
+            if (face.IsShared())
+            {
+               // Indicate shared face by index >= ne
+               f2e(1, 0, f_ind) = ne + el_idx_1;
+            }
+            else
+            {
+               // Face is not shared
+               f2e(1, 0, f_ind) = el_idx_1;
+               elem_indicator[el_idx_1] = 1;
+            }
+            f2e(1, 1, f_ind) = face.element[1].local_face_id;
+
+            if (dim == 3)
+            {
+               f2e(1, 2, f_ind) = face.element[1].orientation;
+            }
+         }
+         else
+         {
+            f2e(1, 0, f_ind) = -1;
+            f2e(1, 1, f_ind) = -1;
+
+            if (dim == 3)
+            {
+               f2e(1, 2, f_ind) = -1;
+            }
+         }
+
+         f_ind++;
+      }
+   }
+
+   // evaluate face to vol map
+   if (dim == 2)
+   {
+      NormalDerivativeSetupFaceIndexMap2D(nf, d, face_to_elem, face_to_vol);
+   }
+   else if (dim == 3)
+   {
+      NormalDerivativeSetupFaceIndexMap3D(nf, d, face_to_elem, face_to_vol);
+   }
+
+   // Number of elements adjacent to faces of face_type
+   ne_type = elem_indicator.Sum();
+
+   // In 2D: (el, f0,f1,f2,f3, s0,s1,s2,s3)
+   // In 3D: (el, f0,f1,f2,f3,f4,f5, s0,s1,s2,s3,s4,s5)
+   const int elem_data_sz = (dim == 2) ? 9 : 13;
+
+   elem_to_face.SetSize(elem_data_sz * ne_type);
+   elem_to_face = -1;
+
+   auto e2f = Reshape(elem_to_face.HostWrite(), elem_data_sz, ne_type);
+   elem_indicator.PartialSum();
+
+   const int nsides = (face_type == FaceType::Interior) ? 2 : 1;
+   const int side_begin = (dim == 2) ? 5 : 7;
+   for (int f = 0; f < nf; ++f)
+   {
+      for (int side = 0; side < nsides; ++side)
+      {
+         const int el = f2e(side, 0, f);
+         // Skip shared faces
+         if (el < ne)
+         {
+            const int face_id = f2e(side, 1, f);
+
+            const int e = elem_indicator[el] - 1;
+            e2f(0, e) = el;
+            e2f(1 + face_id, e) = f;
+            e2f(side_begin + face_id, e) = side;
+         }
+      }
+   }
+}
+
+void L2NormalDerivativeFaceRestriction::Mult(const Vector &x, Vector &y) const
+{
+   if (nf == 0) { return; }
+   switch (dim)
+   {
+      case 2:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: Mult2D<1>(x, y); break;
+            case 2: Mult2D<2>(x, y); break;
+            case 3: Mult2D<3>(x, y); break;
+            case 4: Mult2D<4>(x, y); break;
+            case 5: Mult2D<5>(x, y); break;
+            case 6: Mult2D<6>(x, y); break;
+            case 7: Mult2D<7>(x, y); break;
+            case 8: Mult2D<8>(x, y); break;
+            default: Mult2D(x, y); break;
+         }
+      }
+      break;
+      case 3:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: Mult3D<1>(x, y); break;
+            case 2: Mult3D<2>(x, y); break;
+            case 3: Mult3D<3>(x, y); break;
+            case 4: Mult3D<4>(x, y); break;
+            case 5: Mult3D<5>(x, y); break;
+            case 6: Mult3D<6>(x, y); break;
+            case 7: Mult3D<7>(x, y); break;
+            case 8: Mult3D<8>(x, y); break;
+            default: Mult3D(x, y); break; // fallback
+         }
+         break;
+      }
+      default: MFEM_ABORT("Dimension not supported."); break;
+   }
+}
+
+void L2NormalDerivativeFaceRestriction::AddMultTranspose(
+   const Vector &x, Vector &y, const real_t a) const
+{
+   if (nf == 0) { return; }
+   switch (dim)
+   {
+      case 2:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: AddMultTranspose2D<1>(x, y, a); break;
+            case 2: AddMultTranspose2D<2>(x, y, a); break;
+            case 3: AddMultTranspose2D<3>(x, y, a); break;
+            case 4: AddMultTranspose2D<4>(x, y, a); break;
+            case 5: AddMultTranspose2D<5>(x, y, a); break;
+            case 6: AddMultTranspose2D<6>(x, y, a); break;
+            case 7: AddMultTranspose2D<7>(x, y, a); break;
+            case 8: AddMultTranspose2D<8>(x, y, a); break;
+            default: AddMultTranspose2D(x, y, a); break;
+         }
+      }
+      break;
+      case 3:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: AddMultTranspose3D<1>(x, y, a); break;
+            case 2: AddMultTranspose3D<2>(x, y, a); break;
+            case 3: AddMultTranspose3D<3>(x, y, a); break;
+            case 4: AddMultTranspose3D<4>(x, y, a); break;
+            case 5: AddMultTranspose3D<5>(x, y, a); break;
+            case 6: AddMultTranspose3D<6>(x, y, a); break;
+            case 7: AddMultTranspose3D<7>(x, y, a); break;
+            case 8: AddMultTranspose3D<8>(x, y, a); break;
+            default: AddMultTranspose3D(x, y, a); break; // fallback
+         }
+         break;
+      }
+      default: MFEM_ABORT("Not yet implemented"); break;
+   }
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::Mult2D(const Vector &x, Vector &y) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+   const int num_elem = ne;
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+
+   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, face_type);
+   const int ne_shared = face_nbr_data.Size() / d / d / vd;
+
+   MFEM_VERIFY(q == d, "");
+   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
+
+   // derivative of 1D basis function
+   const auto G_ = Reshape(maps.G.Read(), q, d);
+   // (el0, el1, fid0, fid1)
+   const auto f2e = Reshape(face_to_elem.Read(), 2, 2, nf);
+
+   const auto f2v = Reshape(face_to_vol.Read(), q, 2, nf);
+
+   // if byvdim, d_x has shape (vdim, nddof, nddof, ne)
+   // otherwise, d_x has shape (nddof, nddof, ne, vdim)
+   const auto d_x = Reshape(x.Read(), t?vd:d, d, t?d:ne, t?ne:vd);
+   const auto d_x_shared = Reshape(face_nbr_data.Read(),
+                                   t?vd:d, d, t?d:ne_shared, t?ne_shared:vd);
+   auto d_y = Reshape(y.Write(), q, vd, 2, nf);
+
+   mfem::forall_2D(nf, 2, q, [=] MFEM_HOST_DEVICE (int f) -> void
+   {
+      constexpr int MD = (T_D1D) ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED real_t G_s[MD*MD];
+      DeviceMatrix G(G_s, q, d);
+
+      MFEM_SHARED int E[2];
+      MFEM_SHARED int FID[2];
+      MFEM_SHARED int F2V[2][MD];
+
+      if (MFEM_THREAD_ID(x) == 0)
+      {
+         MFEM_FOREACH_THREAD(j, y, d)
+         {
+            for (int i = 0; i < q; ++i)
+            {
+               G(i, j) = G_(i, j);
+            }
+         }
+      }
+
+      MFEM_FOREACH_THREAD(side, x, 2)
+      {
+         if (MFEM_THREAD_ID(y) == 0)
+         {
+            E[side] = f2e(side, 0, f);
+            FID[side] = f2e(side, 1, f);
+         }
+
+         MFEM_FOREACH_THREAD(j, y, d)
+         {
+            F2V[side][j] = f2v(j, side, f);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, x, 2)
+      {
+         const int el = E[side];
+         const bool shared = (el >= num_elem);
+         const auto &d_x_e = shared ? d_x_shared : d_x;
+         const int el_idx = shared ? el - num_elem : el;
+
+         const int face_id = FID[side];
+
+         MFEM_FOREACH_THREAD(p, y, q)
+         {
+            if (el < 0)
+            {
+               for (int c = 0; c < vd; ++c)
+               {
+                  d_y(p, c, side, f) = 0.0;
+               }
+            }
+            else
+            {
+               const int ij = F2V[side][p];
+               const int i = ij % q;
+               const int j = ij / q;
+
+               for (int c=0; c < vd; ++c)
+               {
+                  real_t grad_n = 0;
+                  for (int kk=0; kk < d; ++kk)
+                  {
+                     const int k = (face_id == 0 || face_id == 2) ? i : kk;
+                     const int l = (face_id == 0 || face_id == 2) ? kk : j;
+                     const real_t g = (face_id == 0 || face_id == 2) ? G(j,l) : G(i,k);
+                     grad_n += g * d_x_e(t?c:k, t?k:l, t?l:el_idx, t?el_idx:c);
+                  }
+                  d_y(p, c, side, f) = grad_n;
+               }
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::Mult3D(const Vector &x, Vector &y) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+   const int num_elem = ne;
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+   const int q2d = q * q;
+
+   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, face_type);
+   const int ne_shared = face_nbr_data.Size() / d / d / d / vd;
+
+   MFEM_VERIFY(q == d, "");
+   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
+
+   const auto G_ = Reshape(maps.G.Read(), q, d);
+   // (el0, el1, fid0, fid1, or0, or1)
+   const auto f2e = Reshape(face_to_elem.Read(), 2, 3, nf);
+   const auto f2v = Reshape(face_to_vol.Read(), q2d, 2, nf);
+
+   // t ? (vdim, d, d, d, ne) : (d, d, d, ne, vdim)
+   const auto d_x = Reshape(x.Read(), t?vd:d, d, d, t?d:ne, t?ne:vd);
+   const auto d_x_shared = Reshape(face_nbr_data.Read(),
+                                   t?vd:d, d, d, t?d:ne_shared, t?ne_shared:vd);
+   auto d_y = Reshape(y.Write(), q2d, vd, 2, nf);
+
+   mfem::forall_2D(nf, q2d, 2, [=] MFEM_HOST_DEVICE (int f) -> void
+   {
+      static constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED real_t G_s[MD*MD];
+      DeviceMatrix G(G_s, d, q);
+
+      MFEM_SHARED int E[2];
+      MFEM_SHARED int FID[2];
+      MFEM_SHARED int F2V[2][MD*MD];
+
+      // Load G matrix into shared memory
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         MFEM_FOREACH_THREAD(j, x, d*q)
+         {
+            const int p = j % q;
+            const int k = j / q;
+            G(k, p) = G_(p, k);
+         }
+      }
+
+      MFEM_FOREACH_THREAD(side, y, 2)
+      {
+         if (MFEM_THREAD_ID(x) == 0)
+         {
+            E[side] = f2e(side, 0, f);
+            FID[side] = f2e(side, 1, f);
+         }
+         MFEM_FOREACH_THREAD(j, x, q2d)
+         {
+            F2V[side][j] = f2v(j, side, f);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, y, 2)
+      {
+         const int el = E[side];
+         const bool shared = (el >= num_elem);
+         const auto &d_x_e = shared ? d_x_shared : d_x;
+         const int el_idx = shared ? el - num_elem : el;
+
+         const int face_id = FID[side];
+
+         // Is this face parallel to the x-y plane in reference coordinates?
+         const bool xy_plane = (face_id == 0 || face_id == 5);
+         const bool xz_plane = (face_id == 1 || face_id == 3);
+         const bool yz_plane = (face_id == 2 || face_id == 4);
+
+         MFEM_FOREACH_THREAD(p, x, q2d)
+         {
+            if (el_idx < 0)
+            {
+               for (int c = 0; c < vd; ++c)
+               {
+                  d_y(p, c, side, f) = 0.0;
+               }
+            }
+            else
+            {
+               const int ijk = F2V[side][p];
+               const int k = ijk / q2d;
+               const int i = ijk % q;
+               const int j = (ijk - q2d*k) / q;
+
+               // the fixed 1D index of the normal component of the face
+               // quadrature point
+               const int g_row = yz_plane ? i : xz_plane ? j : k;
+
+               for (int c = 0; c < vd; ++c)
+               {
+                  real_t grad_n = 0.0;
+
+                  for (int kk = 0; kk < d; ++kk)
+                  {
+                     // (l, m, n) 3D lexicographic index of interior points used
+                     // in evaluating normal derivatives
+                     const int l = yz_plane ? kk : i;
+                     const int m = xz_plane ? kk : j;
+                     const int n = xy_plane ? kk : k;
+
+                     const real_t g = G(kk, g_row);
+
+                     grad_n += g * d_x_e(t?c:l, t?l:m, t?m:n, t?n:el_idx, t?el_idx:c);
+                  }
+                  d_y(p, c, side, f) = grad_n;
+               }
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::AddMultTranspose2D(
+   const Vector &y, Vector &x, const real_t a) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+
+   // derivative of 1D basis function
+   auto G_ = Reshape(maps.G.Read(), q, d);
+
+   // entries of e2f: (el,f0,f1,f2,f3,s0,s1,s2,s3)
+   auto e2f = Reshape(elem_to_face.Read(), 9, ne_type);
+
+   auto f2v = Reshape(face_to_vol.Read(), d, 2, nf);
+
+   // if byvdim, d_x has shape (vdim, nddof, nddof, ne)
+   // otherwise, d_x has shape (nddof, nddof, ne, vdim)
+   auto d_x = Reshape(x.ReadWrite(), t?vd:d, d, t?d:ne, t?ne:vd);
+   auto d_y = Reshape(y.Read(), q, vd, 2, nf);
+
+   mfem::forall_2D(ne_type, d, d, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED real_t y_s[MD];
+      MFEM_SHARED int pp[MD];
+      MFEM_SHARED int jj;
+      if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0) { jj = 0; }
+
+      MFEM_SHARED real_t BG[MD*MD];
+      DeviceMatrix G(BG, q, d);
+
+      MFEM_SHARED real_t x_s[MD*MD];
+      DeviceMatrix xx(x_s, d, d);
+
+      MFEM_SHARED int el; // global element index
+      MFEM_SHARED int faces[4];
+      MFEM_SHARED int sides[4];
+
+      MFEM_FOREACH_THREAD(i,x,d)
+      {
+         MFEM_FOREACH_THREAD(p,y,q)
+         {
+            G(p,i) = a * G_(p,i);
+            xx(p,i) = 0.0;
+         }
+      }
+
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         if (MFEM_THREAD_ID(x) == 0)
+         {
+            el = e2f(0, e);
+         }
+
+         MFEM_FOREACH_THREAD(i, x, 4)
+         {
+            faces[i] = e2f(1 + i, e);
+            sides[i] = e2f(5 + i, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int face_id=0; face_id < 4; ++face_id)
+      {
+         const int f = faces[face_id];
+
+         if (f < 0) { continue; }
+
+         const int side = sides[face_id];
+
+         if (MFEM_THREAD_ID(y) == 0)
+         {
+            MFEM_FOREACH_THREAD(p,x,d)
+            {
+               y_s[p] = d_y(p, 0, side, f);
+
+               const int ij = f2v(p, side, f);
+               const int i = ij % q;
+               const int j = ij / q;
+
+               pp[(face_id == 0 || face_id == 2) ? i : j] = p;
+               if (MFEM_THREAD_ID(x) == 0)
+               {
+                  jj = (face_id == 0 || face_id == 2) ? j : i;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(k,x,d)
+         {
+            MFEM_FOREACH_THREAD(l,y,d)
+            {
+               const int p = (face_id == 0 || face_id == 2) ? pp[k] : pp[l];
+               const int kk = (face_id == 0 || face_id == 2) ? l : k;
+               const real_t g = G(jj, kk);
+               xx(k,l) += g * y_s[p];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(k,x,d)
+      {
+         MFEM_FOREACH_THREAD(l,y,d)
+         {
+            const int c = 0;
+            d_x(t?c:k, t?k:l, t?l:el, t?el:c) += xx(k,l);
+         }
+      }
+   });
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::AddMultTranspose3D(
+   const Vector &y, Vector &x, const real_t a) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+
+   MFEM_VERIFY(vd == 1, "vdim > 1 not supported.");
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+   const int q2d = q * q;
+
+   MFEM_VERIFY(q == d, "");
+   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
+
+   auto G_ = Reshape(maps.G.Read(), q, d);
+
+   // (el, f0,f1,f2,f3,f4,f5, s0,s1,s2,s3,s4,s5)
+   auto e2f = Reshape(elem_to_face.Read(), 13, ne_type);
+
+   auto f2v = Reshape(face_to_vol.Read(), q2d, 2, nf);
+
+   auto d_x = Reshape(x.ReadWrite(), t?vd:d, d, d, t?d:ne, t?ne:vd);
+   const auto d_y = Reshape(y.Read(), q2d, vd, 2, nf);
+
+   mfem::forall_2D(ne_type, q, q, [=] MFEM_HOST_DEVICE (int e) -> void
+   {
+      static constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED int pp[MD][MD];
+      MFEM_SHARED real_t y_s[MD*MD];
+      MFEM_SHARED int jj;
+      if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0) { jj = 0; }
+
+      MFEM_SHARED real_t xx_s[MD*MD*MD];
+      auto xx = Reshape(xx_s, d, d, d);
+
+      MFEM_SHARED real_t G_s[MD*MD];
+      DeviceMatrix G(G_s, q, d);
+
+      MFEM_SHARED int el;
+      MFEM_SHARED int faces[6];
+      MFEM_SHARED int sides[6];
+
+      // Load G into shared memory
+      MFEM_FOREACH_THREAD(j, x, d)
+      {
+         MFEM_FOREACH_THREAD(i, y, q)
+         {
+            G(i, j) = a * G_(i, j);
+            G(i, j) = a * G_(i, j);
+            G(i, j) = a * G_(i, j);
+         }
+      }
+
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         if (MFEM_THREAD_ID(x) == 0)
+         {
+            el = e2f(0, e); // global element index
+         }
+
+         MFEM_FOREACH_THREAD(i, x, 6)
+         {
+            faces[i] = e2f(1 + i, e);
+            sides[i] = e2f(7 + i, e);
+         }
+      }
+
+      MFEM_FOREACH_THREAD(k, x, d)
+      {
+         MFEM_FOREACH_THREAD(j, y, d)
+         {
+            for (int i = 0; i < d; ++i)
+            {
+               xx(i, j, k) = 0.0;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int face_id = 0; face_id < 6; ++face_id)
+      {
+         const int f = faces[face_id];
+
+         if (f < 0)
+         {
+            continue;
+         }
+
+         const int side = sides[face_id];
+
+         // is this face parallel to the x-y plane in reference coordinates?
+         const bool xy_plane = (face_id == 0 || face_id == 5);
+         const bool xz_plane = (face_id == 1 || face_id == 3);
+
+         MFEM_FOREACH_THREAD(p1, x, q)
+         {
+            MFEM_FOREACH_THREAD(p2, y, q)
+            {
+               const int p = p1 + q * p2;
+               y_s[p] = d_y(p, 0, side, f);
+
+               const int ijk = f2v(p, side, f);
+               const int k = ijk / q2d;
+               const int i = ijk % q;
+               const int j = (ijk - q2d*k) / q;
+
+               pp[(xy_plane || xz_plane) ? i : j][(xy_plane) ? j : k] = p;
+               if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0)
+               {
+                  jj = (xy_plane) ? k : (xz_plane) ? j : i;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(n, x, d)
+         {
+            MFEM_FOREACH_THREAD(m, y, d)
+            {
+               for (int l = 0; l < d; ++l)
+               {
+                  const int p = (xy_plane) ? pp[l][m] : (xz_plane) ? pp[l][n] : pp[m][n];
+                  const int kk = (xy_plane) ? n : (xz_plane) ? m : l;
+                  const real_t g = G(jj, kk);
+                  xx(l, m, n) += g * y_s[p];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // map back to global array
+      MFEM_FOREACH_THREAD(n, x, d)
+      {
+         MFEM_FOREACH_THREAD(m, y, d)
+         {
+            for (int l = 0; l < d; ++l)
+            {
+               const int c = 0;
+               d_x(t?c:l, t?l:m, t?m:n, t?n:el, t?el:c) += xx(l, m, n);
+            }
+         }
+      }
+   });
+}
+
+} // namespace mfem
@@ -0,0 +1,85 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_NORMAL_DERIV_RESTRICTION
+#define MFEM_NORMAL_DERIV_RESTRICTION
+
+#include "../mesh/mesh.hpp"
+
+namespace mfem
+{
+
+class FiniteElementSpace;
+enum class ElementDofOrdering;
+
+/// @brief Class to compute face normal derivatives (in reference coordinate) of
+/// an L2 grid function (used internally by L2FaceRestriction).
+class L2NormalDerivativeFaceRestriction
+{
+protected:
+   const FiniteElementSpace &fes; ///< The L2 finite element space.
+   const FaceType face_type; ///< Face type: either boundary or interior.
+   const int dim; ///< Dimension of the mesh.
+   const int nf; ///< Number of faces of the given @a face_type.
+   const int ne; ///< Number of elements.
+   int ne_type; ///< Number of elements with faces of type face type
+
+   Array<int> face_to_elem; ///< Face-wise information array.
+   Array<int> elem_to_face; ///< Element-wise information array.
+   Array<int> face_to_vol; ///< maps face index to volume index
+
+public:
+   /// @brief Constructor.
+   /// @param[in] fes_ The associated FiniteElementSpace (should be L2/DG).
+   /// @param[in] f_ordering Request a specific face dof ordering. Currently
+   /// only ElementDofOrdering::LEXICOGRAPHIC is supported.
+   /// @param[in] face_type_ Type of faces to compute restriction (interior or boundary).
+   L2NormalDerivativeFaceRestriction(const FiniteElementSpace &fes_,
+                                     const ElementDofOrdering f_ordering,
+                                     const FaceType face_type_);
+
+   /// @brief Computes the normal derivatives on the @a face_type faces of the mesh.
+   /// @param[in] x The L-vector degrees of freedom.
+   /// @param[out] y The face E(like)-vector degrees of freedom of the format
+   /// (face_dofs x vdim x 2 x nf) where nf is the number of faces of type @a
+   /// face_type. The face_dofs are ordered according to @a f_ordering specified
+   /// in the constructor.
+   void Mult(const Vector &x, Vector &y) const;
+
+   /// @brief Computes the transpose of the action of Mult(), accumulating into
+   ///        @a y with coefficient @a a.
+   /// @param x Face E-vector layout (face_dofs x vdim x 2 x nf).
+   /// @param y L-vector layout.
+   /// @param a Optional coefficient (y = y + a*R^t*x)
+   void AddMultTranspose(const Vector &x, Vector &y,
+                         const real_t a = 1.0) const;
+
+   /// @name Internal compute kernels. Public because of nvcc restriction.
+   ///@{
+
+   template <int T_D1D = 0>
+   void Mult2D(const Vector &x, Vector &y) const;
+
+   template <int T_D1D = 0>
+   void AddMultTranspose2D(const Vector &x, Vector &y, const real_t a) const;
+
+   template <int T_D1D = 0>
+   void Mult3D(const Vector &x, Vector &y) const;
+
+   template <int T_D1D = 0>
+   void AddMultTranspose3D(const Vector &x, Vector &y, const real_t a) const;
+
+   /// @}
+};
+
+}
+
+#endif // MFEM_RESTRICTION
@@ -368,6 +368,72 @@ const
   y.Add(a, Ytmp);
 }

+real_t ParBilinearForm::ParInnerProduct(const ParGridFunction &x,
+                                        const ParGridFunction &y) const
+{
+   MFEM_ASSERT(mat != NULL, "local matrix must be assembled");
+
+   real_t loc = InnerProduct(x, y);
+   real_t glob = 0.;
+
+   MPI_Allreduce(&loc, &glob, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                 pfes->GetComm());
+
+   return glob;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(const ParGridFunction &x,
+                                         const ParGridFunction &y) const
+{
+   MFEM_ASSERT(x.ParFESpace() == pfes, "the parallel spaces must match");
+   MFEM_ASSERT(y.ParFESpace() == pfes, "the parallel spaces must match");
+
+   HypreParVector *x_p = x.ParallelProject();
+   HypreParVector *y_p = y.ParallelProject();
+
+   real_t res = TrueInnerProduct(*x_p, *y_p);
+
+   delete x_p;
+   delete y_p;
+
+   return res;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(HypreParVector &x,
+                                         HypreParVector &y) const
+{
+   MFEM_VERIFY(p_mat.Ptr() != NULL, "parallel matrix must be assembled");
+
+   if (p_mat->GetType() != Operator::Hypre_ParCSR)
+   {
+      return TrueInnerProduct((const Vector&)x, (const Vector&)y);
+   }
+
+   HypreParVector *Ax = new HypreParVector(pfes);
+   HypreParMatrix *A = p_mat.As<HypreParMatrix>();
+
+   A->Mult(x, *Ax);
+
+   real_t res = mfem::InnerProduct(y, *Ax);
+
+   delete Ax;
+
+   return res;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(const Vector &x,
+                                         const Vector &y) const
+{
+   MFEM_VERIFY(p_mat.Ptr() != NULL, "parallel matrix must be assembled");
+
+   Vector Ax(pfes->GetTrueVSize());
+   p_mat->Mult(x, Ax);
+
+   real_t res = mfem::InnerProduct(pfes->GetComm(), y, Ax);
+
+   return res;
+}
+
 void ParBilinearForm::FormLinearSystem(
   const Array<int> &ess_tdof_list, Vector &x, Vector &b,
   OperatorHandle &A, Vector &X, Vector &B, int copy_interior)
@@ -173,6 +173,37 @@ public:
       vectors on the true dofs. */
   void TrueAddMult(const Vector &x, Vector &y, const real_t a = 1.0) const;

+   /// Compute $ y^T M x $
+   /** @warning The calculation is performed on local dofs, assuming that
+       the local vectors are consistent with the prolongations of the true
+       vectors (see ParGridFunction::Distribute()). If this is not the case,
+       use TrueInnerProduct(const ParGridFunction &, const ParGridFunction &)
+       instead.
+       @note It is assumed that the local matrix is assembled and it has
+       not been replaced by the parallel matrix through FormSystemMatrix().
+       @see TrueInnerProduct(const ParGridFunction&, const ParGridFunction&) */
+   real_t ParInnerProduct(const ParGridFunction &x,
+                          const ParGridFunction &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (grid function version)
+   /** @note The ParGridFunction%s are restricted to the true-vectors for
+       for calculation.
+       @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix().
+       @see ParInnerProduct(const ParGridFunction&, const ParGridFunction&) */
+   real_t TrueInnerProduct(const ParGridFunction &x,
+                           const ParGridFunction &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (Hypre vector version)
+   /** @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix(). */
+   real_t TrueInnerProduct(HypreParVector &x, HypreParVector &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (true-vector version)
+   /** @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix(). */
+   real_t TrueInnerProduct(const Vector &x, const Vector &y) const;
+
   /// Return the parallel FE space associated with the ParBilinearForm.
   ParFiniteElementSpace *ParFESpace() const { return pfes; }

@@ -861,17 +861,17 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
         }
      }

-      HYPRE_Int *i_diag = new HYPRE_Int[ldof+1];
-      HYPRE_Int *j_diag = new HYPRE_Int[ltdof];
-      real_t    *d_diag = new real_t[ltdof];
+      HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
+      real_t    *d_diag = Memory<real_t>(ltdof);
      int diag_counter;

-      HYPRE_Int *i_offd = new HYPRE_Int[ldof+1];
-      HYPRE_Int *j_offd = new HYPRE_Int[nnz_offd];
-      real_t    *d_offd = new real_t[nnz_offd];
+      HYPRE_Int *i_offd = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_offd = Memory<HYPRE_Int>(nnz_offd);
+      real_t    *d_offd = Memory<real_t>(nnz_offd);
      int offd_counter;

-      HYPRE_BigInt *cmap   = new HYPRE_BigInt[ldof-ltdof];
+      HYPRE_BigInt *cmap   = Memory<HYPRE_BigInt>(ldof-ltdof);

      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
      HYPRE_BigInt *row_starts = GetDofOffsets();
@@ -249,6 +249,8 @@ void ParGridFunction::ExchangeFaceNbrData()
   auto send_data_ptr = mpi_gpu_aware ? send_data.Read() : send_data.HostRead();
   auto face_nbr_data_ptr = mpi_gpu_aware ? face_nbr_data.Write() :
                            face_nbr_data.HostWrite();
+   // Wait for the kernel to be done since it updates what's sent and it may be async
+   if (mpi_gpu_aware) { MFEM_STREAM_SYNC; }
   for (int fn = 0; fn < num_face_nbrs; fn++)
   {
      int nbr_rank = pmesh->GetFaceNbrRank(fn);
@@ -518,7 +520,7 @@ void ParGridFunction::CountElementsPerVDof(Array<int> &elem_per_vdof) const
 }

 void ParGridFunction::GetDerivative(int comp, int der_comp,
-                                    ParGridFunction &der)
+                                    ParGridFunction &der) const
 {
   Array<int> overlap;
   AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
@@ -713,10 +715,10 @@ void ParGridFunction::ProjectBdrCoefficient(
         }
      }
   }
+   gcomm.Bcast<int>(values_counter.HostReadWrite());
   for (int i = 0; i < values_counter.Size(); i++)
   {
-      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
-                  bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
                  "internal error");
   }
 #endif
@@ -753,10 +755,10 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
 #ifdef MFEM_DEBUG
   Array<int> ess_vdofs_marker;
   pfes->GetEssentialVDofs(bdr_attr, ess_vdofs_marker);
+   gcomm.Bcast<int>(values_counter.HostReadWrite());
   for (int i = 0; i < values_counter.Size(); i++)
   {
-      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
-                  bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
                  "internal error: " << pfes->GetLocalTDofNumber(i) << ' ' << bool(
                     values_counter[i]));
   }
@@ -231,7 +231,7 @@ public:
   void CountElementsPerVDof(Array<int> &elem_per_vdof) const override;

   /// Parallel version of GridFunction::GetDerivative(); see its documentation.
-   void GetDerivative(int comp, int der_comp, ParGridFunction &der);
+   void GetDerivative(int comp, int der_comp, ParGridFunction &der) const;

   /** Sets the output vector @a dof_vals to the values of the degrees of
       freedom of element @a el. If @a el is greater than or equal to the number
@@ -262,7 +262,6 @@ public:
                              const Array<int> &attr) override
   { ProjectBdrCoefficient(coeff, NULL, attr); }

-   // Only the values in the master are guaranteed to be correct!
   void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                     const Array<int> &bdr_attr) override;

@@ -18,6 +18,7 @@
 #include "pgridfunc.hpp"
 #include "pfespace.hpp"
 #include "fespace.hpp"
+#include "fe/face_map_utils.hpp"
 #include "../general/forall.hpp"

 namespace mfem
@@ -277,21 +278,22 @@ void ParNCH1FaceRestriction::ComputeGatherIndices(
   gather_offsets[0] = 0;
 }

-ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
+ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &pfes_,
                                           ElementDofOrdering f_ordering,
                                           FaceType type,
                                           L2FaceValues m,
                                           bool build)
-   : L2FaceRestriction(fes, f_ordering, type, m, false)
+   : L2FaceRestriction(pfes_, f_ordering, type, m, false),
+     pfes(pfes_)
 {
   if (!build) { return; }
   if (nf==0) { return; }

-   CheckFESpace(f_ordering);
+   CheckFESpace();

-   ComputeScatterIndicesAndOffsets(f_ordering, type);
+   ComputeScatterIndicesAndOffsets();

-   ComputeGatherIndices(f_ordering, type);
+   ComputeGatherIndices();
 }

 ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
@@ -307,14 +309,8 @@ void ParL2FaceRestriction::DoubleValuedConformingMult(
   MFEM_ASSERT(
      m == L2FaceValues::DoubleValued,
      "This method should be called when m == L2FaceValues::DoubleValued.");
-   const ParFiniteElementSpace &pfes =
-      static_cast<const ParFiniteElementSpace&>(this->fes);
-   ParGridFunction x_gf;
-   x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(&pfes),
-                const_cast<Vector&>(x), 0);
-   // Face-neighbor information is only needed for interior faces. For boundary
-   // faces, no communication is required.
-   if (type == FaceType::Interior) { x_gf.ExchangeFaceNbrData(); }
+
+   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, type);

   // Early return only after calling ParGridFunction::ExchangeFaceNbrData,
   // otherwise MPI communication can hang.
@@ -329,7 +325,7 @@ void ParL2FaceRestriction::DoubleValuedConformingMult(
   auto d_indices1 = scatter_indices1.Read();
   auto d_indices2 = scatter_indices2.Read();
   auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
-   auto d_x_shared = Reshape(x_gf.FaceNbrData().Read(),
+   auto d_x_shared = Reshape(face_nbr_data.Read(),
                             t?vd:nsdofs, t?nsdofs:vd);
   auto d_y = Reshape(y.Write(), nface_dofs, vd, 2, nf);
   mfem::forall(nfdofs, [=] MFEM_HOST_DEVICE (int i)
@@ -567,13 +563,9 @@ void ParL2FaceRestriction::FillJAndData(const Vector &ea_data,
   });
 }

-void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();
-   const ParFiniteElementSpace &pfes =
-      static_cast<const ParFiniteElementSpace&>(this->fes);

   // Initialization of the offsets
   for (int i = 0; i <= ndofs; ++i)
@@ -622,9 +614,7 @@ void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets(
 }


-void ParL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParL2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -666,11 +656,11 @@ ParNCL2FaceRestriction::ParNCL2FaceRestriction(const ParFiniteElementSpace &fes,
   if (nf==0) { return; }
   x_interp.UseDevice(true);

-   CheckFESpace(f_ordering);
+   CheckFESpace();

-   ComputeScatterIndicesAndOffsets(f_ordering, type);
+   ComputeScatterIndicesAndOffsets();

-   ComputeGatherIndices(f_ordering, type);
+   ComputeGatherIndices();
 }

 void ParNCL2FaceRestriction::SingleValuedNonconformingMult(
@@ -979,9 +969,7 @@ void ParNCL2FaceRestriction::FillJAndData(const Vector &ea_data,
   MFEM_ABORT("Not yet implemented.");
 }

-void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -1064,9 +1052,7 @@ void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
   interpolations.InitializeNCInterpConfig();
 }

-void ParNCL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParNCL2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -139,9 +139,11 @@ public: // For nvcc
 class ParL2FaceRestriction : virtual public L2FaceRestriction
 {
 protected:
+   const ParFiniteElementSpace &pfes;
+
   /** @brief Constructs an ParL2FaceRestriction.

-       @param[in] fes        The ParFiniteElementSpace on which this operates
+       @param[in] pfes_      The ParFiniteElementSpace on which this operates
       @param[in] f_ordering Request a specific face dof ordering
       @param[in] type       Request internal or boundary faces dofs
       @param[in] m          Request the face dofs for elem1, or both elem1 and
@@ -149,7 +151,7 @@ protected:
       @param[in] build      Request the ParL2FaceRestriction to compute the
                             scatter/gather indices. False should only be used
                             when inheriting from ParL2FaceRestriction. */
-   ParL2FaceRestriction(const ParFiniteElementSpace& fes,
+   ParL2FaceRestriction(const ParFiniteElementSpace &pfes_,
                        ElementDofOrdering f_ordering,
                        FaceType type,
                        L2FaceValues m,
@@ -229,22 +231,14 @@ public:
 private:
   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
       for the gathering: E-vector to L-vector.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();

 public:
   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -381,22 +375,14 @@ private:
   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
       for the gathering: E-vector to L-vector, and the interpolators from
       coarse to fine face for master non-comforming faces.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();

 public:
   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -215,6 +215,10 @@ public:
   /// quadrature point, oriented relative to "element 1".
   int GetPermutedIndex(int idx, int iq) const override;

+   /// @brief Get the face index (in the standard Mesh numbering) associated
+   /// with face @a idx in the FaceQuadratureSpace.
+   int GetMeshFaceIndex(int idx) const { return face_indices[idx]; }
+
   /// @brief Returns the index associated with the face described by @a T.
   ///
   /// The index may differ from the mesh face or boundary element index
@@ -10,17 +10,16 @@
 // CONTRIBUTING.md for details.

 #include "restriction.hpp"
+#include "normal_deriv_restriction.hpp"
 #include "gridfunc.hpp"
 #include "fespace.hpp"
+#include "pgridfunc.hpp"
+#include "qspace.hpp"
+#include "fe/face_map_utils.hpp"
 #include "../general/forall.hpp"
+
 #include <climits>

-#ifdef MFEM_USE_MPI
-
-#include "pfespace.hpp"
-
-#endif
-
 namespace mfem
 {

@@ -907,112 +906,6 @@ void ConformingFaceRestriction::SetFaceDofsGatherIndices(
   }
 }

-static int ToLexOrdering2D(const int face_id, const int size1d, const int i)
-{
-   if (face_id==2 || face_id==3)
-   {
-      return size1d-1-i;
-   }
-   else
-   {
-      return i;
-   }
-}
-
-static int PermuteFace2D(const int face_id1, const int face_id2,
-                         const int orientation,
-                         const int size1d, const int index)
-{
-   int new_index;
-   // Convert from lex ordering
-   if (face_id1==2 || face_id1==3)
-   {
-      new_index = size1d-1-index;
-   }
-   else
-   {
-      new_index = index;
-   }
-   // Permute based on face orientations
-   if (orientation==1)
-   {
-      new_index = size1d-1-new_index;
-   }
-   return ToLexOrdering2D(face_id2, size1d, new_index);
-}
-
-static int ToLexOrdering3D(const int face_id, const int size1d, const int i,
-                           const int j)
-{
-   if (face_id==2 || face_id==1 || face_id==5)
-   {
-      return i + j*size1d;
-   }
-   else if (face_id==3 || face_id==4)
-   {
-      return (size1d-1-i) + j*size1d;
-   }
-   else // face_id==0
-   {
-      return i + (size1d-1-j)*size1d;
-   }
-}
-
-static int PermuteFace3D(const int face_id1, const int face_id2,
-                         const int orientation,
-                         const int size1d, const int index)
-{
-   int i=0, j=0, new_i=0, new_j=0;
-   i = index%size1d;
-   j = index/size1d;
-   // Convert from lex ordering
-   if (face_id1==3 || face_id1==4)
-   {
-      i = size1d-1-i;
-   }
-   else if (face_id1==0)
-   {
-      j = size1d-1-j;
-   }
-   // Permute based on face orientations
-   switch (orientation)
-   {
-      case 0:
-         new_i = i;
-         new_j = j;
-         break;
-      case 1:
-         new_i = j;
-         new_j = i;
-         break;
-      case 2:
-         new_i = j;
-         new_j = (size1d-1-i);
-         break;
-      case 3:
-         new_i = (size1d-1-i);
-         new_j = j;
-         break;
-      case 4:
-         new_i = (size1d-1-i);
-         new_j = (size1d-1-j);
-         break;
-      case 5:
-         new_i = (size1d-1-j);
-         new_j = (size1d-1-i);
-         break;
-      case 6:
-         new_i = (size1d-1-j);
-         new_j = i;
-         break;
-      case 7:
-         new_i = i;
-         new_j = (size1d-1-j);
-         break;
-   }
-   return ToLexOrdering3D(face_id2, size1d, new_i, new_j);
-}
-
 // Permute dofs or quads on a face for e2 to match with the ordering of e1
 int PermuteFaceL2(const int dim, const int face_id1,
                  const int face_id2, const int orientation,
@@ -1023,9 +916,9 @@ int PermuteFaceL2(const int dim, const int face_id1,
      case 1:
         return 0;
      case 2:
-         return PermuteFace2D(face_id1, face_id2, orientation, size1d, index);
+         return internal::PermuteFace2D(face_id1, face_id2, orientation, size1d, index);
      case 3:
-         return PermuteFace3D(face_id1, face_id2, orientation, size1d, index);
+         return internal::PermuteFace3D(face_id1, face_id2, orientation, size1d, index);
      default:
         MFEM_ABORT("Unsupported dimension.");
         return 0;
@@ -1038,6 +931,7 @@ L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
                                     const L2FaceValues m,
                                     bool build)
   : fes(fes),
+     ordering(f_ordering),
     nf(fes.GetNFbyType(type)),
     ne(fes.GetNE()),
     vdim(fes.GetVDim()),
@@ -1060,11 +954,9 @@ L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
   width = fes.GetVSize();
   if (!build) { return; }

-   CheckFESpace(f_ordering);
-
-   ComputeScatterIndicesAndOffsets(f_ordering,type);
-
-   ComputeGatherIndices(f_ordering, type);
+   CheckFESpace();
+   ComputeScatterIndicesAndOffsets();
+   ComputeGatherIndices();
 }

 L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
@@ -1327,7 +1219,7 @@ void L2FaceRestriction::AddFaceMatricesToElementMatrices(const Vector &fea_data,
   }
 }

-void L2FaceRestriction::CheckFESpace(const ElementDofOrdering f_ordering)
+void L2FaceRestriction::CheckFESpace()
 {
 #ifdef MFEM_USE_MPI

@@ -1351,7 +1243,7 @@ void L2FaceRestriction::CheckFESpace(const ElementDofOrdering f_ordering)
               "Only Gauss-Lobatto and Bernstein basis are supported in "
               "L2FaceRestriction.");
   if (nf==0) { return; }
-   const bool dof_reorder = (f_ordering == ElementDofOrdering::LEXICOGRAPHIC);
+   const bool dof_reorder = (ordering == ElementDofOrdering::LEXICOGRAPHIC);
   if (!dof_reorder)
   {
      MFEM_ABORT("Non-Tensor L2FaceRestriction not yet implemented.");
@@ -1371,9 +1263,7 @@ void L2FaceRestriction::CheckFESpace(const ElementDofOrdering f_ordering)
 #endif
 }

-void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType face_type)
+void L2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();
   // Initialization of the offsets
@@ -1389,16 +1279,16 @@ void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
      MFEM_ASSERT(!face.IsShared(),
                  "Unexpected shared face in L2FaceRestriction.");
-      if ( face.IsOfFaceType(face_type) )
+      if ( face.IsOfFaceType(type) )
      {
         SetFaceDofsScatterIndices1(face,f_ind);
         if ( m==L2FaceValues::DoubleValued )
         {
-            if ( face_type==FaceType::Interior && face.IsInterior() )
+            if ( type==FaceType::Interior && face.IsInterior() )
            {
               PermuteAndSetFaceDofsScatterIndices2(face,f_ind);
            }
-            else if ( face_type==FaceType::Boundary && face.IsBoundary() )
+            else if ( type==FaceType::Boundary && face.IsBoundary() )
            {
               SetBoundaryDofsScatterIndices2(face,f_ind);
            }
@@ -1415,9 +1305,7 @@ void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
   }
 }

-void L2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType face_type)
+void L2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();
   // Computation of gather_indices
@@ -1427,11 +1315,11 @@ void L2FaceRestriction::ComputeGatherIndices(
      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
      MFEM_ASSERT(!face.IsShared(),
                  "Unexpected shared face in L2FaceRestriction.");
-      if ( face.IsOfFaceType(face_type) )
+      if ( face.IsOfFaceType(type) )
      {
         SetFaceDofsGatherIndices1(face,f_ind);
         if ( m==L2FaceValues::DoubleValued &&
-              face_type==FaceType::Interior &&
+              type==FaceType::Interior &&
              face.IsLocal())
         {
            PermuteAndSetFaceDofsGatherIndices2(face,f_ind);
@@ -1598,6 +1486,28 @@ void L2FaceRestriction::PermuteAndSetFaceDofsGatherIndices2(
   }
 }

+void L2FaceRestriction::NormalDerivativeMult(const Vector &x, Vector &y) const
+{
+   EnsureNormalDerivativeRestriction();
+   normal_deriv_restr->Mult(x, y);
+}
+
+void L2FaceRestriction::NormalDerivativeAddMultTranspose(const Vector &x,
+                                                         Vector &y) const
+{
+   EnsureNormalDerivativeRestriction();
+   normal_deriv_restr->AddMultTranspose(x, y);
+}
+
+void L2FaceRestriction::EnsureNormalDerivativeRestriction() const
+{
+   if (!normal_deriv_restr)
+   {
+      normal_deriv_restr.reset(
+         new L2NormalDerivativeFaceRestriction(fes, ordering, type));
+   }
+}
+
 InterpolationManager::InterpolationManager(const FiniteElementSpace &fes,
                                           ElementDofOrdering ordering,
                                           FaceType type)
@@ -1775,11 +1685,11 @@ NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
   if (!build) { return; }
   x_interp.UseDevice(true);

-   CheckFESpace(f_ordering);
+   CheckFESpace();

-   ComputeScatterIndicesAndOffsets(f_ordering, type);
+   ComputeScatterIndicesAndOffsets();

-   ComputeGatherIndices(f_ordering, type);
+   ComputeGatherIndices();
 }

 NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
@@ -2259,18 +2169,16 @@ int ToLexOrdering(const int dim, const int face_id, const int size1d,
      case 1:
         return 0;
      case 2:
-         return ToLexOrdering2D(face_id, size1d, index);
+         return internal::ToLexOrdering2D(face_id, size1d, index);
      case 3:
-         return ToLexOrdering3D(face_id, size1d, index%size1d, index/size1d);
+         return internal::ToLexOrdering3D(face_id, size1d, index%size1d, index/size1d);
      default:
         MFEM_ABORT("Unsupported dimension.");
         return 0;
   }
 }

-void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -2334,9 +2242,7 @@ void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
   interpolations.InitializeNCInterpConfig();
 }

-void NCL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void NCL2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();
   // Computation of gather_indices
@@ -2376,4 +2282,33 @@ void NCL2FaceRestriction::ComputeGatherIndices(
   gather_offsets[0] = 0;
 }

+Vector GetLVectorFaceNbrData(
+   const FiniteElementSpace &fes, const Vector &x, FaceType ftype)
+{
+#ifdef MFEM_USE_MPI
+   if (ftype == FaceType::Interior)
+   {
+      if (auto *pfes = const_cast<ParFiniteElementSpace*>
+                       (dynamic_cast<const ParFiniteElementSpace*>(&fes)))
+      {
+         if (auto *x_gf = const_cast<ParGridFunction*>
+                          (dynamic_cast<const ParGridFunction*>(&x)))
+         {
+            Vector &gf_face_nbr = x_gf->FaceNbrData();
+            if (gf_face_nbr.Size() == 0) { x_gf->ExchangeFaceNbrData(); }
+            gf_face_nbr.Read();
+            return Vector(gf_face_nbr, 0, gf_face_nbr.Size());
+         }
+         else
+         {
+            ParGridFunction gf(pfes, const_cast<Vector&>(x));
+            gf.ExchangeFaceNbrData();
+            return std::move(gf.FaceNbrData());
+         }
+      }
+   }
+#endif
+   return Vector();
+}
+
 } // namespace mfem
@@ -14,6 +14,7 @@

 #include "../linalg/operator.hpp"
 #include "../mesh/mesh.hpp"
+#include "normal_deriv_restriction.hpp"

 namespace mfem
 {
@@ -21,6 +22,8 @@ namespace mfem
 class FiniteElementSpace;
 enum class ElementDofOrdering;

+class FaceQuadratureSpace;
+
 /// Abstract base class that defines an interface for element restrictions.
 class ElementRestrictionOperator : public Operator
 {
@@ -220,6 +223,45 @@ public:
      y = 0.0;
      AddMultTranspose(x, y);
   }
+
+   /** @brief For each face, sets @a y to the partial derivative of @a x with
+              respect to the reference coordinate whose direction is
+              perpendicular to the face on the reference element.
+
+    @details This is not the normal derivative in physical coordinates, but can
+             be mapped to the physical normal derivative using the element
+             Jacobian and the tangential derivatives (in reference coordinates)
+             which can be computed from the face values (provided by Mult).
+
+             Note that due to the polynomial degree of the element mapping, the
+             physical normal derivative may be a higher degree polynomial than
+             the restriction of the values to the face. However, the normal
+             derivative in reference coordinates has degree-1, and therefore can
+             be exactly represented with the degrees of freedom of a face
+             E-vector.
+
+    @param[in]     x The L-vector degrees of freedom.
+    @param[in,out] y The reference normal derivative degrees of freedom. Is
+                     E-vector like.
+    */
+   virtual void NormalDerivativeMult(const Vector &x, Vector &y) const
+   {
+      MFEM_ABORT("Not implemented for this restriction operator.");
+   }
+
+   /** @brief Add the face reference-normal derivative degrees of freedom in @a
+              x to the element degrees of freedom in @a y.
+
+       @details see NormalDerivativeMult.
+
+       @param[in]     x The degrees of freedom of the face reference-normal
+                        derivative. Is E-vector like.
+       @param[in,out] y The L-vector degrees of freedom.
+   */
+   virtual void NormalDerivativeAddMultTranspose(const Vector &x, Vector &y) const
+   {
+      MFEM_ABORT("Not implemented for this restriction operator.");
+   }
 };

 /// @brief Operator that extracts face degrees of freedom for H1, ND, or RT
@@ -365,6 +407,7 @@ class L2FaceRestriction : public FaceRestriction
 {
 protected:
   const FiniteElementSpace &fes;
+   const ElementDofOrdering ordering;
   const int nf; // Number of faces of the requested type
   const int ne; // Number of elements
   const int vdim; // vdim
@@ -379,6 +422,7 @@ protected:
   Array<int> scatter_indices2; // Scattering indices for element 2 on each face
   Array<int> gather_offsets; // offsets for the gathering indices of each dof
   Array<int> gather_indices; // gathering indices for each dof
+   mutable std::unique_ptr<L2NormalDerivativeFaceRestriction> normal_deriv_restr;

   /** @brief Constructs an L2FaceRestriction.

@@ -487,34 +531,49 @@ public:
   virtual void AddFaceMatricesToElementMatrices(const Vector &fea_data,
                                                 Vector &ea_data) const;

+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+      face E-Vector.
+
+      @param[in]  x The L-vector degrees of freedom.
+      @param[out] y The face E-Vector degrees of freedom with the given format:
+                    (face_dofs x vdim x 2 x nf) where nf is the number of
+                    interior or boundary faces requested by @a type in the
+                    constructor. The face_dofs are ordered according to the
+                    given ElementDofOrdering. */
+   void NormalDerivativeMult(const Vector &x, Vector &y) const override;
+
+   /** @brief Add the face reference-normal derivative degrees of freedom in @a
+              x to the element degrees of freedom in @a y.
+
+       @details see NormalDerivativeMult.
+
+       @param[in]     x The degrees of freedom of the face reference-normal
+                        derivative. Is E-vector like.
+       @param[in,out] y The L-vector degrees of freedom.
+   */
+   void NormalDerivativeAddMultTranspose(const Vector &x,
+                                         Vector &y) const override;
 private:
   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
       for the gathering: E-vector to L-vector.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();
+
+   /// Create the internal normal derivative restriction operator if needed.
+   void EnsureNormalDerivativeRestriction() const;

 protected:
   mutable Array<int> face_map; // Used in the computation of GetFaceDofs

   /** @brief Verify that L2FaceRestriction is built from an L2 FESpace.
-
-       @param[in] f_ordering The requested face dof ordering.
   */
-   void CheckFESpace(const ElementDofOrdering f_ordering);
+   void CheckFESpace();

   /** @brief Set the scattering indices of elem1, and increment the offsets for
       the face described by the @a face. The ordering of the face dofs of elem1
@@ -938,22 +997,14 @@ private:
   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
       for the gathering: E-vector to L-vector, and the interpolators from
       coarse to fine face for master non-comforming faces.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();

 public:
   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -1015,7 +1066,6 @@ public:
   void DoubleValuedNonconformingTransposeInterpolationInPlace(Vector& x) const;
 };

-
 /** @brief Convert a dof face index from Native ordering to lexicographic
    ordering for quads and hexes.

@@ -1044,6 +1094,20 @@ int PermuteFaceL2(const int dim, const int face_id1,
                  const int face_id2, const int orientation,
                  const int size1d, const int index);

+/// @brief Return the face-neighbor data given the L-vector @a x.
+///
+/// If the input vector @a x is a ParGridFunction with non-empty face-neighbor
+/// data, return an alias to ParGridFunction::FaceNbrData() (avoiding an
+/// unneeded call to ParGridFunction::ExchangeFaceNbrData).
+///
+/// Otherwise, create a temporary ParGridFunction, exchange the face-neighbor
+/// data, and return the resulting vector.
+///
+/// If @a fes is not a parallel space, or if @a ftype is not FaceType::Interior,
+/// return an empty vector.
+Vector GetLVectorFaceNbrData(
+   const FiniteElementSpace &fes, const Vector &x, FaceType ftype);
+
 }

 #endif // MFEM_RESTRICTION
@@ -2949,6 +2949,15 @@ void TMOP_Integrator::EnableSurfaceFitting(const GridFunction &s0,
   MFEM_VERIFY(surf_fit_pos == NULL,
               "Using both fitting approaches is not supported.");

+   const int dim = s0.FESpace()->GetMesh()->Dimension();
+   Mesh *mesh = s0.FESpace()->GetMesh();
+   MFEM_VERIFY(mesh->GetNodes()->Size() == dim*s0.Size(),
+               "Mesh and level-set polynomial order must be the same.");
+   const H1_FECollection *fec = dynamic_cast<const H1_FECollection *>
+                                (s0.FESpace()->FEColl());
+   MFEM_VERIFY(fec, "Only H1_FECollection is supported for the surface fitting "
+               "grid function.");
+
   delete surf_fit_gf;
   surf_fit_gf = new GridFunction(s0);
   surf_fit_gf->CountElementsPerVDof(surf_fit_dof_count);
@@ -2987,12 +2996,24 @@ void TMOP_Integrator::EnableSurfaceFitting(const GridFunction &pos,
 void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
                                           const Array<bool> &smarker,
                                           Coefficient &coeff,
-                                           AdaptivityEvaluator &ae)
+                                           AdaptivityEvaluator &ae,
+                                           AdaptivityEvaluator *aegrad,
+                                           AdaptivityEvaluator *aehess)
 {
   // To have both we must duplicate the markers.
   MFEM_VERIFY(surf_fit_pos == NULL,
               "Using both fitting approaches is not supported.");

+   const int dim = s0.FESpace()->GetMesh()->Dimension();
+   ParMesh *pmesh = s0.ParFESpace()->GetParMesh();
+   MFEM_VERIFY(pmesh->GetNodes()->Size() == dim*s0.Size(),
+               "Mesh and level-set polynomial order must be the same.");
+   const H1_FECollection *fec = dynamic_cast<const H1_FECollection *>
+                                (s0.FESpace()->FEColl());
+   MFEM_VERIFY(fec, "Only H1_FECollection is supported for the surface fitting "
+               "grid function.");
+
+
   delete surf_fit_gf;
   surf_fit_gf = new GridFunction(s0);
   s0.CountElementsPerVDof(surf_fit_dof_count);
@@ -3000,11 +3021,80 @@ void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
   surf_fit_coeff = &coeff;
   surf_fit_eval = &ae;

-   surf_fit_eval->SetParMetaInfo(*s0.ParFESpace()->GetParMesh(),
-                                 *s0.ParFESpace());
+   surf_fit_eval->SetParMetaInfo(*pmesh, *s0.ParFESpace());
   surf_fit_eval->SetInitialField
   (*surf_fit_gf->FESpace()->GetMesh()->GetNodes(), *surf_fit_gf);
-   surf_fit_gf_bg = false;
+
+   if (!aegrad) { return; }
+
+   MFEM_VERIFY(aehess, "AdaptivityEvaluator for Hessians must be provided too.");
+
+   ParFiniteElementSpace *fes = s0.ParFESpace();
+
+   // FE space for gradients.
+   delete surf_fit_grad;
+   H1_FECollection *fec_grad = new H1_FECollection(fec->GetOrder(), dim,
+                                                   fec->GetBasisType());
+   ParFiniteElementSpace *fes_grad = new ParFiniteElementSpace(pmesh, fec_grad,
+                                                               dim);
+   // Initial gradients.
+   surf_fit_grad = new GridFunction(fes_grad);
+   surf_fit_grad->MakeOwner(fec_grad);
+   for (int d = 0; d < dim; d++)
+   {
+      ParGridFunction surf_fit_grad_comp(fes, surf_fit_grad->GetData()+d*s0.Size());
+      s0.GetDerivative(1, d, surf_fit_grad_comp);
+   }
+   surf_fit_eval_grad = aegrad;
+   surf_fit_eval_grad->SetParMetaInfo(*pmesh, *fes_grad);
+   surf_fit_eval_grad->SetInitialField(*pmesh->GetNodes(), *surf_fit_grad);
+
+   // FE space for Hessians.
+   delete surf_fit_hess;
+   H1_FECollection *fec_hess = new H1_FECollection(fec->GetOrder(), dim,
+                                                   fec->GetBasisType());
+   ParFiniteElementSpace *fes_hess = new ParFiniteElementSpace(pmesh, fec_hess,
+                                                               dim*dim);
+   // Initial Hessians.
+   surf_fit_hess = new GridFunction(fes_hess);
+   surf_fit_hess->MakeOwner(fec_hess);
+   int id = 0;
+   for (int d = 0; d < dim; d++)
+   {
+      for (int idir = 0; idir < dim; idir++)
+      {
+         ParGridFunction surf_fit_grad_comp(fes,
+                                            surf_fit_grad->GetData()+d*s0.Size());
+         ParGridFunction surf_fit_hess_comp(fes,
+                                            surf_fit_hess->GetData()+id*s0.Size());
+         surf_fit_grad_comp.GetDerivative(1, idir, surf_fit_hess_comp);
+         id++;
+      }
+   }
+   surf_fit_eval_hess = aehess;
+   surf_fit_eval_hess->SetParMetaInfo(*pmesh, *fes_hess);
+   surf_fit_eval_hess->SetInitialField(*pmesh->GetNodes(), *surf_fit_hess);
+
+   // Store DOF indices that are marked for fitting. Used to reduce work for
+   // transferring information between source/background and current mesh.
+   surf_fit_marker_dof_index.SetSize(0);
+#ifdef MFEM_USE_GSLIB
+   if (dynamic_cast<InterpolatorFP *>(surf_fit_eval) &&
+       dynamic_cast<InterpolatorFP *>(surf_fit_eval_grad) &&
+       dynamic_cast<InterpolatorFP *>(surf_fit_eval_hess))
+   {
+      for (int i = 0; i < surf_fit_marker->Size(); i++)
+      {
+         if ((*surf_fit_marker)[i] == true)
+         {
+            surf_fit_marker_dof_index.Append(i);
+         }
+      }
+   }
+#endif
+
+   *surf_fit_grad = 0.0;
+   *surf_fit_hess = 0.0;
 }

 void TMOP_Integrator::EnableSurfaceFittingFromSource(
@@ -3022,16 +3112,17 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
   // Setup for level set function
   delete surf_fit_gf;
   surf_fit_gf = new GridFunction(s0);
-   *surf_fit_gf = 0.0;
   surf_fit_marker = &smarker;
   surf_fit_coeff = &coeff;
   surf_fit_eval = &ae;

-   surf_fit_gf_bg = true;
   surf_fit_eval->SetParMetaInfo(*s_bg.ParFESpace()->GetParMesh(),
                                 *s_bg.ParFESpace());
   surf_fit_eval->SetInitialField
   (*s_bg.FESpace()->GetMesh()->GetNodes(), s_bg);
+   GridFunction *nodes = s0.FESpace()->GetMesh()->GetNodes();
+   surf_fit_eval->ComputeAtNewPosition(*nodes, *surf_fit_gf,
+                                       nodes->FESpace()->GetOrdering());

   // Setup for gradient on background mesh
   MFEM_VERIFY(s_bg_grad.ParFESpace()->GetOrdering() ==
@@ -3041,11 +3132,11 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
   delete surf_fit_grad;
   surf_fit_grad = new GridFunction(s0_grad);
   *surf_fit_grad = 0.0;
-   surf_fit_eval_bg_grad = &age;
-   surf_fit_eval_bg_hess = &ahe;
-   surf_fit_eval_bg_grad->SetParMetaInfo(*s_bg_grad.ParFESpace()->GetParMesh(),
-                                         *s_bg_grad.ParFESpace());
-   surf_fit_eval_bg_grad->SetInitialField
+   surf_fit_eval_grad = &age;
+   surf_fit_eval_hess = &ahe;
+   surf_fit_eval_grad->SetParMetaInfo(*s_bg_grad.ParFESpace()->GetParMesh(),
+                                      *s_bg_grad.ParFESpace());
+   surf_fit_eval_grad->SetInitialField
   (*s_bg_grad.FESpace()->GetMesh()->GetNodes(), s_bg_grad);

   // Setup for Hessian on background mesh
@@ -3056,9 +3147,9 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
   delete surf_fit_hess;
   surf_fit_hess = new GridFunction(s0_hess);
   *surf_fit_hess = 0.0;
-   surf_fit_eval_bg_hess->SetParMetaInfo(*s_bg_hess.ParFESpace()->GetParMesh(),
-                                         *s_bg_hess.ParFESpace());
-   surf_fit_eval_bg_hess->SetInitialField
+   surf_fit_eval_hess->SetParMetaInfo(*s_bg_hess.ParFESpace()->GetParMesh(),
+                                      *s_bg_hess.ParFESpace());
+   surf_fit_eval_hess->SetInitialField
   (*s_bg_hess.FESpace()->GetMesh()->GetNodes(), s_bg_hess);

   // Count number of zones that share each of the DOFs
@@ -3863,7 +3954,7 @@ void TMOP_Integrator::AssembleElemVecSurfFit(const FiniteElement &el_x,

   Vector sigma_e(dof_s);
   DenseMatrix surf_fit_grad_e(dof_s, dim);
-   if (surf_fit_gf || surf_fit_gf_bg)
+   if (surf_fit_gf)
   {
      surf_fit_gf->GetSubVector(vdofs, sigma_e);

@@ -3871,7 +3962,7 @@ void TMOP_Integrator::AssembleElemVecSurfFit(const FiniteElement &el_x,
      // The FE coefficients of the gradient go in surf_fit_grad_e.
      Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
      DenseMatrix grad_phys; // This will be (dof x dim, dof).
-      if (surf_fit_gf_bg)
+      if (surf_fit_grad)
      {
         surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
         surf_fit_grad->GetSubVector(dofs, grad_ptr);
@@ -3945,7 +4036,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
   Vector sigma_e(dof_s);
   DenseMatrix surf_fit_grad_e(dof_s, dim);
   DenseMatrix surf_fit_hess_e(dof_s, dim*dim);
-   if (surf_fit_gf || surf_fit_gf_bg)
+   if (surf_fit_gf)
   {
      surf_fit_gf->GetSubVector(vdofs, sigma_e);

@@ -3953,7 +4044,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
      // The FE coefficients of the gradient go in surf_fit_grad_e.
      Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
      DenseMatrix grad_phys; // This will be (dof x dim, dof).
-      if (surf_fit_gf_bg)
+      if (surf_fit_grad)
      {
         surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
         surf_fit_grad->GetSubVector(dofs, grad_ptr);
@@ -3967,7 +4058,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
      // Project the Hessian of sigma in the same space.
      // The FE coefficients of the Hessian go in surf_fit_hess_e.
      Vector hess_ptr(surf_fit_hess_e.GetData(), dof_s*dim*dim);
-      if (surf_fit_gf_bg)
+      if (surf_fit_hess)
      {
         surf_fit_hess->FESpace()->GetElementVDofs(el_id, dofs);
         surf_fit_hess->GetSubVector(dofs, hess_ptr);
@@ -3994,7 +4085,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
      Tpr.SetIntPoint(&ip);
      real_t w = surf_fit_normal * surf_fit_coeff->Eval(Tpr, ip);

-      if (surf_fit_gf || surf_fit_gf_bg)
+      if (surf_fit_gf)
      {
         Vector gg_ptr(surf_fit_hess_s.GetData(), dim * dim);
         surf_fit_hess_e.GetRow(s, gg_ptr);
@@ -4376,6 +4467,130 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
   dx = detv_avg_min / dxscale;
 }

+void TMOP_Integrator::RemapSurfaceFittingLevelSetAtNodes(const Vector &new_x,
+                                                         int new_x_ordering)
+{
+   if (!surf_fit_gf) { return; }
+
+   if (surf_fit_marker_dof_index.Size())
+   {
+      // Interpolate information only at DOFs marked for fitting.
+      const int dim = surf_fit_gf->FESpace()->GetMesh()->Dimension();
+      const int cnt = surf_fit_marker_dof_index.Size();
+      const int total_cnt = new_x.Size()/dim;
+      Vector new_x_sorted(cnt*dim);
+      if (new_x_ordering == 0)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               new_x_sorted(i + d*cnt) = new_x(dof_index + d*total_cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < dim; d++)
+            {
+               new_x_sorted(d + i*dim) = new_x(d + dof_index*dim);
+            }
+         }
+      }
+
+      // Interpolate values of the LS.
+      Vector surf_fit_gf_int, surf_fit_grad_int, surf_fit_hess_int;
+      surf_fit_eval->ComputeAtNewPosition(new_x_sorted, surf_fit_gf_int,
+                                          new_x_ordering);
+      for (int i = 0; i < cnt; i++)
+      {
+         int dof_index = surf_fit_marker_dof_index[i];
+         (*surf_fit_gf)[dof_index] = surf_fit_gf_int(i);
+      }
+
+      // Interpolate gradients of the LS.
+      surf_fit_eval_grad->ComputeAtNewPosition(new_x_sorted, surf_fit_grad_int,
+                                               new_x_ordering);
+      // Assumes surf_fit_grad and surf_fit_gf share the same space
+      const int grad_dim = surf_fit_grad->VectorDim();
+      const int grad_cnt = surf_fit_grad->Size()/grad_dim;
+      if (surf_fit_grad->FESpace()->GetOrdering() == Ordering::byNODES)
+      {
+         for (int d = 0; d < grad_dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               (*surf_fit_grad)[dof_index + d*grad_cnt] =
+                  surf_fit_grad_int(i + d*cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < grad_dim; d++)
+            {
+               (*surf_fit_grad)[dof_index*grad_dim + d] =
+                  surf_fit_grad_int(i*grad_dim + d);
+            }
+         }
+      }
+
+      // Interpolate Hessians of the LS.
+      surf_fit_eval_hess->ComputeAtNewPosition(new_x_sorted, surf_fit_hess_int,
+                                               new_x_ordering);
+      // Assumes surf_fit_hess and surf_fit_gf share the same space
+      const int hess_dim = surf_fit_hess->VectorDim();
+      const int hess_cnt = surf_fit_hess->Size()/hess_dim;
+      if (surf_fit_hess->FESpace()->GetOrdering() == Ordering::byNODES)
+      {
+         for (int d = 0; d < hess_dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               (*surf_fit_hess)[dof_index + d*hess_cnt] =
+                  surf_fit_hess_int(i + d*cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < hess_dim; d++)
+            {
+               (*surf_fit_hess)[dof_index*hess_dim + d] =
+                  surf_fit_hess_int(i*hess_dim + d);
+            }
+         }
+      }
+
+   }
+   else
+   {
+      surf_fit_eval->ComputeAtNewPosition(new_x, *surf_fit_gf, new_x_ordering);
+      if (surf_fit_eval_grad)
+      {
+         surf_fit_eval_grad->ComputeAtNewPosition(new_x, *surf_fit_grad,
+                                                  new_x_ordering);
+      }
+      if (surf_fit_eval_hess)
+      {
+         surf_fit_eval_hess->ComputeAtNewPosition(new_x, *surf_fit_hess,
+                                                  new_x_ordering);
+      }
+   }
+}
+
 void TMOP_Integrator::
 UpdateAfterMeshPositionChange(const Vector &x_new,
                              const FiniteElementSpace &x_fes)
@@ -4406,112 +4621,11 @@ UpdateAfterMeshPositionChange(const Vector &x_new,
      adapt_lim_eval->ComputeAtNewPosition(x_new, *adapt_lim_gf, ordering);
   }

-   // Update surf_fit_gf if surface fitting is enabled.
+   // Update surf_fit_gf (and optionally its gradients) if surface
+   // fitting is enabled.
   if (surf_fit_gf)
   {
-      if (surf_fit_gf_bg)
-      {
-         // Interpolate information for only DOFs marked for fitting.
-         const int dim = surf_fit_gf->FESpace()->GetMesh()->Dimension();
-         const int cnt = surf_fit_marker_dof_index.Size();
-         const int total_cnt = x_new.Size()/dim;
-         Vector new_x_sorted(cnt*dim);
-         if (ordering == 0)
-         {
-            for (int d = 0; d < dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  new_x_sorted(i + d*cnt) = x_new(dof_index + d*total_cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < dim; d++)
-               {
-                  new_x_sorted(d + i*dim) = x_new(d + dof_index*dim);
-               }
-            }
-         }
-
-         Vector surf_fit_gf_int, surf_fit_grad_int, surf_fit_hess_int;
-         surf_fit_eval->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_gf_int, ordering);
-         for (int i = 0; i < cnt; i++)
-         {
-            int dof_index = surf_fit_marker_dof_index[i];
-            (*surf_fit_gf)[dof_index] = surf_fit_gf_int(i);
-         }
-
-         surf_fit_eval_bg_grad->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_grad_int, ordering);
-         // Assumes surf_fit_grad and surf_fit_gf share the same space
-         const int grad_dim = surf_fit_grad->VectorDim();
-         const int grad_cnt = surf_fit_grad->Size()/grad_dim;
-         if (surf_fit_grad->FESpace()->GetOrdering() == Ordering::byNODES)
-         {
-            for (int d = 0; d < grad_dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  (*surf_fit_grad)[dof_index + d*grad_cnt] =
-                     surf_fit_grad_int(i + d*cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < grad_dim; d++)
-               {
-                  (*surf_fit_grad)[dof_index*dim + d] =
-                     surf_fit_grad_int(i*dim + d);
-               }
-            }
-         }
-
-         surf_fit_eval_bg_hess->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_hess_int, ordering);
-         // Assumes surf_fit_hess and surf_fit_gf share the same space
-         const int hess_dim = surf_fit_hess->VectorDim();
-         const int hess_cnt = surf_fit_hess->Size()/hess_dim;
-         if (surf_fit_hess->FESpace()->GetOrdering() == Ordering::byNODES)
-         {
-            for (int d = 0; d < hess_dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  (*surf_fit_hess)[dof_index + d*hess_cnt] =
-                     surf_fit_hess_int(i + d*cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < hess_dim; d++)
-               {
-                  (*surf_fit_hess)[dof_index*dim + d] =
-                     surf_fit_hess_int(i*dim + d);
-               }
-            }
-         }
-      }
-      else
-      {
-         surf_fit_eval->ComputeAtNewPosition(x_new, *surf_fit_gf, ordering);
-      }
+      RemapSurfaceFittingLevelSetAtNodes(x_new, ordering);
   }
 }

@@ -1784,12 +1784,11 @@ protected:
   // Fitting to given physical positions.
   TMOP_QuadraticLimiter *surf_fit_limiter; // Owned. Created internally.
   const GridFunction *surf_fit_pos;        // Not owned. Positions to fit.
-   real_t surf_fit_normal;
-   bool surf_fit_gf_bg;
-   GridFunction *surf_fit_grad, *surf_fit_hess;
-   AdaptivityEvaluator *surf_fit_eval_bg_grad, *surf_fit_eval_bg_hess;
-   Array<int> surf_fit_dof_count;
-   Array<int> surf_fit_marker_dof_index;
+   real_t surf_fit_normal;                  // Normalization factor.
+   GridFunction *surf_fit_grad, *surf_fit_hess; // Owned. Created internally.
+   AdaptivityEvaluator *surf_fit_eval_grad, *surf_fit_eval_hess; // Not owned.
+   Array<int> surf_fit_dof_count;            // Number of dofs per node.
+   Array<int> surf_fit_marker_dof_index;     // Indices of nodes to fit.

   DiscreteAdaptTC *discr_tc;

@@ -1985,6 +1984,10 @@ protected:
   real_t ComputeUntanglerMaxMuBarrier(const Vector &x,
                                       const FiniteElementSpace &fes);

+   // Remaps the internal surface fitting gridfunction object at provided
+   // locations.
+   void RemapSurfaceFittingLevelSetAtNodes(const Vector &new_x,
+                                           int new_x_ordering);
 public:
   /** @param[in] m    TMOP_QualityMetric for r-adaptivity (not owned).
       @param[in] tc   Target-matrix construction algorithm to use (not owned).
@@ -2000,9 +2003,8 @@ public:
        surf_fit_marker(NULL), surf_fit_coeff(NULL),
        surf_fit_gf(NULL), surf_fit_eval(NULL),
        surf_fit_limiter(NULL), surf_fit_pos(NULL),
-        surf_fit_normal(1.0),
-        surf_fit_gf_bg(false), surf_fit_grad(NULL), surf_fit_hess(NULL),
-        surf_fit_eval_bg_grad(NULL), surf_fit_eval_bg_hess(NULL),
+        surf_fit_normal(1.0), surf_fit_grad(NULL), surf_fit_hess(NULL),
+        surf_fit_eval_grad(NULL), surf_fit_eval_hess(NULL),
        discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
        fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
   { PA.enabled = false; }
@@ -2103,9 +2105,15 @@ public:

 #ifdef MFEM_USE_MPI
   /// Parallel support for surface fitting to the zero level set of a function.
+   /// Here, we add two optional inputs: @a aegrad and @a aehess. When provided,
+   /// the first and second derivative of the input level set are computed on
+   /// the initial mesh, and @a aegrad and @a aehess are used to remap grad_s(x)
+   /// from grad_s0(x0) and hess_s(x) from hess_s0(x0), respectively.
   void EnableSurfaceFitting(const ParGridFunction &s0,
                             const Array<bool> &smarker, Coefficient &coeff,
-                             AdaptivityEvaluator &ae);
+                             AdaptivityEvaluator &ae,
+                             AdaptivityEvaluator *aegrad = NULL,
+                             AdaptivityEvaluator *aehess = NULL);

   /** @brief Fitting of certain DOFs in the current mesh to the zero level set
       of a function defined on another (finer) source mesh.
@@ -429,11 +429,13 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 #endif

   real_t scale = 1.0;
-   real_t avg_surf_fit_err, max_surf_fit_err = 0.0;
-   if (surf_fit_max_threshold > 0.0)
+   bool fitting = IsSurfaceFittingEnabled();
+   real_t init_fit_avg_err, init_fit_max_err = 0.0;
+   if (fitting && surf_fit_converge_error)
   {
-      GetSurfaceFittingError(x_out_loc, avg_surf_fit_err, max_surf_fit_err);
-      if (max_surf_fit_err < surf_fit_max_threshold)
+      GetSurfaceFittingError(x_out_loc, init_fit_avg_err, init_fit_max_err);
+      // Check for convergence
+      if (init_fit_max_err < surf_fit_max_err_limit)
      {
         if (print_options.iterations)
         {
@@ -444,11 +446,12 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
         return scale;
      }
   }
-   if (adapt_inc_count >= max_adapt_inc_count)
+
+   if (surf_fit_adapt_count >= surf_fit_adapt_count_limit)
   {
      if (print_options.iterations)
      {
-         mfem::out << "TMOPNewtonSolver converged "
+         mfem::out << "TMOPNewtonSolver terminated "
                   "based on max number of times surface fitting weight can"
                   "be increased. \n";
      }
@@ -467,7 +470,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
      // reference to detect deteriorations.
      MFEM_VERIFY(min_det_ptr != NULL, " Initial mesh was valid, but"
                  " intermediate mesh is invalid. Contact TMOP Developers.");
-      MFEM_VERIFY(min_detJ_threshold == 0.0,
+      MFEM_VERIFY(min_detJ_limit == 0.0,
                  "This setup is not supported. Contact TMOP Developers.");
      *min_det_ptr = untangle_factor * min_detT_in;
   }
@@ -478,6 +481,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
   bool x_out_ok = false;
   real_t energy_out = 0.0, min_detT_out;
   const real_t norm_in = Norm(r);
+   real_t avg_fit_err, max_fit_err = 0.0;

   const real_t detJ_factor = (solver_type == 1) ? 0.25 : 0.5;
   compute_metric_quantile_flag = false;
@@ -488,6 +492,9 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
   // Perform the line search.
   for (int i = 0; i < 12; i++)
   {
+      avg_fit_err = 0.0;
+      max_fit_err = 0.0;
+
      // Update the mesh and get the L-vector in x_out_loc.
      add(x, -scale, c, x_out);
      if (serial)
@@ -502,7 +509,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,

      // Check the changes in detJ.
      min_detT_out = ComputeMinDet(x_out_loc, *fes);
-      if (untangling == false && min_detT_out <= min_detJ_threshold)
+      if (untangling == false && min_detT_out <= min_detJ_limit)
      {
         // No untangling, and detJ got negative (or small) -- no good.
         if (print_options.iterations)
@@ -529,18 +536,19 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
      // Check the changes in total energy.
      ProcessNewState(x_out);

-      real_t avg_fit_err, max_fit_err = 0.0;
-      if (surf_fit_max_threshold > 0.0)
+      // Ensure sufficient decrease in fitting error if we are trying to
+      // converge based on error.
+      if (fitting && surf_fit_converge_error)
      {
         GetSurfaceFittingError(x_out_loc, avg_fit_err, max_fit_err);
-      }
-      if (surf_fit_max_threshold > 0.0 && max_fit_err >= 1.2*max_surf_fit_err)
-      {
-         if (print_options.iterations)
+         if (max_fit_err >= 1.2*init_fit_max_err)
         {
-            mfem::out << "Scale = " << scale << " Surf fit err increased.\n";
+            if (print_options.iterations)
+            {
+               mfem::out << "Scale = " << scale << " Surf fit err increased.\n";
+            }
+            scale *= 0.5; continue;
         }
-         scale *= 0.5; continue;
      }

      if (serial)
@@ -614,7 +622,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,

   if (x_out_ok == false) { scale = 0.0; }

-   if (surf_fit_scale_factor > 0.0) { update_surf_fit_coeff = true; }
+   if (surf_fit_scale_factor > 0.0) { surf_fit_coeff_update = true; }
   compute_metric_quantile_flag = true;

   return scale;
@@ -657,7 +665,7 @@ void TMOPNewtonSolver::GetSurfaceFittingWeight(Array<real_t> &weights) const
   for (int i = 0; i < integs.Size(); i++)
   {
      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
-      if (ti)
+      if (ti && ti->IsSurfaceFittingEnabled())
      {
         weight = ti->GetSurfaceFittingWeight();
         weights.Append(weight);
@@ -668,8 +676,11 @@ void TMOPNewtonSolver::GetSurfaceFittingWeight(Array<real_t> &weights) const
         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
         for (int j = 0; j < ati.Size(); j++)
         {
-            weight = ati[j]->GetSurfaceFittingWeight();
-            weights.Append(weight);
+            if (ati[j]->IsSurfaceFittingEnabled())
+            {
+               weight = ati[j]->GetSurfaceFittingWeight();
+               weights.Append(weight);
+            }
         }
      }
   }
@@ -716,6 +727,39 @@ void TMOPNewtonSolver::GetSurfaceFittingError(const Vector &x_loc,
   }
 }

+bool TMOPNewtonSolver::IsSurfaceFittingEnabled() const
+{
+   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
+   const Array<NonlinearFormIntegrator*> &integs = *nlf->GetDNFI();
+   TMOP_Integrator *ti  = NULL;
+   TMOPComboIntegrator *co = NULL;
+
+   for (int i = 0; i < integs.Size(); i++)
+   {
+      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+      if (ti)
+      {
+         if (ti->IsSurfaceFittingEnabled())
+         {
+            return true;
+         }
+      }
+      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+      if (co)
+      {
+         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+         for (int j = 0; j < ati.Size(); j++)
+         {
+            if (ati[j]->IsSurfaceFittingEnabled())
+            {
+               return true;
+            }
+         }
+      }
+   }
+   return false;
+}
+
 void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
 {
   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
@@ -801,38 +845,46 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
   // adaptive surface fitting is enabled. The idea is to increase the
   // coefficient if the surface fitting error does not sufficiently
   // decrease between subsequent TMOPNewtonSolver iterations.
-   if (update_surf_fit_coeff)
+   if (surf_fit_coeff_update)
   {
      // Get surface fitting errors.
-      GetSurfaceFittingError(x_loc, surf_fit_err_avg, surf_fit_err_max);
+      GetSurfaceFittingError(x_loc, surf_fit_avg_err, surf_fit_max_err);
      // Get array with surface fitting weights.
-      Array<real_t> weights;
-      GetSurfaceFittingWeight(weights);
+      Array<real_t> fitweights;
+      GetSurfaceFittingWeight(fitweights);

      if (print_options.iterations)
      {
         mfem::out << "Avg/Max surface fitting error: " <<
-                   surf_fit_err_avg << " " <<
-                   surf_fit_err_max << "\n";
+                   surf_fit_avg_err << " " <<
+                   surf_fit_max_err << "\n";
         mfem::out << "Min/Max surface fitting weight: " <<
-                   weights.Min() << " " << weights.Max() << "\n";
+                   fitweights.Min() << " " << fitweights.Max() << "\n";
      }

-      real_t change_surf_fit_err = surf_fit_err_avg_prvs-surf_fit_err_avg;
-      real_t rel_change_surf_fit_err = change_surf_fit_err/surf_fit_err_avg_prvs;
+      real_t change_surf_fit_err = surf_fit_avg_err_prvs-surf_fit_avg_err;
+      real_t rel_change_surf_fit_err = change_surf_fit_err/surf_fit_avg_err_prvs;
+
      // Increase the surface fitting coefficient if the surface fitting error
-      // does not decrease sufficiently.
-      if (rel_change_surf_fit_err < surf_fit_rel_change_threshold)
+      // does not decrease sufficiently. If we are converging based on residual,
+      // also make sure we have not reached the maximum fitting weight and
+      // error threshold.
+      if (rel_change_surf_fit_err < surf_fit_err_rel_change_limit &&
+          (surf_fit_converge_error ||
+           (fitweights.Max() < surf_fit_weight_limit &&
+            surf_fit_max_err > surf_fit_max_err_limit)))
      {
-         UpdateSurfaceFittingWeight(surf_fit_scale_factor);
-         adapt_inc_count += 1;
+         real_t scale_factor = std::min(surf_fit_scale_factor,
+                                        surf_fit_weight_limit/fitweights.Max());
+         UpdateSurfaceFittingWeight(scale_factor);
+         surf_fit_adapt_count += 1;
      }
      else
      {
-         adapt_inc_count = 0;
+         surf_fit_adapt_count = 0;
      }
-      surf_fit_err_avg_prvs = surf_fit_err_avg;
-      update_surf_fit_coeff = false;
+      surf_fit_avg_err_prvs = surf_fit_avg_err;
+      surf_fit_coeff_update = false;
   }
 }

@@ -134,18 +134,20 @@ protected:
   int solver_type;
   bool parallel;

-   // Line search step is rejected if min(detJ) <= min_detJ_threshold.
-   real_t min_detJ_threshold = 0.0;
+   // Line search step is rejected if min(detJ) <= min_detJ_limit.
+   real_t min_detJ_limit = 0.0;

   // Surface fitting variables.
-   mutable real_t surf_fit_err_avg_prvs = 10000.0;
-   mutable real_t surf_fit_err_avg, surf_fit_err_max;
-   mutable bool update_surf_fit_coeff = false;
-   real_t surf_fit_max_threshold = -1.0;
-   real_t surf_fit_rel_change_threshold = 0.001;
+   mutable real_t surf_fit_avg_err_prvs = 10000.0;
+   mutable real_t surf_fit_avg_err, surf_fit_max_err;
+   mutable bool surf_fit_coeff_update = false;
+   real_t surf_fit_max_err_limit = -1.0;
+   real_t surf_fit_err_rel_change_limit = 0.001;
   real_t surf_fit_scale_factor = 0.0;
-   mutable int adapt_inc_count = 0;
-   mutable int max_adapt_inc_count = 10;
+   mutable int surf_fit_adapt_count = 0;
+   mutable int surf_fit_adapt_count_limit = 10;
+   mutable real_t surf_fit_weight_limit = 1e10;
+   bool surf_fit_converge_error = false;

   // Minimum determinant over the whole mesh. Used for mesh untangling.
   real_t *min_det_ptr = nullptr;
@@ -191,6 +193,9 @@ protected:
   void GetSurfaceFittingWeight(Array<real_t> &weights) const;
   ///@}

+   /// Check if surface fitting is enabled.
+   bool IsSurfaceFittingEnabled() const;
+
 public:
 #ifdef MFEM_USE_MPI
   TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule, int type = 0)
@@ -224,38 +229,94 @@ public:
   /// (ii) surface fitting weight.
   virtual void ProcessNewState(const Vector &x) const;

-   /** @name Methods for adaptive surface fitting weight. (Experimental) */
-   /// Enable/Disable adaptive surface fitting weight.
-   /// The weight is modified after each TMOPNewtonSolver iteration as:
-   /// w_{k+1} = w_{k} * @a surf_fit_scale_factor if relative change in
-   /// max surface fitting error < @a surf_fit_rel_change_threshold.
-   /// The solver terminates if the maximum surface fitting error does
-   /// not sufficiently decrease for @a max_adapt_inc_count consecutive
-   /// solver iterations or if the max error falls below @a surf_fit_max_threshold.
-   void EnableAdaptiveSurfaceFitting()
-   {
-      surf_fit_scale_factor = 10.0;
-      surf_fit_rel_change_threshold = 0.001;
-   }
+   /** @name Methods for adaptive surface fitting.
+       \brief These methods control the behavior of the weight and the
+       termination of the solver. (Experimental)
+
+       Adaptive fitting weight: The weight is modified after each
+       TMOPNewtonSolver iteration as:
+       w_{k+1} = w_{k} * \ref surf_fit_scale_factor if the relative
+       change in average fitting error < \ref surf_fit_err_rel_change_limit.
+       When converging based on the residual, we enforce the fitting weight
+       to be at-most \ref surf_fit_weight_limit, and increase it only if the
+       fitting error is below user prescribed threshold
+       (\ref surf_fit_max_err_limit).
+       See \ref SetAdaptiveSurfaceFittingScalingFactor and
+       \ref SetAdaptiveSurfaceFittingRelativeChangeThreshold.
+
+       Note that the solver stops if the maximum surface fitting error
+       does not sufficiently decrease for \ref surf_fit_adapt_count_limit (default 10)
+       consecutive increments of the fitting weight during weight adaptation.
+       This typically occurs when the mesh cannot align with the level-set
+       without degrading element quality.
+       See \ref SetMaxNumberofIncrementsForAdaptiveFitting.
+
+       Convergence criterion: There are two modes, residual- and error-based,
+       which can be toggled using \ref SetSurfaceFittingConvergenceBasedOnError.
+
+       (i) Residual based (default): Stop when the norm of the gradient of the
+       TMOP objective reaches the prescribed tolerance. This method is best used
+       with a reasonable value for \ref surf_fit_weight_limit when the
+       adaptive surface fitting scheme is used. See method
+       \ref SetSurfaceFittingWeightLimit.
+
+       (ii) Error based: Stop when the maximum fitting error
+       reaches the user-prescribed threshold, \ref surf_fit_max_err_limit.
+       In this case, \ref surf_fit_weight_limit is ignored during weight
+       adaptation.
+   */
+   ///@{
   void SetAdaptiveSurfaceFittingScalingFactor(real_t factor)
   {
+      MFEM_VERIFY(factor > 1.0, "Scaling factor must be greater than 1.");
      surf_fit_scale_factor = factor;
   }
   void SetAdaptiveSurfaceFittingRelativeChangeThreshold(real_t threshold)
   {
-      surf_fit_rel_change_threshold = threshold;
+      surf_fit_err_rel_change_limit = threshold;
   }
+   /// Used for stopping based on the number of consecutive failed weight
+   /// adaptation iterations.
+   // TODO: Rename to SetMaxNumberofIncrementsForAdaptiveSurfaceFitting
+   // in future.
   void SetMaxNumberofIncrementsForAdaptiveFitting(int count)
   {
-      max_adapt_inc_count = count;
+      surf_fit_adapt_count_limit = count;
   }
+   /// Used for error-based surface fitting termination.
   void SetTerminationWithMaxSurfaceFittingError(real_t max_error)
   {
-      surf_fit_max_threshold = max_error;
+      surf_fit_max_err_limit = max_error;
+      surf_fit_converge_error = true;
   }
+   /// Could be used with both error-based or residual-based convergence.
+   void SetSurfaceFittingMaxErrorLimit(real_t max_error)
+   {
+      surf_fit_max_err_limit = max_error;
+   }
+   /// Used for residual-based surface fitting termination.
+   void SetSurfaceFittingWeightLimit(real_t weight)
+   {
+      surf_fit_weight_limit = weight;
+   }
+   /// Toggle convergence based on residual or error.
+   void SetSurfaceFittingConvergenceBasedOnError(bool mode)
+   {
+      surf_fit_converge_error = mode;
+      if (surf_fit_converge_error)
+      {
+         MFEM_VERIFY(surf_fit_max_err_limit >= 0,
+                     "Fitting error based convergence requires the user to "
+                     "first set the error threshold."
+                     "See SetTerminationWithMaxSurfaceFittingError");
+      }
+   }
+   ///@}
+
+   /// Set minimum determinant enforced during line-search.
   void SetMinimumDeterminantThreshold(real_t threshold)
   {
-      min_detJ_threshold = threshold;
+      min_detJ_limit = threshold;
   }

   virtual void Mult(const Vector &b, Vector &x) const
@@ -124,7 +124,7 @@ T Array<T>::Sum()
 }

 template <class T>
-int Array<T>::IsSorted()
+int Array<T>::IsSorted() const
 {
   T val_prev = operator[](0), val;
   for (int i = 1; i < size; i++)
@@ -74,10 +74,14 @@ public:
   inline Array(int asize, MemoryType mt)
      : size(asize) { asize > 0 ? data.New(asize, mt) : data.Reset(mt); }

-   /** @brief Creates array using an externally allocated pointer @a data_ to
-       @a asize elements. The data pointer will not be deleted by Array. */
-   inline Array(T *data_, int asize)
-   { data.Wrap(data_, asize, false); size = asize; }
+   /** @brief Creates array using an externally allocated host pointer @a data_
+       to @a asize elements. If @a own_data is true, the array takes ownership
+       of the pointer.
+
+       When @a own_data is true, the pointer @a data_ must be allocated with
+       MemoryType given by MemoryManager::GetHostMemoryType(). */
+   inline Array(T *data_, int asize, bool own_data = false)
+   { data.Wrap(data_, asize, own_data); size = asize; }

   /// Copy constructor: deep copy from @a src
   /** This method supports source arrays using any MemoryType. */
@@ -205,7 +209,14 @@ public:
   inline void Copy(Array &copy) const;

   /// Make this Array a reference to a pointer.
-   inline void MakeRef(T *, int);
+   /** When @a own_data is true, the pointer @a data_ must be allocated with
+       MemoryType given by MemoryManager::GetHostMemoryType(). */
+   inline void MakeRef(T *data_, int size_, bool own_data = false);
+
+   /// Make this Array a reference to a pointer.
+   /** When @a own_data is true, the pointer @a data_ must be allocated with
+       MemoryType given by @a mt. */
+   inline void MakeRef(T *data_, int size, MemoryType mt, bool own_data);

   /// Make this Array a reference to 'master'.
   inline void MakeRef(const Array &master);
@@ -262,7 +273,7 @@ public:
   }

   /// Return 1 if the array is sorted from lowest to highest.  Otherwise return 0.
-   int IsSorted();
+   int IsSorted() const;

   /// Fill the entries of the array with the cumulative sum of the entries.
   void PartialSum();
@@ -868,11 +879,19 @@ inline void Array<T>::Copy(Array &copy) const
 }

 template <class T>
-inline void Array<T>::MakeRef(T *p, int s)
+inline void Array<T>::MakeRef(T *data_, int size_, bool own_data)
 {
   data.Delete();
-   data.Wrap(p, s, false);
-   size = s;
+   data.Wrap(data_, size_, own_data);
+   size = size_;
+}
+
+template <class T>
+inline void Array<T>::MakeRef(T *data_, int size_, MemoryType mt, bool own_data)
+{
+   data.Delete();
+   data.Wrap(data_, size_, mt, own_data);
+   size = size_;
 }

 template <class T>
@@ -288,4 +288,3 @@ void ArraysByName<T>::Load(std::istream &in)
 }

 #endif
-
@@ -275,7 +275,7 @@ void GroupTopology::Save(ostream &os) const
   os << "\ncommunication_groups\n";
   os << "number_of_groups " << NGroups() << "\n\n";

-   os << "# number of entities in each group, followed by group ids in group\n";
+   os << "# number of entities in each group, followed by ranks in group\n";
   for (int group_id = 0; group_id < NGroups(); ++group_id)
   {
      int group_size = GetGroupSize(group_id);
@@ -14,6 +14,9 @@
 #ifdef MFEM_USE_CEED
 #include "../fem/ceed/interface/util.hpp"
 #endif
+#ifdef MFEM_USE_MPI
+#include "../linalg/hypre.hpp"
+#endif

 #include <unordered_map>
 #include <string>
@@ -250,6 +253,10 @@ void Device::Configure(const std::string &device, const int device_id)

   // Only '*this' will call the MemoryManager::Destroy() method.
   destroy_mm = true;
+
+#ifdef MFEM_USE_MPI
+   Hypre::InitDevice();
+#endif
 }

 // static method
@@ -19,6 +19,9 @@
 #include "device.hpp"
 #include "mem_manager.hpp"
 #include "../linalg/dtensor.hpp"
+#ifdef MFEM_USE_MPI
+#include <_hypre_utilities.h>
+#endif

 namespace mfem
 {
@@ -780,6 +783,63 @@ inline void forall_3D_grid(int N, int X, int Y, int Z, int G, lambda &&body)
   ForallWrap<3>(true, N, body, X, Y, Z, G);
 }

+#ifdef MFEM_USE_MPI
+
+// Function mfem::hypre_forall_cpu() similar to mfem::forall, but it always
+// executes on the CPU using sequential or OpenMP-parallel execution based on
+// the hypre build time configuration.
+template<typename lambda>
+inline void hypre_forall_cpu(int N, lambda &&body)
+{
+#ifdef HYPRE_USING_OPENMP
+   #pragma omp parallel for HYPRE_SMP_SCHEDULE
+#endif
+   for (int i = 0; i < N; i++) { body(i); }
+}
+
+// Function mfem::hypre_forall_gpu() similar to mfem::forall, but it always
+// executes on the GPU device that hypre was configured with at build time.
+#if defined(HYPRE_USING_GPU)
+template<typename lambda>
+inline void hypre_forall_gpu(int N, lambda &&body)
+{
+#if defined(HYPRE_USING_CUDA)
+   CuWrap1D(N, body);
+#elif defined(HYPRE_USING_HIP)
+   HipWrap1D(N, body);
+#else
+#error Unknown HYPRE GPU backend!
+#endif
+}
+#endif
+
+// Function mfem::hypre_forall() similar to mfem::forall, but it executes on the
+// device, CPU or GPU, that hypre was configured with at build time (when the
+// HYPRE version is < 2.31.0) or at runtime (when HYPRE was configured with GPU
+// support at build time and HYPRE's version is >= 2.31.0). This selection is
+// generally independent of what device was selected in MFEM's runtime
+// configuration.
+template<typename lambda>
+inline void hypre_forall(int N, lambda &&body)
+{
+#if !defined(HYPRE_USING_GPU)
+   hypre_forall_cpu(N, body);
+#elif MFEM_HYPRE_VERSION < 23100
+   hypre_forall_gpu(N, body);
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   if (!HypreUsingGPU())
+   {
+      hypre_forall_cpu(N, body);
+   }
+   else
+   {
+      hypre_forall_gpu(N, body);
+   }
+#endif
+}
+
+#endif // MFEM_USE_MPI
+
 } // namespace mfem

 #endif // MFEM_FORALL_HPP
@@ -1154,6 +1154,10 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
   //  dest   d  | h2d   d2d   d2d
   //        hd  | h2h   d2d   d2d

+   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
+   MFEM_ASSERT(dst_h_ptr != nullptr, "invalid dst_h_ptr = nullptr");
+   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
+
   const bool dst_on_host =
      (dst_flags & Mem::VALID_HOST) &&
      (!(dst_flags & Mem::VALID_DEVICE) ||
@@ -1229,6 +1233,10 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
 void MemoryManager::CopyToHost_(void *dest_h_ptr, const void *src_h_ptr,
                                size_t bytes, unsigned src_flags)
 {
+   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
+   MFEM_ASSERT(dest_h_ptr != nullptr, "invalid dest_h_ptr = nullptr");
+   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
+
   const bool src_on_host = src_flags & Mem::VALID_HOST;
   if (src_on_host)
   {
@@ -1255,6 +1263,10 @@ void MemoryManager::CopyToHost_(void *dest_h_ptr, const void *src_h_ptr,
 void MemoryManager::CopyFromHost_(void *dest_h_ptr, const void *src_h_ptr,
                                  size_t bytes, unsigned &dest_flags)
 {
+   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
+   MFEM_ASSERT(dest_h_ptr != nullptr, "invalid dest_h_ptr = nullptr");
+   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
+
   const bool dest_on_host = dest_flags & Mem::VALID_HOST;
   if (dest_on_host)
   {
@@ -18,8 +18,14 @@
 #include <cstring> // std::memcpy
 #include <type_traits> // std::is_const
 #include <cstddef> // std::max_align_t
+
 #ifdef MFEM_USE_MPI
-#include <HYPRE_config.h> // HYPRE_USING_GPU
+// Enable internal hypre timing routines
+#define HYPRE_TIMING
+#include <HYPRE_utilities.h> // for HYPRE_GetMemoryLocation() and others
+#if (21400 <= MFEM_HYPRE_VERSION) && (MFEM_HYPRE_VERSION < 21900)
+#include <_hypre_utilities.h> // for HYPRE_MEMORY_HOST and others
+#endif
 #endif

 namespace mfem
@@ -869,6 +875,45 @@ public:
 };


+#ifdef MFEM_USE_MPI
+
+#if MFEM_HYPRE_VERSION < 21400
+#define HYPRE_MEMORY_DEVICE (0)
+#define HYPRE_MEMORY_HOST   (1)
+#endif
+#if MFEM_HYPRE_VERSION < 21900
+typedef int HYPRE_MemoryLocation;
+#endif
+
+/// Return the configured HYPRE_MemoryLocation
+inline HYPRE_MemoryLocation GetHypreMemoryLocation()
+{
+#if !defined(HYPRE_USING_GPU)
+   return HYPRE_MEMORY_HOST;
+#elif MFEM_HYPRE_VERSION < 23100
+   return HYPRE_MEMORY_DEVICE;
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   HYPRE_MemoryLocation loc;
+   HYPRE_GetMemoryLocation(&loc);
+   return loc;
+#endif
+}
+
+/// Return true if HYPRE is configured to use GPU
+inline bool HypreUsingGPU()
+{
+#if !defined(HYPRE_USING_GPU)
+   return false;
+#elif MFEM_HYPRE_VERSION < 23100
+   return true;
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   return GetHypreMemoryLocation() != HYPRE_MEMORY_HOST;
+#endif
+}
+
+#endif // MFEM_USE_MPI
+
+
 // Inline methods

 template <typename T>
@@ -1004,10 +1049,12 @@ inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
         // If the following condition is true then MemoryManager::Exists()
         // should also be true:
         IsDeviceMemory(MemoryManager::GetDeviceMemoryType())
-#else
-         // When HYPRE_USING_GPU is defined we always register the 'base' if
-         // the MemoryManager::Exists():
+#elif MFEM_HYPRE_VERSION < 23100
+         // When HYPRE_USING_GPU is defined and HYPRE < 2.31.0, we always
+         // register the 'base' if the MemoryManager::Exists():
         MemoryManager::Exists()
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+         MemoryManager::Exists() && HypreUsingGPU()
 #endif
      )
      {
@@ -1213,9 +1260,10 @@ template <typename T>
 inline void Memory<T>::CopyFrom(const Memory &src, int size)
 {
   MFEM_VERIFY(src.capacity>=size && capacity>=size, "Incorrect size");
+   if (size <= 0) { return; }
   if (!(flags & Registered) && !(src.flags & Registered))
   {
-      if (h_ptr != src.h_ptr && size != 0)
+      if (h_ptr != src.h_ptr)
      {
         MFEM_ASSERT(h_ptr + size <= src.h_ptr || src.h_ptr + size <= h_ptr,
                     "data overlaps!");
@@ -1233,9 +1281,10 @@ template <typename T>
 inline void Memory<T>::CopyFromHost(const T *src, int size)
 {
   MFEM_VERIFY(capacity>=size, "Incorrect size");
+   if (size <= 0) { return; }
   if (!(flags & Registered))
   {
-      if (h_ptr != src && size != 0)
+      if (h_ptr != src)
      {
         MFEM_ASSERT(h_ptr + size <= src || src + size <= h_ptr,
                     "data overlaps!");
@@ -1252,7 +1301,6 @@ inline void Memory<T>::CopyFromHost(const T *src, int size)
 template <typename T>
 inline void Memory<T>::CopyTo(Memory &dest, int size) const
 {
-   MFEM_VERIFY(capacity>=size, "Incorrect size");
   dest.CopyFrom(*this, size);
 }

@@ -1260,9 +1308,10 @@ template <typename T>
 inline void Memory<T>::CopyToHost(T *dest, int size) const
 {
   MFEM_VERIFY(capacity>=size, "Incorrect size");
+   if (size <= 0) { return; }
   if (!(flags & Registered))
   {
-      if (h_ptr != dest && size != 0)
+      if (h_ptr != dest)
      {
         MFEM_ASSERT(h_ptr + size <= dest || dest + size <= h_ptr,
                     "data overlaps!");
@@ -134,7 +134,7 @@ int socketbuf::open(const char hostname[], int port)
      {
         closesocket(socket_descriptor);
         socket_descriptor = -2;
-         return -1;
+         continue;
      }
 #endif

@@ -148,7 +148,7 @@ int socketbuf::open(const char hostname[], int port)
   }

   freeaddrinfo(res);
-   return 0;
+   return (socket_descriptor < 0) ? -1 : 0;
 }

 int socketbuf::close()
@@ -95,7 +95,7 @@ public:
       not called, it returns the number of possible connections established
       by the used constructor. Otherwise, it is exactly the number of
       established connections before calling Finalize(). */
-   inline int Size_of_connections() const { return I[size]; }
+   inline int Size_of_connections() const { HostReadI(); return I[size]; }

   /** Returns index of the connection between element i of TYPE I and
       element j of TYPE II. If there is no connection between element i
@@ -207,7 +207,12 @@ template <> inline void Swap<Table>(Table &a, Table &b)
 void Transpose (const Table &A, Table &At, int ncols_A_ = -1);
 Table * Transpose (const Table &A);

-///  Transpose an Array<int>
+///  @brief Transpose an Array<int>.
+///
+/// The array @a A represents a table where each row @a i has exactly one
+/// connection to the column (TYPE II) index specified by @a A[i].
+///
+/// @note The column (TYPE II) indices in each row of @a At will be sorted.
 void Transpose(const Array<int> &A, Table &At, int ncols_A_ = -1);

 ///  C = A * B  (as boolean matrices)
@@ -400,6 +400,9 @@ inline double StopWatch::SystTime()

 StopWatch::StopWatch() : M(new internal::StopWatch) { }

+StopWatch::StopWatch(const StopWatch &sw)
+   : M(new internal::StopWatch(*(sw.M))) { }
+
 void StopWatch::Clear()
 {
   M->Clear();
@@ -40,6 +40,7 @@ private:
 public:
   /// Creates a new (stopped) StopWatch object.
   StopWatch();
+   StopWatch(const StopWatch &);

   /// Clear the elapsed time on the stopwatch and restart it if it's running.
   void Clear();
@@ -23,6 +23,10 @@
 #include "amgxsolver.hpp"
 #ifdef MFEM_USE_AMGX

+#ifdef MFEM_USE_MPI
+#include "../general/communication.hpp"
+#endif
+
 namespace mfem
 {

@@ -81,7 +81,7 @@ void BlockOperator::Mult (const Vector & x, Vector & y) const
      tmp.SetSize(row_offsets[iRow+1] - row_offsets[iRow]);
      for (int jCol=0; jCol < nColBlocks; ++jCol)
      {
-         if (op(iRow,jCol))
+         if (op(iRow,jCol) && coef(iRow,jCol) != 0.)
         {
            op(iRow,jCol)->Mult(xblock.GetBlock(jCol), tmp);
            yblock.GetBlock(iRow).Add(coef(iRow,jCol), tmp);
@@ -112,7 +112,7 @@ void BlockOperator::MultTranspose (const Vector & x, Vector & y) const
      tmp.SetSize(col_offsets[iRow+1] - col_offsets[iRow]);
      for (int jCol=0; jCol < nRowBlocks; ++jCol)
      {
-         if (op(jCol,iRow))
+         if (op(jCol,iRow) && coef(jCol,iRow) != 0.)
         {
            op(jCol,iRow)->MultTranspose(xblock.GetBlock(jCol), tmp);
            yblock.GetBlock(iRow).Add(coef(jCol,iRow), tmp);
@@ -1,3 +1,14 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
 #include "cpardiso.hpp"
 #include "hypre.hpp"
 #include <algorithm>
@@ -532,6 +532,69 @@ MatrixInverse *DenseMatrix::Inverse() const
   return new DenseMatrixInverse(*this);
 }

+void DenseMatrix::Exponential()
+{
+   MFEM_ASSERT(Height() == Width() && Height() <= 2,
+               "The matrix must be square and "
+               << "of size less than or equal to 2."
+               << "  Height() = " << Height()
+               << ", Width() = " << Width());
+
+   switch (Height())
+   {
+      case 1:
+      {
+         data[0] = std::exp(data[0]);
+         break;
+      }
+      case 2:
+      {
+         /// Formulas from Corollary 2.4 of doi:10.1109/9.233156
+         /// Note typo in the paper, in the prefactor in the equation under (i).
+         const real_t a = data[0];
+         const real_t b = data[1];
+         const real_t c = data[2];
+         const real_t d = data[3];
+         const real_t e = (a - d)*(a - d) + 4*b*c;
+         const real_t f = std::exp((a + d)/2.0);
+         const real_t g = std::sqrt(std::abs(e)) / 2.0;
+
+         if (e == 0)
+         {
+            data[0] = 1.0 + (a - d)/2.0;
+            data[3] = 1.0 - (a - d)/2.0;
+         }
+         else if (e > 0)
+         {
+            data[0] = std::cosh(g) + (a - d)/2 * std::sinh(g) / g;
+            data[1] = b * std::sinh(g) / g;
+            data[2] = c * std::sinh(g) / g;
+            data[3] = std::cosh(g) - (a - d)/2 * std::sinh(g) / g;
+         }
+         else
+         {
+            data[0] = std::cos(g) + (a - d)/2 * std::sin(g) / g;
+            data[1] = b * std::sin(g) / g;
+            data[2] = c * std::sin(g) / g;
+            data[3] = std::cos(g) - (a - d)/2 * std::sin(g) / g;
+         }
+         for (int i = 0; i < 4; i++)
+         {
+            data[i] *= f;
+         }
+         break;
+      }
+      case 3:
+      {
+         MFEM_ABORT("3x3 matrices are not currently supported");
+      }
+      default:
+      {
+         MFEM_ABORT("Only 1x1 and 2x2 matrices are currently supported");
+      }
+   }
+}
+
 real_t DenseMatrix::Det() const
 {
   MFEM_ASSERT(Height() == Width() && Height() > 0,
@@ -3217,6 +3280,93 @@ void MultAtB(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &AtB)
 #endif
 }

+void AddMultAtB(const DenseMatrix &A, const DenseMatrix &B,
+                DenseMatrix &AtB)
+{
+   MFEM_ASSERT(AtB.Height() == A.Width() && AtB.Width() == B.Width() &&
+               A.Height() == B.Height(), "incompatible dimensions");
+
+#ifdef MFEM_USE_LAPACK
+   static char transa = 'T', transb = 'N';
+   static real_t alpha = 1.0, beta = 1.0;
+   int m = A.Width(), n = B.Width(), k = A.Height();
+
+#ifdef MFEM_USE_SINGLE
+   sgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
+#elif defined MFEM_USE_DOUBLE
+   dgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
+#endif
+          B.Data(), &k, &beta, AtB.Data(), &m);
+#else
+   const int ah = A.Height();
+   const int aw = A.Width();
+   const int bw = B.Width();
+   const real_t *ad = A.Data();
+   const real_t *bd = B.Data();
+   real_t *cd = AtB.Data();
+
+   for (int j = 0; j < bw; j++)
+   {
+      const real_t *ap = ad;
+      for (int i = 0; i < aw; i++)
+      {
+         real_t d = 0.0;
+         for (int k = 0; k < ah; k++)
+         {
+            d += ap[k] * bd[k];
+         }
+         *(cd++) += d;
+         ap += ah;
+      }
+      bd += ah;
+   }
+#endif
+}
+
+void AddMult_a_AtB(real_t a, const DenseMatrix &A, const DenseMatrix &B,
+                   DenseMatrix &AtB)
+{
+   MFEM_ASSERT(AtB.Height() == A.Width() && AtB.Width() == B.Width() &&
+               A.Height() == B.Height(), "incompatible dimensions");
+
+#ifdef MFEM_USE_LAPACK
+   static char transa = 'T', transb = 'N';
+   real_t alpha = a;
+   static real_t beta = 1.0;
+   int m = A.Width(), n = B.Width(), k = A.Height();
+
+#ifdef MFEM_USE_SINGLE
+   sgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
+#elif defined MFEM_USE_DOUBLE
+   dgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
+#endif
+          B.Data(), &k, &beta, AtB.Data(), &m);
+#else
+   const int ah = A.Height();
+   const int aw = A.Width();
+   const int bw = B.Width();
+   const real_t *ad = A.Data();
+   const real_t *bd = B.Data();
+   real_t *cd = AtB.Data();
+
+   for (int j = 0; j < bw; j++)
+   {
+      const real_t *ap = ad;
+      for (int i = 0; i < aw; i++)
+      {
+         real_t d = 0.0;
+         for (int k = 0; k < ah; k++)
+         {
+            d += ap[k] * bd[k];
+         }
+         *(cd++) += a * d;
+         ap += ah;
+      }
+      bd += ah;
+   }
+#endif
+}
+
 void AddMult_a_AAt(real_t a, const DenseMatrix &A, DenseMatrix &AAt)
 {
   real_t d;
@@ -207,6 +207,10 @@ public:
   /// Replaces the current matrix with its square root inverse
   void SquareRootInverse();

+   /// Replaces the current matrix with its exponential
+   /// (currently only supports 2x2 matrices)
+   void Exponential();
+
   /// Calculates the determinant of the matrix
   /// (optimized for 2x2, 3x3, and 4x4 matrices)
   real_t Det() const;
@@ -580,6 +584,13 @@ void AddMult_a_ABt(real_t a, const DenseMatrix &A, const DenseMatrix &B,
 /// Multiply the transpose of a matrix A with a matrix B:   At*B
 void MultAtB(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &AtB);

+/// AtB += A^t * B
+void AddMultAtB(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &AtB);
+
+/// AtB += a * A^t * B
+void AddMult_a_AtB(real_t a, const DenseMatrix &A, const DenseMatrix &B,
+                   DenseMatrix &AtB);
+
 /// AAt += a * A * A^t
 void AddMult_a_AAt(real_t a, const DenseMatrix &A, DenseMatrix &AAt);

@@ -16,17 +16,19 @@

 #ifdef MFEM_USE_MPI

+#include "../general/globals.hpp"
+#include "sparsemat.hpp"
+#include "hypre_parcsr.hpp"
 #include <mpi.h>

 // Enable internal hypre timing routines
 #define HYPRE_TIMING

 // hypre header files
-#include "seq_mv.h"
-#include "_hypre_parcsr_mv.h"
-#include "_hypre_parcsr_ls.h"
-#include "temp_multivector.h"
-#include "../general/globals.hpp"
+#include <seq_mv.h>
+#include <temp_multivector.h>
+#include <_hypre_parcsr_mv.h>
+#include <_hypre_parcsr_ls.h>

 #ifdef HYPRE_COMPLEX
 #error "MFEM does not work with HYPRE's complex numbers support"
@@ -51,22 +53,6 @@
 #error "MFEM_USE_HIP=YES is required when HYPRE is built with HIP!"
 #endif

-// MFEM_HYPRE_FORALL is a macro similar to mfem::forall, but it executes on the
-// device that hypre was configured with (no matter what device was selected
-// in MFEM's runtime configuration).
-#if defined(HYPRE_USING_CUDA)
-#define MFEM_HYPRE_FORALL(i, N,...) CuWrap1D(N, [=] MFEM_DEVICE      \
-                                       (int i) {__VA_ARGS__})
-#elif defined(HYPRE_USING_HIP)
-#define MFEM_HYPRE_FORALL(i, N,...) HipWrap1D(N, [=] MFEM_DEVICE     \
-                                        (int i) {__VA_ARGS__})
-#else
-#define MFEM_HYPRE_FORALL(i, N,...) for (int i = 0; i < N; i++) { __VA_ARGS__ }
-#endif
-
-#include "sparsemat.hpp"
-#include "hypre_parcsr.hpp"
-
 namespace mfem
 {

@@ -87,6 +73,22 @@ public:
   /// Calling HYPRE_Finalize() manually is not compatible with this class.
   static void Init() { Instance(); }

+   /// @brief Configure HYPRE's compute and memory policy.
+   ///
+   /// By default HYPRE will be configured with the same policy as MFEM unless
+   /// `Hypre::configure_runtime_policy_from_mfem` is false, in which case
+   /// HYPRE's default will be used; if HYPRE is built for the GPU and the
+   /// aforementioned variable is false then HYPRE will use the GPU even if MFEM
+   /// is not.
+   ///
+   /// This function is no-op if HYPRE is built without GPU support or the HYPRE
+   /// version is less than 2.31.0.
+   ///
+   /// This function is NOT called by Init(). Instead it is called by
+   /// Device::Configure() (when MFEM_USE_MPI=YES) after the MFEM device
+   /// configuration is complete.
+   static void InitDevice();
+
   /// @brief Finalize hypre (called automatically at program exit if
   /// Hypre::Init() has been called).
   ///
@@ -94,6 +96,13 @@ public:
   /// called manually to more precisely control when hypre is finalized.
   static void Finalize();

+   /// @brief Use MFEM's device policy to configure HYPRE's device policy, true
+   /// by default. This variable is used by InitDevice().
+   ///
+   /// This value is not used if HYPRE is build without GPU support or the HYPRE
+   /// version is less than 2.31.0.
+   static bool configure_runtime_policy_from_mfem;
+
 private:
   /// Calls HYPRE_Init() when the singleton is constructed.
   Hypre();
@@ -142,15 +151,28 @@ inline int to_int(HYPRE_Int i)


 /// The MemoryClass used by Hypre objects.
-inline constexpr MemoryClass GetHypreMemoryClass()
+inline MemoryClass GetHypreMemoryClass()
 {
 #if !defined(HYPRE_USING_GPU)
   return MemoryClass::HOST;
-#elif defined(HYPRE_USING_UNIFIED_MEMORY)
+#elif MFEM_HYPRE_VERSION < 23100
+#if defined(HYPRE_USING_UNIFIED_MEMORY)
   return MemoryClass::MANAGED;
 #else
   return MemoryClass::DEVICE;
 #endif
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   if (GetHypreMemoryLocation() == HYPRE_MEMORY_HOST)
+   {
+      return MemoryClass::HOST;
+   }
+   // Return the actual memory location, see hypre_GetActualMemLocation():
+#if defined(HYPRE_USING_UNIFIED_MEMORY)
+   return MemoryClass::MANAGED;
+#else
+   return MemoryClass::DEVICE;
+#endif
+#endif
 }

 /// The MemoryType used by MFEM when allocating arrays for Hypre objects.
@@ -158,13 +180,27 @@ inline MemoryType GetHypreMemoryType()
 {
 #if !defined(HYPRE_USING_GPU)
   return Device::GetHostMemoryType();
-#elif defined(HYPRE_USING_UNIFIED_MEMORY)
+#elif MFEM_HYPRE_VERSION < 23100
+#if defined(HYPRE_USING_UNIFIED_MEMORY)
   return MemoryType::MANAGED;
 #else
   return MemoryType::DEVICE;
 #endif
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   if (GetHypreMemoryLocation() == HYPRE_MEMORY_HOST)
+   {
+      return Device::GetHostMemoryType();
+   }
+   // Return the actual memory location, see hypre_GetActualMemLocation():
+#if defined(HYPRE_USING_UNIFIED_MEMORY)
+   return MemoryType::MANAGED;
+#else
+   return MemoryType::DEVICE;
+#endif
+#endif
 }

+
 /// Wrapper for hypre's parallel vector class
 class HypreParVector : public Vector
 {
@@ -328,10 +364,10 @@ public:
   HYPRE_Int Randomize(HYPRE_Int seed);

   /// Prints the locally owned rows in parallel
-   void Print(const char *fname) const;
+   void Print(const std::string &fname) const;

   /// Reads a HypreParVector from files saved with HypreParVector::Print
-   void Read(MPI_Comm comm, const char *fname);
+   void Read(MPI_Comm comm, const std::string &fname);

   /// Calls hypre's destroy function
   ~HypreParVector();
@@ -883,12 +919,14 @@ public:
   const Memory<HYPRE_Int> &GetDiagMemoryJ() const { return mem_diag.J; }
   const Memory<real_t> &GetDiagMemoryData() const { return mem_diag.data; }

-   /// Prints the locally owned rows in parallel
-   void Print(const char *fname, HYPRE_Int offi = 0, HYPRE_Int offj = 0) const;
+   /// @brief Prints the locally owned rows in parallel. The resulting files can
+   /// be read with Read_IJMatrix().
+   void Print(const std::string &fname, HYPRE_Int offi = 0,
+              HYPRE_Int offj = 0) const;
   /// Reads the matrix from a file
-   void Read(MPI_Comm comm, const char *fname);
+   void Read(MPI_Comm comm, const std::string &fname);
   /// Read a matrix saved as a HYPRE_IJMatrix
-   void Read_IJMatrix(MPI_Comm comm, const char *fname);
+   void Read_IJMatrix(MPI_Comm comm, const std::string &fname);

   /// Print information about the hypre_ParCSRCommPkg of the HypreParMatrix.
   void PrintCommPkg(std::ostream &out = mfem::out) const;
@@ -1037,29 +1075,40 @@ protected:
   bool A_is_symmetric;

 public:
-   /** Hypre smoother types:
-       0    = Jacobi
-       1    = l1-scaled Jacobi
-       2    = l1-scaled block Gauss-Seidel/SSOR
-       4    = truncated l1-scaled block Gauss-Seidel/SSOR
-       5    = lumped Jacobi
-       6    = Gauss-Seidel
-       10   = On-processor forward solve for matrix w/ triangular structure
-       16   = Chebyshev
-       1001 = Taubin polynomial smoother
-       1002 = FIR polynomial smoother. */
-   enum Type { Jacobi = 0, l1Jacobi = 1, l1GS = 2, l1GStr = 4, lumpedJacobi = 5,
-               GS = 6, OPFS = 10, Chebyshev = 16, Taubin = 1001, FIR = 1002
-             };
+   /// HYPRE smoother types
+   enum Type
+   {
+      Jacobi = 0,       ///< Jacobi
+      l1Jacobi = 1,     ///< l1-scaled Jacobi
+      l1GS = 2,         ///< l1-scaled block Gauss-Seidel/SSOR
+      l1GStr = 4,       ///< truncated l1-scaled block Gauss-Seidel/SSOR
+      lumpedJacobi = 5, ///< lumped Jacobi
+      GS = 6,           ///< Gauss-Seidel
+      OPFS = 10,        /**< On-processor forward solve for matrix w/ triangular
+                             structure */
+      Chebyshev = 16,   ///< Chebyshev
+      Taubin = 1001,    ///< Taubin polynomial smoother
+      FIR = 1002        ///< FIR polynomial smoother
+   };
+
+   /// @deprecated Use DefaultType() instead
 #if !defined(HYPRE_USING_GPU)
-   static constexpr Type default_type = l1GS;
+   MFEM_DEPRECATED static constexpr Type default_type = l1GS;
 #else
-   static constexpr Type default_type = l1Jacobi;
+   MFEM_DEPRECATED static constexpr Type default_type = l1Jacobi;
 #endif

+   /** @brief Default value for the smoother type used by the constructors:
+       Type::l1GS when HYPRE is running on CPU and Type::l1Jacobi when HYPRE is
+       running on GPU. */
+   static Type DefaultType()
+   {
+      return HypreUsingGPU() ? l1Jacobi : l1GS;
+   }
+
   HypreSmoother();

-   HypreSmoother(const HypreParMatrix &A_, int type = default_type,
+   HypreSmoother(const HypreParMatrix &A_, int type = DefaultType(),
                 int relax_times = 1, real_t relax_weight = 1.0,
                 real_t omega = 1.0, int poly_order = 2,
                 real_t poly_fraction = .3, int eig_est_cg_iter = 10);
@@ -1474,13 +1523,54 @@ public:

   virtual void SetOperator(const Operator &op);

-   void SetParams(real_t threshold, int max_levels);
+   /// Set the threshold and levels parameters
+   /** The accuracy and cost of ParaSails are parametrized by the real
+    * @a thresh and integer @a nlevels parameters (0<=thresh<=1,  0<=nlevels).
+    * Lower values of @a thresh and higher values of @a nlevels lead to
+    * more accurate, but more expensive preconditioners. More accurate
+    * preconditioners are also more expensive per iteration. The default
+    * values are @a thresh = 0.1 and @a nlevels = 1.
+    */
+   void SetParams(real_t thresh, int nlevels);
+
+   /// Set the filter parameter
+   /** The filter parameter is used to drop small nonzeros in the preconditioner,
+    * to reduce the cost of applying the preconditioner. Values from 0.055
+    * to 0.1 are recommended. The default value is 0.1.
+    */
   void SetFilter(real_t filter);
-   void SetLoadBal(real_t loadbal);
-   void SetReuse(int reuse);
-   void SetLogging(int logging);
+
+   /// Set symmetry parameter
+   /** The recognized options are:
+    *  0 = nonsymmetric and/or indefinite problem, and nonsymmetric preconditioner
+    *  1 = SPD problem, and SPD (factored) preconditioner
+    *  2 = nonsymmetric, definite problem, and SPD (factored) preconditioner
+    */
   void SetSymmetry(int sym);

+   /// Set the load balance parameter
+   /** A zero value indicates that no load balance is attempted; a value
+    * of unity indicates that perfect load balance will be attempted. The
+    * recommended value is 0.9 to balance the overhead of data exchanges
+    * for load balancing. No load balancing is needed if the preconditioner
+    * is very sparse and fast to construct. The default value is 0.
+    */
+   void SetLoadBal(real_t loadbal);
+
+   /// Set the pattern reuse parameter
+   /** A nonzero value indicates that the pattern of the preconditioner
+    * should be reused for subsequent constructions of the proconditioner.
+    * A zero value inicates that the peconditioner should be constructed
+    * from scratch. The default value is 0.
+    */
+   void SetReuse(int reuse);
+
+   /// Set the logging parameter
+   /** A nonzero value prints statistics of the setup procedure to stdout.
+    * The default value of this parameter is 1.
+    */
+   void SetLogging(int logging);
+
   /// The typecast to HYPRE_Solver returns the internal sai_precond
   virtual operator HYPRE_Solver() const { return sai_precond; }

@@ -16,11 +16,10 @@

 #ifdef MFEM_USE_MPI

-// Enable internal hypre timing routines
-#define HYPRE_TIMING
-
 #include "../general/mem_manager.hpp"
-#include "_hypre_parcsr_mv.h"
+
+// hypre header files
+#include <_hypre_parcsr_mv.h>

 // Older hypre versions do not define HYPRE_BigInt and HYPRE_MPI_BIG_INT, so we
 // define them here for backward compatibility.
@@ -46,10 +45,10 @@ typedef HYPRE_Int HYPRE_BigInt;
 #else // MFEM_HYPRE_VERSION >= 21400

 #define mfem_hypre_TAlloc(type, size) \
-   hypre_TAlloc(type, size, HYPRE_MEMORY_DEVICE)
+   hypre_TAlloc(type, size, mfem::GetHypreMemoryLocation())
 #define mfem_hypre_CTAlloc(type, size) \
-   hypre_CTAlloc(type, size, HYPRE_MEMORY_DEVICE)
-#define mfem_hypre_TFree(ptr) hypre_TFree(ptr, HYPRE_MEMORY_DEVICE)
+   hypre_CTAlloc(type, size, mfem::GetHypreMemoryLocation())
+#define mfem_hypre_TFree(ptr) hypre_TFree(ptr, mfem::GetHypreMemoryLocation())

 #define mfem_hypre_TAlloc_host(type, size) \
   hypre_TAlloc(type, size, HYPRE_MEMORY_HOST)
@@ -309,18 +309,51 @@ public:


 /// Base abstract class for first order time dependent operators.
-/** Operator of the form: (x,t) -> f(x,t), where k = f(x,t) generally solves the
-    algebraic equation F(x,k,t) = G(x,t). The functions F and G represent the
-    _implicit_ and _explicit_ parts of the operator, respectively. For explicit
-    operators, F(x,k,t) = k, so f(x,t) = G(x,t). */
+/** Operator of the form: (u,t) -> k(u,t), where k generally solves the
+    algebraic equation F(u,k,t) = G(u,t). The functions F and G represent the
+    _implicit_ and _explicit_ parts of the operator, respectively.
+
+    A common use for this class is representing a differential algebraic
+    equation of the form $ F(y,\frac{dy}{dt},t) = G(y,t) $.
+
+    For example, consider an ordinary differential equation of the form
+    $ M \frac{dy}{dt} = g(y,t) $. There are various ways of expressing this ODE
+    as a TimeDependentOperator depending on the choices for F and G. Here are
+    some common choices:
+
+      1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t),
+      2. F(u,k,t) = M k and G(u,t) = g(u,t),
+      3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0.
+
+    Note that depending on the ODE solver, some of the above choices may be
+    preferable to the others.
+*/
 class TimeDependentOperator : public Operator
 {
 public:
+   /// Enum used to describe the form of the time-dependent operator.
+   /** The type should be set by classes derived from TimeDependentOperator to
+       describe the form, in terms of the functions F and G, used by the
+       specific derived class. This information can be queried by classes or
+       functions (like time stepping algorithms) to make choices about the
+       algorithm to use, or to ensure that the TimeDependentOperator uses the
+       form expected by the class/function.
+
+       For example, assume that a derived class is implementing the ODE
+       $M \frac{dy}{dt} = g(y,t)$ and chooses to define $F(u,k,t) = M k$ and
+       $G(u,t) = g(u,t)$. Then it cannot use type EXPLICIT, unless $M = I$, or
+       type HOMOGENEOUS, unless $g(u,t) = 0$. If, on the other hand, the derived
+       class chooses to define $F(u,k,t) = k$ and $G(u,t) = M^{-1} g(y,t)$, then
+       the natural choice is to set the type to EXPLICIT, even though setting it
+       to IMPLICIT is also not wrong -- doing so will simply fail to inform
+       methods that query this information that it uses a more specific
+       implementation, EXPLICIT, that may allow the use of algorithms that
+       support only the EXPLICIT type. */
   enum Type
   {
-      EXPLICIT,   ///< This type assumes F(x,k,t) = k, i.e. k = f(x,t) = G(x,t).
+      EXPLICIT,   ///< This type assumes F(u,k,t) = k.
      IMPLICIT,   ///< This is the most general type, no assumptions on F and G.
-      HOMOGENEOUS ///< This type assumes that G(x,t) = 0.
+      HOMOGENEOUS ///< This type assumes that G(u,t) = 0.
   };

   /// Evaluation mode. See SetEvalMode() for details.
@@ -328,29 +361,30 @@ public:
   {
      /** Normal evaluation. */
      NORMAL,
-      /** Assuming additive split, f(x,t) = f1(x,t) + f2(x,t), evaluate the
-          first term, f1. */
+      /** Assuming additive split, k(u,t) = k1(u,t) + k2(u,t), evaluate the
+          first term, k1. */
      ADDITIVE_TERM_1,
-      /** Assuming additive split, f(x,t) = f1(x,t) + f2(x,t), evaluate the
-          second term, f2. */
+      /** Assuming additive split, k(u,t) = k1(u,t) + k2(u,t), evaluate the
+          second term, k2. */
      ADDITIVE_TERM_2
   };

 protected:
   real_t t;  ///< Current time.
-   Type type; ///< Describes the form of the TimeDependentOperator.
+   Type type; /**< @brief Describes the form of the TimeDependentOperator, see
+                   the documentation of #Type. */
   EvalMode eval_mode; ///< Current evaluation mode.

 public:
-   /** @brief Construct a "square" TimeDependentOperator y = f(x,t), where x and
-       y have the same dimension @a n. */
+   /** @brief Construct a "square" TimeDependentOperator (u,t) -> k(u,t), where
+       u and k have the same dimension @a n. */
   explicit TimeDependentOperator(int n = 0, real_t t_ = 0.0,
                                  Type type_ = EXPLICIT)
      : Operator(n) { t = t_; type = type_; eval_mode = NORMAL; }

-   /** @brief Construct a TimeDependentOperator y = f(x,t), where x and y have
-       dimensions @a w and @a h, respectively. */
-   TimeDependentOperator(int h, int w, real_t t_ = 0.0, Type type_ = EXPLICIT)
+   /** @brief Construct a TimeDependentOperator (u,t) -> k(u,t), where u and k
+       have dimensions @a w and @a h, respectively. */
+   TimeDependentOperator(int h, int w, double t_ = 0.0, Type type_ = EXPLICIT)
      : Operator(h, w) { t = t_; type = type_; eval_mode = NORMAL; }

   /// Read the currently set time.
@@ -373,7 +407,7 @@ public:
   /** The evaluation mode is a switch that allows time-stepping methods to
       request evaluation of separate components/terms of the time-dependent
       operator. For example, IMEX methods typically assume additive split of
-       the operator: f(x,t) = f1(x,t) + f2(x,t) and they rely on the ability to
+       the operator: k(u,t) = k1(u,t) + k2(u,t) and they rely on the ability to
       evaluate the two terms separately.

       Generally, setting the evaluation mode should affect the behavior of all
@@ -384,62 +418,104 @@ public:
   { eval_mode = new_eval_mode; }

   /** @brief Perform the action of the explicit part of the operator, G:
-       @a y = G(@a x, t) where t is the current time.
+       @a v = G(@a u, t) where t is the current time.

       Presently, this method is used by some PETSc ODE solvers, for more
       details, see the PETSc Manual. */
-   virtual void ExplicitMult(const Vector &x, Vector &y) const;
+   virtual void ExplicitMult(const Vector &u, Vector &v) const;

   /** @brief Perform the action of the implicit part of the operator, F:
-       @a y = F(@a x, @a k, t) where t is the current time.
+       @a v = F(@a u, @a k, t) where t is the current time.

       Presently, this method is used by some PETSc ODE solvers, for more
       details, see the PETSc Manual.*/
-   virtual void ImplicitMult(const Vector &x, const Vector &k, Vector &y) const;
+   virtual void ImplicitMult(const Vector &u, const Vector &k, Vector &v) const;

-   /** @brief Perform the action of the operator: @a y = k = f(@a x, t), where
-       k solves the algebraic equation F(@a x, k, t) = G(@a x, t) and t is the
-       current time. */
-   virtual void Mult(const Vector &x, Vector &y) const;
+   /** @brief Perform the action of the operator (u,t) -> k(u,t) where t is the
+       current time set by SetTime() and @a k satisfies
+       F(@a u, @a k, t) = G(@a u, t).

-   /** @brief Solve the equation: @a k = f(@a x + @a dt @a k, t), for the
-       unknown @a k at the current time t.
+       For solving an ordinary differential equation of the form
+       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined in
+       various ways, e.g.:

-       For general F and G, the equation for @a k becomes:
-       F(@a x + @a dt @a k, @a k, t) = G(@a x + @a dt @a k, t).
+         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
+         2. F(u,k,t) = M k and G(u,t) = g(u,t)
+         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0.

-       The input vector @a x corresponds to time index (or cycle) n, while the
-       currently set time, #t, and the result vector @a k correspond to time
-       index n+1. The time step @a dt corresponds to the time interval between
-       cycles n and n+1.
+       Regardless of the choice of F and G, this function should always compute
+       @a k = inv(M) g(@a u, t). */
+   virtual void Mult(const Vector &u, Vector &v) const override;

-       This method allows for the abstract implementation of some time
-       integration methods, including diagonal implicit Runge-Kutta (DIRK)
-       methods and the backward Euler method in particular.
+   /** @brief Solve for the unknown @a k, at the current time t, the following
+       equation:
+       F(@a u + @a gamma @a k, @a k, t) = G(@a u + @a gamma @a k, t).
+
+       For solving an ordinary differential equation of the form
+       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined in
+       various ways, e.g.:
+
+         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
+         2. F(u,k,t) = M k and G(u,t) = g(u,t)
+         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0
+
+       Regardless of the choice of F and G, this function should solve for @a k
+       in M @a k = g(@a u + @a gamma @a k, t).
+
+       To see how @a k can be useful, consider the backward Euler method defined
+       by $ y(t + \Delta t) = y(t) + \Delta t k_0 $ where
+       $ M k_0 = g \big( y(t) + \Delta t k_0, t + \Delta t \big) $. A backward
+       Euler integrator can use @a k from this function for $k_0$, with the call
+       using @a u set to $ y(t) $, @a gamma set to $ \Delta t$, and time set to
+       $t + \Delta t$. See class BackwardEulerSolver.
+
+       Generalizing further, consider a diagonally implicit Runge-Kutta (DIRK)
+       method defined by
+       $ y(t + \Delta t) = y(t) + \Delta t \sum_{i=1}^s b_i k_i $ where
+       $ M k_i = g \big( y(t) + \Delta t \sum_{j=1}^i a_{ij} k_j,
+                         t + c_i \Delta t \big) $.
+       A DIRK integrator can use @a k from this function, with @a u set to
+       $ y(t) + \Delta t \sum_{j=1}^{i-1} a_{ij} k_j $ and @a gamma set to
+       $ a_{ii} \Delta t $, for $ k_i $. For example, see class SDIRK33Solver.

       If not re-implemented, this method simply generates an error. */
-   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);
+   virtual void ImplicitSolve(const real_t gamma, const Vector &u, Vector &k);

-   /** @brief Return an Operator representing (dF/dk @a shift + dF/dx) at the
-       given @a x, @a k, and the currently set time.
+   /** @brief Return an Operator representing (dF/dk @a shift + dF/du) at the
+       given @a u, @a k, and the currently set time.

       Presently, this method is used by some PETSc ODE solvers, for more
       details, see the PETSc Manual. */
-   virtual Operator& GetImplicitGradient(const Vector &x, const Vector &k,
+   virtual Operator& GetImplicitGradient(const Vector &u, const Vector &k,
                                         real_t shift) const;

-   /** @brief Return an Operator representing dG/dx at the given point @a x and
+   /** @brief Return an Operator representing dG/du at the given point @a u and
       the currently set time.

       Presently, this method is used by some PETSc ODE solvers, for more
       details, see the PETSc Manual. */
-   virtual Operator& GetExplicitGradient(const Vector &x) const;
+   virtual Operator& GetExplicitGradient(const Vector &u) const;

-   /** @brief Setup the ODE linear system $ A(x,t) = (I - gamma J) $ or
-       $ A = (M - gamma J) $, where $ J(x,t) = \frac{df}{dt(x,t)} $.
+   /** @brief Setup a linear system as needed by some SUNDIALS ODE solvers.

-       @param[in]  x     The state at which $A(x,t)$ should be evaluated.
-       @param[in]  fx    The current value of the ODE rhs function, $f(x,t)$.
+       For solving an ordinary differential equation of the form
+       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined as one
+       of the following:
+
+         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
+         2. F(u,k,t) = M k and G(u,t) = g(u,t)
+         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0
+
+       This function performs setup to solve $ A x = b $ where A is either
+
+         1. A(@a y,t) = I - @a gamma inv(M) J(@a y,t)
+         2. A(@a y,t) = M - @a gamma J(@a y,t)
+         3. A(@a y,t) = M - @a gamma J(@a y,t)
+
+       with J = dg/dy (or a reasonable approximation thereof).
+
+       @param[in]  y     The state at which A(@a y,t) should be evaluated.
+       @param[in]  v     The value of inv(M) g(y,t) for 1 or g(y,t) for 2 & 3.
       @param[in]  jok   Flag indicating if the Jacobian should be updated.
       @param[out] jcur  Flag to signal if the Jacobian was updated.
       @param[in]  gamma The scaled time step value.
@@ -448,10 +524,10 @@ public:

       Presently, this method is used by SUNDIALS ODE solvers, for more
       details, see the SUNDIALS User Guides. */
-   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
+   virtual int SUNImplicitSetup(const Vector &y, const Vector &v,
                                int jok, int *jcur, real_t gamma);

-   /** @brief Solve the ODE linear system $ A x = b $ as setup by
+   /** @brief Solve the ODE linear system A @a x = @a b, where A is defined by
       the method SUNImplicitSetup().

       @param[in]      b   The linear system right-hand side.
@@ -464,7 +540,8 @@ public:
       details, see the SUNDIALS User Guides. */
   virtual int SUNImplicitSolve(const Vector &b, Vector &x, real_t tol);

-   /** @brief Setup the mass matrix in the ODE system $ M y' = f(y,t) $ .
+   /** @brief Setup the mass matrix in the ODE system
+       $ M \frac{dy}{dt} = g(y,t) $ .

       If not re-implemented, this method simply generates an error.

@@ -472,8 +549,8 @@ public:
       details, see the ARKode User Guide. */
   virtual int SUNMassSetup();

-   /** @brief Solve the mass matrix linear system $ M x = b $
-       as setup by the method SUNMassSetup().
+   /** @brief Solve the mass matrix linear system  M @a x = @a b, where M is
+       defined by the method SUNMassSetup().

       @param[in]      b   The linear system right-hand side.
       @param[in,out]  x   On input, the initial guess. On output, the solution.
@@ -485,7 +562,8 @@ public:
       details, see the ARKode User Guide. */
   virtual int SUNMassSolve(const Vector &b, Vector &x, real_t tol);

-   /** @brief Compute the mass matrix-vector product $ v = M x $ .
+   /** @brief Compute the mass matrix-vector product @a v = M @a x, where M is
+       defined by the method SUNMassSetup().

       @param[in]   x The vector to multiply.
       @param[out]  v The result of the matrix-vector product.
@@ -1188,6 +1188,8 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const

   final_norm = std::max(rel_tol*beta, abs_tol);

+   converged = false;
+
   if (beta <= final_norm)
   {
      final_norm = beta;
@@ -1303,8 +1305,6 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
      MFEM_ASSERT(IsFinite(beta), "beta = " << beta);
      if (beta <= final_norm)
      {
-         final_norm = beta;
-         final_iter = j;
         converged = true;

         break;
@@ -1317,7 +1317,9 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
      if (v[i]) { delete v[i]; }
      if (z[i]) { delete z[i]; }
   }
-   converged = false;
+
+   final_norm = beta;
+   final_iter = converged ? j : max_iter;

   // Note: j is off by one when we arrive here
   if (!print_options.iterations && print_options.first_and_last)
@@ -1328,7 +1330,7 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
   }
   if (print_options.summary || (print_options.warnings && !converged))
   {
-      mfem::out << "FGMRES: Number of iterations: " << j-1 << '\n';
+      mfem::out << "FGMRES: Number of iterations: " << final_iter << '\n';
   }
   if (print_options.warnings && !converged)
   {
@@ -58,6 +58,20 @@ DenseSymmetricMatrix &DenseSymmetricMatrix::operator=(real_t c)
   return *this;
 }

+DenseSymmetricMatrix &DenseSymmetricMatrix::operator=(const DenseSymmetricMatrix
+                                                      &m)
+{
+   SetSize(m.height);
+
+   const int hw = m.GetStoredSize();
+   for (int i = 0; i < hw; i++)
+   {
+      data[i] = m.data[i];
+   }
+
+   return *this;
+}
+
 real_t &DenseSymmetricMatrix::Elem(int i, int j)
 {
   return (*this)(i,j);
@@ -89,11 +103,6 @@ MatrixInverse *DenseSymmetricMatrix::Inverse() const
   return nullptr;
 }

-void DenseSymmetricMatrix::Print (std::ostream & os, int width_) const
-{
-   mfem_error("DenseSymmetricMatrix::Print() not implemented!");
-}
-
 DenseSymmetricMatrix::~DenseSymmetricMatrix()
 {
   data.Delete();
@@ -102,6 +102,9 @@ public:

   DenseSymmetricMatrix &operator*=(real_t c);

+   /// Sets the matrix size and elements equal to those of m
+   DenseSymmetricMatrix &operator=(const DenseSymmetricMatrix &m);
+
   std::size_t MemoryUsage() const { return data.Capacity() * sizeof(real_t); }

   /// Shortcut for mfem::Read( GetMemory(), TotalSize(), on_dev).
@@ -134,9 +137,6 @@ public:
   /// Returns a pointer to (an approximation) of the matrix inverse.
   virtual MatrixInverse *Inverse() const;

-   /// Prints matrix to stream out.
-   virtual void Print (std::ostream & out = mfem::out, int width_ = 4) const;
-
   /// Destroys the symmetric matrix.
   virtual ~DenseSymmetricMatrix();
 };
@@ -716,13 +716,7 @@ inline real_t InnerProduct(MPI_Comm comm, const Vector &x, const Vector &y)
 {
   real_t loc_prod = x * y;
   real_t glb_prod;
-#ifdef MFEM_USE_SINGLE
-   MPI_Allreduce(&loc_prod, &glb_prod, 1, MPI_FLOAT, MPI_SUM, comm);
-#elif defined MFEM_USE_DOUBLE
-   MPI_Allreduce(&loc_prod, &glb_prod, 1, MPI_DOUBLE, MPI_SUM, comm);
-#else
-   MFEM_ABORT("Floating point type undefined");
-#endif
+   MPI_Allreduce(&loc_prod, &glb_prod, 1, MFEM_MPI_REAL_T, MPI_SUM, comm);
   return glb_prod;
 }
 #endif
@@ -10,7 +10,7 @@
 # CONTRIBUTING.md for details.

 # The current MFEM version as an integer, see also `CMakeLists.txt`.
-MFEM_VERSION = 40601
+MFEM_VERSION = 40701
 MFEM_VERSION_STRING = $(shell printf "%06d" $(MFEM_VERSION) | \
  sed -e 's/^0*\(.*.\)\(..\)\(..\)$$/\1.\2.\3/' -e 's/\.0/./g' -e 's/\.0$$//')

@@ -205,16 +205,12 @@ MFEM_SHARED_BUILD = $(MFEM_SHARED)
 override static = $(if $(MFEM_STATIC:YES=),,YES)
 override shared = $(if $(MFEM_SHARED:YES=),,YES)

-# Process MFEM_PRECISION -> MFEM_USE_SINGLE, MFEM_USE_DOUBLE
-ifneq ($(filter double Double DOUBLE,$(MFEM_PRECISION)),)
-   MFEM_USE_DOUBLE ?= YES
-   MFEM_USE_SINGLE ?= NO
-else ifneq ($(filter single Single SINGLE,$(MFEM_PRECISION)),)
-   MFEM_USE_DOUBLE ?= NO
-   MFEM_USE_SINGLE ?= YES
-else ifeq ($(MAKECMDGOALS),config)
-   $(error Invalid floating-point precision: \
-     MFEM_PRECISION = $(MFEM_PRECISION))
+# Error for package integrations that currently don't support single precision
+ifeq ($(MFEM_USE_SINGLE),YES)
+   PKGS_NO_SINGLE = SUNDIALS SUITESPARSE SUPERLU STRUMPACK GINKGO AMGX SLEPC\
+	 PUMI GSLIB ALGOIM CEED MOONOLITH TRIBOL
+   $(foreach pkg,$(PKGS_NO_SINGLE),$(if $(MFEM_USE_$(pkg):NO=),\
+     $(error Package $(pkg) is NOT supported with single precision)))
 endif

 # The default value of CXXFLAGS is based on the value of MFEM_DEBUG
--- a/Show More
+++ b/Show More