small updates prior to passing the code along for comparison

updating contact and parproblems files
cleanup on finite difference test code. Making it easier for others to use
2024-07-29 16:34:06 -07:00 · 2024-07-15 10:47:54 -07:00 · 2024-07-15 08:08:00 -07:00 · 2024-07-14 20:43:48 -07:00 · 2024-06-17 13:15:59 -07:00 · 2024-06-01 14:03:22 -07:00
208 changed files with 69546 additions and 2202 deletions
@@ -33,6 +33,7 @@ env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
  HYPRE_TOP_DIR: hypre-2.19.0
  METIS_ARCHIVE: metis-4.0.3.tar.gz
+  METIS_ARCHIVE_MAC: metis-4.0.3-mac.tgz
  METIS_TOP_DIR: metis-4.0.3
  MFEM_TOP_DIR: mfem

@@ -52,6 +53,7 @@ jobs:
        mpi: [seq, par]
        build-system: [make, cmake]
        hypre-target: [int32]
+        precision: [fp64]
        exclude:
          - os: ubuntu-latest
            build-system: cmake
@@ -75,6 +77,8 @@ jobs:
          - os: ubuntu-latest
            target: dbg
            config-opts: 'CPPFLAGS+=-Og'
+          - os: macos-latest
+            codecov: NO
          - os: windows-latest
            codecov: NO
          - os: windows-latest
@@ -87,6 +91,7 @@ jobs:
            mpi: par
            build-system: cmake
            hypre-target: int32
+            precision: fp64
            # This option can be set to pass additional configuration options to
            # the MFEM configuration command.
            # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
@@ -96,7 +101,15 @@ jobs:
            mpi: par
            build-system: make
            hypre-target: int64
-    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+            precision: fp64
+          - os: ubuntu-latest
+            target: opt
+            codecov: NO
+            mpi: par
+            build-system: make
+            hypre-target: int32
+            precision: fp32
+    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}

    runs-on: ${{ matrix.os }}

@@ -126,6 +139,17 @@ jobs:
          # Fetch the complete history for codecov to access commits ID
          fetch-depth: 0

+      - name: Xcode version setup (MacOS)
+        if: matrix.os == 'macos-latest'
+        run: |
+          XCODE_PATH="/Applications/Xcode_15.3.app"
+          echo "> sudo xcode-select -s ${XCODE_PATH}"
+          sudo xcode-select -s ${XCODE_PATH}
+          echo "> g++ -v"
+          g++ -v
+          echo "> clang++ -v"
+          clang++ -v
+
      # Only get MPI if defined for the job.
      # TODO: It would be nice to have only one step, e.g. with a dedicated
      # action, but I (@adrienbernede) don't see how at the moment.
@@ -169,25 +193,27 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5

      - name: get hypre
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
          target: ${{ matrix.hypre-target }}
          build-system: make
+          precision: ${{ matrix.precision }}

      - name: get hypre (Windows)
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
          target: ${{ matrix.hypre-target }}
          build-system: cmake
+          precision: ${{ matrix.precision }}

      # Get Metis through cache, or build it.
      # Install will only run on cache miss.
@@ -197,13 +223,13 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: install metis
        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
+        uses: mfem/github-actions/build-metis@v2.5
        with:
-          archive: ${{ env.METIS_ARCHIVE }}
+          archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
          dir: ${{ env.METIS_TOP_DIR }}

      - name: cache vcpkg (Windows)
@@ -228,7 +254,7 @@ jobs:

      # MFEM build and test
      - name: build
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        env:
          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
        with:
@@ -240,6 +266,7 @@ jobs:
          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
          metis-dir: ${{ env.METIS_TOP_DIR }}
          mfem-dir: ${{ env.MFEM_TOP_DIR }}
+          precision: ${{ matrix.precision }}
          config-options: ${{ matrix.config-opts }}
          library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}

@@ -282,7 +309,7 @@ jobs:
      # Code coverage (process and upload reports)
      - name: codecov
        if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.4
+        uses: mfem/github-actions/upload-coverage@v2.5
        with:
          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
          project_dir: ${{ env.MFEM_TOP_DIR }}
@@ -53,11 +53,11 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.5

      - name: Get Hypre
        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
@@ -68,18 +68,18 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: Install Metis
        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
+        uses: mfem/github-actions/build-metis@v2.5
        with:
          archive: ${{ env.METIS_ARCHIVE }}
          dir: ${{ env.METIS_TOP_DIR }}

      # MFEM build and test
      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        with:
          os: ${{ runner.os }}
          target: opt
@@ -44,7 +44,7 @@ jobs:
          path: mfem

      - name: MFEM Build
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        with:
          os: ${{ runner.os }}
          target: opt
@@ -57,6 +57,8 @@ examples/ex2[0-9]
 examples/ex2[0-9]p
 examples/ex3[0-9]
 examples/ex3[0-9]p
+examples/ex4[0-9]
+examples/ex4[0-9]p

 examples/refined.mesh
 examples/displaced.mesh
@@ -232,7 +234,7 @@ miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
 miniapps/meshing/twist-*.mesh
-miniapps/meshing/mesh-explorer.mesh
+miniapps/meshing/mesh-explorer.mesh*
 miniapps/meshing/partitioning.txt
 miniapps/meshing/mesh-explorer-visit*
 miniapps/meshing/mesh-explorer-paraview/
@@ -369,6 +371,8 @@ miniapps/dpg/ParaView
 miniapps/spde/generate_random_field
 miniapps/spde/ParaView

+miniapps/tribol/contact-patch-test
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -13,6 +13,9 @@
 # at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
 # LLNL-specific!

+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'

 # The pipeline is divided into stages. Usually, jobs in a given stage wait for
 # the preceding stages to complete before to start. However, we sometimes use
@@ -9,6 +9,10 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
 # We define the following GitLab pipeline variables:
 variables:

@@ -35,9 +35,8 @@ variables:
    - when: on_success

 # Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen
-# to speed-up the allocation.  However this would not be scalable to
-# multiple builds.
+# pre-allocation the same way slurm does.  We use the pci queue on lassen
+# to speed-up the allocation.
 .build_and_test_on_lassen:
  extends: [.on_lassen]
  stage: build_and_test
@@ -45,5 +44,5 @@ variables:
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 45 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
  needs: [setup]
@@ -52,4 +52,4 @@ variables:
    - echo ${JOBID}
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -14,14 +14,14 @@ stages:
  - build_and_test
  - report

-opt_mpi_cuda_xl_16_1_1_12:
+opt_mpi_cuda_gcc:
  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70"
  extends: .build_and_test_on_lassen

-opt_mpi_cuda_hypre_cuda_xl:
+opt_mpi_cuda_hypre_cuda_gcc:
  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
  extends: .build_and_test_on_lassen

 # Jobs report
@@ -32,11 +32,11 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}

 # run
 if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  lalloc 1 -q pci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
  exit 1
@@ -8,74 +8,110 @@
                               https://mfem.org


-Version 4.6.1 (development)
+Version 4.7.1 (development)
 ===========================

+- Added an MFEM example for the eikonal equation. This new solver is based on
+  the proximal Galerkin method introduced by Keith and Surowiec.
+
+
+Version 4.7, released on May 7, 2024
+====================================
+
+- Added support for single precision (with corresponding hypre build). The MFEM
+  floating point type was generalized from `double` to `real_t`. For details see
+  https://github.com/orgs/mfem/discussions/4207.
+
 Meshing improvements
 --------------------
+- Added the capability to partition (big) serial meshes in serial code, see the
+  new classes MeshPartitioner and MeshPart. This capability is also exposed as a
+  menu option in the mesh-explorer miniapp in miniapps/meshing.
+
+- Added named attribute sets and basic supporting methods to the Mesh class as a
+  convenient means of referring to sets of domain or boundary attribute numbers.
+  See the new Example 39/39p and data/compass.mesh.
+
 - Introduced formulas for refinement of patches in NURBS meshes. Refinement by
  arbitrary integer factors is also enabled, e.g. in the mesh-explorer miniapp.
  NURBS coarsening and knot removal are also introduced.

-Discretization improvements
---------------------------
- Introduced support for higher order non conformal Nedelec elements on
-  simplices in ParMesh.
- Introduced support for internal boundary elements in nonconformal adapted
-  meshes.
-
- Added functionality for construction of cut-surface and cut-volume
-  IntegrationRules through a moment-fitting approach. The cut is specified by
-  the zero level set of a Coefficient. See fem/intrules_cut.hpp and Example 38.
-
- Added a new nonlinear integrator, `HyperbolicFormIntegrator`. This implements 
-  both element-wise weak divergence and face-wise numerical flux for a general
-  system of hyperbolic conservation laws. To use this integrator for a specific
-  flux function, users can define a derived class of `FluxFunction`. Currently,
-  advection, Burgers', shallow-water, Euler equations (see, Example 18) are
-  available.
-
-GPU support
----------------------------
- Added support for full assembly on simplices.
- Added functionality for BilinearFormIntegrators to use kernels that work for both
-  tensor and unstructured elements.
- Added partial assembly for linear elasticity. Does not use sum factorization for now.
-
-New and updated examples and miniapps
-------------------------------------
- Added a new block solver in miniapp/solvers for the Darcy problem.
-  The new solver is based on a Bramble-Pasciak preconditioning. User can
-  use and implement their own preconditioner for the mass matrix.
-
- Added miniapp to demonstrate new elasticity integrator and unstructured element GPU support,
-  and a block diagonal preconditioner using low order refinement. Allows comparison with
-  currently existing legacy mode integrator. See miniapps/solvers/lor_elast.
-
-Miscellaneous
-------------
- Added support for single and double precision, with corresponding hypre build.
-  Generalized the floating point type from `double` to `real_t`. For more
-  details see https://github.com/orgs/mfem/discussions/4207.
+- Added support for internal boundary elements in nonconforming meshes.

 - The ReadCubit Genesis mesh importer has been rewritten to improve readability.

+Discretization improvements
+---------------------------
+- Added a new nonlinear integrator, `HyperbolicFormIntegrator` that implements
+  both element-wise weak divergence and face-wise numerical flux for a general
+  system of hyperbolic conservation laws. To use the integrator for a specific
+  flux function, users can define a derived class of `FluxFunction`. Currently,
+  advection, Burgers, shallow-water and Euler equations (see Example 18/18p) are
+  available.
+
+- Added a capability to construct cut-surface and cut-volume IntegrationRules
+  through a moment-fitting approach. The cut is specified by the zero level set
+  of a Coefficient. See fem/intrules_cut.hpp and the new Example 38.
+
+- Introduced support for high-order nonconforming Nedelec elements on simplices.
+
+GPU computing
+-------------
+- Added partial assembly and GPU support for the DG diffusion integrator.
+
+- Efficient GPU-accelerated LOR assembly is now supported on surface meshes.
+
+- Added functionality to automatically configure hypre's compute policy to match
+  MFEM's compute policy when hypre is built with GPU support. Requires version
+  hypre-2.31.0 or later.
+
+- Added support for full assembly on simplices.
+
+- Added partial assembly for linear elasticity (no sum factorization for now).
+
+- Added functionality for BilinearFormIntegrators to use kernels that work for
+  both tensor and unstructured elements.
+
+- The RAJA backend will use `seq_exec` for serial loop execution when RAJA
+  v2023.06.00 and beyond is detected as `loop_exec` is deprecated.
+
+- API change: The macro MFEM_HYPRE_FORALL (from hypre.hpp) which was intended
+  for internal use, has been removed and replaced by the function template
+  mfem::hypre_forall in general/forall.hpp.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new miniapp illustrating elastic contact based on the Tribol library,
+  (https://github.com/LLNL/Tribol). See miniapps/tribol.
+
+- Added a miniapp to demonstrate low order refined (LOR) block preconditioning
+  for linear elasticity on GPUs. See miniapps/solvers/lor_elast.
+
+- Added a new block solver in miniapp/solvers for the Darcy problem. The new
+  solver is based on a Bramble-Pasciak preconditioning. User can use and
+  implement their own preconditioner for the mass matrix.
+
+- Added a small miniapp for printing the shape functions of a KnotVector. See
+  miniapps/nurbs/nurbs_printfunc.cpp.
+
+- Added two new example codes: 38 and 39/39p described above. Substantially
+  updated Example 18/18p.
+
+Miscellaneous
+-------------
 - Updated the Doxygen documentation style, which now requires Doxygen version
  1.9.8 or later. See the doc/ directory.

- Improved thread safety for global variables in the library, for example
-  IntegrationRules IntRules, RefinedIntRules, GeometryRefiner
-  GlobGeometryRefiner, and FiniteElement::dof2quad_array.
+- Improved thread safety for global variables in the library, e.g. for IntRules,
+  RefinedIntRules, GlobGeometryRefiner, and FiniteElement::dof2quad_array.

 - PETSc integration now generally requires PETSc version 3.21 or later, though
  depending on the functionality older versions may still work.

- RAJA backend will use seq_exec for serial loop execution when RAJA
-  v2023.06.00 and beyond is detected as loop_exec is deprecated.
+- Various other simplifications, extensions, and bugfixes in the code.
+
+- Added GSLIB-based gather-scatter operator.

- Adding named attribute sets and basic supporting methods to the Mesh class as
-  a convenient means of referring to sets of domain or boundary attribute
-  numbers. Also adding related serial and parallel examples which illustrate.

 Version 4.6, released on September 27, 2023
 ===========================================
@@ -96,7 +132,6 @@ Meshing improvements
  * The edge to knot map for NURBS meshes can be determined automatically. It is
    no longer needed to specify this in the NURBS mesh.
  * Added curve interpolation method for NURBS.
-  * Added new small miniapp for printing of shape functions of a KnotVector
  * See miniapps/nurbs for example meshes and miniapps.

 Discretization improvements
@@ -143,8 +178,6 @@ Linear and nonlinear solvers

 - Added HIP support to the PETSc and SUNDIALS interfaces.

- Efficient GPU-accelerated LOR assembly now supports surface meshes.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new H(div) solver miniapp demonstrating the use of a matrix-free
@@ -58,7 +58,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.6.1)
+set(${PROJECT_NAME}_VERSION 4.7.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -87,10 +87,11 @@ if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
  # Just needed to find the MPI_Fortran libraries to link with
  set(XSDK_ENABLE_Fortran ON)
 endif()
-# SUNDIALS, STRUMPACK, Ginkgo, RAJA and Umpire require C++14:
+# SUNDIALS, STRUMPACK, Ginkgo, Tribol, RAJA and Umpire require C++14:
 if ((MFEM_USE_SUNDIALS OR
     MFEM_USE_STRUMPACK OR
     MFEM_USE_GINKGO OR
+     MFEM_USE_TRIBOL OR
     MFEM_USE_RAJA OR
     MFEM_USE_UMPIRE) AND
    ("${CMAKE_CXX_STANDARD}" LESS "14"))
@@ -503,6 +504,15 @@ if (MFEM_USE_PARELAG)
   find_package(PARELAG REQUIRED)
 endif()

+# Tribol
+if (MFEM_USE_TRIBOL)
+  if (MFEM_USE_MPI)
+    find_package(Tribol REQUIRED tribol redecomp)
+  else()
+    message(FATAL_ERROR " *** Tribol requires that MPI be enabled.")
+  endif()
+endif()
+
 # Enzyme
 if (MFEM_USE_ENZYME)
   find_package(ENZYME REQUIRED)
@@ -548,7 +558,7 @@ set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
    ADIOS2 CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
-    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
+    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -151,7 +151,8 @@ The MFEM source code has the following structure:
  │   ├── solvers
  │   ├── spde
  │   ├── tools
-  │   └── toys
+  │   ├── toys
+  │   └── tribol
  └── tests
      ├── benchmarks
      ├── convergence
@@ -75,6 +75,8 @@ and miniapps. See https://glvis.org and https://mfem.org/building.

 Quick start with GNU make
 =========================
+See also: https://mfem.org/building
+
 Serial build:
   make serial -j 4

@@ -83,6 +85,7 @@ Parallel build:
   (build METIS 4 in ../metis-4.0 relative to mfem/)
   (build hypre in ../hypre relative to mfem/)
   make parallel -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   make cuda -j 4
@@ -116,6 +119,7 @@ Parallel build:
   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
   make -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   (this build requires CMake 3.8 or newer)
@@ -571,6 +575,11 @@ MFEM_USE_PARELAG = YES/NO
   use ParELAG. In fact, ParELAG is dependent on MFEM. Therefore, this option
   currently only concerns the miniapps.

+MFEM_USE_TRIBOL = YES/NO
+   Enables the miniapps that use the Tribol library. MFEM does not currently
+   use Tribol. In fact, Tribol is dependent on MFEM. Therefore, this option
+   currently only concerns the miniapps.
+
 MFEM_USE_ENZYME = YES/NO
   Enables automatic differentiation support through the LLVM plugin Enzyme.
   This requires the compiler to be set to clang (>=14.0.0). We also advise to
@@ -607,9 +616,13 @@ The specific libraries and their options are:
            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
            HYPRE >= 2.22.1   (HYPRE built with CUDA)
            HYPRE >= 2.23.0   (HYPRE built with HIP)
+            HYPRE >= 2.31.0   (runtime selectable HYPRE execution on CPU/GPU)

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
-  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
+  MFEM_USE_METIS_5 = YES (default is to use METIS 4). For building instructions,
+  see the following:
+    - METIS 4.0.3: https://mfem.org/building/#parallel-mpi-version-of-mfem
+    - METIS 5.1.0: https://mfem.org/building/#parallel-build-using-metis-5
  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
  Options: METIS_OPT, METIS_LIB.
  Versions: METIS 4.0.3 or 5.1.0.
@@ -857,6 +870,10 @@ The specific libraries and their options are:
  URL: https://github.com/LLNL/parelag
  Options: PARELAG_DIR, PARELAG_OPT, PARELAG_LIB.

+- Tribol, used when MFEM_USE_TRIBOL = YES.
+  URL: https://github.com/LLNL/Tribol
+  Options: TRIBOL_DIR, TRIBOL_OPT, TRIBOL_LIB.
+
 - Enzyme, used when MFEM_USE_ENZYME = YES. Requires LLVM/Clang >= 14.0.0.
  URL: https://github.com/EnzymeAD/Enzyme
  Options: ENZYME_DIR, ENZYME_OPT, ENZYME_LIB.
@@ -1001,6 +1018,7 @@ MFEM_USE_CALIPER
 MFEM_USE_FMS
 MFEM_USE_BENCHMARK
 MFEM_USE_PARELAG
+MFEM_USE_TRIBOL
 MFEM_USE_ENZYME

 The following options are CMake specific:
@@ -287,3 +287,7 @@ ENDIF()
 IF (DEFINED TPL_ENABLE_PARELAG)
  SET(MFEM_USE_PARELAG ${TPL_ENABLE_PARELAG} CACHE BOOL "Enable ParELAG" FORCE)
 ENDIF()
+
+IF (DEFINED TPL_ENABLE_TRIBOL)
+  SET(MFEM_USE_TRIBOL ${TPL_ENABLE_TRIBOL} CACHE BOOL "Enable Tribol" FORCE)
+ENDIF()
@@ -64,6 +64,7 @@ set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)
 set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
 set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)
+set(MFEM_USE_TRIBOL @MFEM_USE_TRIBOL@)
 set(MFEM_USE_ENZYME @MFEM_USE_ENZYME@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
@@ -18,4 +18,13 @@ include(MfemCmakeUtilities)
 # Note: components are enabled based on the find_package() parameters.
 mfem_find_package(Axom AXOM AXOM_DIR "include" "" "lib" ""
  "Paths to headers required by Axom." "Libraries required by Axom."
-  ADD_COMPONENT Axom "include" axom/config.hpp "lib" axom)
+  ADD_COMPONENT core "include" axom/core.hpp "lib" axom_core
+  ADD_COMPONENT inlet "include" axom/inlet.hpp "lib" axom_inlet
+  ADD_COMPONENT klee "include" axom/klee.hpp "lib" axom_klee
+  ADD_COMPONENT lumberjack "include" axom/lumberjack.hpp "lib" axom_lumberjack
+  ADD_COMPONENT mint "include" axom/mint.hpp "lib" axom_mint
+  ADD_COMPONENT multimat "include" axom/multimat.hpp "lib" axom_multimat
+  ADD_COMPONENT quest "include" axom/quest.hpp "lib" axom_quest
+  ADD_COMPONENT sidre "include" axom/sidre.hpp "lib" axom_sidre
+  ADD_COMPONENT slam "include" axom/slam.hpp "lib" axom_slam
+  ADD_COMPONENT slic "include" axom/slic.hpp "lib" axom_slic)
@@ -36,7 +36,11 @@ include(MfemCmakeUtilities)
 mfem_find_package(Conduit CONDUIT CONDUIT_DIR
  "include;include/conduit" conduit.hpp "lib" conduit
  "Paths to headers required by Conduit." "Libraries required by Conduit."
+  ADD_COMPONENT blueprint
+    "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint
+  ADD_COMPONENT blueprint_mpi
+    "include;include/conduit" conduit_blueprint_mpi.hpp "lib" conduit_blueprint_mpi
  ADD_COMPONENT relay
    "include;include/conduit" conduit_relay.hpp "lib" conduit_relay
-  ADD_COMPONENT blueprint
-    "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint)
+  ADD_COMPONENT relay_mpi
+    "include;include/conduit" conduit_relay_mpi.hpp "lib" conduit_relay_mpi)
@@ -16,12 +16,22 @@
 #   - MUMPS_VERSION

 include(MfemCmakeUtilities)
+
+# Toggle which precision of MUMPS to use depending on the precision of MFEM.
+if (MFEM_USE_DOUBLE)
+  set(_mumps_header dmumps_c.h)
+  set(_mumps_lib dmumps)
+elseif(MFEM_USE_SINGLE)
+  set(_mumps_header smumps_c.h)
+  set(_mumps_lib smumps)
+endif()
+
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
-  "include" dmumps_c.h "lib" dmumps
+  "include" ${_mumps_header} "lib" ${_mumps_lib}
  "Paths to headers required by MUMPS."
  "Libraries required by MUMPS."
-  ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
-  ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)
+  ADD_COMPONENT mumps_common "include" ${_mumps_header} "lib" mumps_common
+  ADD_COMPONENT pord "include" ${_mumps_header} "lib" pord)

 if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
  try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
@@ -0,0 +1,22 @@
+# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - TRIBOL_FOUND
+#   - TRIBOL_LIBRARIES
+#   - TRIBOL_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+# Note: components are enabled based on the find_package() parameters.
+mfem_find_package(Tribol TRIBOL TRIBOL_DIR "include" tribol/config.hpp "lib" tribol
+  "Paths to headers required by Tribol." "Libraries required by Tribol."
+  ADD_COMPONENT redecomp
+    "include" redecomp/redecomp.hpp "lib" redecomp)
@@ -852,8 +852,8 @@ function(mfem_export_mk_files)
      MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED
      MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2
      MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO MFEM_USE_ADFORWARD
-      MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_MOONOLITH
-      MFEM_USE_ALGOIM MFEM_USE_ENZYME)
+      MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_TRIBOL
+      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -120,6 +120,15 @@ constexpr real_t operator""_r(unsigned long long v)

 // Check dependencies:

+// Define MFEM_MPI_REAL_T to be the appropriate MPI real type
+#ifdef MFEM_USE_MPI
+#ifdef MFEM_USE_SINGLE
+#define MFEM_MPI_REAL_T MPI_FLOAT
+#elif defined MFEM_USE_DOUBLE
+#define MFEM_MPI_REAL_T MPI_DOUBLE
+#endif
+#endif
+
 // Options that require MPI
 #ifndef MFEM_USE_MPI
 #ifdef MFEM_USE_SUPERLU
@@ -65,6 +65,7 @@ MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
 MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
 MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@
+MFEM_USE_TRIBOL        = @MFEM_USE_TRIBOL@
 MFEM_USE_ENZYME        = @MFEM_USE_ENZYME@

 # Compiler, compile options, and link options
@@ -67,6 +67,7 @@ option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
 option(MFEM_USE_PARELAG "Enable ParELAG" OFF)
+option(MFEM_USE_TRIBOL "Enable Tribol" OFF)
 option(MFEM_USE_ENZYME "Enable Enzyme" OFF)

 # Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
@@ -212,8 +213,15 @@ set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH

 set(AXOM_DIR "${MFEM_DIR}/../axom" CACHE PATH "Path to the Axom library.")
 # May need to add "Boost" as requirement.
-set(Axom_REQUIRED_PACKAGES "Conduit/relay/blueprint" CACHE STRING
-    "Additional packages required by Axom.")
+if (MFEM_USE_SIDRE)
+    if (MFEM_USE_MPI)
+        set(Axom_REQUIRED_PACKAGES "Conduit/blueprint/blueprint_mpi/relay/relay_mpi" CACHE STRING
+            "Additional packages required by Axom.")
+    elseif()
+        set(Axom_REQUIRED_PACKAGES "Conduit/blueprint/relay" CACHE STRING
+            "Additional packages required by Axom.")
+    endif()
+endif()

 set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING
    "Directory where PUMI is installed")
@@ -250,6 +258,10 @@ set(PARELAG_INCLUDE_DIRS "${PARELAG_DIR}/src;${PARELAG_DIR}/build/src" CACHE
 set(PARELAG_LIBRARIES "${PARELAG_DIR}/build/src/libParELAG.a" CACHE STRING
    "The ParELAG library.")

+set(TRIBOL_DIR "${MFEM_DIR}/../tribol" CACHE PATH "Path to Tribol")
+set(Tribol_REQUIRED_PACKAGES "Axom/core/mint/slam/slic" CACHE STRING 
+    "Additional packages required by Tribol")
+
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
@@ -167,8 +167,21 @@ MFEM_USE_ADFORWARD     = NO
 MFEM_USE_CODIPACK      = NO
 MFEM_USE_BENCHMARK     = NO
 MFEM_USE_PARELAG       = NO
+MFEM_USE_TRIBOL        = NO
 MFEM_USE_ENZYME        = NO

+# Process MFEM_PRECISION -> MFEM_USE_SINGLE, MFEM_USE_DOUBLE
+ifneq ($(filter double Double DOUBLE,$(MFEM_PRECISION)),)
+   MFEM_USE_DOUBLE = YES
+   MFEM_USE_SINGLE = NO
+else ifneq ($(filter single Single SINGLE,$(MFEM_PRECISION)),)
+   MFEM_USE_DOUBLE = NO
+   MFEM_USE_SINGLE = YES
+else ifeq ($(MAKECMDGOALS),config)
+   $(error Invalid floating-point precision: \
+     MFEM_PRECISION = $(MFEM_PRECISION))
+endif
+
 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
 ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
@@ -318,13 +331,13 @@ MPI_FORTRAN_LIB = -lmpifort
 # MUMPS library configuration
 MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib \
- -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
+MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib
 ifeq ($(MFEM_USE_SINGLE),YES)
   MUMPS_LIB += -lsmumps
 else
   MUMPS_LIB += -ldmumps
 endif
+MUMPS_LIB += -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)

 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
@@ -375,7 +388,7 @@ GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LINK_LIB_DIR) -L$(GINKGO_LINK_LIB_DIR)\
 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
 AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx
+AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt

 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -576,6 +589,16 @@ PARELAG_DIR = @MFEM_DIR@/../parelag
 PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
 PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG

+# Tribol library configuration
+ifeq ($(MFEM_USE_TRIBOL),YES)
+   BASE_FLAGS = -std=c++14
+endif
+AXOM_DIR = @MFEM_DIR@/../axom
+TRIBOL_DIR = @MFEM_DIR@/../tribol
+TRIBOL_OPT = -I$(TRIBOL_DIR)/include -I$(AXOM_DIR)/include
+TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
+   -laxom_slam -laxom_slic -laxom_core
+
 # Enzyme configuration

 # If you want to enable automatic differentiation at compile time, use the
@@ -110,4 +110,4 @@ config-mk:

 clean:
 	rm -f $(CONFIG_HPP) $(CONFIG_MK) sample-runs-build.log
-	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out
+	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out *.dSYM
@@ -92,4 +92,5 @@ vertices
 -0.70710678 -0.70710678
 0 -1
 0.70710678 -0.70710678
+
 mfem_mesh_end
@@ -48,7 +48,7 @@ PROJECT_NAME           = MFEM
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.6.1
+PROJECT_NUMBER         = v4.7.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -987,6 +987,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/solvers \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
                         @MFEM_SOURCE_DIR@/miniapps/toys \
+                         @MFEM_SOURCE_DIR@/miniapps/tribol \
                         @MFEM_SOURCE_DIR@/miniapps/spde \
                         @MFEM_SOURCE_DIR@/miniapps/dpg \
                         @MFEM_SOURCE_DIR@/miniapps/dpg/util
@@ -110,9 +110,13 @@ namespace mfem {
 * - <a class="el" href="ex35p_8cpp_source.html">Example 35p</a>: parallel multi-domain damped harmonic oscillators
 * - <a class="el" href="ex36_8cpp_source.html">Example 36</a>: Proximal Galerkin FEM for the obstacle problem
 * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
- * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: Topology optimization
+ * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: topology optimization
 * - <a class="el" href="ex37p_8cpp_source.html">Example 37p</a>: parallel topology optimization
 * - <a class="el" href="ex38_8cpp_source.html">Example 38</a>: cut-surface and cut-volume integration
+ * - <a class="el" href="ex39_8cpp_source.html">Example 39</a>: named mesh attributes
+ * - <a class="el" href="ex39p_8cpp_source.html">Example 39p</a>: parallel named mesh attributes
+ * - <a class="el" href="ex40_8cpp_source.html">Example 40</a>: eikonal equation
+ * - <a class="el" href="ex40p_8cpp_source.html">Example 40p</a>: parallel eikonal equation
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -214,6 +218,8 @@ namespace mfem {
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
+ * - <a class="el" href="contact-patch-test_8cpp_source.html">Contact</a>: mortar contact patch test for elasticity
+ * - <a class="el" href="multidomain_8cpp_source.html">Multidomain miniapp</a>: Multidomain and Submesh demonstration miniapp
 * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
 * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
 * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
@@ -46,7 +46,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
            DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
        })
        // Update the color scheme when the tab is made visible again.
-        // It is possible that the appearance was changed in another tab 
+        // It is possible that the appearance was changed in another tab
        // while this tab was in the background.
        document.addEventListener("visibilitychange", visibilityState => {
            if (document.visibilityState === 'visible') {
@@ -97,7 +97,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
     * @returns `true` for dark-mode, `false` for light-mode user preference
     */
    static get userPreference() {
-        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) || 
+        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) ||
        (DoxygenAwesomeDarkModeToggle.systemPreference && !localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey))
    }

@@ -148,10 +148,10 @@ if (MFEM_ENABLE_TESTING)
  # Add CUDA/HIP tests.
  set(DEVICE_EXAMPLES
      # serial examples with device support:
-      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
+      ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
      # parallel examples with device support:
-      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p
-      ex34p ex35p)
+      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p ex22p ex24p ex25p
+      ex26p ex34p ex35p)
  set(MFEM_TEST_DEVICE)
  if (MFEM_USE_CUDA)
    set(MFEM_TEST_DEVICE "cuda")
@@ -161,6 +161,11 @@ if (MFEM_ENABLE_TESTING)
  if (MFEM_TEST_DEVICE)
    foreach(TEST_NAME ${DEVICE_EXAMPLES})
      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+      if (${TEST_NAME} MATCHES "ex14p")
+        list(APPEND THIS_TEST_OPTIONS "-rs" "2" "-rp" "0" "-pa")
+      elseif (${TEST_NAME} MATCHES "ex14")
+        list(APPEND THIS_TEST_OPTIONS "-r" "2" "-pa")
+      endif()
      if (NOT (${TEST_NAME} MATCHES ".*p$"))
        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
          COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
@@ -646,10 +646,7 @@ real_t HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 real_t HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   real_t loc_energy = 0.5*M.InnerProduct(v, v);
-   real_t energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPITypeMap<real_t>::mpi_type,
-                 MPI_SUM, fespace.GetComm());
+   real_t energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -18,6 +18,12 @@
 //               ex14 -m ../data/amr-quad.mesh -r 3
 //               ex14 -m ../data/amr-hex.mesh
 //               ex14 -m ../data/fichera-amr.mesh
+//               ex14 -pa -r 1 -o 3
+//               ex14 -pa -r 1 -o 3 -m ../data/fichera.mesh
+//
+// Device sample runs:
+//               ex14 -pa -r 2 -d cuda -o 3
+//               ex14 -pa -r 2 -d cuda -o 3 -m ../data/fichera.mesh
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -46,7 +52,9 @@ int main(int argc, char *argv[])
   real_t sigma = -1.0;
   real_t kappa = -1.0;
   real_t eta = 0.0;
+   bool pa = false;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -62,9 +70,13 @@ int main(int argc, char *argv[])
                  "One of the three DG penalty parameters, should be positive."
                  " Negative values are replaced with (order+1)^2.");
   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -77,117 +89,129 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
   //    NURBS meshes are projected to second order meshes.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   Mesh mesh(mesh_file);
+   const int dim = mesh.Dimension();

-   // 3. Refine the mesh to increase the resolution. In this example we do
+   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. By default, or if ref_levels < 0,
   //    we choose it to be the largest number that gives a final mesh with no
   //    more than 50,000 elements.
   {
      if (ref_levels < 0)
      {
-         ref_levels = (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+         ref_levels = (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
      }
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh->UniformRefinement();
+         mesh.UniformRefinement();
      }
   }
-   if (mesh->NURBSext)
+   if (mesh.NURBSext)
   {
-      mesh->SetCurvature(max(order, 1));
+      mesh.SetCurvature(max(order, 1));
   }

-   // 4. Define a finite element space on the mesh. Here we use discontinuous
+   // 5. Define a finite element space on the mesh. Here we use discontinuous
   //    finite elements of the specified order >= 0.
-   FiniteElementCollection *fec = new DG_FECollection(order, dim);
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of unknowns: " << fespace->GetVSize() << endl;
+   const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
+   DG_FECollection fec(order, dim, bt);
+   FiniteElementSpace fespace(&mesh, &fec);
+   cout << "Number of unknowns: " << fespace.GetVSize() << endl;

-   // 5. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system.
-   LinearForm *b = new LinearForm(fespace);
+   LinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   ConstantCoefficient zero(0.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->AddBdrFaceIntegrator(
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.AddBdrFaceIntegrator(
      new DGDirichletLFIntegrator(zero, one, sigma, kappa));
-   b->Assemble();
+   b.Assemble();

-   // 6. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
-   GridFunction x(fespace);
+   GridFunction x(&fespace);
   x = 0.0;

-   // 7. Set up the bilinear form a(.,.) on the finite element space
+   // 8. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    domain integrator and the interior and boundary DG face integrators.
   //    Note that boundary conditions are imposed weakly in the form, so there
   //    is no need for dof elimination. After assembly and finalizing we
   //    extract the corresponding sparse matrix A.
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   BilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      MFEM_VERIFY(!pa, "BR2 not yet compatible with partial assembly.");
+      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
-   a->Assemble();
-   a->Finalize();
-   const SparseMatrix &A = a->SpMat();
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.Assemble();
+   a.Finalize();

-#ifndef MFEM_USE_SUITESPARSE
-   // 8. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   // 9. Define a simple symmetric Gauss-Seidel preconditioner and use it to
   //    solve the system Ax=b with PCG in the symmetric case, and GMRES in the
-   //    non-symmetric one.
-   GSSmoother M(A);
-   if (sigma == -1.0)
+   //    non-symmetric one. (Note that tolerances are squared: 1e-12 corresponds
+   //    to a relative tolerance of 1e-6).
+   //
+   //    If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   if (pa)
   {
-      PCG(A, M, *b, x, 1, 500, 1e-12, 0.0);
+      MFEM_VERIFY(sigma == -1.0,
+                  "The case of PA with sigma != -1 is not yet supported.");
+      CG(a, b, x, 1, 500, 1e-12, 0.0);
   }
   else
   {
-      GMRES(A, M, *b, x, 1, 500, 10, 1e-12, 0.0);
-   }
+      const SparseMatrix &A = a.SpMat();
+#ifndef MFEM_USE_SUITESPARSE
+      GSSmoother M(A);
+      if (sigma == -1.0)
+      {
+         PCG(A, M, b, x, 1, 500, 1e-12, 0.0);
+      }
+      else
+      {
+         GMRES(A, M, b, x, 1, 500, 10, 1e-12, 0.0);
+      }
 #else
-   // 8. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(A);
-   umf_solver.Mult(*b, x);
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(A);
+      umf_solver.Mult(b, x);
 #endif
+   }

-   // 9. Save the refined mesh and the solution. This output can be viewed later
-   //    using GLVis: "glvis -m refined.mesh -g sol.gf".
+   // 10. Save the refined mesh and the solution. This output can be viewed
+   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
   ofstream mesh_ofs("refined.mesh");
   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
+   mesh.Print(mesh_ofs);
   ofstream sol_ofs("sol.gf");
   sol_ofs.precision(8);
   x.Save(sol_ofs);

-   // 10. Send the solution by socket to a GLVis server.
+   // 11. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
+      sol_sock << "solution\n" << mesh << x << flush;
   }

-   // 11. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete mesh;
-
   return 0;
 }
@@ -17,6 +17,12 @@
 //               mpirun -np 4 ex14p -m ../data/inline-segment.mesh -rs 5
 //               mpirun -np 4 ex14p -m ../data/amr-quad.mesh -rs 3
 //               mpirun -np 4 ex14p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -o 3
+//               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -m ../data/fichera.mesh -o 3
+//
+// Device sample runs:
+//               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -o 3
+//               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -m ../data/fichera.mesh -o 3
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -38,11 +44,14 @@ using namespace mfem;

 class CustomSolverMonitor : public IterativeSolverMonitor
 {
+private:
+   const ParMesh &pmesh;
+   ParGridFunction &pgf;
 public:
-   CustomSolverMonitor(const ParMesh *m,
-                       ParGridFunction *f) :
-      pmesh(m),
-      pgf(f) {}
+   CustomSolverMonitor(const ParMesh &pmesh_,
+                       ParGridFunction &pgf_) :
+      pmesh(pmesh_),
+      pgf(pgf_) {}

   void MonitorSolution(int i, real_t norm, const Vector &x, bool final)
   {
@@ -50,30 +59,24 @@ public:
      int  visport   = 19916;
      int  num_procs, myid;

-      MPI_Comm_size(pmesh->GetComm(),&num_procs);
-      MPI_Comm_rank(pmesh->GetComm(),&myid);
+      MPI_Comm_size(pmesh.GetComm(), &num_procs);
+      MPI_Comm_rank(pmesh.GetComm(), &myid);

-      pgf->SetFromTrueDofs(x);
+      pgf.SetFromTrueDofs(x);

      socketstream sol_sock(vishost, visport);
      sol_sock << "parallel " << num_procs << " " << myid << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << *pgf
+      sol_sock << "solution\n" << pmesh << pgf
               << "window_title 'Iteration no " << i << "'"
               << "keys rRjlc\n" << flush;
   }
-
-private:
-   const ParMesh *pmesh;
-   ParGridFunction *pgf;
 };

 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI and HYPRE.
   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
   Hypre::Init();

   // 2. Parse command-line options.
@@ -84,7 +87,9 @@ int main(int argc, char *argv[])
   real_t sigma = -1.0;
   real_t kappa = -1.0;
   real_t eta = 0.0;
+   bool pa = false;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -103,13 +108,17 @@ int main(int argc, char *argv[])
                  "One of the three DG penalty parameters, should be positive."
                  " Negative values are replaced with (order+1)^2.");
   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
-      if (myid == 0)
+      if (Mpi::Root())
      {
         args.PrintUsage(cout);
      }
@@ -119,16 +128,19 @@ int main(int argc, char *argv[])
   {
      kappa = (order+1)*(order+1);
   }
-   if (myid == 0)
+   if (Mpi::Root())
   {
      args.PrintOptions(cout);
   }

+   Device device(device_config);
+   if (Mpi::Root()) { device.Print(); }
+
   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral and hexahedral meshes
   //    with the same code. NURBS meshes are projected to second order meshes.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   Mesh mesh(mesh_file);
+   int dim = mesh.Dimension();

   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ser_ref_levels' of uniform refinement. By default,
@@ -137,53 +149,54 @@ int main(int argc, char *argv[])
   {
      if (ser_ref_levels < 0)
      {
-         ser_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+         ser_ref_levels = (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
      }
      for (int l = 0; l < ser_ref_levels; l++)
      {
-         mesh->UniformRefinement();
+         mesh.UniformRefinement();
      }
   }
-   if (mesh->NURBSext)
+   if (mesh.NURBSext)
   {
-      mesh->SetCurvature(max(order, 1));
+      mesh.SetCurvature(max(order, 1));
   }

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
   {
      for (int l = 0; l < par_ref_levels; l++)
      {
-         pmesh->UniformRefinement();
+         pmesh.UniformRefinement();
      }
   }

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use discontinuous finite elements of the specified order >= 0.
-   FiniteElementCollection *fec = new DG_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
-   if (myid == 0)
+   const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
+   DG_FECollection fec(order, dim, bt);
+   ParFiniteElementSpace fespace(&pmesh, &fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   if (Mpi::Root())
   {
      cout << "Number of unknowns: " << size << endl;
   }

   // 7. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system.
-   ParLinearForm *b = new ParLinearForm(fespace);
+   ParLinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   ConstantCoefficient zero(0.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->AddBdrFaceIntegrator(
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.AddBdrFaceIntegrator(
      new DGDirichletLFIntegrator(zero, one, sigma, kappa));
-   b->Assemble();
+   b.Assemble();

   // 8. Define the solution vector x as a parallel finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
-   ParGridFunction x(fespace);
+   ParGridFunction x(&fespace);
   x = 0.0;

   // 9. Set up the bilinear form a(.,.) on the finite element space
@@ -192,42 +205,51 @@ int main(int argc, char *argv[])
   //    Note that boundary conditions are imposed weakly in the form, so there
   //    is no need for dof elimination. After serial and parallel assembly we
   //    extract the corresponding parallel matrix A.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      MFEM_VERIFY(!pa, "BR2 not yet compatible with partial assembly.");
+      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
-   a->Assemble();
-   a->Finalize();
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.Assemble();
+   a.Finalize();

   // 10. Define the parallel (hypre) matrix and vectors representing a(.,.),
   //     b(.) and the finite element approximation.
-   HypreParMatrix *A = a->ParallelAssemble();
-   HypreParVector *B = b->ParallelAssemble();
-   HypreParVector *X = x.ParallelProject();
+   OperatorHandle A;

-   delete a;
-   delete b;
-
-   // 11. Depending on the symmetry of A, define and apply a parallel PCG or
-   //     GMRES solver for AX=B using the BoomerAMG preconditioner from hypre.
-   HypreSolver *amg = new HypreBoomerAMG(*A);
-   if (sigma == -1.0)
+   std::unique_ptr<HypreBoomerAMG> amg;
+   if (pa)
   {
-      HyprePCG pcg(*A);
-      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(500);
-      pcg.SetPrintLevel(2);
-      pcg.SetPreconditioner(*amg);
-      pcg.Mult(*B, *X);
+      A.Reset(&a, false);
   }
   else
   {
-      CustomSolverMonitor monitor(pmesh, &x);
+      A.SetType(Operator::Hypre_ParCSR);
+      a.ParallelAssemble(A);
+      amg.reset(new HypreBoomerAMG(*A.As<HypreParMatrix>()));
+   }
+
+   // 11. Depending on the symmetry of A, define and apply a parallel PCG or
+   //     GMRES solver for AX=B using the BoomerAMG preconditioner from hypre.
+   if (sigma == -1.0)
+   {
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(500);
+      cg.SetPrintLevel(1);
+      cg.SetOperator(*A);
+      if (amg) { cg.SetPreconditioner(*amg); }
+      cg.Mult(b, x);
+   }
+   else
+   {
+      CustomSolverMonitor monitor(pmesh, x);
      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetAbsTol(0.0);
      gmres.SetRelTol(1e-12);
@@ -235,51 +257,37 @@ int main(int argc, char *argv[])
      gmres.SetKDim(10);
      gmres.SetPrintLevel(1);
      gmres.SetOperator(*A);
-      gmres.SetPreconditioner(*amg);
+      if (amg) { gmres.SetPreconditioner(*amg); }
      gmres.SetMonitor(monitor);
-      gmres.Mult(*B, *X);
+      gmres.Mult(b, x);
   }
-   delete amg;

-   // 12. Extract the parallel grid function corresponding to the finite element
-   //     approximation X. This is the local solution on each processor.
-   x = *X;
-
-   // 13. Save the refined mesh and the solution in parallel. This output can
+   // 12. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      mesh_name << "mesh." << setfill('0') << setw(6) << Mpi::WorldRank();
+      sol_name << "sol." << setfill('0') << setw(6) << Mpi::WorldRank();

      ofstream mesh_ofs(mesh_name.str().c_str());
      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
+      pmesh.Print(mesh_ofs);

      ofstream sol_ofs(sol_name.str().c_str());
      sol_ofs.precision(8);
      x.Save(sol_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock << "parallel " << Mpi::WorldSize() << " " << Mpi::WorldRank() << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
+      sol_sock << "solution\n" << pmesh << x << flush;
   }

-   // 15. Free the used memory.
-   delete X;
-   delete B;
-   delete A;
-
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
   return 0;
 }
@@ -39,8 +39,8 @@ private:
   // Base Nonlinear Form
   std::unique_ptr<NonlinearForm> nonlinearForm;
   // element-wise inverse mass matrix
-   std::vector<DenseMatrix> invmass; // local scalar inverse mass.
-   std::vector<DenseMatrix> weakdiv; // local weakdivergence. Trial space is ByDim.
+   std::vector<DenseMatrix> invmass; // local scalar inverse mass
+   std::vector<DenseMatrix> weakdiv; // local weak divergence (trial space ByDim)
   // global maximum characteristic speed. Updated by form integrators
   mutable real_t max_char_speed;
   // auxiliary variable used in Mult
@@ -169,9 +169,9 @@ void DGHyperbolicConservationLaws::Mult(const Vector &x, Vector &y) const
 {
   // 0. Reset wavespeed computation before operator application.
   formIntegrator->ResetMaxCharSpeed();
-   // 1. Apply Nonlinear form to obtain an axiliary result
+   // 1. Apply Nonlinear form to obtain an auxiliary result
   //         z = - <F̂(u_h,n), [[v]]>_e
-   //    If weak-divergencee is not preassembled, we also have weak-divergence
+   //    If weak-divergence is not preassembled, we also have weak-divergence
   //         z = - <F̂(u_h,n), [[v]]>_e + (F(u_h), ∇v)
   nonlinearForm->Mult(x, z);
   if (!weakdiv.empty()) // if weak divergence is pre-assembled
@@ -19,8 +19,11 @@
 //    ex33 -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
 //    ex33 -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1
 //
-//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
-//        for all alpha.
+//  Note: The manufactured solution used in this problem is
+//
+//            u = ∏_{i=0}^{dim-1} sin(π x_i) ,
+//
+//        regardless of the value of alpha.
 //
 // Description:
 //
@@ -114,7 +117,8 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
                  "--no-verification",
-                  "Use sinusoidal function (f) for analytic comparison.");
+                  "Use sinusoidal function (f) for manufactured "
+                  "solution test.");
   args.Parse();
   if (!args.Good())
   {
@@ -163,7 +167,7 @@ int main(int argc, char *argv[])
   // 5. Define a finite element space on the mesh.
   H1_FECollection fec(order, dim);
   FiniteElementSpace fespace(&mesh, &fec);
-   cout << "Number of finite element unknowns: "
+   cout << "Number of degrees of freedom: "
        << fespace.GetTrueVSize() << endl;

   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
@@ -379,29 +383,29 @@ int main(int argc, char *argv[])
      FunctionCoefficient sol(solution);
      real_t l2_error = u.ComputeL2Error(sol);

-      string analytic_solution,expected_mesh;
+      string manufactured_solution,expected_mesh;
      switch (dim)
      {
         case 1:
-            analytic_solution = "sin(π x)";
+            manufactured_solution = "sin(π x)";
            expected_mesh = "inline_segment.mesh";
            break;
         case 2:
-            analytic_solution = "sin(π x) sin(π y)";
+            manufactured_solution = "sin(π x) sin(π y)";
            expected_mesh = "inline_quad.mesh";
            break;
         default:
-            analytic_solution = "sin(π x) sin(π y) sin(π z)";
+            manufactured_solution = "sin(π x) sin(π y) sin(π z)";
            expected_mesh = "inline_hex.mesh";
            break;
      }

      mfem::out << "\n" << string(80,'=')
                << "\n\nSolution Verification in "<< dim << "D \n\n"
-                << "Analytic solution : " << analytic_solution << "\n"
-                << "Expected mesh     : " << expected_mesh <<"\n"
-                << "Your mesh         : " << mesh_file << "\n"
-                << "L2 error          : " << l2_error << "\n\n"
+                << "Manufactured solution : " << manufactured_solution << "\n"
+                << "Expected mesh         : " << expected_mesh <<"\n"
+                << "Your mesh             : " << mesh_file << "\n"
+                << "L2 error              : " << l2_error << "\n\n"
                << string(80,'=') << endl;
   }

@@ -131,7 +131,7 @@ void RationalApproximation_AAA(const Vector &val, const Vector &pt,
      }

 #ifdef MFEM_USE_LAPACK
-      DenseMatrixSVD svd(Am,false,true);
+      DenseMatrixSVD svd(Am,'N','A');
      svd.Eval(Am);
      DenseMatrix &v = svd.RightSingularvectors();
      v.GetRow(k,w);
@@ -346,7 +346,7 @@ void ComputePartialFractionApproximation(real_t & alpha,
   }
   else
   {
-      if (abs(alpha - 0.5) > eps && print_warning)
+      if (abs(alpha - 0.5) > eps)
      {
         alpha = 0.5;
      }
@@ -368,6 +368,8 @@ void ComputePartialFractionApproximation(real_t & alpha,


   return;
+#else
+   MFEM_CONTRACT_VAR(print_warning);
 #endif

   Vector x(npoints);
@@ -19,8 +19,11 @@
 //    mpirun -np 4 ex33p -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
 //    mpirun -np 4 ex33p -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1

-//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
-//        for all alpha.
+//  Note: The manufactured solution used in this problem is
+//
+//            u = ∏_{i=0}^{dim-1} sin(π x_i) ,
+//
+//        regardless of the value of alpha.
 //
 // Description:
 //
@@ -120,7 +123,8 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
                  "--no-verification",
-                  "Use sinusoidal function (f) for analytic comparison.");
+                  "Use sinusoidal function (f) for manufactured "
+                  "solution test.");
   args.Parse();
   if (!args.Good())
   {
@@ -180,10 +184,11 @@ int main(int argc, char *argv[])
   // 5. Define a finite element space on the mesh.
   H1_FECollection fec(order, dim);
   ParFiniteElementSpace fespace(&pmesh, &fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
   if (Mpi::Root())
   {
-      cout << "Number of finite element unknowns: "
-           << fespace.GetTrueVSize() << endl;
+      cout << "Number of degrees of freedom: "
+           << size << endl;
   }

   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
@@ -223,7 +228,7 @@ int main(int argc, char *argv[])
   if (verification)
   {
      // This statement is only relevant for the verification of the code. It
-      // uses a different f such that an analytic solution is known and easy
+      // uses a different f such that an manufactured solution is known and easy
      // to compare with the numerical one. The FPDE becomes:
      // (-Δ)^α u = (2\pi ^2)^α sin(\pi x) sin(\pi y) on [0,1]^2
      // -> u(x,y) = sin(\pi x) sin(\pi y)
@@ -415,29 +420,29 @@ int main(int argc, char *argv[])

      if (Mpi::Root())
      {
-         string analytic_solution,expected_mesh;
+         string manufactured_solution,expected_mesh;
         switch (dim)
         {
            case 1:
-               analytic_solution = "sin(π x)";
+               manufactured_solution = "sin(π x)";
               expected_mesh = "inline_segment.mesh";
               break;
            case 2:
-               analytic_solution = "sin(π x) sin(π y)";
+               manufactured_solution = "sin(π x) sin(π y)";
               expected_mesh = "inline_quad.mesh";
               break;
            default:
-               analytic_solution = "sin(π x) sin(π y) sin(π z)";
+               manufactured_solution = "sin(π x) sin(π y) sin(π z)";
               expected_mesh = "inline_hex.mesh";
               break;
         }

         mfem::out << "\n" << string(80,'=')
                   << "\n\nSolution Verification in "<< dim << "D \n\n"
-                   << "Analytic solution : " << analytic_solution << "\n"
-                   << "Expected mesh     : " << expected_mesh <<"\n"
-                   << "Your mesh         : " << mesh_file << "\n"
-                   << "L2 error          : " << l2_error << "\n\n"
+                   << "Manufactured solution : " << manufactured_solution << "\n"
+                   << "Expected mesh         : " << expected_mesh <<"\n"
+                   << "Your mesh             : " << mesh_file << "\n"
+                   << "L2 error              : " << l2_error << "\n\n"
                   << string(80,'=') << endl;
      }
   }
@@ -199,7 +199,6 @@ public:
      {
         mesh->GetElementTransformation(elem, &Tr);
         MFIRs.GetSurfaceIntegrationRule(Tr, ir);
-         Vector w;
         MFIRs.GetSurfaceWeights(Tr, ir, w);
         SurfaceWeights.SetCol(elem, w);

@@ -2,39 +2,53 @@
 //
 // Compile with: make ex40
 //
-// Sample runs: ex40 -o 2
-//              ex40 -o 2 -r 4
+// Sample runs: ex40 -step 10 -gr 2.0
+//              ex40 -step 10 -gr 2.0 -o 3 -r 1
+//              ex40 -step 10 -gr 2.0 -r 4 -m ../data/l-shape.mesh
+//              ex40 -step 10 -gr 2.0 -r 2 -m ../data/fichera.mesh
 //
-// Description: This example code demonstrates to how to use MFEM to solve
-//              the Monge–Ampère equation
+// Description: This example code demonstrates how to use MFEM to solve the
+//              eikonal equation,
 //
-//                      det(∇²u) = f in Ω,   u = 0 on ∂Ω.
+//                      |∇𝑢| = 1 in Ω,  𝑢 = g on ∂Ω.
 //
-//              This example highlights the ExponentialMatrixCoefficient
-//              class, which is used in Newton's method to solve the
-//              variational formulation
+//              The solution of this problem coincides with the unique optimum of
+//              the nonlinear program
 //
-//               Find M ∈ H₀(div,Ω)ⁿ and u ∈ H₀¹(Ω) such that
-//                 (exp(M), N) + (∇u, ∇⋅N) = 0           ∀ N ∈ H₀(div,Ω)ⁿ
-//                 (tr(M), v)              = (ln f, v)   ∀ v ∈ H₀¹(Ω)
+//                   maximize ∫_Ω 𝑢 d𝑥 subject to |∇𝑢| ≤ 1,   𝑢 = g on Ω,      (⋆)
 //
-//              where n is the spatial dimension of the domain Ω.
+//              which is the foundation for method implemented below.
 //
+//              Following the proximal Galerkin methodology [1] (see also Example
+//              36), we construct a Legendre function for the unit ball
+//              𝐵₁ := {𝑥 ∈ Rⁿ | |𝑥| < 1}. Our choice is the Hellinger entropy,
 //
-//              The linearized subproblem is
+//                    h(𝑥) = −( 1 − |𝑥|² )^{1/2},
 //
-//               Find δM ∈ H₀(div,Ω)ⁿ and u ∈ H₀¹(Ω) such that
-//                 (exp(M) δM, N) + (∇u, ∇⋅N) = -(exp(M), N)        ∀ N ∈ H₀(div,Ω)ⁿ
-//                 (tr(δM), v)                = (ln f - tr(M), v)   ∀ v ∈ H₀¹(Ω)
+//              although other choices are possible, each leading to a slightly
+//              different algorithm. We then adaptively regularize the optimization
+//              problem (⋆) with the Bregman divergence of the Hellinger entropy,
 //
+//                 maximize  ∫_Ω 𝑢 d𝑥 - αₖ⁻¹ Dₕ(∇𝑢,∇𝑢ₖ₋₁)  subject to  𝑢 = g on Ω.
 //
-//              (exp(M) δM, N)    :::   VectorFEMassIntegrator
-//              (∇u, ∇⋅N)         :::   MixedGradDivIntegrator
-//              (tr(δM), v)       :::   MixedDotProductIntegrator
-//              (exp(M), N)       :::   VectorFEDomainLFIntegrator
-//              (ln f - tr(M), v) :::   DomainLFIntegrator
+//              This results in a sequence of functions ( 𝜓ₖ , 𝑢ₖ ),
 //
+//                      𝑢ₖ → 𝑢,    𝜓ₖ/|𝜓ₖ| → ∇𝑢    as k → \infty,
 //
+//              defined by the nonlinear saddle-point problems
+//
+//               Find 𝜓ₖ ∈ H(div,Ω) and 𝑢ₖ ∈ L²(Ω) such that
+//               ( Zₖ(𝜓ₖ) , τ ) + ( 𝑢ₖ , ∇⋅τ ) = ⟨ g , τ⋅n ⟩         ∀ τ ∈ H(div,Ω)
+//               ( ∇⋅𝜓ₖ , v )                 = ( ∇⋅𝜓ₖ₋₁ - 1 , v )   ∀ v ∈ L²(Ω)
+//
+//              where Zₖ(𝜓) := ∇h⁻¹(αₖ 𝜓) = 𝜓 / ( αₖ⁻² + |𝜓|² )^{1/2} and step size
+//              αₖ > 0. These saddle-point problems are solved using a damped Newton's
+//              method. This example assumes that g = 0 and allows the step size to
+//              grow geometrically, αₖ = α₀rᵏ, where r ≥ 1 is the growth rate.
+//
+//              [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//                  preserving finite element method for pointwise bound constraints.
+//                  arXiv:2307.12444 [math.NA]

 #include "mfem.hpp"
 #include <fstream>
@@ -43,23 +57,53 @@
 using namespace std;
 using namespace mfem;

-real_t exact_solution(const Vector &pt);
-void exact_solution_gradient(const Vector &pt, Vector &grad);
+class ZCoefficient : public VectorCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   ZCoefficient(int vdim, GridFunction &psi_, real_t alpha_ = 1.0)
+      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+class DZCoefficient : public MatrixCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   DZCoefficient(int height, GridFunction &psi_, real_t alpha_ = 1.0)
+      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};

 int main(int argc, char *argv[])
 {
   // 1. Parse command-line options.
-   const char *mesh_file = "../data/disc-nurbs.mesh";
-   // const char *mesh_file = "../data/star.mesh";
-   int order = 2;
-   int max_it = 10;
-   int ref_levels = 1;
-   real_t tol = 1e-5;
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int max_it = 5;
+   int ref_levels = 3;
+   real_t alpha = 1.0;
+   real_t growth_rate = 1.0;
+   real_t newton_scaling = 0.9;
+   real_t tichonov = 1e-1;
+   real_t tol = 1e-4;
   bool visualization = true;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file.");
+                  "Mesh file to use.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
   args.AddOption(&ref_levels, "-r", "--refs",
@@ -69,6 +113,10 @@ int main(int argc, char *argv[])
   args.AddOption(&tol, "-tol", "--tol",
                  "Stopping criteria based on the difference between"
                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Initial size alpha");
+   args.AddOption(&growth_rate, "-gr", "--growth-rate",
+                  "Growth rate of the step size alpha");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -83,11 +131,12 @@ int main(int argc, char *argv[])
   // 2. Read the mesh from the mesh file.
   Mesh mesh(mesh_file, 1, 1);
   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();

-   if (dim != 2)
-   {
-      MFEM_ABORT("Example 40 currently only supports 2D problems")
-   }
+   MFEM_ASSERT(mesh.bdr_attributes.Size(),
+               "This example does not currently support meshes"
+               " without boundary attributes."
+              )

   // 3. Postprocess the mesh.
   // 3A. Refine the mesh to increase the resolution.
@@ -102,73 +151,45 @@ int main(int argc, char *argv[])
   mesh.SetCurvature(curvature_order);

   // 4. Define the necessary finite element spaces on the mesh.
-   H1_FECollection H1fec(order, dim);
-   FiniteElementSpace H1fes(&mesh, &H1fec);
-
-   RT_FECollection RTfec(order-1, dim);
+   RT_FECollection RTfec(order, dim);
   FiniteElementSpace RTfes(&mesh, &RTfec);

-   cout << "Number of H¹ degrees of freedom: "
-        << H1fes.GetTrueVSize() << endl;
-   cout << "Number of H(div) degrees of freedom: "
-        << RTfes.GetTrueVSize() * dim << endl;
+   L2_FECollection L2fec(order, dim);
+   FiniteElementSpace L2fes(&mesh, &L2fec);

-   Array<int> offsets(4);
+   cout << "Number of H(div) dofs: "
+        << RTfes.GetTrueVSize() << endl;
+   cout << "Number of L² dofs: "
+        << L2fes.GetTrueVSize() << endl;
+
+   // 5. Define the offsets for the block matrices
+   Array<int> offsets(3);
   offsets[0] = 0;
   offsets[1] = RTfes.GetVSize();
-   offsets[2] = RTfes.GetVSize();
-   offsets[3] = H1fes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
   offsets.PartialSum();

   BlockVector x(offsets), rhs(offsets);
   x = 0.0; rhs = 0.0;

-   // 5. Determine the list of true (i.e., conforming) essential boundary dofs.
-   Array<int> ess_bdr;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-   }
-
-   // 6. Define constants to be used later.
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient neg_one(-1.0);
-   ConstantCoefficient zero(0.0);
-   Vector V1(2), V2(2);
-   V1(0) = 1.0; V1(1) = 0.0;
-   V2(0) = 0.0; V2(1) = 1.0;
-   VectorConstantCoefficient onezero(V1);
-   VectorConstantCoefficient zeroone(V2);
-   ScalarVectorProductCoefficient neg_onezero(-1.0, onezero);
-   ScalarVectorProductCoefficient neg_zeroone(-1.0, zeroone);
-
-   // 7. Define the solution vectors as finite element grid functions
+   // 6. Define the solution vectors as a finite element grid functions
   //    corresponding to the fespaces.
-   GridFunction delta_M1_gf, delta_M2_gf, delta_u_gf;
+   GridFunction u_gf, delta_psi_gf;
+   delta_psi_gf.MakeRef(&RTfes,x,offsets[0]);
+   u_gf.MakeRef(&L2fes,x,offsets[1]);

-   delta_M1_gf.MakeRef(&RTfes,x,offsets[0]);
-   delta_M2_gf.MakeRef(&RTfes,x,offsets[1]);
-   delta_u_gf.MakeRef(&H1fes,x,offsets[2]);
+   GridFunction psi_old_gf(&RTfes);
+   GridFunction psi_gf(&RTfes);
+   GridFunction u_old_gf(&L2fes);

-   GridFunction M1_gf(&RTfes);
-   GridFunction M2_gf(&RTfes);
-   GridFunction u_gf(&H1fes);
-
-   // 8. Define the function coefficients for the solution and use them to
-   //    initialize the initial guess
-   FunctionCoefficient exact_coef(exact_solution);
-   VectorFunctionCoefficient exact_grad_coef(dim,exact_solution_gradient);
-   ConstantCoefficient ln_rhs_coef(0.0);
-   u_gf.ProjectCoefficient(exact_coef);
-   // u_gf.ProjectCoefficient(zero);
-   M1_gf = 0.0;
-   M2_gf = 0.0;
-
-   delta_M1_gf = 0.0;
-   delta_M2_gf = 0.0;
-   delta_u_gf = 0.0;
+   // 7. Define initial guesses for the solution variables.
+   delta_psi_gf = 0.0;
+   psi_gf = 0.0;
+   u_gf = 0.0;
+   psi_old_gf = psi_gf;
+   u_old_gf = u_gf;

+   // 8. Prepare for glvis output.
   char vishost[] = "localhost";
   int  visport   = 19916;
   socketstream sol_sock;
@@ -178,234 +199,176 @@ int main(int argc, char *argv[])
      sol_sock.precision(8);
   }

-   // 10. Iterate
+   // 9. Coefficients to be used later.
+   ConstantCoefficient neg_one(-1.0);
+   ConstantCoefficient zero(0.0);
+   ConstantCoefficient tichonov_cf(tichonov);
+   ConstantCoefficient neg_tichonov_cf(-1.0*tichonov);
+   ZCoefficient Z(sdim, psi_gf, alpha);
+   DZCoefficient DZ(sdim, psi_gf, alpha);
+   ScalarVectorProductCoefficient neg_Z(-1.0, Z);
+   DivergenceGridFunctionCoefficient div_psi_cf(&psi_gf);
+   DivergenceGridFunctionCoefficient div_psi_old_cf(&psi_old_gf);
+   SumCoefficient psi_old_minus_psi(div_psi_old_cf, div_psi_cf, 1.0, -1.0);
+
+   // 10. Assemble constant matrices/vectors to avoid reassembly in the loop.
+   LinearForm b0, b1;
+   b0.MakeRef(&RTfes,rhs.GetBlock(0),0);
+   b1.MakeRef(&L2fes,rhs.GetBlock(1),0);
+
+   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_Z));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+
+   BilinearForm a00(&RTfes);
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(DZ));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(tichonov_cf));
+
+   MixedBilinearForm a10(&RTfes,&L2fes);
+   a10.AddDomainIntegrator(new VectorFEDivergenceIntegrator());
+   a10.Assemble();
+   a10.Finalize();
+   SparseMatrix &A10 = a10.SpMat();
+   SparseMatrix *A01 = Transpose(A10);
+
+   BilinearForm a11(&L2fes);
+   a11.AddDomainIntegrator(new MassIntegrator(neg_tichonov_cf));
+   a11.Assemble();
+   a11.Finalize();
+   SparseMatrix &A11 = a11.SpMat();
+
+   // 11. Iterate.
   int k;
+   int total_iterations = 0;
+   real_t increment_u = 0.1;
+   GridFunction u_tmp(&L2fes);
   for (k = 0; k < max_it; k++)
   {
-      mfem::out << "\nITERATION " << k+1 << endl;
+      u_tmp = u_old_gf;
+      Z.SetAlpha(alpha);
+      DZ.SetAlpha(alpha);

-      LinearForm b0,b1,b2;
-      b0.Update(&RTfes,rhs.GetBlock(0),0);
-      b1.Update(&RTfes,rhs.GetBlock(1),0);
-      b2.Update(&H1fes,rhs.GetBlock(2),0);
+      mfem::out << "\nOUTER ITERATION " << k+1 << endl;

-      VectorGridFunctionCoefficient M1(&M1_gf);
-      VectorGridFunctionCoefficient M2(&M2_gf);
-
-      MatrixArrayVectorCoefficient M(dim);
-      M.Set(0, &M1, false);
-      M.Set(1, &M2, false);
-      ExponentialMatrixCoefficient exp_M(M);
-
-      MatrixVectorProductCoefficient exp_M1(exp_M, onezero);
-      MatrixVectorProductCoefficient exp_M2(exp_M, zeroone);
-      InnerProductCoefficient exp_M11(exp_M1, onezero);
-      InnerProductCoefficient exp_M12(exp_M1, zeroone);
-      InnerProductCoefficient exp_M21(exp_M2, onezero);
-      InnerProductCoefficient exp_M22(exp_M2, zeroone);
-
-      GradientGridFunctionCoefficient grad_u(&u_gf);
-      InnerProductCoefficient neg_dudx(neg_onezero, grad_u);
-      ScalarVectorProductCoefficient neg_exp_M1(-1.0, exp_M1);
-      b0.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(neg_dudx));
-      b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_exp_M1));
-      b0.Assemble();
-
-      InnerProductCoefficient neg_dudy(neg_zeroone, grad_u);
-      b1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(neg_dudy));
-      ScalarVectorProductCoefficient neg_exp_M2(-1.0, exp_M2);
-      b1.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_exp_M2));
-      b1.Assemble();
-
-      InnerProductCoefficient M11(M1, onezero);
-      InnerProductCoefficient M22(M2, zeroone);
-      SumCoefficient trace_M(M11, M22);
-      SumCoefficient rhs2(ln_rhs_coef, trace_M, 1.0, -1.0);
-      b2.AddDomainIntegrator(new DomainLFIntegrator(rhs2));
-      b2.Assemble();
-
-      cout << "b0.Norml2() = " << b0.Norml2() << endl;
-      cout << "b1.Norml2() = " << b1.Norml2() << endl;
-      cout << "b2.Norml2() = " << b2.Norml2() << endl;
-
-      BilinearForm a00(&RTfes);
-      a00.AddDomainIntegrator(new VectorFEMassIntegrator());
-      // a00.AddDomainIntegrator(new VectorFEMassIntegrator(exp_M11));
-      a00.Assemble();
-      a00.EliminateEssentialBC(ess_bdr,x.GetBlock(0),rhs.GetBlock(0),mfem::Operator::DIAG_ONE);
-      a00.Finalize();
-      SparseMatrix &A00 = a00.SpMat();
-
-      BilinearForm a01(&RTfes);
-      a01.AddDomainIntegrator(new VectorFEMassIntegrator(zero));
-      // a01.AddDomainIntegrator(new VectorFEMassIntegrator(exp_M12));
-      a01.Assemble();
-      a01.EliminateEssentialBC(ess_bdr,mfem::Operator::DIAG_ZERO);
-      a01.Finalize();
-      SparseMatrix &A01 = a01.SpMat();
-
-      MixedBilinearForm a02(&H1fes,&RTfes);
-      a02.AddDomainIntegrator(new MixedGradDivIntegrator(neg_onezero));
-      a02.Assemble(false);
-      a02.EliminateTrialDofs(ess_bdr,x.GetBlock(2),rhs.GetBlock(0));
-      a02.EliminateTestDofs(ess_bdr);
-      a02.Finalize();
-      SparseMatrix &A02 = a02.SpMat();
-
-      BilinearForm a10(&RTfes);
-      a10.AddDomainIntegrator(new VectorFEMassIntegrator(zero));
-      // a10.AddDomainIntegrator(new VectorFEMassIntegrator(exp_M21));
-      a10.Assemble();
-      a10.EliminateEssentialBC(ess_bdr,mfem::Operator::DIAG_ZERO);
-      a10.Finalize();
-      SparseMatrix &A10 = a10.SpMat();
-
-      BilinearForm a11(&RTfes);
-      a11.AddDomainIntegrator(new VectorFEMassIntegrator());
-      // a11.AddDomainIntegrator(new VectorFEMassIntegrator(exp_M22));
-      a11.Assemble();
-      a11.EliminateEssentialBC(ess_bdr,x.GetBlock(1),rhs.GetBlock(1),mfem::Operator::DIAG_ONE);
-      a11.Finalize();
-      SparseMatrix &A11 = a11.SpMat();
-
-      MixedBilinearForm a12(&H1fes,&RTfes);
-      a12.AddDomainIntegrator(new MixedGradDivIntegrator(neg_zeroone));
-      a12.Assemble(false);
-      a12.EliminateTrialDofs(ess_bdr,x.GetBlock(2),rhs.GetBlock(1));
-      a12.EliminateTestDofs(ess_bdr);
-      a12.Finalize();
-      SparseMatrix &A12 = a12.SpMat();
-
-      MixedBilinearForm a20(&RTfes,&H1fes);
-      a20.AddDomainIntegrator(new MixedDotProductIntegrator(onezero));
-      a20.Assemble();
-      a20.EliminateTrialDofs(ess_bdr,x.GetBlock(0),rhs.GetBlock(2));
-      a20.EliminateTestDofs(ess_bdr);
-      a20.Finalize();
-      SparseMatrix &A20 = a20.SpMat();
-
-      MixedBilinearForm a21(&RTfes,&H1fes);
-      a21.AddDomainIntegrator(new MixedDotProductIntegrator(zeroone));
-      a21.Assemble();
-      a21.EliminateTrialDofs(ess_bdr,x.GetBlock(1),rhs.GetBlock(2));
-      a21.EliminateTestDofs(ess_bdr);
-      a21.Finalize();
-      SparseMatrix &A21 = a21.SpMat();
-
-      BilinearForm a22(&H1fes);
-      // a22.AddDomainIntegrator(new MassIntegrator(neg_one));
-      a22.AddDomainIntegrator(new MassIntegrator(zero));
-      a22.Assemble(false);
-      a22.EliminateEssentialBC(ess_bdr,x.GetBlock(2),rhs.GetBlock(2),mfem::Operator::DIAG_ONE);
-      a22.Finalize();
-      SparseMatrix &A22 = a22.SpMat();
-
-      cout << "b0.Norml2() = " << b0.Norml2() << endl;
-      cout << "b1.Norml2() = " << b1.Norml2() << endl;
-      cout << "b2.Norml2() = " << b2.Norml2() << endl;
-
-      // BlockOperator A(offsets);
-      // A.SetBlock(0,0,&A00);
-      // A.SetBlock(0,1,&A01);
-      // A.SetBlock(0,2,&A02);
-      // A.SetBlock(1,0,&A10);
-      // A.SetBlock(1,1,&A11);
-      // A.SetBlock(1,2,&A12);
-      // A.SetBlock(2,0,&A20);
-      // A.SetBlock(2,1,&A21);
-      // A.SetBlock(2,2,&A22);
-
-      // BlockDiagonalPreconditioner prec(offsets);
-      // prec.SetDiagonalBlock(0,new GSSmoother(A00));
-      // prec.SetDiagonalBlock(1,new GSSmoother(A11));
-      // prec.SetDiagonalBlock(1,new GSSmoother(A22));
-      // prec.owns_blocks = 1;
-
-      // GMRES(A,prec,rhs,x,1,10000,500,1e-12,0.0);
-
-      BlockMatrix A(offsets);
-      A.SetBlock(0,0,&A00);
-      A.SetBlock(0,1,&A01);
-      A.SetBlock(0,2,&A02);
-      A.SetBlock(1,0,&A10);
-      A.SetBlock(1,1,&A11);
-      A.SetBlock(1,2,&A12);
-      A.SetBlock(2,0,&A20);
-      A.SetBlock(2,1,&A21);
-      A.SetBlock(2,2,&A22);
-
-      SparseMatrix * A_mono = A.CreateMonolithic();
-      UMFPackSolver umf(*A_mono);
-      umf.Mult(rhs,x);
-
-      delta_M1_gf.MakeRef(&RTfes, x.GetBlock(0), 0);
-      delta_M2_gf.MakeRef(&RTfes, x.GetBlock(1), 0);
-      delta_u_gf.MakeRef(&H1fes, x.GetBlock(2), 0);
-
-      real_t Newton_update_size = delta_u_gf.ComputeL2Error(zero);
-
-      real_t gamma = 0.3;
-      delta_M1_gf *= gamma;
-      delta_M2_gf *= gamma;
-      delta_u_gf *= gamma;
-      M1_gf += delta_M1_gf;
-      M2_gf += delta_M2_gf;
-      u_gf += delta_u_gf;
-
-      if (visualization)
+      int j;
+      for ( j = 0; j < 5; j++)
      {
-         // sol_sock << "solution\n" << mesh << delta_M1_gf << "window_title 'Discrete solution'"
-         sol_sock << "solution\n" << mesh << u_gf << "window_title 'Discrete solution'"
-                  << flush;
-         mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << Newton_update_size <<
-                   endl;
+         total_iterations++;
+
+         b0.Assemble();
+         b1.Assemble();
+
+         a00.Assemble(false);
+         a00.Finalize(false);
+         SparseMatrix &A00 = a00.SpMat();
+
+         // Construct Schur-complement preconditioner
+         Vector A00_diag(a00.Height());
+         A00.GetDiag(A00_diag);
+         A00_diag.Reciprocal();
+         SparseMatrix *S = Mult_AtDA(*A01, A00_diag);
+
+         BlockDiagonalPreconditioner prec(offsets);
+         prec.SetDiagonalBlock(0,new DSmoother(A00));
+#ifndef MFEM_USE_SUITESPARSE
+         prec.SetDiagonalBlock(1,new GSSmoother(*S));
+#else
+         prec.SetDiagonalBlock(1,new UMFPackSolver(*S));
+#endif
+         prec.owns_blocks = 1;
+
+         BlockOperator A(offsets);
+         A.SetBlock(0,0,&A00);
+         A.SetBlock(1,0,&A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,&A11);
+
+         GMRES(A,prec,rhs,x,0,2000,500,1e-12,0.0);
+         delete S;
+
+         u_tmp -= u_gf;
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         // Damped Newton update
+         psi_gf.Add(newton_scaling, delta_psi_gf);
+         a00.Update();
+
+         if (visualization)
+         {
+            sol_sock << "solution\n" << mesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+         }
+
+         mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
      }

-      // if (Newton_update_size < tol || k == max_it-1)
-      // {
-      //    break;
-      // }
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);

-      real_t H1_error = u_gf.ComputeH1Error(&exact_coef,&exact_grad_coef);
-      real_t L2_error = u_gf.ComputeL2Error(exact_coef);
-      mfem::out << "L2-error  (|| u - uₕᵏ||)       = " << L2_error << endl;
-      // mfem::out << "H1-error  (|| u - uₕᵏ||)       = " << H1_error << endl;
+      mfem::out << "Number of Newton iterations = " << j+1 << endl;
+      mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;

-      cin.get();
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      alpha *= max(growth_rate, 1_r);

   }

-   mfem::out << "\n Total iterations: " << k+1
-             << "\n Total dofs:       " << RTfes.GetTrueVSize() * 2 + H1fes.GetTrueVSize()
+   mfem::out << "\n Outer iterations: " << k+1
+             << "\n Total iterations: " << total_iterations
+             << "\n Total dofs:       " << RTfes.GetTrueVSize() + L2fes.GetTrueVSize()
             << endl;

-   // 11. Exact solution.
-   // if (visualization)
-   // {
-   //    socketstream err_sock(vishost, visport);
-   //    err_sock.precision(8);
-
-   //    GridFunction error_gf(&H1fes);
-   //    error_gf.ProjectCoefficient(exact_coef);
-   //    error_gf -= u_gf;
-
-   //    err_sock << "solution\n" << mesh << error_gf << "window_title 'Error'"  <<
-   //             flush;
-   // }
-
+   delete A01;
   return 0;
 }

-real_t exact_solution(const Vector &pt)
+void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
+                        const IntegrationPoint &ip)
 {
-   real_t x = pt(0), y = pt(1);
-   return (x*x + y*y) / 2.0 - 4.0;
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(vdim);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   V = psi_vals;
+   V *= phi;
 }

-void exact_solution_gradient(const Vector &pt, Vector &grad)
+void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                         const IntegrationPoint &ip)
 {
-   real_t x = pt(0), y = pt(1);
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");

-   grad(0) = x;
-   grad(1) = y;
+   Vector psi_vals(height);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   K = 0.0;
+   for (int i = 0; i < height; i++)
+   {
+      K(i,i) = phi;
+      for (int j = 0; j < height; j++)
+      {
+         K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
+      }
+   }
 }
@@ -0,0 +1,436 @@
+//                       MFEM Example 40 - Parallel Version
+//
+// Compile with: make ex40p
+//
+// Sample runs: mpirun -np 4 ex40p -step 10 -gr 2.0
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -o 3 -r 1
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -r 4 -m ../data/l-shape.mesh
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -r 2 -m ../data/fichera.mesh
+//
+// Description: This example code demonstrates how to use MFEM to solve the
+//              eikonal equation,
+//
+//                      |∇𝑢| = 1 in Ω,  𝑢 = g on ∂Ω.
+//
+//              The solution of this problem coincides with the unique optimum of
+//              the nonlinear program
+//
+//                   maximize ∫_Ω 𝑢 d𝑥 subject to |∇𝑢| ≤ 1,   𝑢 = g on Ω,      (⋆)
+//
+//              which is the foundation for method implemented below.
+//
+//              Following the proximal Galerkin methodology [1] (see also Example
+//              36), we construct a Legendre function for the unit ball
+//              𝐵₁ := {𝑥 ∈ Rⁿ | |𝑥| < 1}. Our choice is the Hellinger entropy,
+//
+//                    h(𝑥) = −( 1 − |𝑥|² )^{1/2},
+//
+//              although other choices are possible, each leading to a slightly
+//              different algorithm. We then adaptively regularize the optimization
+//              problem (⋆) with the Bregman divergence of the Hellinger entropy,
+//
+//                 maximize  ∫_Ω 𝑢 d𝑥 - αₖ⁻¹ Dₕ(∇𝑢,∇𝑢ₖ₋₁)  subject to  𝑢 = g on Ω.
+//
+//              This results in a sequence of functions ( 𝜓ₖ , 𝑢ₖ ),
+//
+//                      𝑢ₖ → 𝑢,    𝜓ₖ/|𝜓ₖ| → ∇𝑢    as k → \infty,
+//
+//              defined by the nonlinear saddle-point problems
+//
+//               Find 𝜓ₖ ∈ H(div,Ω) and 𝑢ₖ ∈ L²(Ω) such that
+//               ( Zₖ(𝜓ₖ) , τ ) + ( 𝑢ₖ , ∇⋅τ ) = ⟨ g , τ⋅n ⟩         ∀ τ ∈ H(div,Ω)
+//               ( ∇⋅𝜓ₖ , v )                 = ( ∇⋅𝜓ₖ₋₁ - 1 , v )   ∀ v ∈ L²(Ω)
+//
+//              where Zₖ(𝜓) := ∇h⁻¹(αₖ 𝜓) = 𝜓 / ( αₖ⁻² + |𝜓|² )^{1/2} and step size
+//              αₖ > 0. These saddle-point problems are solved using a damped Newton's
+//              method. This example assumes that g = 0 and allows the step size to
+//              grow geometrically, αₖ = α₀rᵏ, where r ≥ 1 is the growth rate.
+//
+//              [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//                  preserving finite element method for pointwise bound constraints.
+//                  arXiv:2307.12444 [math.NA]
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ZCoefficient : public VectorCoefficient
+{
+protected:
+   ParGridFunction *psi;
+   real_t alpha;
+
+public:
+   ZCoefficient(int vdim, ParGridFunction &psi_, real_t alpha_ = 1.0)
+      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+class DZCoefficient : public MatrixCoefficient
+{
+protected:
+   ParGridFunction *psi;
+   real_t alpha;
+
+public:
+   DZCoefficient(int height, ParGridFunction &psi_, real_t alpha_ = 1.0)
+      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int max_it = 5;
+   int ref_levels = 3;
+   real_t alpha = 1.0;
+   real_t growth_rate = 1.0;
+   real_t newton_scaling = 0.9;
+   real_t tichonov = 1e-1;
+   real_t tol = 1e-4;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Initial size alpha");
+   args.AddOption(&growth_rate, "-gr", "--growth-rate",
+                  "Growth rate of the step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2. Read the mesh from the mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+
+   MFEM_ASSERT(mesh.bdr_attributes.Size(),
+               "This example does not currently support meshes"
+               " without boundary attributes."
+              )
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   RT_FECollection RTfec(order, dim);
+   ParFiniteElementSpace RTfes(&pmesh, &RTfec);
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2fes(&pmesh, &L2fec);
+
+   int num_dofs_RT = RTfes.GlobalTrueVSize();
+   int num_dofs_L2 = L2fes.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of H(div) dofs: "
+           << num_dofs_RT << endl;
+      cout << "Number of L² dofs: "
+           << num_dofs_L2 << endl;
+   }
+
+   // 5. Define the offsets for the block matrices
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = RTfes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   offsets.PartialSum();
+
+   Array<int> toffsets(3);
+   toffsets[0] = 0;
+   toffsets[1] = RTfes.GetTrueVSize();
+   toffsets[2] = L2fes.GetTrueVSize();
+   toffsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   BlockVector tx(toffsets), trhs(toffsets);
+   tx = 0.0; trhs = 0.0;
+
+   // 6. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   ParGridFunction u_gf, delta_psi_gf;
+   delta_psi_gf.MakeRef(&RTfes,x,offsets[0]);
+   u_gf.MakeRef(&L2fes,x,offsets[1]);
+
+   ParGridFunction psi_old_gf(&RTfes);
+   ParGridFunction psi_gf(&RTfes);
+   ParGridFunction u_old_gf(&L2fes);
+
+   // 7. Define initial guesses for the solution variables.
+   delta_psi_gf = 0.0;
+   psi_gf = 0.0;
+   u_gf = 0.0;
+   psi_old_gf = psi_gf;
+   u_old_gf = u_gf;
+
+   // 8. Prepare for glvis output.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 9. Coefficients to be used later.
+   ConstantCoefficient neg_one(-1.0);
+   ConstantCoefficient zero(0.0);
+   ConstantCoefficient tichonov_cf(tichonov);
+   ConstantCoefficient neg_tichonov_cf(-1.0*tichonov);
+   ZCoefficient Z(sdim, psi_gf, alpha);
+   DZCoefficient DZ(sdim, psi_gf, alpha);
+   ScalarVectorProductCoefficient neg_Z(-1.0, Z);
+   DivergenceGridFunctionCoefficient div_psi_cf(&psi_gf);
+   DivergenceGridFunctionCoefficient div_psi_old_cf(&psi_old_gf);
+   SumCoefficient psi_old_minus_psi(div_psi_old_cf, div_psi_cf, 1.0, -1.0);
+
+   // 10. Assemble constant matrices/vectors to avoid reassembly in the loop.
+   ParLinearForm b0, b1;
+   b0.MakeRef(&RTfes,rhs.GetBlock(0),0);
+   b1.MakeRef(&L2fes,rhs.GetBlock(1),0);
+
+   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_Z));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+
+   ParBilinearForm a00(&RTfes);
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(DZ));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(tichonov_cf));
+
+   ParMixedBilinearForm a10(&RTfes,&L2fes);
+   a10.AddDomainIntegrator(new VectorFEDivergenceIntegrator());
+   a10.Assemble();
+   a10.Finalize();
+   HypreParMatrix *A10 = a10.ParallelAssemble();
+
+   HypreParMatrix *A01 = A10->Transpose();
+
+   ParBilinearForm a11(&L2fes);
+   a11.AddDomainIntegrator(new MassIntegrator(neg_tichonov_cf));
+   a11.Assemble();
+   a11.Finalize();
+   HypreParMatrix *A11 = a11.ParallelAssemble();
+
+   // 11. Iterate.
+   int k;
+   int total_iterations = 0;
+   real_t increment_u = 0.1;
+   ParGridFunction u_tmp(&L2fes);
+   for (k = 0; k < max_it; k++)
+   {
+      u_tmp = u_old_gf;
+      Z.SetAlpha(alpha);
+      DZ.SetAlpha(alpha);
+
+      if (myid == 0)
+      {
+         mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+      }
+
+      int j;
+      for ( j = 0; j < 5; j++)
+      {
+         total_iterations++;
+
+         b0.Assemble();
+         b0.ParallelAssemble(trhs.GetBlock(0));
+
+         b1.Assemble();
+         b1.ParallelAssemble(trhs.GetBlock(1));
+
+         a00.Assemble(false);
+         a00.Finalize(false);
+         HypreParMatrix *A00 = a00.ParallelAssemble();
+
+         // Construct Schur-complement preconditioner
+         HypreParVector A00_diag(MPI_COMM_WORLD, A00->GetGlobalNumRows(),
+                                 A00->GetRowStarts());
+         A00->GetDiag(A00_diag);
+         HypreParMatrix S_tmp(*A01);
+         S_tmp.InvScaleRows(A00_diag);
+         HypreParMatrix *S = ParMult(A10, &S_tmp, true);
+
+         BlockDiagonalPreconditioner prec(toffsets);
+         HypreBoomerAMG P00(*A00);
+         P00.SetPrintLevel(0);
+         HypreBoomerAMG P11(*S);
+         P11.SetPrintLevel(0);
+         prec.SetDiagonalBlock(0,&P00);
+         prec.SetDiagonalBlock(1,&P11);
+
+         BlockOperator A(toffsets);
+         A.SetBlock(0,0,A00);
+         A.SetBlock(1,0,A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,A11);
+
+         GMRESSolver gmres(MPI_COMM_WORLD);
+         gmres.SetPrintLevel(-1);
+         gmres.SetRelTol(1e-8);
+         gmres.SetMaxIter(2000);
+         gmres.SetKDim(500);
+         gmres.SetOperator(A);
+         gmres.SetPreconditioner(prec);
+         gmres.Mult(trhs,tx);
+         delete S;
+         delete A00;
+
+         delta_psi_gf.SetFromTrueDofs(tx.GetBlock(0));
+         u_gf.SetFromTrueDofs(tx.GetBlock(1));
+
+         u_tmp -= u_gf;
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         // Damped Newton update
+         psi_gf.Add(newton_scaling, delta_psi_gf);
+         a00.Update();
+
+         if (visualization)
+         {
+            sol_sock << "parallel " << num_procs << " " << myid << "\n";
+            sol_sock << "solution\n" << pmesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+         }
+
+         if (myid == 0)
+         {
+            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+         }
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      if (myid == 0)
+      {
+         mfem::out << "Number of Newton iterations = " << j+1 << endl;
+         mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+      }
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      alpha *= max(growth_rate, 1_r);
+
+   }
+
+   // 12. Print stats.
+   if (myid == 0)
+   {
+      mfem::out << "\n Outer iterations: " << k+1
+                << "\n Total iterations: " << total_iterations
+                << "\n Total dofs:       " << RTfes.GetTrueVSize() + L2fes.GetTrueVSize()
+                << endl;
+   }
+
+   // 13. Free the used memory.
+   delete A01;
+   delete A10;
+   delete A11;
+   return 0;
+}
+
+void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
+                        const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(vdim);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   V = psi_vals;
+   V *= phi;
+}
+
+void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                         const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(height);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   K = 0.0;
+   for (int i = 0; i < height; i++)
+   {
+      K(i,i) = phi;
+      for (int j = 0; j < height; j++)
+      {
+         K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
+      }
+   }
+}
@@ -28,9 +28,9 @@ PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
   ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
   ex37p ex39p ex40p
-SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
-PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-   ex24p ex25p ex26p ex34p ex35p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
+PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
+   ex22p ex24p ex25p ex26p ex34p ex35p

 ifeq ($(MFEM_USE_LAPACK),YES)
   SEQ_EXAMPLES += ex38
@@ -138,6 +138,14 @@ ex10-test-seq: ex10
 	@$(call mfem-test,$<,, Serial example,-tf 5)
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-tf 5)
+ex14-test-seq-cuda: ex14
+	@$(call mfem-test,$<,, Serial CUDA example,-r 2 -pa -d cuda)
+ex14p-test-par-cuda: ex14p
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-rs 2 -rp 0 -pa -d cuda)
+ex14-test-seq-hip: ex14
+	@$(call mfem-test,$<,, Serial HIP example,-r 2 -pa -d hip)
+ex14p-test-par-hip: ex14p
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-rs 2 -rp 0 -pa -d hip)
 ex15-test-seq: ex15
 	@$(call mfem-test,$<,, Serial example,-e 1)
 ex15p-test-par: ex15p
@@ -1,3 +1,14 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
 #include <algorithm>
 #include <assert.h>
 #include <cstdlib>
@@ -709,10 +709,7 @@ real_t HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 real_t HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   real_t loc_energy = 0.5*M.InnerProduct(v, v);
-   real_t energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
-                 fespace.GetComm());
+   real_t energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -856,10 +856,7 @@ double HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 double HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   double loc_energy = 0.5*M.InnerProduct(v, v);
-   double energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPI_DOUBLE, MPI_SUM,
-                 fespace.GetComm());
+   double energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -18,6 +18,7 @@ set(SRCS
  integ/bilininteg_convection_pa.cpp
  integ/bilininteg_convection_ea.cpp
  integ/bilininteg_curlcurl_pa.cpp
+  integ/bilininteg_dgdiffusion_pa.cpp
  integ/bilininteg_dgtrace_pa.cpp
  integ/bilininteg_dgtrace_ea.cpp
  integ/bilininteg_diffusion_mf.cpp
@@ -117,6 +118,7 @@ set(SRCS
  quadinterpolator.cpp
  quadinterpolator_face.cpp
  restriction.cpp
+  normal_deriv_restriction.cpp
  staticcond.cpp
  tmop.cpp
  tmop/tmop_pa.cpp
@@ -228,6 +230,7 @@ set(HDRS
  quadinterpolator.hpp
  quadinterpolator_face.hpp
  restriction.hpp
+  normal_deriv_restriction.hpp
  fespacehierarchy.hpp
  staticcond.hpp
  tbilinearform.hpp
@@ -340,9 +340,9 @@ public:
       $ M^{-1} $ (currently returns NULL) */
   virtual MatrixInverse *Inverse() const;

-   /** @brief  Finalizes the matrix initialization if the ::AssemblyLevel is
+   /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
       AssemblyLevel::LEGACY.
-       THe matrix that gets finalized is different if you are using static
+       The matrix that gets finalized is different if you are using static
       condensation or hybridization.*/
   virtual void Finalize(int skip_zeros = 1);

@@ -643,7 +643,7 @@ public:
   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
                       DiagonalPolicy dpolicy = DIAG_ONE);

-   /** @brief  Eliminate the given @a vdofs, storing the eliminated part
+   /** @brief Eliminate the given @a vdofs, storing the eliminated part
       internally in $ M_e $.

       This method works in conjunction with EliminateVDofsInRHS() and allows
@@ -826,7 +826,7 @@ public:
       $ M^{-1} $ (currently unimplemented and returns NULL)*/
   virtual MatrixInverse *Inverse() const;

-   /** @brief  Finalizes the matrix initialization if the ::AssemblyLevel is
+   /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
       AssemblyLevel::LEGACY.*/
   virtual void Finalize(int skip_zeros = 1);

@@ -282,6 +282,22 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
      int_face_X.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
      int_face_Y.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
      int_face_Y.UseDevice(true); // ensure 'int_face_Y = 0.0' is done on device
+
+      bool needs_normal_derivs = false;
+      auto &integs = *a->GetFBFI();
+      for (int i = 0; i < integs.Size(); ++i)
+      {
+         if (integs[i]->RequiresFaceNormalDerivatives())
+         {
+            needs_normal_derivs = true;
+            break;
+         }
+      }
+      if (needs_normal_derivs)
+      {
+         int_face_dXdn.SetSize(int_face_restrict_lex->Height());
+         int_face_dYdn.SetSize(int_face_restrict_lex->Height());
+      }
   }

   const bool has_bdr_integs = (a->GetBFBFI()->Size() > 0 ||
@@ -296,6 +312,22 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
      bdr_face_Y.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
      bdr_face_Y.UseDevice(true); // ensure 'faceBoundY = 0.0' is done on device

+      bool needs_normal_derivs = false;
+      auto &integs = *a->GetBFBFI();
+      for (int i = 0; i < integs.Size(); ++i)
+      {
+         if (integs[i]->RequiresFaceNormalDerivatives())
+         {
+            needs_normal_derivs = true;
+            break;
+         }
+      }
+      if (needs_normal_derivs)
+      {
+         bdr_face_dXdn.SetSize(bdr_face_restrict_lex->Height());
+         bdr_face_dYdn.SetSize(bdr_face_restrict_lex->Height());
+      }
+
      const Mesh &mesh = *trial_fes->GetMesh();
      // See LinearFormExtension::Update for explanation of f_to_be logic.
      std::unordered_map<int,int> f_to_be;
@@ -542,8 +574,8 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         localY = 0.0;
         for (int i = 0; i < iSz; ++i)
         {
-            AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes,
-                               false, localY);
+            AddMultWithMarkers(*integrators[i], localX, elem_markers[i],
+                               elem_attributes, false, localY);
         }
         elem_restrict->MultTranspose(localY, y);
      }
@@ -557,15 +589,57 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
   const int iFISz = intFaceIntegrators.Size();
   if (int_face_restrict_lex && iFISz>0)
   {
-      int_face_restrict_lex->Mult(x, int_face_X);
-      if (int_face_X.Size()>0)
+      // When assembling interior face integrators for DG spaces, we need to
+      // exchange the face-neighbor information. This happens inside member
+      // functions of the 'int_face_restrict_lex'. To avoid repeated calls to
+      // ParGridFunction::ExchangeFaceNbrData, if we have a parallel space
+      // with interior face integrators, we create a ParGridFunction that
+      // will be used to cache the face-neighbor data. x_dg should be passed
+      // to any restriction operator that may need to use face-neighbor data.
+      const Vector *x_dg = &x;
+#ifdef MFEM_USE_MPI
+      ParGridFunction x_pgf;
+      if (auto *pfes = dynamic_cast<ParFiniteElementSpace*>(a->FESpace()))
+      {
+         x_pgf.MakeRef(pfes, const_cast<Vector&>(x), 0);
+         x_dg = &x_pgf;
+      }
+#endif
+
+      int_face_restrict_lex->Mult(*x_dg, int_face_X);
+      if (int_face_dXdn.Size() > 0)
+      {
+         int_face_restrict_lex->NormalDerivativeMult(*x_dg, int_face_dXdn);
+      }
+      if (int_face_X.Size() > 0)
      {
         int_face_Y = 0.0;
+
+         // if normal derivatives are needed by at least one integrator...
+         if (int_face_dYdn.Size() > 0)
+         {
+            int_face_dYdn = 0.0;
+         }
+
         for (int i = 0; i < iFISz; ++i)
         {
-            intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
+            if (intFaceIntegrators[i]->RequiresFaceNormalDerivatives())
+            {
+               intFaceIntegrators[i]->AddMultPAFaceNormalDerivatives(
+                  int_face_X, int_face_dXdn,
+                  int_face_Y, int_face_dYdn);
+            }
+            else
+            {
+               intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
+            }
         }
         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         if (int_face_dYdn.Size() > 0)
+         {
+            int_face_restrict_lex->NormalDerivativeAddMultTranspose(
+               int_face_dYdn, y);
+         }
      }
   }

@@ -579,9 +653,19 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
      Array<Array<int>*> &bdr_markers = *a->GetBBFI_Marker();
      Array<Array<int>*> &bdr_face_markers = *a->GetBFBFI_Marker();
      bdr_face_restrict_lex->Mult(x, bdr_face_X);
-      if (bdr_face_X.Size()>0)
+      if (bdr_face_dXdn.Size() > 0)
+      {
+         bdr_face_restrict_lex->NormalDerivativeMult(x, bdr_face_dXdn);
+      }
+      if (bdr_face_X.Size() > 0)
      {
         bdr_face_Y = 0.0;
+
+         // if normal derivatives are needed by at least one integrator...
+         if (bdr_face_dYdn.Size() > 0)
+         {
+            bdr_face_dYdn = 0.0;
+         }
         for (int i = 0; i < n_bdr_integs; ++i)
         {
            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
@@ -589,10 +673,23 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         }
         for (int i = 0; i < n_bdr_face_integs; ++i)
         {
-            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
-                               bdr_attributes, false, bdr_face_Y);
+            if (bdr_face_integs[i]->RequiresFaceNormalDerivatives())
+            {
+               AddMultNormalDerivativesWithMarkers(
+                  *bdr_face_integs[i], bdr_face_X, bdr_face_dXdn,
+                  bdr_face_markers[i], bdr_attributes, bdr_face_Y, bdr_face_dYdn);
+            }
+            else
+            {
+               AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
+                                  bdr_attributes, false, bdr_face_Y);
+            }
         }
         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         if (bdr_face_dYdn.Size() > 0)
+         {
+            bdr_face_restrict_lex->NormalDerivativeAddMultTranspose(bdr_face_dYdn, y);
+         }
      }
   }
 }
@@ -693,6 +790,37 @@ static void AddWithMarkers_(
   });
 }

+void PABilinearFormExtension::AddMultNormalDerivativesWithMarkers(
+   const BilinearFormIntegrator &integ,
+   const Vector &x,
+   const Vector &dxdn,
+   const Array<int> *markers,
+   const Array<int> &attributes,
+   Vector &y,
+   Vector &dydn) const
+{
+   if (markers)
+   {
+      tmp_evec.SetSize(y.Size() + dydn.Size());
+      tmp_evec = 0.0;
+      Vector tmp_y(tmp_evec, 0, y.Size());
+      Vector tmp_dydn(tmp_evec, y.Size(), dydn.Size());
+
+      integ.AddMultPAFaceNormalDerivatives(x, dxdn, tmp_y, tmp_dydn);
+
+      const int ne = attributes.Size();
+      const int nd_1 = x.Size() / ne;
+      const int nd_2 = dxdn.Size() / ne;
+
+      AddWithMarkers_(ne, nd_1, tmp_y, *markers, attributes, y);
+      AddWithMarkers_(ne, nd_2, tmp_dydn, *markers, attributes, dydn);
+   }
+   else
+   {
+      integ.AddMultPAFaceNormalDerivatives(x, dxdn, y, dydn);
+   }
+}
+
 void PABilinearFormExtension::AddMultWithMarkers(
   const BilinearFormIntegrator &integ,
   const Vector &x,
@@ -74,6 +74,8 @@ protected:
   mutable Vector localX, localY;
   mutable Vector int_face_X, int_face_Y;
   mutable Vector bdr_face_X, bdr_face_Y;
+   mutable Vector int_face_dXdn, int_face_dYdn;
+   mutable Vector bdr_face_dXdn, bdr_face_dYdn;
   const Operator *elem_restrict; // Not owned
   const FaceRestriction *int_face_restrict_lex; // Not owned
   const FaceRestriction *bdr_face_restrict_lex; // Not owned
@@ -113,6 +115,23 @@ protected:
                           const Array<int> &attributes,
                           const bool transpose,
                           Vector &y) const;
+
+   /// @brief Performs the same function as AddMultWithMarkers, but takes as
+   /// input and output face normal derivatives.
+   ///
+   /// This is required when the integrator requires face normal derivatives,
+   /// for example, DGDiffusionIntegrator.
+   ///
+   /// This is called when the integrator's member function
+   /// BilinearFormIntegrator::RequiresFaceNormalDerivatives() returns true.
+   void AddMultNormalDerivativesWithMarkers(
+      const BilinearFormIntegrator &integ,
+      const Vector &x,
+      const Vector &dxdn,
+      const Array<int> *markers,
+      const Array<int> &attributes,
+      Vector &y,
+      Vector &dydn) const;
 };

 /// Data and methods for element-assembled bilinear forms
@@ -189,6 +189,12 @@ void BilinearFormIntegrator::AssembleTraceFaceMatrix (int elem,
              " Integrator class.");
 }

+void BilinearFormIntegrator::AddMultPAFaceNormalDerivatives(
+   const Vector &x, const Vector &dxdn, Vector &y, Vector &dydn) const
+{
+   MFEM_ABORT("Not implemented.");
+}
+
 void BilinearFormIntegrator::AssembleElementVector(
   const FiniteElement &el, ElementTransformation &Tr, const Vector &elfun,
   Vector &elvect)
@@ -3423,7 +3429,7 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   const FiniteElement &el1, const FiniteElement &el2,
   FaceElementTransformations &Trans, DenseMatrix &elmat)
 {
-   int dim, ndof1, ndof2, ndofs;
+   int ndof1, ndof2, ndofs;
   bool kappa_is_nonzero = (kappa != 0.);
   real_t w, wq = 0.0;

@@ -3466,17 +3472,9 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
   {
-      // a simple choice for the integration order; is this OK?
-      int order;
-      if (ndof2)
-      {
-         order = 2*max(el1.GetOrder(), el2.GetOrder());
-      }
-      else
-      {
-         order = 2*el1.GetOrder();
-      }
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      const int order = (ndof2) ? max(el1.GetOrder(),
+                                      el2.GetOrder()) : el1.GetOrder();
+      ir = &GetRule(order, Trans);
   }

   // assemble: < {(Q \nabla u).n},[v] >      --> elmat
@@ -3654,6 +3652,13 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   }
 }

+const IntegrationRule &DGDiffusionIntegrator::GetRule(
+   int order, FaceElementTransformations &T)
+{
+   // order is typically the maximum of the order of the left and right elements
+   // neighboring the given face.
+   return IntRules.Get(T.GetGeometryType(), 2*order);
+}

 // static method
 void DGElasticityIntegrator::AssembleBlock(
@@ -266,6 +266,39 @@ public:
                                    Vector &flux, Vector *d_energy = NULL)
   { return 0.0; }

+   /** @brief For bilinear forms on element faces, specifies if the normal
+              derivatives are needed on the faces or just the face restriction.
+
+       @details if RequiresFaceNormalDerivatives() == true, then
+                AddMultPAFaceNormalDerivatives(...) should be invoked in place
+                of AddMultPA(...) and L2NormalDerivativeFaceRestriction should
+                be used to compute the normal derivatives. This is used for some
+                DG integrators, for example DGDiffusionIntegrator.
+
+       @returns whether normal derivatives appear in the bilinear form.
+   */
+   virtual bool RequiresFaceNormalDerivatives() const { return false; }
+
+   /// Method for partially assembled action.
+   /** @brief For bilinear forms on element faces that depend on the normal
+              derivative on the faces, computes the action of integrator to the
+              face values @a x and reference-normal derivatives @a dxdn and adds
+              the result to @a y and @a dydn.
+
+      @details This method can be called only after the method AssemblePA() has
+               been called.
+
+      @param[in]     x E-vector of face values (provided by
+                       FaceRestriction::Mult)
+      @param[in]     dxdn E-vector of face reference-normal derivatives
+                          (provided by FaceRestriction::NormalDerivativeMult)
+      @param[in,out] y E-vector of face values to add action to.
+      @param[in,out] dydn E-vector of face reference-normal derivative values to
+                          add action to.
+   */
+   virtual void AddMultPAFaceNormalDerivatives(const Vector &x, const Vector &dxdn,
+                                               Vector &y, Vector &dydn) const;
+
   virtual ~BilinearFormIntegrator() { }
 };

@@ -3229,6 +3262,13 @@ protected:
   Vector shape1, shape2, dshape1dn, dshape2dn, nor, nh, ni;
   DenseMatrix jmat, dshape1, dshape2, mq, adjJ;

+
+   // PA extension
+   Vector pa_data; // (Q, h, dot(n,J)|el0, dot(n,J)|el1)
+   const DofToQuad *maps; ///< Not owned
+   int dim, nf, nq, dofs1D, quad1D;
+   IntegrationRules irs{0, Quadrature1D::GaussLobatto};
+
 public:
   DGDiffusionIntegrator(const real_t s, const real_t k)
      : Q(NULL), MQ(NULL), sigma(s), kappa(k) { }
@@ -3237,10 +3277,26 @@ public:
   DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k)
      : Q(NULL), MQ(&q), sigma(s), kappa(k) { }
   using BilinearFormIntegrator::AssembleFaceMatrix;
-   virtual void AssembleFaceMatrix(const FiniteElement &el1,
-                                   const FiniteElement &el2,
-                                   FaceElementTransformations &Trans,
-                                   DenseMatrix &elmat);
+   void AssembleFaceMatrix(const FiniteElement &el1,
+                           const FiniteElement &el2,
+                           FaceElementTransformations &Trans,
+                           DenseMatrix &elmat) override;
+
+   bool RequiresFaceNormalDerivatives() const override { return true; }
+
+   using BilinearFormIntegrator::AssemblePA;
+
+   void AssemblePAInteriorFaces(const FiniteElementSpace &fes) override;
+
+   void AssemblePABoundaryFaces(const FiniteElementSpace &fes) override;
+
+   void AddMultPAFaceNormalDerivatives(const Vector &x, const Vector &dxdn,
+                                       Vector &y, Vector &dydn) const override;
+
+   const IntegrationRule &GetRule(int order, FaceElementTransformations &T);
+
+private:
+   void SetupPA(const FiniteElementSpace &fes, FaceType type);
 };

 /** Integrator for the "BR2" diffusion stabilization term
@@ -807,6 +807,7 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)

   QuadratureSpaceBase &qspace = *qf.GetSpace();
   const int ne = qspace.GetNE();
+   qf.HostWrite();
   DenseMatrix values;
   DenseSymmetricMatrix matrix;
   for (int iel = 0; iel < ne; ++iel)
@@ -818,7 +819,7 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
      {
         const IntegrationPoint &ip = ir[iq];
         T.SetIntPoint(&ip);
-         matrix.UseExternalData(&values(0, iq), vdim);
+         matrix.UseExternalData(&values(0, iq), height);
         Eval(matrix, T, ip);
      }
   }
@@ -828,13 +829,12 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
 void SymmetricMatrixCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
-   mat.SetSize(height);
-   Eval(mat, T, ip);
+   Eval(mat_aux, T, ip);
   for (int j = 0; j < width; ++j)
   {
      for (int i = 0; i < height; ++ i)
      {
-         K(i, j) = mat(i, j);
+         K(i, j) = mat_aux(i, j);
      }
   }
 }
@@ -989,10 +989,7 @@ void MatrixArrayVectorCoefficient::Eval(DenseMatrix &K,
   for (int i = 0; i < height; i++)
   {
      this->Eval(i, V, T, ip);
-      for (int j = 0; j < width; j++)
-      {
-         K(i,j) = V(j);
-      }
+      K.SetRow(i, V);
   }
 }

@@ -1352,17 +1352,17 @@ public:
   /// Set the time for internally stored coefficients
   void SetTime(real_t t) override;

-   /// Get the coefficient located at the iᵗʰ row of the matrix.
+   /// Get the vector coefficient located at the i-th row of the matrix
   VectorCoefficient* GetCoeff (int i) { return Coeff[i]; }

-   /** @brief Set the coefficient located at the iᵗʰ row of the matrix.
+   /** @brief Set the coefficient located at the i-th row of the matrix.
       By this will take ownership of the Coefficient passed in, but this
       can be overridden with the @a own parameter. */
   void Set(int i, VectorCoefficient * c, bool own=true);

   using MatrixCoefficient::Eval;

-   /// Evaluate coefficient located at the iᵗʰ row of the matrix using integration
+   /// Evaluate coefficient located at the i-th row of the matrix using integration
   /// point @a ip.
   void Eval(int i, Vector &V, ElementTransformation &T,
             const IntegrationPoint &ip);
@@ -1466,12 +1466,13 @@ public:
 class SymmetricMatrixCoefficient : public MatrixCoefficient
 {
 protected:
+
   /// Internal matrix used when evaluating this coefficient as a DenseMatrix.
-   DenseSymmetricMatrix mat;
+   mutable DenseSymmetricMatrix mat_aux;
 public:
   /// Construct a dim x dim matrix coefficient.
   explicit SymmetricMatrixCoefficient(int dimension)
-      : MatrixCoefficient(dimension, true) { }
+      : MatrixCoefficient(dimension, true), mat_aux(height) { }

   /// Get the size of the matrix.
   int GetSize() const { return height; }
@@ -1504,8 +1505,9 @@ public:
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip);

-   /// Return a reference to the constant matrix.
-   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
+   /// @deprecated Return a reference to the internal matrix used when evaluating this coefficient as a DenseMatrix.
+   MFEM_DEPRECATED const DenseSymmetricMatrix& GetMatrix() { return mat_aux; }

   virtual ~SymmetricMatrixCoefficient() { }
 };
@@ -1525,6 +1527,10 @@ public:
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseSymmetricMatrix &M, ElementTransformation &T,
                     const IntegrationPoint &ip) { M = mat; }
+
+   /// Return a reference to the constant matrix.
+   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
 };


@@ -1243,25 +1243,16 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
         HypreParMatrix * Ah;
         A_i.Get(Ah);
         hypre_ParCSRMatrix *Aih = *Ah;
-#if !defined(HYPRE_USING_GPU)
-         ess_tdof_list.HostRead();
-         for (int k = 0; k < n; k++)
-         {
-            const int j = ess_tdof_list[k];
-            Aih->diag->data[Aih->diag->i[j]] = 0.0;
-         }
-#else
         Ah->HypreReadWrite();
         const int *d_ess_tdof_list =
-            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
-         const int *d_diag_i = Aih->diag->i;
+            ess_tdof_list.GetMemory().Read(GetHypreMemoryClass(), n);
+         HYPRE_Int *d_diag_i = Aih->diag->i;
         real_t *d_diag_data = Aih->diag->data;
-         MFEM_GPU_FORALL(k, n,
+         mfem::hypre_forall(n, [=] MFEM_HOST_DEVICE (int k)
         {
            const int j = d_ess_tdof_list[k];
            d_diag_data[d_diag_i[j]] = 0.0;
         });
-#endif
      }
      else
      {
@@ -1,8 +1,18 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
 #include "convergence.hpp"

 using namespace std;

-
 namespace mfem
 {

@@ -101,7 +101,7 @@ void DGMassInverse::SetRelTol(const real_t rel_tol_) { rel_tol = rel_tol_; }

 void DGMassInverse::SetAbsTol(const real_t abs_tol_) { abs_tol = abs_tol_; }

-void DGMassInverse::SetMaxIter(const real_t max_iter_) { max_iter = max_iter_; }
+void DGMassInverse::SetMaxIter(const int max_iter_) { max_iter = max_iter_; }

 void DGMassInverse::Update()
 {
@@ -137,7 +137,7 @@ void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const

   const real_t RELTOL = rel_tol;
   const real_t ABSTOL = abs_tol;
-   const real_t MAXIT = max_iter;
+   const int MAXIT = max_iter;
   const bool IT_MODE = iterative_mode;
   const bool CHANGE_BASIS = (d2q != nullptr);

@@ -96,7 +96,7 @@ public:
   /// Set the absolute tolerance.
   void SetAbsTol(const real_t abs_tol_);
   /// Set the maximum number of iterations.
-   void SetMaxIter(const real_t max_iter_);
+   void SetMaxIter(const int max_iter_);
   /// Recompute operator and preconditioner (when coefficient or mesh changes).
   void Update();

@@ -13,6 +13,7 @@
 #define MFEM_FACE_MAP_UTILS_HPP

 #include "../../general/array.hpp"
+#include "../../general/backends.hpp"
 #include <utility> // std::pair
 #include <vector>

@@ -51,6 +52,189 @@ void FillFaceMap(const int n_face_dofs_per_component,
 void GetTensorFaceMap(const int dim, const int order, const int face_id,
                      Array<int> &face_map);

+/// @brief Given a face DOF index in native (counter-clockwise) ordering, return
+/// the corresponding DOF index in lexicographic ordering (for a quadrilateral
+/// element).
+MFEM_HOST_DEVICE
+inline int ToLexOrdering2D(const int face_id, const int size1d, const int i)
+{
+   if (face_id==2 || face_id==3)
+   {
+      return size1d-1-i;
+   }
+   else
+   {
+      return i;
+   }
+}
+
+/// @brief Given a face DOF index on a shared face, ordered lexicographically
+/// relative to element 1, return the corresponding face DOF index ordered
+/// lexicographically relative to element 2.
+MFEM_HOST_DEVICE
+inline int PermuteFace2D(const int face_id1, const int face_id2,
+                         const int orientation, const int size1d,
+                         const int index)
+{
+   int new_index;
+   // Convert from element 1 lex ordering to native ordering
+   if (face_id1 == 2 || face_id1 == 3)
+   {
+      new_index = size1d-1-index;
+   }
+   else
+   {
+      new_index = index;
+   }
+   // Permute based on face orientations
+   if (orientation == 1)
+   {
+      new_index = size1d-1-new_index;
+   }
+   // Covert to element 2 lex ordering
+   return ToLexOrdering2D(face_id2, size1d, new_index);
+}
+
+/// @brief Given a face DOF index in native (counter-clockwise) ordering, return
+/// the corresponding DOF index in lexicographic ordering (for a hexahedral
+/// element).
+MFEM_HOST_DEVICE
+inline int ToLexOrdering3D(const int face_id, const int size1d, const int i,
+                           const int j)
+{
+   if (face_id==2 || face_id==1 || face_id==5)
+   {
+      return i + j*size1d;
+   }
+   else if (face_id==3 || face_id==4)
+   {
+      return (size1d-1-i) + j*size1d;
+   }
+   else // face_id==0
+   {
+      return i + (size1d-1-j)*size1d;
+   }
+}
+
+/// @brief Given the index of a face DOF in lexicographic ordering relative
+/// element 1, permute the index so that it is lexicographically ordered
+/// relative to element 2.
+///
+/// The given face corresponds to local face index @a face_id1 relative to
+/// element 1, and @a face_id2 (with @a orientation) relative to element 2.
+MFEM_HOST_DEVICE
+inline int PermuteFace3D(const int face_id1, const int face_id2,
+                         const int orientation,
+                         const int size1d, const int index)
+{
+   int i=0, j=0, new_i=0, new_j=0;
+   i = index%size1d;
+   j = index/size1d;
+   // Convert from lex ordering
+   if (face_id1==3 || face_id1==4)
+   {
+      i = size1d-1-i;
+   }
+   else if (face_id1==0)
+   {
+      j = size1d-1-j;
+   }
+   // Permute based on face orientations
+   switch (orientation)
+   {
+      case 0:
+         new_i = i;
+         new_j = j;
+         break;
+      case 1:
+         new_i = j;
+         new_j = i;
+         break;
+      case 2:
+         new_i = j;
+         new_j = (size1d-1-i);
+         break;
+      case 3:
+         new_i = (size1d-1-i);
+         new_j = j;
+         break;
+      case 4:
+         new_i = (size1d-1-i);
+         new_j = (size1d-1-j);
+         break;
+      case 5:
+         new_i = (size1d-1-j);
+         new_j = (size1d-1-i);
+         break;
+      case 6:
+         new_i = (size1d-1-j);
+         new_j = i;
+         break;
+      case 7:
+         new_i = i;
+         new_j = (size1d-1-j);
+         break;
+   }
+   return ToLexOrdering3D(face_id2, size1d, new_i, new_j);
+}
+
+/// @brief Given a face DOF (or quadrature) index ordered lexicographically
+/// relative to element 1, return the associated (i, j) coordinates.
+///
+/// The returned coordinates will be relative to element 1 or element 2
+/// according to the value of side (side == 0 corresponds element 1).
+MFEM_HOST_DEVICE
+inline void FaceIdxToVolIdx2D(const int qi, const int nq, const int face_id0,
+                              const int face_id1, const int side, int &i, int &j)
+{
+   // Note: in 2D, a consistently ordered mesh will always have the element 2
+   // face reversed relative to element 1, so orientation is determined entirely
+   // by side. (In 3D, separate orientation information is needed).
+   const int orientation = side;
+
+   const int face_id = (side == 0) ? face_id0 : face_id1;
+   const int edge_idx = (side == 0) ? qi : PermuteFace2D(face_id0, face_id1,
+                                                         orientation, nq, qi);
+
+   const int level = (face_id == 0 || face_id == 3) ? 0 : (nq-1);
+   const bool x_axis = (face_id == 0 || face_id == 2);
+
+   i = x_axis ? edge_idx : level;
+   j = x_axis ? level : edge_idx;
+}
+
+/// @brief Given a face DOF (or quadrature) index ordered lexicographically
+/// relative to element 1, return the associated (i, j, k) coordinates.
+///
+/// The returned coordinates will be relative to element 1 or element 2
+/// according to the value of side (side == 0 corresponds element 1).
+MFEM_HOST_DEVICE
+inline void FaceIdxToVolIdx3D(const int index, const int size1d,
+                              const int face_id0, const int face_id1,
+                              const int side, const int orientation,
+                              int& i, int& j, int& k)
+{
+   MFEM_VERIFY_KERNEL(face_id1 >= 0 || side == 0,
+                      "Accessing second side but face_id1 is not valid.");
+
+   const int face_id = (side == 0) ? face_id0 : face_id1;
+   const int fidx = (side == 0) ? index
+                    : PermuteFace3D(face_id0, face_id1, orientation, size1d, index);
+
+   const bool xy_plane = (face_id == 0 || face_id == 5);
+   const bool yz_plane = (face_id == 2 || face_id == 4);
+
+   const int level = (face_id == 0 || face_id == 1 || face_id == 4)
+                     ? 0 : (size1d-1);
+
+   const int _i = fidx % size1d;
+   const int _j = fidx / size1d;
+
+   k = xy_plane ? level : _j;
+   j = yz_plane ? _i : xy_plane ? _j : level;
+   i = yz_plane ? level : _i;
+}
+
 } // namespace internal

 } // namespace mfem
@@ -316,7 +316,7 @@ public:
   int GetDim() const { return dim; }

   /** @brief Returns the vector dimension for vector-valued finite elements,
-       which is also the dimension of the interpolation operatrion. */
+       which is also the dimension of the interpolation operation. */
   int GetRangeDim() const { return vdim; }

   /// Returns the dimension of the curl for vector-valued finite elements.
@@ -1321,9 +1321,9 @@ void GridFunction::ProjectVectorFieldOn(GridFunction &vec_field, int comp)
   }
 }

-void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
-                                                      GridFunction &der,
-                                                      Array<int> &zones_per_dof)
+void GridFunction::AccumulateAndCountDerivativeValues(
+   int comp, int der_comp, GridFunction &der,
+   Array<int> &zones_per_dof) const
 {
   FiniteElementSpace * der_fes = der.FESpace();
   ElementTransformation * transf;
@@ -1374,7 +1374,8 @@ void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
   }
 }

-void GridFunction::GetDerivative(int comp, int der_comp, GridFunction &der)
+void GridFunction::GetDerivative(int comp, int der_comp,
+                                 GridFunction &der) const
 {
   Array<int> overlap;
   AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
@@ -2061,41 +2062,37 @@ void GridFunction::AccumulateAndCountBdrValues(
   Coefficient *coeff[], VectorCoefficient *vcoeff, const Array<int> &attr,
   Array<int> &values_counter)
 {
-   int i, j, fdof, d, ind, vdim;
-   real_t val;
-   const FiniteElement *fe;
-   ElementTransformation *transf;
   Array<int> vdofs;
   Vector vc;

   values_counter.SetSize(Size());
   values_counter = 0;

-   vdim = fes->GetVDim();
-
+   const int vdim = fes->GetVDim();
   HostReadWrite();

-   for (i = 0; i < fes->GetNBE(); i++)
+   for (int i = 0; i < fes->GetNBE(); i++)
   {
      if (attr[fes->GetBdrAttribute(i) - 1] == 0) { continue; }

-      fe = fes->GetBE(i);
-      fdof = fe->GetDof();
-      transf = fes->GetBdrElementTransformation(i);
+      const FiniteElement *fe = fes->GetBE(i);
+      const int fdof = fe->GetDof();
+      ElementTransformation *transf = fes->GetBdrElementTransformation(i);
      const IntegrationRule &ir = fe->GetNodes();
      fes->GetBdrElementVDofs(i, vdofs);

-      for (j = 0; j < fdof; j++)
+      for (int j = 0; j < fdof; j++)
      {
         const IntegrationPoint &ip = ir.IntPoint(j);
         transf->SetIntPoint(&ip);
         if (vcoeff) { vcoeff->Eval(vc, *transf, ip); }
-         for (d = 0; d < vdim; d++)
+         for (int d = 0; d < vdim; d++)
         {
            if (!vcoeff && !coeff[d]) { continue; }

-            val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
-            if ( (ind = vdofs[fdof*d+j]) < 0 )
+            real_t val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
+            int ind = vdofs[fdof*d+j];
+            if ( ind < 0 )
            {
               val = -val, ind = -1-ind;
            }
@@ -2117,10 +2114,11 @@ void GridFunction::AccumulateAndCountBdrValues(
   // iff A_ij != 0. It is sufficient to resolve just the first level of
   // dependency, since A is a projection matrix: A^n = A due to cR.cP = I.
   // Cases like these arise in 3D when boundary edges are constrained by
-   // (depend on) internal faces/elements. We use the virtual method
-   // GetBoundaryClosure from NCMesh to resolve the dependencies.
-
-   if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3)
+   // (depend on) internal faces/elements, or for internal boundaries in 2 or
+   // 3D. We use the virtual method GetBoundaryClosure from NCMesh to resolve
+   // the dependencies.
+   if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 ||
+                                fes->GetMesh()->Dimension() == 3))
   {
      Vector vals;
      Mesh *mesh = fes->GetMesh();
@@ -2128,26 +2126,19 @@ void GridFunction::AccumulateAndCountBdrValues(
      Array<int> bdr_edges, bdr_vertices, bdr_faces;
      ncmesh->GetBoundaryClosure(attr, bdr_vertices, bdr_edges, bdr_faces);

-      for (i = 0; i < bdr_edges.Size(); i++)
+      auto mark_dofs = [&](ElementTransformation &transf, const FiniteElement &fe)
      {
-         int edge = bdr_edges[i];
-         fes->GetEdgeVDofs(edge, vdofs);
-         if (vdofs.Size() == 0) { continue; }
-
-         transf = mesh->GetEdgeTransformation(edge);
-         transf->Attribute = -1; // TODO: set the boundary attribute
-         fe = fes->GetEdgeElement(edge);
         if (!vcoeff)
         {
-            vals.SetSize(fe->GetDof());
-            for (d = 0; d < vdim; d++)
+            vals.SetSize(fe.GetDof());
+            for (int d = 0; d < vdim; d++)
            {
               if (!coeff[d]) { continue; }

-               fe->Project(*coeff[d], *transf, vals);
+               fe.Project(*coeff[d], transf, vals);
               for (int k = 0; k < vals.Size(); k++)
               {
-                  ind = vdofs[d*vals.Size()+k];
+                  const int ind = vdofs[d*vals.Size()+k];
                  if (++values_counter[ind] == 1)
                  {
                     (*this)(ind) = vals(k);
@@ -2161,11 +2152,11 @@ void GridFunction::AccumulateAndCountBdrValues(
         }
         else // vcoeff != NULL
         {
-            vals.SetSize(vdim*fe->GetDof());
-            fe->Project(*vcoeff, *transf, vals);
+            vals.SetSize(vdim*fe.GetDof());
+            fe.Project(*vcoeff, transf, vals);
            for (int k = 0; k < vals.Size(); k++)
            {
-               ind = vdofs[k];
+               const int ind = vdofs[k];
               if (++values_counter[ind] == 1)
               {
                  (*this)(ind) = vals(k);
@@ -2176,6 +2167,26 @@ void GridFunction::AccumulateAndCountBdrValues(
               }
            }
         }
+      };
+
+      for (auto edge : bdr_edges)
+      {
+         fes->GetEdgeVDofs(edge, vdofs);
+         if (vdofs.Size() == 0) { continue; }
+
+         ElementTransformation *transf = mesh->GetEdgeTransformation(edge);
+         const FiniteElement *fe = fes->GetEdgeElement(edge);
+         mark_dofs(*transf, *fe);
+      }
+
+      for (auto face : bdr_faces)
+      {
+         fes->GetFaceVDofs(face, vdofs);
+         if (vdofs.Size() == 0) { continue; }
+
+         ElementTransformation *transf = mesh->GetFaceTransformation(face);
+         const FiniteElement *fe = fes->GetFaceElement(face);
+         mark_dofs(*transf, *fe);
      }
   }
 }
@@ -2228,26 +2239,37 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
      accumulate_dofs(dofs, lvec, *this, values_counter);
   }

-   if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3)
+   if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 ||
+                                fes->GetMesh()->Dimension() == 3))
   {
      Mesh *mesh = fes->GetMesh();
      NCMesh *ncmesh = mesh->ncmesh;
      Array<int> bdr_edges, bdr_vertices, bdr_faces;
      ncmesh->GetBoundaryClosure(bdr_attr, bdr_vertices, bdr_edges, bdr_faces);

-      for (int i = 0; i < bdr_edges.Size(); i++)
+      for (auto edge : bdr_edges)
      {
-         int edge = bdr_edges[i];
         fes->GetEdgeDofs(edge, dofs);
         if (dofs.Size() == 0) { continue; }

         T = mesh->GetEdgeTransformation(edge);
-         T->Attribute = -1; // TODO: set the boundary attribute
         fe = fes->GetEdgeElement(edge);
         lvec.SetSize(fe->GetDof());
         fe->Project(vcoeff, *T, lvec);
         accumulate_dofs(dofs, lvec, *this, values_counter);
      }
+
+      for (auto face : bdr_faces)
+      {
+         fes->GetFaceDofs(face, dofs);
+         if (dofs.Size() == 0) { continue; }
+
+         T = mesh->GetFaceTransformation(face);
+         fe = fes->GetFaceElement(face);
+         lvec.SetSize(fe->GetDof());
+         fe->Project(vcoeff, *T, lvec);
+         accumulate_dofs(dofs, lvec, *this, values_counter);
+      }
   }
 }

@@ -321,7 +321,7 @@ public:
       @param[out] der       The resulting derivative (scalar function). The
                             FiniteElementSpace of this function must be set
                             before the call. */
-   void GetDerivative(int comp, int der_comp, GridFunction &der);
+   void GetDerivative(int comp, int der_comp, GridFunction &der) const;

   real_t GetDivergence(ElementTransformation &tr) const;

@@ -443,7 +443,7 @@ protected:
       GetDerivative() method; see its documentation. */
   void AccumulateAndCountDerivativeValues(int comp, int der_comp,
                                           GridFunction &der,
-                                           Array<int> &zones_per_dof);
+                                           Array<int> &zones_per_dof) const;

   void AccumulateAndCountBdrValues(Coefficient *coeff[],
                                    VectorCoefficient *vcoeff,
@@ -1352,6 +1352,85 @@ void OversetFindPointsGSLIB::Interpolate(const Vector &point_pos,
   Interpolate(field_in, field_out);
 }

+GSOPGSLIB::GSOPGSLIB(Array<long long> &ids)
+{
+   gsl_comm = new gslib::comm;
+   cr       = new gslib::crystal;
+#ifdef MFEM_USE_MPI
+   int initialized;
+   MPI_Initialized(&initialized);
+   if (!initialized) { MPI_Init(NULL, NULL); }
+   MPI_Comm comm = MPI_COMM_WORLD;
+   comm_init(gsl_comm, comm);
+#else
+   comm_init(gsl_comm, 0);
+#endif
+   crystal_init(cr, gsl_comm);
+   UpdateIdentifiers(ids);
+}
+
+#ifdef MFEM_USE_MPI
+GSOPGSLIB::GSOPGSLIB(MPI_Comm comm_, Array<long long> &ids)
+   : cr(NULL), gsl_comm(NULL)
+{
+   gsl_comm = new gslib::comm;
+   cr      = new gslib::crystal;
+   comm_init(gsl_comm, comm_);
+   crystal_init(cr, gsl_comm);
+   UpdateIdentifiers(ids);
+}
+#endif
+
+GSOPGSLIB::~GSOPGSLIB()
+{
+   crystal_free(cr);
+   gslib_gs_free(gsl_data);
+   comm_free(gsl_comm);
+   delete gsl_comm;
+   delete cr;
+}
+
+void GSOPGSLIB::UpdateIdentifiers(const Array<long long> &ids)
+{
+   long long minval = ids.Min();
+#ifdef MFEM_USE_MPI
+   MPI_Allreduce(MPI_IN_PLACE, &minval, 1, MPI_LONG_LONG_INT,
+                 MPI_MIN, gsl_comm->c);
+#endif
+   MFEM_VERIFY(minval >= 0, "Unique identifier cannot be negative.");
+   if (gsl_data != NULL) { gslib_gs_free(gsl_data); }
+   num_ids = ids.Size();
+   gsl_data = gslib_gs_setup(ids.GetData(),
+                             ids.Size(),
+                             gsl_comm, 0,
+                             gslib::gs_crystal_router, 0);
+}
+
+void GSOPGSLIB::GS(Vector &senddata, GSOp op)
+{
+   MFEM_VERIFY(senddata.Size() == num_ids,
+               "Incompatible setup and GOP operation.");
+   if (op == GSOp::ADD)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_add,0,gsl_data,0);
+   }
+   else if (op == GSOp::MUL)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_mul,0,gsl_data,0);
+   }
+   else if (op == GSOp::MAX)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_max,0,gsl_data,0);
+   }
+   else if (op == GSOp::MIN)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_min,0,gsl_data,0);
+   }
+   else
+   {
+      MFEM_ABORT("Invalid GSOp operation.");
+   }
+}

 } // namespace mfem

@@ -23,13 +23,16 @@ struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
 struct crystal;
+struct gs_data;
 }

 namespace mfem
 {

 /** \brief FindPointsGSLIB can robustly evaluate a GridFunction on an arbitrary
- *  collection of points. There are three key functions in FindPointsGSLIB:
+ *  collection of points.
+ *
+ *  There are three key functions in FindPointsGSLIB:
 *
 *  1. Setup - constructs the internal data structures of gslib.
 *
@@ -226,6 +229,7 @@ public:

 /** \brief OversetFindPointsGSLIB enables use of findpts for arbitrary number of
    overlapping grids.
+
    The parameters in this class are the same as FindPointsGSLIB with the
    difference of additional inputs required to account for more than 1 mesh. */
 class OversetFindPointsGSLIB : public FindPointsGSLIB
@@ -290,6 +294,63 @@ public:
   using FindPointsGSLIB::Interpolate;
 };

+/** \brief  Class for gather-scatter (gs) operations on Vectors based on
+    corresponding global identifiers.
+
+    This functionality is useful for gs-ops on DOF values across processor
+    boundary, where the global identifier would be the corresponding true DOF
+    index. Operations currently supported are min, max, sum, and multiplication.
+    Note: identifier 0 does not participate in the gather-scatter operation and
+    a given identifier can be included multiple times on a given rank.
+    For example, consider a vector, v:
+    - v = [0.3, 0.4, 0.25, 0.7] on rank1,
+    - v = [0.6, 0.1] on rank 2,
+    - v = [-0.2, 0.3, 0.7, 0.] on rank 3.
+
+    Consider a corresponding Array<int>, a:
+    - a = [1, 2, 3, 1] on rank 1,
+    - a = [3, 2] on rank 2,
+    - a = [1, 2, 0, 3] on rank 3.
+
+    A gather-scatter "minimum" operation, done as follows:
+    GSOPGSLIB gs = GSOPGSLIB(MPI_COMM_WORLD, a);
+    gs.GS(v, GSOp::MIN);
+    would return into v:
+    - v = [-0.2, 0.1, 0., -0.2] on rank 1,
+    - v = [0., 0.1] on rank 2,
+    - v = [-0.2, 0.1, 0.7, 0.] on rank 3,
+    where the values have been compared across all processors based on the
+    integer identifier. */
+class GSOPGSLIB
+{
+protected:
+   struct gslib::crystal *cr;               // gslib's internal data
+   struct gslib::comm *gsl_comm;            // gslib's internal data
+   struct gslib::gs_data *gsl_data = NULL;
+   int num_ids;
+
+public:
+   GSOPGSLIB(Array<long long> &ids);
+
+#ifdef MFEM_USE_MPI
+   GSOPGSLIB(MPI_Comm comm_, Array<long long> &ids);
+#endif
+
+   virtual ~GSOPGSLIB();
+
+   /// Supported operation types. See class description.
+   enum GSOp {ADD, MUL, MIN, MAX};
+
+   /// Update the identifiers used for the gather-scatter operator.
+   /// Same @a ids get grouped together and id == 0 does not participate.
+   /// See class description.
+   void UpdateIdentifiers(const Array<long long> &ids);
+
+   /// Gather-Scatter operation on senddata. Must match length of unique
+   /// identifiers used in the constructor. See class description.
+   void GS(Vector &senddata, GSOp op);
+};
+
 } // namespace mfem

 #endif // MFEM_USE_GSLIB
@@ -18,7 +18,6 @@
 namespace mfem
 {

-
 void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
                                                     ElementTransformation &Tr,
                                                     const Vector &elfun,
@@ -29,7 +28,7 @@ void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
   const int dof = el.GetDof();

 #ifdef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration

   // shape function value at an integration point
   Vector shape(dof);
@@ -62,7 +61,7 @@ void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
      ir = &IntRules.Get(Tr.GetGeometryType(), order);
   }

-   // loop over interation points
+   // loop over integration points
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
@@ -92,7 +91,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
   const int dof2 = el2.GetDof();

 #ifdef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration

   // shape function value at an integration point - first elem
   Vector shape1(dof1);
@@ -122,7 +121,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
   DenseMatrix elvect2_mat(elvect.GetData() + dof1 * num_equations, dof2,
                           num_equations);

-   // obtain integration rule. If integration is rule is given, then use it.
+   // Obtain integration rule. If integration is rule is given, then use it.
   // Otherwise, get (2*p + IntOrderOffset) order integration rule
   const IntegrationRule *ir = IntRule;
   if (!ir)
@@ -149,7 +148,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
      if (nor.Size() == 1)  // if 1D, use 1 or -1.
      {
         // This assume the 1D integration point is in (0,1). This may not work
-         // if this chages.
+         // if this changes.
         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
      }
      else
@@ -18,43 +18,36 @@
 namespace mfem
 {

-//                     MFEM Hyperbolic Conservation Laws
+// This file contains general hyperbolic conservation element/face form
+// integrators. HyperbolicFormIntegrator and RiemannSolver are defined.
 //
-// Description:
+// HyperbolicFormIntegrator is a NonlinearFormIntegrator that implements
+// element weak divergence and interface flux
 //
-//    This file contains general hyperbolic conservation element/face form
-//    integrators.
+//     ∫_T F(u):∇v,   -∫_e F̂(u)⋅[[v]]
 //
-//    HyperbolicFormIntegrator and RiemannSolver are defined.
-//    HyperbolicFormIntegrator is a NonlinearFormIntegrator that implements
-//    element weak divergence and interface flux
+// Here, T is an element, e is an edge, and [[⋅]] is jump. This form integrator
+// is coupled with RiemannSolver that implements the numerical flux F̂. For
+// RiemannSolver, the Rusanov flux, also known as local Lax-Friedrichs flux, is
+// provided.
 //
-//       ∫_T F(u):∇v,   -∫_e F̂(u)⋅[[v]]
+// To implement a specific hyperbolic conservation laws, users can create
+// derived classes from FluxFunction with overloaded ComputeFlux. One can
+// optionally overload ComputeFluxDotN to avoid creating dense matrix when
+// computing normal flux. Several example equations are also defined including:
+// advection, Burgers', shallow water, and Euler equations. Users can control
+// the quadrature rule by either providing the integration rule, or integration
+// order offset. Integration will use 2*p + IntOrderOffset order quadrature
+// rule.
 //
-//    Here, T is an element, e is an edge, and [[⋅]] is jump. This form
-//    integrator is coupled with RiemannSolver that implements the numerical
-//    flux F̂. For RiemannSolver, the Rusanov flux, also known as local
-//    Lax-Friedrichs flux, is provided.
-//
-//    To implement a specific hyperbolic conservation laws, users can create
-//    derived classes from FluxFunction with overloaded ComputeFlux. One can
-//    optionally overload ComputeFluxDotN to avoid creating dense matrix when
-//    computing normal flux. Several example equations are also defined
-//    including: advection, Burgers', shallow water, and Euler equations. Users
-//    can control the quadrature rule by either providing the integration rule,
-//    or integration order offset. Integration will use 2*p + IntOrderOffset
-//    order quadrature rule.
-//
-//    At each call of HyperbolicFormIntegrator::AssembleElementVector
-//    HyperbolicFormIntegrator::AssembleFaceVector, the maximum characteristic
-//    speed will be updated. This will not be reinitialized automatically.
-//    To reinitialize, use HyperbolicFormIntegrator::ResetMaxCharSpeed. See,
-//    ex18.hpp.
-//
-//    Note: To avoid communication overhead, we update the maximum
-//    characteristic speed within each process. Use a proper MPI routine to
-//    gather the information.
+// At each call of HyperbolicFormIntegrator::AssembleElementVector
+// HyperbolicFormIntegrator::AssembleFaceVector, the maximum characteristic
+// speed will be updated. This will not be reinitialized automatically. To
+// reinitialize, use HyperbolicFormIntegrator::ResetMaxCharSpeed. See, ex18.hpp.
 //
+// Note: To avoid communication overhead, we update the maximum characteristic
+// speed within each MPI process only. Use the appropriate MPI routine to gather
+// the information.

 /**
 * @brief Abstract class for hyperbolic flux for a system of hyperbolic
@@ -88,7 +81,7 @@ public:
   virtual real_t ComputeFlux(const Vector &state, ElementTransformation &Tr,
                              DenseMatrix &flux) const = 0;
   /**
-    * @brief Compute normal flux. Optionally overloadded in the
+    * @brief Compute normal flux. Optionally overloaded in the
    * derived class to avoid creating full dense matrix for flux.
    *
    * @param[in] state state at the current integration point
@@ -168,13 +161,13 @@ protected:
 class HyperbolicFormIntegrator : public NonlinearFormIntegrator
 {
 private:
-   // The maximum characterstic speed, updated during element/face vector assembly
+   // The maximum characteristic speed, updated during element/face vector assembly
   real_t max_char_speed;
   const RiemannSolver &rsolver;   // Numerical flux that maps F(u±,x) to hat(F)
   const FluxFunction &fluxFunction;
   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
 #ifndef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration
   Vector shape;              // shape function value at an integration point
   Vector state;              // state value at an integration point
   DenseMatrix flux;          // flux value at an integration point
@@ -563,7 +563,7 @@ void DiffusionIntegrator::AssemblePatchMatrix_fullQuadrature(
   cdofs.SetSize(maxw[0], maxw[1], maxw[2]);

   // Compute sparsity of the sparse matrix
-   smati = new int[ndof+1];
+   smati = Memory<int>(ndof+1);
   smati[0] = 0;

   for (int dof_j=0; dof_j<ndof; ++dof_j)
@@ -586,8 +586,8 @@ void DiffusionIntegrator::AssemblePatchMatrix_fullQuadrature(
      nnz += ndd;
   }

-   smatj = new int[nnz];
-   smata = new real_t[nnz];
+   smatj = Memory<int>(nnz);
+   smata = Memory<real_t>(nnz);

   for (int i=0; i<nnz; ++i)
   {
@@ -973,7 +973,7 @@ void DiffusionIntegrator::AssemblePatchMatrix_reducedQuadrature(
   cdofs.SetSize(maxw[0], maxw[1], maxw[2]);

   // Compute sparsity of the sparse matrix
-   smati = new int[ndof+1];
+   smati = Memory<int>(ndof+1);
   smati[0] = 0;

   for (int dof_j=0; dof_j<ndof; ++dof_j)
@@ -996,8 +996,8 @@ void DiffusionIntegrator::AssemblePatchMatrix_reducedQuadrature(
      nnz += ndd;
   }

-   smatj = new int[nnz];
-   smata = new real_t[nnz];
+   smatj = Memory<int>(nnz);
+   smata = Memory<real_t>(nnz);

   for (int i=0; i<nnz; ++i)
   {
@@ -157,7 +157,7 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
   static constexpr int aSize = aUpper-aLower;
   static constexpr bool isComponent = (i_block >= 0);

-   //Assuming all elements are the same
+   // Assuming all elements are the same
   const auto &ir = QVec.GetIntRule(0);
   const QuadratureInterpolator *E_To_Q_Map = fespace.GetQuadratureInterpolator(
                                                 ir);
@@ -180,7 +180,7 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
         auto invJ = inv(make_tensor<d, d>(
         [&](int i, int j) { return J(p, i, j, e); }));
         tensor<real_t, aSize, d> gradx;
-         //load grad(x) into gradx
+         // load grad(x) into gradx
         if (isComponent)
         {
            for (int i = 0; i < d; i++)
@@ -198,11 +198,11 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
               }
            }
         }
-         //compute divergence
+         // compute divergence
         real_t div = 0.;
         for (int i = aLower; i < aUpper; i++)
         {
-            //take size of gradx into account
+            // take size of gradx into account
            const int iIndex = isComponent ? 0 : i;
            div += gradx(iIndex,i);
         }
@@ -211,11 +211,11 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
         {
            for (int q = qLower; q < qUpper; q++)
            {
-               //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-               //this contraction could be made slightly cheaper using Voigt
-               //notation, but repeated entries are summed for simplicity.
+               // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+               // this contraction could be made slightly cheaper using Voigt
+               // notation, but repeated entries are summed for simplicity.
               real_t contraction = 0.;
-               //not sure how to combine cases
+               // not sure how to combine cases
               if (isComponent)
               {
                  for (int a = 0; a < d; a++)
@@ -276,7 +276,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
                                   const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
-   //Assuming all elements are the same
+   // Assuming all elements are the same
   const auto &ir = QVec.GetIntRule(0);
   static constexpr int d = dim;
   const int numPoints = ir.GetNPoints();
@@ -299,9 +299,9 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
            {
               for (int q = 0; q < d; q++)
               {
-                  //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-                  //this contraction could be made slightly cheaper using Voigt
-                  //notation, but repeated entries are summed for simplicity.
+                  // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                  // this contraction could be made slightly cheaper using Voigt
+                  // notation, but repeated entries are summed for simplicity.
                  real_t contraction = 0.;
                  for (int a = 0; a < d; a++)
                  {
@@ -321,7 +321,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
      }
   });

-   //Reduce quadrature function to an E-Vector
+   // Reduce quadrature function to an E-Vector
   const auto QRead = Reshape(QVec.Read(), numPoints, d, d, d, numEls);
   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
@@ -348,7 +348,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
   });
 }

-//Templated implementation of ElasticityAssembleEA.
+// Templated implementation of ElasticityAssembleEA.
 template<int dim>
 void ElasticityAssembleEA_(const int i_block,
                           const int j_block,
@@ -360,7 +360,7 @@ void ElasticityAssembleEA_(const int i_block,
                           const DofToQuad &maps,
                           Vector &emat)
 {
-   //Assuming all elements are the same
+   // Assuming all elements are the same
   static constexpr int d = dim;
   const int numPoints = ir.GetNPoints();
   const int numEls = lambda.Size()/numPoints;
@@ -386,7 +386,7 @@ void ElasticityAssembleEA_(const int i_block,
               {
                  for (int m = 0; m < d; m++)
                  {
-                     //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                     // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
                     real_t contraction = 0.;
                     for (int a = 0; a < d; a++)
                     {
@@ -101,7 +101,7 @@ void MomentFittingIntRules::InitVolume(int order, Coefficient& levelset,
         }
      }

-      // assamble the matrix
+      // assemble the matrix
      DenseMatrix Mat(nBasisVolume, ir.GetNPoints());
      for (int ip = 0; ip < ir.GetNPoints(); ip++)
      {
@@ -118,7 +118,7 @@ void MomentFittingIntRules::InitVolume(int order, Coefficient& levelset,
         Mat.SetCol(ip, shape);
      }

-      // compute the svd for the matrix
+      // compute the SVD for the matrix
      VolumeSVD = new DenseMatrixSVD(Mat, 'A', 'A');
      VolumeSVD->Eval(Mat);
   }
@@ -1239,7 +1239,7 @@ void MomentFittingIntRules::OrthoBasis2D(const IntegrationPoint& ip,

   shape.SetSize(nBasis, 2);

-   // evaluate basis inthe point
+   // evaluate basis in the point
   DenseMatrix preshape(nBasis, 2);
   DivFreeBasis2D(ip, shape);

@@ -1597,6 +1597,6 @@ void MomentFittingIntRules::GetSurfaceWeights(ElementTransformation& Tr,
   }
 }

-#endif //MFEM_USE_LAPACK
+#endif // MFEM_USE_LAPACK

 }
@@ -291,7 +291,7 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   const auto ltdof_ldof = HypreRead(R->GetMemoryJ());

   // Go from E-vector format directly to T-vector format
-   MFEM_HYPRE_FORALL(i, ntdofs,
+   mfem::hypre_forall(ntdofs, [=] MFEM_HOST_DEVICE (int i)
   {
      const int j = d_offsets[ltdof_ldof[i]];
      for (int c = 0; c < sdim; ++c)
@@ -269,13 +269,13 @@ void BatchedLOR_H1::Assemble3D()
               real_t vx[8], vy[8], vz[8];
               LORVertexCoordinates3D<ORDER>(X, iel_ho, kx, ky, kz, vx, vy, vz);

-               //MFEM_UNROLL(2)
+               // MFEM_UNROLL(2)
               for (int iqz=0; iqz<2; ++iqz)
               {
-                  //MFEM_UNROLL(2)
+                  // MFEM_UNROLL(2)
                  for (int iqy=0; iqy<2; ++iqy)
                  {
-                     //MFEM_UNROLL(2)
+                     // MFEM_UNROLL(2)
                     for (int iqx=0; iqx<2; ++iqx)
                     {
                        const real_t x = iqx;
@@ -307,21 +307,21 @@ void BatchedLOR_H1::Assemble3D()
                  }
               }

-               //MFEM_UNROLL(2)
+               // MFEM_UNROLL(2)
               for (int iqx=0; iqx<2; ++iqx)
               {
-                  //MFEM_UNROLL(2)
+                  // MFEM_UNROLL(2)
                  for (int jz=0; jz<2; ++jz)
                  {
                     // Note loop starts at iz=jz here, taking advantage of
                     // symmetries.
-                     //MFEM_UNROLL(2)
+                     // MFEM_UNROLL(2)
                     for (int iz=jz; iz<2; ++iz)
                     {
-                        //MFEM_UNROLL(2)
+                        // MFEM_UNROLL(2)
                        for (int iqy=0; iqy<2; ++iqy)
                        {
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int iqz=0; iqz<2; ++iqz)
                           {
                              const real_t mq = const_mq ? MQ(0,0,0,0) : MQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
@@ -356,10 +356,10 @@ void BatchedLOR_H1::Assemble3D()
                              real_t wdetJ = Q(6,iqz,iqy,iqx);
                              mass_A(iqy,iz,jz,iqx) += mq*wdetJ*biz*bjz;
                           }
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int jy=0; jy<2; ++jy)
                           {
-                              //MFEM_UNROLL(2)
+                              // MFEM_UNROLL(2)
                              for (int iy=0; iy<2; ++iy)
                              {
                                 const real_t biy = (iy == iqy) ? 1.0 : 0.0;
@@ -382,16 +382,16 @@ void BatchedLOR_H1::Assemble3D()
                              }
                           }
                        }
-                        //MFEM_UNROLL(2)
+                        // MFEM_UNROLL(2)
                        for (int jy=0; jy<2; ++jy)
                        {
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int jx=0; jx<2; ++jx)
                           {
-                              //MFEM_UNROLL(2)
+                              // MFEM_UNROLL(2)
                              for (int iy=0; iy<2; ++iy)
                              {
-                                 //MFEM_UNROLL(2)
+                                 // MFEM_UNROLL(2)
                                 for (int ix=0; ix<2; ++ix)
                                 {
                                    const real_t bix = (ix == iqx) ? 1.0 : 0.0;
@@ -431,7 +431,7 @@ void BatchedLOR_H1::Assemble3D()
               // Assemble the local matrix into the macro-element sparse matrix
               // in a format similar to coordinate format. The (I,J) arrays
               // are implicit (not stored explicitly).
-               //MFEM_UNROLL(8)
+               // MFEM_UNROLL(8)
               for (int ii_loc=0; ii_loc<nv; ++ii_loc)
               {
                  const int ix = ii_loc%2;
@@ -0,0 +1,828 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "normal_deriv_restriction.hpp"
+#include "fespace.hpp"
+#include "pgridfunc.hpp"
+#include "fe/face_map_utils.hpp"
+#include "../general/forall.hpp"
+
+namespace mfem
+{
+
+/// Compute the face index to volume index map "face_to_vol" in 2D
+static void NormalDerivativeSetupFaceIndexMap2D(
+   int nf, int d, const Array<int>& face_to_elem, Array<int>& face_to_vol)
+{
+   const auto f2e = Reshape(face_to_elem.HostRead(), 2, 2, nf);
+   auto f2v = Reshape(face_to_vol.HostWrite(), d, 2, nf);
+
+   for (int f = 0; f < nf; ++f)
+   {
+      const int fid0 = f2e(0, 1, f);
+      const int fid1 = f2e(1, 1, f);
+      for (int side = 0; side < 2; ++side)
+      {
+         const int el = f2e(side, 0, f);
+
+         if (el < 0)
+         {
+            for (int p = 0; p < d; ++p)
+            {
+               f2v(p, side, f) = -1;
+            }
+         }
+         else
+         {
+            for (int p = 0; p < d; ++p)
+            {
+               int i, j;
+               internal::FaceIdxToVolIdx2D(p, d, fid0, fid1, side, i, j);
+
+               f2v(p, side, f) = i + d * j;
+            }
+         }
+      }
+   }
+}
+
+/// Compute the face index to volume index map "face_to_vol" in 3D
+static void NormalDerivativeSetupFaceIndexMap3D(
+   int nf, int d, const Array<int>& face_to_elem, Array<int>& face_to_vol)
+{
+   const auto f2e = Reshape(face_to_elem.HostRead(), 2, 3, nf);
+   auto f2v = Reshape(face_to_vol.HostWrite(), d*d, 2, nf);
+
+   for (int f = 0; f < nf; ++f)
+   {
+      const int fid0 = f2e(0, 1, f);
+      const int fid1 = f2e(1, 1, f);
+      for (int side = 0; side < 2; ++side)
+      {
+         const int el = f2e(side, 0, f);
+         const int orientation = f2e(side, 2, f);
+
+         if (el < 0)
+         {
+            for (int p = 0; p < d*d; ++p)
+            {
+               f2v(p, side, f) = -1;
+            }
+         }
+         else
+         {
+            for (int p = 0; p < d*d; ++p)
+            {
+               int i, j, k; // 3D lexicographic index of quad point p
+               internal::FaceIdxToVolIdx3D(p, d, fid0, fid1, side, orientation, i, j, k);
+
+               f2v(p, side, f) = i + d * (j + d * k);
+            }
+         }
+      }
+   }
+}
+
+L2NormalDerivativeFaceRestriction::L2NormalDerivativeFaceRestriction(
+   const FiniteElementSpace &fes_,
+   const ElementDofOrdering f_ordering,
+   const FaceType face_type_)
+   : fes(fes_),
+     face_type(face_type_),
+     dim(fes.GetMesh()->Dimension()),
+     nf(fes.GetNFbyType(face_type)),
+     ne(fes.GetNE())
+{
+   MFEM_VERIFY(f_ordering == ElementDofOrdering::LEXICOGRAPHIC,
+               "Non-lexicographic ordering not currently supported in "
+               "L2NormalDerivativeFaceRestriction.");
+
+   Mesh &mesh = *fes.GetMesh();
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const int d = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR).ndof;
+
+   if (dim == 2)
+   {
+      // (el0, el1, fid0, fid1)
+      face_to_elem.SetSize(nf * 4);
+      face_to_vol.SetSize(2 * nf * d);
+   }
+   else if (dim == 3)
+   {
+      // (el0, el1, fid0, fid1, or0, or1)
+      face_to_elem.SetSize(nf * 6);
+      face_to_vol.SetSize(2 * nf * d * d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension.");
+   }
+   auto f2e = Reshape(face_to_elem.HostWrite(), 2, (dim == 2) ? 2 : 3, nf);
+
+   // Populate the face_to_elem array. The elem_indicator will be used to count
+   // the number of elements that are adjacent to faces of the given type.
+   Array<int> elem_indicator(ne);
+   elem_indicator = 0;
+
+   int f_ind = 0;
+   for (int f = 0; f < fes.GetNF(); ++f)
+   {
+      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
+
+      if (face.IsOfFaceType(face_type))
+      {
+         f2e(0, 0, f_ind) = face.element[0].index;
+         f2e(0, 1, f_ind) = face.element[0].local_face_id;
+         if (dim == 3)
+         {
+            f2e(0, 2, f_ind) = face.element[0].orientation;
+         }
+
+         elem_indicator[face.element[0].index] = 1;
+
+         if (face_type == FaceType::Interior)
+         {
+            const int el_idx_1 = face.element[1].index;
+            if (face.IsShared())
+            {
+               // Indicate shared face by index >= ne
+               f2e(1, 0, f_ind) = ne + el_idx_1;
+            }
+            else
+            {
+               // Face is not shared
+               f2e(1, 0, f_ind) = el_idx_1;
+               elem_indicator[el_idx_1] = 1;
+            }
+            f2e(1, 1, f_ind) = face.element[1].local_face_id;
+
+            if (dim == 3)
+            {
+               f2e(1, 2, f_ind) = face.element[1].orientation;
+            }
+         }
+         else
+         {
+            f2e(1, 0, f_ind) = -1;
+            f2e(1, 1, f_ind) = -1;
+
+            if (dim == 3)
+            {
+               f2e(1, 2, f_ind) = -1;
+            }
+         }
+
+         f_ind++;
+      }
+   }
+
+   // evaluate face to vol map
+   if (dim == 2)
+   {
+      NormalDerivativeSetupFaceIndexMap2D(nf, d, face_to_elem, face_to_vol);
+   }
+   else if (dim == 3)
+   {
+      NormalDerivativeSetupFaceIndexMap3D(nf, d, face_to_elem, face_to_vol);
+   }
+
+   // Number of elements adjacent to faces of face_type
+   ne_type = elem_indicator.Sum();
+
+   // In 2D: (el, f0,f1,f2,f3, s0,s1,s2,s3)
+   // In 3D: (el, f0,f1,f2,f3,f4,f5, s0,s1,s2,s3,s4,s5)
+   const int elem_data_sz = (dim == 2) ? 9 : 13;
+
+   elem_to_face.SetSize(elem_data_sz * ne_type);
+   elem_to_face = -1;
+
+   auto e2f = Reshape(elem_to_face.HostWrite(), elem_data_sz, ne_type);
+   elem_indicator.PartialSum();
+
+   const int nsides = (face_type == FaceType::Interior) ? 2 : 1;
+   const int side_begin = (dim == 2) ? 5 : 7;
+   for (int f = 0; f < nf; ++f)
+   {
+      for (int side = 0; side < nsides; ++side)
+      {
+         const int el = f2e(side, 0, f);
+         // Skip shared faces
+         if (el < ne)
+         {
+            const int face_id = f2e(side, 1, f);
+
+            const int e = elem_indicator[el] - 1;
+            e2f(0, e) = el;
+            e2f(1 + face_id, e) = f;
+            e2f(side_begin + face_id, e) = side;
+         }
+      }
+   }
+}
+
+void L2NormalDerivativeFaceRestriction::Mult(const Vector &x, Vector &y) const
+{
+   if (nf == 0) { return; }
+   switch (dim)
+   {
+      case 2:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: Mult2D<1>(x, y); break;
+            case 2: Mult2D<2>(x, y); break;
+            case 3: Mult2D<3>(x, y); break;
+            case 4: Mult2D<4>(x, y); break;
+            case 5: Mult2D<5>(x, y); break;
+            case 6: Mult2D<6>(x, y); break;
+            case 7: Mult2D<7>(x, y); break;
+            case 8: Mult2D<8>(x, y); break;
+            default: Mult2D(x, y); break;
+         }
+      }
+      break;
+      case 3:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: Mult3D<1>(x, y); break;
+            case 2: Mult3D<2>(x, y); break;
+            case 3: Mult3D<3>(x, y); break;
+            case 4: Mult3D<4>(x, y); break;
+            case 5: Mult3D<5>(x, y); break;
+            case 6: Mult3D<6>(x, y); break;
+            case 7: Mult3D<7>(x, y); break;
+            case 8: Mult3D<8>(x, y); break;
+            default: Mult3D(x, y); break; // fallback
+         }
+         break;
+      }
+      default: MFEM_ABORT("Dimension not supported."); break;
+   }
+}
+
+void L2NormalDerivativeFaceRestriction::AddMultTranspose(
+   const Vector &x, Vector &y, const real_t a) const
+{
+   if (nf == 0) { return; }
+   switch (dim)
+   {
+      case 2:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: AddMultTranspose2D<1>(x, y, a); break;
+            case 2: AddMultTranspose2D<2>(x, y, a); break;
+            case 3: AddMultTranspose2D<3>(x, y, a); break;
+            case 4: AddMultTranspose2D<4>(x, y, a); break;
+            case 5: AddMultTranspose2D<5>(x, y, a); break;
+            case 6: AddMultTranspose2D<6>(x, y, a); break;
+            case 7: AddMultTranspose2D<7>(x, y, a); break;
+            case 8: AddMultTranspose2D<8>(x, y, a); break;
+            default: AddMultTranspose2D(x, y, a); break;
+         }
+      }
+      break;
+      case 3:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: AddMultTranspose3D<1>(x, y, a); break;
+            case 2: AddMultTranspose3D<2>(x, y, a); break;
+            case 3: AddMultTranspose3D<3>(x, y, a); break;
+            case 4: AddMultTranspose3D<4>(x, y, a); break;
+            case 5: AddMultTranspose3D<5>(x, y, a); break;
+            case 6: AddMultTranspose3D<6>(x, y, a); break;
+            case 7: AddMultTranspose3D<7>(x, y, a); break;
+            case 8: AddMultTranspose3D<8>(x, y, a); break;
+            default: AddMultTranspose3D(x, y, a); break; // fallback
+         }
+         break;
+      }
+      default: MFEM_ABORT("Not yet implemented"); break;
+   }
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::Mult2D(const Vector &x, Vector &y) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+   const int num_elem = ne;
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+
+   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, face_type);
+   const int ne_shared = face_nbr_data.Size() / d / d / vd;
+
+   MFEM_VERIFY(q == d, "");
+   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
+
+   // derivative of 1D basis function
+   const auto G_ = Reshape(maps.G.Read(), q, d);
+   // (el0, el1, fid0, fid1)
+   const auto f2e = Reshape(face_to_elem.Read(), 2, 2, nf);
+
+   const auto f2v = Reshape(face_to_vol.Read(), q, 2, nf);
+
+   // if byvdim, d_x has shape (vdim, nddof, nddof, ne)
+   // otherwise, d_x has shape (nddof, nddof, ne, vdim)
+   const auto d_x = Reshape(x.Read(), t?vd:d, d, t?d:ne, t?ne:vd);
+   const auto d_x_shared = Reshape(face_nbr_data.Read(),
+                                   t?vd:d, d, t?d:ne_shared, t?ne_shared:vd);
+   auto d_y = Reshape(y.Write(), q, vd, 2, nf);
+
+   mfem::forall_2D(nf, 2, q, [=] MFEM_HOST_DEVICE (int f) -> void
+   {
+      constexpr int MD = (T_D1D) ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED real_t G_s[MD*MD];
+      DeviceMatrix G(G_s, q, d);
+
+      MFEM_SHARED int E[2];
+      MFEM_SHARED int FID[2];
+      MFEM_SHARED int F2V[2][MD];
+
+      if (MFEM_THREAD_ID(x) == 0)
+      {
+         MFEM_FOREACH_THREAD(j, y, d)
+         {
+            for (int i = 0; i < q; ++i)
+            {
+               G(i, j) = G_(i, j);
+            }
+         }
+      }
+
+      MFEM_FOREACH_THREAD(side, x, 2)
+      {
+         if (MFEM_THREAD_ID(y) == 0)
+         {
+            E[side] = f2e(side, 0, f);
+            FID[side] = f2e(side, 1, f);
+         }
+
+         MFEM_FOREACH_THREAD(j, y, d)
+         {
+            F2V[side][j] = f2v(j, side, f);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, x, 2)
+      {
+         const int el = E[side];
+         const bool shared = (el >= num_elem);
+         const auto &d_x_e = shared ? d_x_shared : d_x;
+         const int el_idx = shared ? el - num_elem : el;
+
+         const int face_id = FID[side];
+
+         MFEM_FOREACH_THREAD(p, y, q)
+         {
+            if (el < 0)
+            {
+               for (int c = 0; c < vd; ++c)
+               {
+                  d_y(p, c, side, f) = 0.0;
+               }
+            }
+            else
+            {
+               const int ij = F2V[side][p];
+               const int i = ij % q;
+               const int j = ij / q;
+
+               for (int c=0; c < vd; ++c)
+               {
+                  real_t grad_n = 0;
+                  for (int kk=0; kk < d; ++kk)
+                  {
+                     const int k = (face_id == 0 || face_id == 2) ? i : kk;
+                     const int l = (face_id == 0 || face_id == 2) ? kk : j;
+                     const real_t g = (face_id == 0 || face_id == 2) ? G(j,l) : G(i,k);
+                     grad_n += g * d_x_e(t?c:k, t?k:l, t?l:el_idx, t?el_idx:c);
+                  }
+                  d_y(p, c, side, f) = grad_n;
+               }
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::Mult3D(const Vector &x, Vector &y) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+   const int num_elem = ne;
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+   const int q2d = q * q;
+
+   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, face_type);
+   const int ne_shared = face_nbr_data.Size() / d / d / d / vd;
+
+   MFEM_VERIFY(q == d, "");
+   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
+
+   const auto G_ = Reshape(maps.G.Read(), q, d);
+   // (el0, el1, fid0, fid1, or0, or1)
+   const auto f2e = Reshape(face_to_elem.Read(), 2, 3, nf);
+   const auto f2v = Reshape(face_to_vol.Read(), q2d, 2, nf);
+
+   // t ? (vdim, d, d, d, ne) : (d, d, d, ne, vdim)
+   const auto d_x = Reshape(x.Read(), t?vd:d, d, d, t?d:ne, t?ne:vd);
+   const auto d_x_shared = Reshape(face_nbr_data.Read(),
+                                   t?vd:d, d, d, t?d:ne_shared, t?ne_shared:vd);
+   auto d_y = Reshape(y.Write(), q2d, vd, 2, nf);
+
+   mfem::forall_2D(nf, q2d, 2, [=] MFEM_HOST_DEVICE (int f) -> void
+   {
+      static constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED real_t G_s[MD*MD];
+      DeviceMatrix G(G_s, d, q);
+
+      MFEM_SHARED int E[2];
+      MFEM_SHARED int FID[2];
+      MFEM_SHARED int F2V[2][MD*MD];
+
+      // Load G matrix into shared memory
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         MFEM_FOREACH_THREAD(j, x, d*q)
+         {
+            const int p = j % q;
+            const int k = j / q;
+            G(k, p) = G_(p, k);
+         }
+      }
+
+      MFEM_FOREACH_THREAD(side, y, 2)
+      {
+         if (MFEM_THREAD_ID(x) == 0)
+         {
+            E[side] = f2e(side, 0, f);
+            FID[side] = f2e(side, 1, f);
+         }
+         MFEM_FOREACH_THREAD(j, x, q2d)
+         {
+            F2V[side][j] = f2v(j, side, f);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, y, 2)
+      {
+         const int el = E[side];
+         const bool shared = (el >= num_elem);
+         const auto &d_x_e = shared ? d_x_shared : d_x;
+         const int el_idx = shared ? el - num_elem : el;
+
+         const int face_id = FID[side];
+
+         // Is this face parallel to the x-y plane in reference coordinates?
+         const bool xy_plane = (face_id == 0 || face_id == 5);
+         const bool xz_plane = (face_id == 1 || face_id == 3);
+         const bool yz_plane = (face_id == 2 || face_id == 4);
+
+         MFEM_FOREACH_THREAD(p, x, q2d)
+         {
+            if (el_idx < 0)
+            {
+               for (int c = 0; c < vd; ++c)
+               {
+                  d_y(p, c, side, f) = 0.0;
+               }
+            }
+            else
+            {
+               const int ijk = F2V[side][p];
+               const int k = ijk / q2d;
+               const int i = ijk % q;
+               const int j = (ijk - q2d*k) / q;
+
+               // the fixed 1D index of the normal component of the face
+               // quadrature point
+               const int g_row = yz_plane ? i : xz_plane ? j : k;
+
+               for (int c = 0; c < vd; ++c)
+               {
+                  real_t grad_n = 0.0;
+
+                  for (int kk = 0; kk < d; ++kk)
+                  {
+                     // (l, m, n) 3D lexicographic index of interior points used
+                     // in evaluating normal derivatives
+                     const int l = yz_plane ? kk : i;
+                     const int m = xz_plane ? kk : j;
+                     const int n = xy_plane ? kk : k;
+
+                     const real_t g = G(kk, g_row);
+
+                     grad_n += g * d_x_e(t?c:l, t?l:m, t?m:n, t?n:el_idx, t?el_idx:c);
+                  }
+                  d_y(p, c, side, f) = grad_n;
+               }
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::AddMultTranspose2D(
+   const Vector &y, Vector &x, const real_t a) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+
+   // derivative of 1D basis function
+   auto G_ = Reshape(maps.G.Read(), q, d);
+
+   // entries of e2f: (el,f0,f1,f2,f3,s0,s1,s2,s3)
+   auto e2f = Reshape(elem_to_face.Read(), 9, ne_type);
+
+   auto f2v = Reshape(face_to_vol.Read(), d, 2, nf);
+
+   // if byvdim, d_x has shape (vdim, nddof, nddof, ne)
+   // otherwise, d_x has shape (nddof, nddof, ne, vdim)
+   auto d_x = Reshape(x.ReadWrite(), t?vd:d, d, t?d:ne, t?ne:vd);
+   auto d_y = Reshape(y.Read(), q, vd, 2, nf);
+
+   mfem::forall_2D(ne_type, d, d, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED real_t y_s[MD];
+      MFEM_SHARED int pp[MD];
+      MFEM_SHARED int jj;
+      if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0) { jj = 0; }
+
+      MFEM_SHARED real_t BG[MD*MD];
+      DeviceMatrix G(BG, q, d);
+
+      MFEM_SHARED real_t x_s[MD*MD];
+      DeviceMatrix xx(x_s, d, d);
+
+      MFEM_SHARED int el; // global element index
+      MFEM_SHARED int faces[4];
+      MFEM_SHARED int sides[4];
+
+      MFEM_FOREACH_THREAD(i,x,d)
+      {
+         MFEM_FOREACH_THREAD(p,y,q)
+         {
+            G(p,i) = a * G_(p,i);
+            xx(p,i) = 0.0;
+         }
+      }
+
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         if (MFEM_THREAD_ID(x) == 0)
+         {
+            el = e2f(0, e);
+         }
+
+         MFEM_FOREACH_THREAD(i, x, 4)
+         {
+            faces[i] = e2f(1 + i, e);
+            sides[i] = e2f(5 + i, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int face_id=0; face_id < 4; ++face_id)
+      {
+         const int f = faces[face_id];
+
+         if (f < 0) { continue; }
+
+         const int side = sides[face_id];
+
+         if (MFEM_THREAD_ID(y) == 0)
+         {
+            MFEM_FOREACH_THREAD(p,x,d)
+            {
+               y_s[p] = d_y(p, 0, side, f);
+
+               const int ij = f2v(p, side, f);
+               const int i = ij % q;
+               const int j = ij / q;
+
+               pp[(face_id == 0 || face_id == 2) ? i : j] = p;
+               if (MFEM_THREAD_ID(x) == 0)
+               {
+                  jj = (face_id == 0 || face_id == 2) ? j : i;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(k,x,d)
+         {
+            MFEM_FOREACH_THREAD(l,y,d)
+            {
+               const int p = (face_id == 0 || face_id == 2) ? pp[k] : pp[l];
+               const int kk = (face_id == 0 || face_id == 2) ? l : k;
+               const real_t g = G(jj, kk);
+               xx(k,l) += g * y_s[p];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(k,x,d)
+      {
+         MFEM_FOREACH_THREAD(l,y,d)
+         {
+            const int c = 0;
+            d_x(t?c:k, t?k:l, t?l:el, t?el:c) += xx(k,l);
+         }
+      }
+   });
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::AddMultTranspose3D(
+   const Vector &y, Vector &x, const real_t a) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+
+   MFEM_VERIFY(vd == 1, "vdim > 1 not supported.");
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+   const int q2d = q * q;
+
+   MFEM_VERIFY(q == d, "");
+   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
+
+   auto G_ = Reshape(maps.G.Read(), q, d);
+
+   // (el, f0,f1,f2,f3,f4,f5, s0,s1,s2,s3,s4,s5)
+   auto e2f = Reshape(elem_to_face.Read(), 13, ne_type);
+
+   auto f2v = Reshape(face_to_vol.Read(), q2d, 2, nf);
+
+   auto d_x = Reshape(x.ReadWrite(), t?vd:d, d, d, t?d:ne, t?ne:vd);
+   const auto d_y = Reshape(y.Read(), q2d, vd, 2, nf);
+
+   mfem::forall_2D(ne_type, q, q, [=] MFEM_HOST_DEVICE (int e) -> void
+   {
+      static constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED int pp[MD][MD];
+      MFEM_SHARED real_t y_s[MD*MD];
+      MFEM_SHARED int jj;
+      if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0) { jj = 0; }
+
+      MFEM_SHARED real_t xx_s[MD*MD*MD];
+      auto xx = Reshape(xx_s, d, d, d);
+
+      MFEM_SHARED real_t G_s[MD*MD];
+      DeviceMatrix G(G_s, q, d);
+
+      MFEM_SHARED int el;
+      MFEM_SHARED int faces[6];
+      MFEM_SHARED int sides[6];
+
+      // Load G into shared memory
+      MFEM_FOREACH_THREAD(j, x, d)
+      {
+         MFEM_FOREACH_THREAD(i, y, q)
+         {
+            G(i, j) = a * G_(i, j);
+            G(i, j) = a * G_(i, j);
+            G(i, j) = a * G_(i, j);
+         }
+      }
+
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         if (MFEM_THREAD_ID(x) == 0)
+         {
+            el = e2f(0, e); // global element index
+         }
+
+         MFEM_FOREACH_THREAD(i, x, 6)
+         {
+            faces[i] = e2f(1 + i, e);
+            sides[i] = e2f(7 + i, e);
+         }
+      }
+
+      MFEM_FOREACH_THREAD(k, x, d)
+      {
+         MFEM_FOREACH_THREAD(j, y, d)
+         {
+            for (int i = 0; i < d; ++i)
+            {
+               xx(i, j, k) = 0.0;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int face_id = 0; face_id < 6; ++face_id)
+      {
+         const int f = faces[face_id];
+
+         if (f < 0)
+         {
+            continue;
+         }
+
+         const int side = sides[face_id];
+
+         // is this face parallel to the x-y plane in reference coordinates?
+         const bool xy_plane = (face_id == 0 || face_id == 5);
+         const bool xz_plane = (face_id == 1 || face_id == 3);
+
+         MFEM_FOREACH_THREAD(p1, x, q)
+         {
+            MFEM_FOREACH_THREAD(p2, y, q)
+            {
+               const int p = p1 + q * p2;
+               y_s[p] = d_y(p, 0, side, f);
+
+               const int ijk = f2v(p, side, f);
+               const int k = ijk / q2d;
+               const int i = ijk % q;
+               const int j = (ijk - q2d*k) / q;
+
+               pp[(xy_plane || xz_plane) ? i : j][(xy_plane) ? j : k] = p;
+               if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0)
+               {
+                  jj = (xy_plane) ? k : (xz_plane) ? j : i;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(n, x, d)
+         {
+            MFEM_FOREACH_THREAD(m, y, d)
+            {
+               for (int l = 0; l < d; ++l)
+               {
+                  const int p = (xy_plane) ? pp[l][m] : (xz_plane) ? pp[l][n] : pp[m][n];
+                  const int kk = (xy_plane) ? n : (xz_plane) ? m : l;
+                  const real_t g = G(jj, kk);
+                  xx(l, m, n) += g * y_s[p];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // map back to global array
+      MFEM_FOREACH_THREAD(n, x, d)
+      {
+         MFEM_FOREACH_THREAD(m, y, d)
+         {
+            for (int l = 0; l < d; ++l)
+            {
+               const int c = 0;
+               d_x(t?c:l, t?l:m, t?m:n, t?n:el, t?el:c) += xx(l, m, n);
+            }
+         }
+      }
+   });
+}
+
+} // namespace mfem
@@ -0,0 +1,85 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_NORMAL_DERIV_RESTRICTION
+#define MFEM_NORMAL_DERIV_RESTRICTION
+
+#include "../mesh/mesh.hpp"
+
+namespace mfem
+{
+
+class FiniteElementSpace;
+enum class ElementDofOrdering;
+
+/// @brief Class to compute face normal derivatives (in reference coordinate) of
+/// an L2 grid function (used internally by L2FaceRestriction).
+class L2NormalDerivativeFaceRestriction
+{
+protected:
+   const FiniteElementSpace &fes; ///< The L2 finite element space.
+   const FaceType face_type; ///< Face type: either boundary or interior.
+   const int dim; ///< Dimension of the mesh.
+   const int nf; ///< Number of faces of the given @a face_type.
+   const int ne; ///< Number of elements.
+   int ne_type; ///< Number of elements with faces of type face type
+
+   Array<int> face_to_elem; ///< Face-wise information array.
+   Array<int> elem_to_face; ///< Element-wise information array.
+   Array<int> face_to_vol; ///< maps face index to volume index
+
+public:
+   /// @brief Constructor.
+   /// @param[in] fes_ The associated FiniteElementSpace (should be L2/DG).
+   /// @param[in] f_ordering Request a specific face dof ordering. Currently
+   /// only ElementDofOrdering::LEXICOGRAPHIC is supported.
+   /// @param[in] face_type_ Type of faces to compute restriction (interior or boundary).
+   L2NormalDerivativeFaceRestriction(const FiniteElementSpace &fes_,
+                                     const ElementDofOrdering f_ordering,
+                                     const FaceType face_type_);
+
+   /// @brief Computes the normal derivatives on the @a face_type faces of the mesh.
+   /// @param[in] x The L-vector degrees of freedom.
+   /// @param[out] y The face E(like)-vector degrees of freedom of the format
+   /// (face_dofs x vdim x 2 x nf) where nf is the number of faces of type @a
+   /// face_type. The face_dofs are ordered according to @a f_ordering specified
+   /// in the constructor.
+   void Mult(const Vector &x, Vector &y) const;
+
+   /// @brief Computes the transpose of the action of Mult(), accumulating into
+   ///        @a y with coefficient @a a.
+   /// @param x Face E-vector layout (face_dofs x vdim x 2 x nf).
+   /// @param y L-vector layout.
+   /// @param a Optional coefficient (y = y + a*R^t*x)
+   void AddMultTranspose(const Vector &x, Vector &y,
+                         const real_t a = 1.0) const;
+
+   /// @name Internal compute kernels. Public because of nvcc restriction.
+   ///@{
+
+   template <int T_D1D = 0>
+   void Mult2D(const Vector &x, Vector &y) const;
+
+   template <int T_D1D = 0>
+   void AddMultTranspose2D(const Vector &x, Vector &y, const real_t a) const;
+
+   template <int T_D1D = 0>
+   void Mult3D(const Vector &x, Vector &y) const;
+
+   template <int T_D1D = 0>
+   void AddMultTranspose3D(const Vector &x, Vector &y, const real_t a) const;
+
+   /// @}
+};
+
+}
+
+#endif // MFEM_RESTRICTION
@@ -368,6 +368,72 @@ const
   y.Add(a, Ytmp);
 }

+real_t ParBilinearForm::ParInnerProduct(const ParGridFunction &x,
+                                        const ParGridFunction &y) const
+{
+   MFEM_ASSERT(mat != NULL, "local matrix must be assembled");
+
+   real_t loc = InnerProduct(x, y);
+   real_t glob = 0.;
+
+   MPI_Allreduce(&loc, &glob, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                 pfes->GetComm());
+
+   return glob;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(const ParGridFunction &x,
+                                         const ParGridFunction &y) const
+{
+   MFEM_ASSERT(x.ParFESpace() == pfes, "the parallel spaces must match");
+   MFEM_ASSERT(y.ParFESpace() == pfes, "the parallel spaces must match");
+
+   HypreParVector *x_p = x.ParallelProject();
+   HypreParVector *y_p = y.ParallelProject();
+
+   real_t res = TrueInnerProduct(*x_p, *y_p);
+
+   delete x_p;
+   delete y_p;
+
+   return res;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(HypreParVector &x,
+                                         HypreParVector &y) const
+{
+   MFEM_VERIFY(p_mat.Ptr() != NULL, "parallel matrix must be assembled");
+
+   if (p_mat->GetType() != Operator::Hypre_ParCSR)
+   {
+      return TrueInnerProduct((const Vector&)x, (const Vector&)y);
+   }
+
+   HypreParVector *Ax = new HypreParVector(pfes);
+   HypreParMatrix *A = p_mat.As<HypreParMatrix>();
+
+   A->Mult(x, *Ax);
+
+   real_t res = mfem::InnerProduct(y, *Ax);
+
+   delete Ax;
+
+   return res;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(const Vector &x,
+                                         const Vector &y) const
+{
+   MFEM_VERIFY(p_mat.Ptr() != NULL, "parallel matrix must be assembled");
+
+   Vector Ax(pfes->GetTrueVSize());
+   p_mat->Mult(x, Ax);
+
+   real_t res = mfem::InnerProduct(pfes->GetComm(), y, Ax);
+
+   return res;
+}
+
 void ParBilinearForm::FormLinearSystem(
   const Array<int> &ess_tdof_list, Vector &x, Vector &b,
   OperatorHandle &A, Vector &X, Vector &B, int copy_interior)
@@ -173,6 +173,37 @@ public:
       vectors on the true dofs. */
   void TrueAddMult(const Vector &x, Vector &y, const real_t a = 1.0) const;

+   /// Compute $ y^T M x $
+   /** @warning The calculation is performed on local dofs, assuming that
+       the local vectors are consistent with the prolongations of the true
+       vectors (see ParGridFunction::Distribute()). If this is not the case,
+       use TrueInnerProduct(const ParGridFunction &, const ParGridFunction &)
+       instead.
+       @note It is assumed that the local matrix is assembled and it has
+       not been replaced by the parallel matrix through FormSystemMatrix().
+       @see TrueInnerProduct(const ParGridFunction&, const ParGridFunction&) */
+   real_t ParInnerProduct(const ParGridFunction &x,
+                          const ParGridFunction &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (grid function version)
+   /** @note The ParGridFunction%s are restricted to the true-vectors for
+       for calculation.
+       @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix().
+       @see ParInnerProduct(const ParGridFunction&, const ParGridFunction&) */
+   real_t TrueInnerProduct(const ParGridFunction &x,
+                           const ParGridFunction &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (Hypre vector version)
+   /** @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix(). */
+   real_t TrueInnerProduct(HypreParVector &x, HypreParVector &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (true-vector version)
+   /** @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix(). */
+   real_t TrueInnerProduct(const Vector &x, const Vector &y) const;
+
   /// Return the parallel FE space associated with the ParBilinearForm.
   ParFiniteElementSpace *ParFESpace() const { return pfes; }

@@ -861,17 +861,17 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
         }
      }

-      HYPRE_Int *i_diag = new HYPRE_Int[ldof+1];
-      HYPRE_Int *j_diag = new HYPRE_Int[ltdof];
-      real_t    *d_diag = new real_t[ltdof];
+      HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
+      real_t    *d_diag = Memory<real_t>(ltdof);
      int diag_counter;

-      HYPRE_Int *i_offd = new HYPRE_Int[ldof+1];
-      HYPRE_Int *j_offd = new HYPRE_Int[nnz_offd];
-      real_t    *d_offd = new real_t[nnz_offd];
+      HYPRE_Int *i_offd = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_offd = Memory<HYPRE_Int>(nnz_offd);
+      real_t    *d_offd = Memory<real_t>(nnz_offd);
      int offd_counter;

-      HYPRE_BigInt *cmap   = new HYPRE_BigInt[ldof-ltdof];
+      HYPRE_BigInt *cmap   = Memory<HYPRE_BigInt>(ldof-ltdof);

      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
      HYPRE_BigInt *row_starts = GetDofOffsets();
@@ -249,6 +249,8 @@ void ParGridFunction::ExchangeFaceNbrData()
   auto send_data_ptr = mpi_gpu_aware ? send_data.Read() : send_data.HostRead();
   auto face_nbr_data_ptr = mpi_gpu_aware ? face_nbr_data.Write() :
                            face_nbr_data.HostWrite();
+   // Wait for the kernel to be done since it updates what's sent and it may be async
+   if (mpi_gpu_aware) { MFEM_STREAM_SYNC; }
   for (int fn = 0; fn < num_face_nbrs; fn++)
   {
      int nbr_rank = pmesh->GetFaceNbrRank(fn);
@@ -518,7 +520,7 @@ void ParGridFunction::CountElementsPerVDof(Array<int> &elem_per_vdof) const
 }

 void ParGridFunction::GetDerivative(int comp, int der_comp,
-                                    ParGridFunction &der)
+                                    ParGridFunction &der) const
 {
   Array<int> overlap;
   AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
@@ -713,10 +715,10 @@ void ParGridFunction::ProjectBdrCoefficient(
         }
      }
   }
+   gcomm.Bcast<int>(values_counter.HostReadWrite());
   for (int i = 0; i < values_counter.Size(); i++)
   {
-      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
-                  bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
                  "internal error");
   }
 #endif
@@ -753,10 +755,10 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
 #ifdef MFEM_DEBUG
   Array<int> ess_vdofs_marker;
   pfes->GetEssentialVDofs(bdr_attr, ess_vdofs_marker);
+   gcomm.Bcast<int>(values_counter.HostReadWrite());
   for (int i = 0; i < values_counter.Size(); i++)
   {
-      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
-                  bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
                  "internal error: " << pfes->GetLocalTDofNumber(i) << ' ' << bool(
                     values_counter[i]));
   }
@@ -231,7 +231,7 @@ public:
   void CountElementsPerVDof(Array<int> &elem_per_vdof) const override;

   /// Parallel version of GridFunction::GetDerivative(); see its documentation.
-   void GetDerivative(int comp, int der_comp, ParGridFunction &der);
+   void GetDerivative(int comp, int der_comp, ParGridFunction &der) const;

   /** Sets the output vector @a dof_vals to the values of the degrees of
       freedom of element @a el. If @a el is greater than or equal to the number
@@ -262,7 +262,6 @@ public:
                              const Array<int> &attr) override
   { ProjectBdrCoefficient(coeff, NULL, attr); }

-   // Only the values in the master are guaranteed to be correct!
   void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                     const Array<int> &bdr_attr) override;

@@ -18,6 +18,7 @@
 #include "pgridfunc.hpp"
 #include "pfespace.hpp"
 #include "fespace.hpp"
+#include "fe/face_map_utils.hpp"
 #include "../general/forall.hpp"

 namespace mfem
@@ -277,21 +278,22 @@ void ParNCH1FaceRestriction::ComputeGatherIndices(
   gather_offsets[0] = 0;
 }

-ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
+ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &pfes_,
                                           ElementDofOrdering f_ordering,
                                           FaceType type,
                                           L2FaceValues m,
                                           bool build)
-   : L2FaceRestriction(fes, f_ordering, type, m, false)
+   : L2FaceRestriction(pfes_, f_ordering, type, m, false),
+     pfes(pfes_)
 {
   if (!build) { return; }
   if (nf==0) { return; }

-   CheckFESpace(f_ordering);
+   CheckFESpace();

-   ComputeScatterIndicesAndOffsets(f_ordering, type);
+   ComputeScatterIndicesAndOffsets();

-   ComputeGatherIndices(f_ordering, type);
+   ComputeGatherIndices();
 }

 ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
@@ -307,14 +309,8 @@ void ParL2FaceRestriction::DoubleValuedConformingMult(
   MFEM_ASSERT(
      m == L2FaceValues::DoubleValued,
      "This method should be called when m == L2FaceValues::DoubleValued.");
-   const ParFiniteElementSpace &pfes =
-      static_cast<const ParFiniteElementSpace&>(this->fes);
-   ParGridFunction x_gf;
-   x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(&pfes),
-                const_cast<Vector&>(x), 0);
-   // Face-neighbor information is only needed for interior faces. For boundary
-   // faces, no communication is required.
-   if (type == FaceType::Interior) { x_gf.ExchangeFaceNbrData(); }
+
+   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, type);

   // Early return only after calling ParGridFunction::ExchangeFaceNbrData,
   // otherwise MPI communication can hang.
@@ -329,7 +325,7 @@ void ParL2FaceRestriction::DoubleValuedConformingMult(
   auto d_indices1 = scatter_indices1.Read();
   auto d_indices2 = scatter_indices2.Read();
   auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
-   auto d_x_shared = Reshape(x_gf.FaceNbrData().Read(),
+   auto d_x_shared = Reshape(face_nbr_data.Read(),
                             t?vd:nsdofs, t?nsdofs:vd);
   auto d_y = Reshape(y.Write(), nface_dofs, vd, 2, nf);
   mfem::forall(nfdofs, [=] MFEM_HOST_DEVICE (int i)
@@ -567,13 +563,9 @@ void ParL2FaceRestriction::FillJAndData(const Vector &ea_data,
   });
 }

-void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();
-   const ParFiniteElementSpace &pfes =
-      static_cast<const ParFiniteElementSpace&>(this->fes);

   // Initialization of the offsets
   for (int i = 0; i <= ndofs; ++i)
@@ -622,9 +614,7 @@ void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets(
 }


-void ParL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParL2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -666,11 +656,11 @@ ParNCL2FaceRestriction::ParNCL2FaceRestriction(const ParFiniteElementSpace &fes,
   if (nf==0) { return; }
   x_interp.UseDevice(true);

-   CheckFESpace(f_ordering);
+   CheckFESpace();

-   ComputeScatterIndicesAndOffsets(f_ordering, type);
+   ComputeScatterIndicesAndOffsets();

-   ComputeGatherIndices(f_ordering, type);
+   ComputeGatherIndices();
 }

 void ParNCL2FaceRestriction::SingleValuedNonconformingMult(
@@ -979,9 +969,7 @@ void ParNCL2FaceRestriction::FillJAndData(const Vector &ea_data,
   MFEM_ABORT("Not yet implemented.");
 }

-void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -1064,9 +1052,7 @@ void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
   interpolations.InitializeNCInterpConfig();
 }

-void ParNCL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParNCL2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -139,9 +139,11 @@ public: // For nvcc
 class ParL2FaceRestriction : virtual public L2FaceRestriction
 {
 protected:
+   const ParFiniteElementSpace &pfes;
+
   /** @brief Constructs an ParL2FaceRestriction.

-       @param[in] fes        The ParFiniteElementSpace on which this operates
+       @param[in] pfes_      The ParFiniteElementSpace on which this operates
       @param[in] f_ordering Request a specific face dof ordering
       @param[in] type       Request internal or boundary faces dofs
       @param[in] m          Request the face dofs for elem1, or both elem1 and
@@ -149,7 +151,7 @@ protected:
       @param[in] build      Request the ParL2FaceRestriction to compute the
                             scatter/gather indices. False should only be used
                             when inheriting from ParL2FaceRestriction. */
-   ParL2FaceRestriction(const ParFiniteElementSpace& fes,
+   ParL2FaceRestriction(const ParFiniteElementSpace &pfes_,
                        ElementDofOrdering f_ordering,
                        FaceType type,
                        L2FaceValues m,
@@ -229,22 +231,14 @@ public:
 private:
   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
       for the gathering: E-vector to L-vector.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();

 public:
   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -381,22 +375,14 @@ private:
   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
       for the gathering: E-vector to L-vector, and the interpolators from
       coarse to fine face for master non-comforming faces.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();

 public:
   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -215,6 +215,10 @@ public:
   /// quadrature point, oriented relative to "element 1".
   int GetPermutedIndex(int idx, int iq) const override;

+   /// @brief Get the face index (in the standard Mesh numbering) associated
+   /// with face @a idx in the FaceQuadratureSpace.
+   int GetMeshFaceIndex(int idx) const { return face_indices[idx]; }
+
   /// @brief Returns the index associated with the face described by @a T.
   ///
   /// The index may differ from the mesh face or boundary element index
@@ -10,17 +10,16 @@
 // CONTRIBUTING.md for details.

 #include "restriction.hpp"
+#include "normal_deriv_restriction.hpp"
 #include "gridfunc.hpp"
 #include "fespace.hpp"
+#include "pgridfunc.hpp"
+#include "qspace.hpp"
+#include "fe/face_map_utils.hpp"
 #include "../general/forall.hpp"
+
 #include <climits>

-#ifdef MFEM_USE_MPI
-
-#include "pfespace.hpp"
-
-#endif
-
 namespace mfem
 {

@@ -907,112 +906,6 @@ void ConformingFaceRestriction::SetFaceDofsGatherIndices(
   }
 }

-static int ToLexOrdering2D(const int face_id, const int size1d, const int i)
-{
-   if (face_id==2 || face_id==3)
-   {
-      return size1d-1-i;
-   }
-   else
-   {
-      return i;
-   }
-}
-
-static int PermuteFace2D(const int face_id1, const int face_id2,
-                         const int orientation,
-                         const int size1d, const int index)
-{
-   int new_index;
-   // Convert from lex ordering
-   if (face_id1==2 || face_id1==3)
-   {
-      new_index = size1d-1-index;
-   }
-   else
-   {
-      new_index = index;
-   }
-   // Permute based on face orientations
-   if (orientation==1)
-   {
-      new_index = size1d-1-new_index;
-   }
-   return ToLexOrdering2D(face_id2, size1d, new_index);
-}
-
-static int ToLexOrdering3D(const int face_id, const int size1d, const int i,
-                           const int j)
-{
-   if (face_id==2 || face_id==1 || face_id==5)
-   {
-      return i + j*size1d;
-   }
-   else if (face_id==3 || face_id==4)
-   {
-      return (size1d-1-i) + j*size1d;
-   }
-   else // face_id==0
-   {
-      return i + (size1d-1-j)*size1d;
-   }
-}
-
-static int PermuteFace3D(const int face_id1, const int face_id2,
-                         const int orientation,
-                         const int size1d, const int index)
-{
-   int i=0, j=0, new_i=0, new_j=0;
-   i = index%size1d;
-   j = index/size1d;
-   // Convert from lex ordering
-   if (face_id1==3 || face_id1==4)
-   {
-      i = size1d-1-i;
-   }
-   else if (face_id1==0)
-   {
-      j = size1d-1-j;
-   }
-   // Permute based on face orientations
-   switch (orientation)
-   {
-      case 0:
-         new_i = i;
-         new_j = j;
-         break;
-      case 1:
-         new_i = j;
-         new_j = i;
-         break;
-      case 2:
-         new_i = j;
-         new_j = (size1d-1-i);
-         break;
-      case 3:
-         new_i = (size1d-1-i);
-         new_j = j;
-         break;
-      case 4:
-         new_i = (size1d-1-i);
-         new_j = (size1d-1-j);
-         break;
-      case 5:
-         new_i = (size1d-1-j);
-         new_j = (size1d-1-i);
-         break;
-      case 6:
-         new_i = (size1d-1-j);
-         new_j = i;
-         break;
-      case 7:
-         new_i = i;
-         new_j = (size1d-1-j);
-         break;
-   }
-   return ToLexOrdering3D(face_id2, size1d, new_i, new_j);
-}
-
 // Permute dofs or quads on a face for e2 to match with the ordering of e1
 int PermuteFaceL2(const int dim, const int face_id1,
                  const int face_id2, const int orientation,
@@ -1023,9 +916,9 @@ int PermuteFaceL2(const int dim, const int face_id1,
      case 1:
         return 0;
      case 2:
-         return PermuteFace2D(face_id1, face_id2, orientation, size1d, index);
+         return internal::PermuteFace2D(face_id1, face_id2, orientation, size1d, index);
      case 3:
-         return PermuteFace3D(face_id1, face_id2, orientation, size1d, index);
+         return internal::PermuteFace3D(face_id1, face_id2, orientation, size1d, index);
      default:
         MFEM_ABORT("Unsupported dimension.");
         return 0;
@@ -1038,6 +931,7 @@ L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
                                     const L2FaceValues m,
                                     bool build)
   : fes(fes),
+     ordering(f_ordering),
     nf(fes.GetNFbyType(type)),
     ne(fes.GetNE()),
     vdim(fes.GetVDim()),
@@ -1060,11 +954,9 @@ L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
   width = fes.GetVSize();
   if (!build) { return; }

-   CheckFESpace(f_ordering);
-
-   ComputeScatterIndicesAndOffsets(f_ordering,type);
-
-   ComputeGatherIndices(f_ordering, type);
+   CheckFESpace();
+   ComputeScatterIndicesAndOffsets();
+   ComputeGatherIndices();
 }

 L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
@@ -1327,7 +1219,7 @@ void L2FaceRestriction::AddFaceMatricesToElementMatrices(const Vector &fea_data,
   }
 }

-void L2FaceRestriction::CheckFESpace(const ElementDofOrdering f_ordering)
+void L2FaceRestriction::CheckFESpace()
 {
 #ifdef MFEM_USE_MPI

@@ -1351,7 +1243,7 @@ void L2FaceRestriction::CheckFESpace(const ElementDofOrdering f_ordering)
               "Only Gauss-Lobatto and Bernstein basis are supported in "
               "L2FaceRestriction.");
   if (nf==0) { return; }
-   const bool dof_reorder = (f_ordering == ElementDofOrdering::LEXICOGRAPHIC);
+   const bool dof_reorder = (ordering == ElementDofOrdering::LEXICOGRAPHIC);
   if (!dof_reorder)
   {
      MFEM_ABORT("Non-Tensor L2FaceRestriction not yet implemented.");
@@ -1371,9 +1263,7 @@ void L2FaceRestriction::CheckFESpace(const ElementDofOrdering f_ordering)
 #endif
 }

-void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType face_type)
+void L2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();
   // Initialization of the offsets
@@ -1389,16 +1279,16 @@ void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
      MFEM_ASSERT(!face.IsShared(),
                  "Unexpected shared face in L2FaceRestriction.");
-      if ( face.IsOfFaceType(face_type) )
+      if ( face.IsOfFaceType(type) )
      {
         SetFaceDofsScatterIndices1(face,f_ind);
         if ( m==L2FaceValues::DoubleValued )
         {
-            if ( face_type==FaceType::Interior && face.IsInterior() )
+            if ( type==FaceType::Interior && face.IsInterior() )
            {
               PermuteAndSetFaceDofsScatterIndices2(face,f_ind);
            }
-            else if ( face_type==FaceType::Boundary && face.IsBoundary() )
+            else if ( type==FaceType::Boundary && face.IsBoundary() )
            {
               SetBoundaryDofsScatterIndices2(face,f_ind);
            }
@@ -1415,9 +1305,7 @@ void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
   }
 }

-void L2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType face_type)
+void L2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();
   // Computation of gather_indices
@@ -1427,11 +1315,11 @@ void L2FaceRestriction::ComputeGatherIndices(
      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
      MFEM_ASSERT(!face.IsShared(),
                  "Unexpected shared face in L2FaceRestriction.");
-      if ( face.IsOfFaceType(face_type) )
+      if ( face.IsOfFaceType(type) )
      {
         SetFaceDofsGatherIndices1(face,f_ind);
         if ( m==L2FaceValues::DoubleValued &&
-              face_type==FaceType::Interior &&
+              type==FaceType::Interior &&
              face.IsLocal())
         {
            PermuteAndSetFaceDofsGatherIndices2(face,f_ind);
@@ -1598,6 +1486,28 @@ void L2FaceRestriction::PermuteAndSetFaceDofsGatherIndices2(
   }
 }

+void L2FaceRestriction::NormalDerivativeMult(const Vector &x, Vector &y) const
+{
+   EnsureNormalDerivativeRestriction();
+   normal_deriv_restr->Mult(x, y);
+}
+
+void L2FaceRestriction::NormalDerivativeAddMultTranspose(const Vector &x,
+                                                         Vector &y) const
+{
+   EnsureNormalDerivativeRestriction();
+   normal_deriv_restr->AddMultTranspose(x, y);
+}
+
+void L2FaceRestriction::EnsureNormalDerivativeRestriction() const
+{
+   if (!normal_deriv_restr)
+   {
+      normal_deriv_restr.reset(
+         new L2NormalDerivativeFaceRestriction(fes, ordering, type));
+   }
+}
+
 InterpolationManager::InterpolationManager(const FiniteElementSpace &fes,
                                           ElementDofOrdering ordering,
                                           FaceType type)
@@ -1775,11 +1685,11 @@ NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
   if (!build) { return; }
   x_interp.UseDevice(true);

-   CheckFESpace(f_ordering);
+   CheckFESpace();

-   ComputeScatterIndicesAndOffsets(f_ordering, type);
+   ComputeScatterIndicesAndOffsets();

-   ComputeGatherIndices(f_ordering, type);
+   ComputeGatherIndices();
 }

 NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
@@ -2259,18 +2169,16 @@ int ToLexOrdering(const int dim, const int face_id, const int size1d,
      case 1:
         return 0;
      case 2:
-         return ToLexOrdering2D(face_id, size1d, index);
+         return internal::ToLexOrdering2D(face_id, size1d, index);
      case 3:
-         return ToLexOrdering3D(face_id, size1d, index%size1d, index/size1d);
+         return internal::ToLexOrdering3D(face_id, size1d, index%size1d, index/size1d);
      default:
         MFEM_ABORT("Unsupported dimension.");
         return 0;
   }
 }

-void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -2334,9 +2242,7 @@ void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
   interpolations.InitializeNCInterpConfig();
 }

-void NCL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void NCL2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();
   // Computation of gather_indices
@@ -2376,4 +2282,33 @@ void NCL2FaceRestriction::ComputeGatherIndices(
   gather_offsets[0] = 0;
 }

+Vector GetLVectorFaceNbrData(
+   const FiniteElementSpace &fes, const Vector &x, FaceType ftype)
+{
+#ifdef MFEM_USE_MPI
+   if (ftype == FaceType::Interior)
+   {
+      if (auto *pfes = const_cast<ParFiniteElementSpace*>
+                       (dynamic_cast<const ParFiniteElementSpace*>(&fes)))
+      {
+         if (auto *x_gf = const_cast<ParGridFunction*>
+                          (dynamic_cast<const ParGridFunction*>(&x)))
+         {
+            Vector &gf_face_nbr = x_gf->FaceNbrData();
+            if (gf_face_nbr.Size() == 0) { x_gf->ExchangeFaceNbrData(); }
+            gf_face_nbr.Read();
+            return Vector(gf_face_nbr, 0, gf_face_nbr.Size());
+         }
+         else
+         {
+            ParGridFunction gf(pfes, const_cast<Vector&>(x));
+            gf.ExchangeFaceNbrData();
+            return std::move(gf.FaceNbrData());
+         }
+      }
+   }
+#endif
+   return Vector();
+}
+
 } // namespace mfem
@@ -14,6 +14,7 @@

 #include "../linalg/operator.hpp"
 #include "../mesh/mesh.hpp"
+#include "normal_deriv_restriction.hpp"

 namespace mfem
 {
@@ -21,6 +22,8 @@ namespace mfem
 class FiniteElementSpace;
 enum class ElementDofOrdering;

+class FaceQuadratureSpace;
+
 /// Abstract base class that defines an interface for element restrictions.
 class ElementRestrictionOperator : public Operator
 {
@@ -220,6 +223,45 @@ public:
      y = 0.0;
      AddMultTranspose(x, y);
   }
+
+   /** @brief For each face, sets @a y to the partial derivative of @a x with
+              respect to the reference coordinate whose direction is
+              perpendicular to the face on the reference element.
+
+    @details This is not the normal derivative in physical coordinates, but can
+             be mapped to the physical normal derivative using the element
+             Jacobian and the tangential derivatives (in reference coordinates)
+             which can be computed from the face values (provided by Mult).
+
+             Note that due to the polynomial degree of the element mapping, the
+             physical normal derivative may be a higher degree polynomial than
+             the restriction of the values to the face. However, the normal
+             derivative in reference coordinates has degree-1, and therefore can
+             be exactly represented with the degrees of freedom of a face
+             E-vector.
+
+    @param[in]     x The L-vector degrees of freedom.
+    @param[in,out] y The reference normal derivative degrees of freedom. Is
+                     E-vector like.
+    */
+   virtual void NormalDerivativeMult(const Vector &x, Vector &y) const
+   {
+      MFEM_ABORT("Not implemented for this restriction operator.");
+   }
+
+   /** @brief Add the face reference-normal derivative degrees of freedom in @a
+              x to the element degrees of freedom in @a y.
+
+       @details see NormalDerivativeMult.
+
+       @param[in]     x The degrees of freedom of the face reference-normal
+                        derivative. Is E-vector like.
+       @param[in,out] y The L-vector degrees of freedom.
+   */
+   virtual void NormalDerivativeAddMultTranspose(const Vector &x, Vector &y) const
+   {
+      MFEM_ABORT("Not implemented for this restriction operator.");
+   }
 };

 /// @brief Operator that extracts face degrees of freedom for H1, ND, or RT
@@ -365,6 +407,7 @@ class L2FaceRestriction : public FaceRestriction
 {
 protected:
   const FiniteElementSpace &fes;
+   const ElementDofOrdering ordering;
   const int nf; // Number of faces of the requested type
   const int ne; // Number of elements
   const int vdim; // vdim
@@ -379,6 +422,7 @@ protected:
   Array<int> scatter_indices2; // Scattering indices for element 2 on each face
   Array<int> gather_offsets; // offsets for the gathering indices of each dof
   Array<int> gather_indices; // gathering indices for each dof
+   mutable std::unique_ptr<L2NormalDerivativeFaceRestriction> normal_deriv_restr;

   /** @brief Constructs an L2FaceRestriction.

@@ -487,34 +531,49 @@ public:
   virtual void AddFaceMatricesToElementMatrices(const Vector &fea_data,
                                                 Vector &ea_data) const;

+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+      face E-Vector.
+
+      @param[in]  x The L-vector degrees of freedom.
+      @param[out] y The face E-Vector degrees of freedom with the given format:
+                    (face_dofs x vdim x 2 x nf) where nf is the number of
+                    interior or boundary faces requested by @a type in the
+                    constructor. The face_dofs are ordered according to the
+                    given ElementDofOrdering. */
+   void NormalDerivativeMult(const Vector &x, Vector &y) const override;
+
+   /** @brief Add the face reference-normal derivative degrees of freedom in @a
+              x to the element degrees of freedom in @a y.
+
+       @details see NormalDerivativeMult.
+
+       @param[in]     x The degrees of freedom of the face reference-normal
+                        derivative. Is E-vector like.
+       @param[in,out] y The L-vector degrees of freedom.
+   */
+   void NormalDerivativeAddMultTranspose(const Vector &x,
+                                         Vector &y) const override;
 private:
   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
       for the gathering: E-vector to L-vector.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();
+
+   /// Create the internal normal derivative restriction operator if needed.
+   void EnsureNormalDerivativeRestriction() const;

 protected:
   mutable Array<int> face_map; // Used in the computation of GetFaceDofs

   /** @brief Verify that L2FaceRestriction is built from an L2 FESpace.
-
-       @param[in] f_ordering The requested face dof ordering.
   */
-   void CheckFESpace(const ElementDofOrdering f_ordering);
+   void CheckFESpace();

   /** @brief Set the scattering indices of elem1, and increment the offsets for
       the face described by the @a face. The ordering of the face dofs of elem1
@@ -938,22 +997,14 @@ private:
   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
       for the gathering: E-vector to L-vector, and the interpolators from
       coarse to fine face for master non-comforming faces.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();

 public:
   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -1015,7 +1066,6 @@ public:
   void DoubleValuedNonconformingTransposeInterpolationInPlace(Vector& x) const;
 };

-
 /** @brief Convert a dof face index from Native ordering to lexicographic
    ordering for quads and hexes.

@@ -1044,6 +1094,20 @@ int PermuteFaceL2(const int dim, const int face_id1,
                  const int face_id2, const int orientation,
                  const int size1d, const int index);

+/// @brief Return the face-neighbor data given the L-vector @a x.
+///
+/// If the input vector @a x is a ParGridFunction with non-empty face-neighbor
+/// data, return an alias to ParGridFunction::FaceNbrData() (avoiding an
+/// unneeded call to ParGridFunction::ExchangeFaceNbrData).
+///
+/// Otherwise, create a temporary ParGridFunction, exchange the face-neighbor
+/// data, and return the resulting vector.
+///
+/// If @a fes is not a parallel space, or if @a ftype is not FaceType::Interior,
+/// return an empty vector.
+Vector GetLVectorFaceNbrData(
+   const FiniteElementSpace &fes, const Vector &x, FaceType ftype);
+
 }

 #endif // MFEM_RESTRICTION
@@ -503,7 +503,7 @@ public:

      Array<int> vdofs;
      const Array<int> *dof_map = sol_fe.GetDofMap();
-      const int *dof_map_ = dof_map->GetData();
+      const int *dof_map_ = (dof_map) ? dof_map->GetData() : NULL;
      DenseMatrix M_loc_perm(dofs*vdim,dofs*vdim); // initialized with zeros

      const int NE = mesh.GetNE();
@@ -98,7 +98,7 @@ protected:
      const int vsize = sizeof(vint_t)/sizeof(attrib[0][0]);
      for (int i = 0; i < NE; i++)
      {
-         for (int j = 0; j < vsize; i++)
+         for (int j = 0; j < vsize; j++)
         {
            attrib[i][j] = elements[el+j+i*vsize]->GetAttribute();
         }
@@ -2949,6 +2949,15 @@ void TMOP_Integrator::EnableSurfaceFitting(const GridFunction &s0,
   MFEM_VERIFY(surf_fit_pos == NULL,
               "Using both fitting approaches is not supported.");

+   const int dim = s0.FESpace()->GetMesh()->Dimension();
+   Mesh *mesh = s0.FESpace()->GetMesh();
+   MFEM_VERIFY(mesh->GetNodes()->Size() == dim*s0.Size(),
+               "Mesh and level-set polynomial order must be the same.");
+   const H1_FECollection *fec = dynamic_cast<const H1_FECollection *>
+                                (s0.FESpace()->FEColl());
+   MFEM_VERIFY(fec, "Only H1_FECollection is supported for the surface fitting "
+               "grid function.");
+
   delete surf_fit_gf;
   surf_fit_gf = new GridFunction(s0);
   surf_fit_gf->CountElementsPerVDof(surf_fit_dof_count);
@@ -2987,12 +2996,24 @@ void TMOP_Integrator::EnableSurfaceFitting(const GridFunction &pos,
 void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
                                           const Array<bool> &smarker,
                                           Coefficient &coeff,
-                                           AdaptivityEvaluator &ae)
+                                           AdaptivityEvaluator &ae,
+                                           AdaptivityEvaluator *aegrad,
+                                           AdaptivityEvaluator *aehess)
 {
   // To have both we must duplicate the markers.
   MFEM_VERIFY(surf_fit_pos == NULL,
               "Using both fitting approaches is not supported.");

+   const int dim = s0.FESpace()->GetMesh()->Dimension();
+   ParMesh *pmesh = s0.ParFESpace()->GetParMesh();
+   MFEM_VERIFY(pmesh->GetNodes()->Size() == dim*s0.Size(),
+               "Mesh and level-set polynomial order must be the same.");
+   const H1_FECollection *fec = dynamic_cast<const H1_FECollection *>
+                                (s0.FESpace()->FEColl());
+   MFEM_VERIFY(fec, "Only H1_FECollection is supported for the surface fitting "
+               "grid function.");
+
+
   delete surf_fit_gf;
   surf_fit_gf = new GridFunction(s0);
   s0.CountElementsPerVDof(surf_fit_dof_count);
@@ -3000,11 +3021,80 @@ void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
   surf_fit_coeff = &coeff;
   surf_fit_eval = &ae;

-   surf_fit_eval->SetParMetaInfo(*s0.ParFESpace()->GetParMesh(),
-                                 *s0.ParFESpace());
+   surf_fit_eval->SetParMetaInfo(*pmesh, *s0.ParFESpace());
   surf_fit_eval->SetInitialField
   (*surf_fit_gf->FESpace()->GetMesh()->GetNodes(), *surf_fit_gf);
-   surf_fit_gf_bg = false;
+
+   if (!aegrad) { return; }
+
+   MFEM_VERIFY(aehess, "AdaptivityEvaluator for Hessians must be provided too.");
+
+   ParFiniteElementSpace *fes = s0.ParFESpace();
+
+   // FE space for gradients.
+   delete surf_fit_grad;
+   H1_FECollection *fec_grad = new H1_FECollection(fec->GetOrder(), dim,
+                                                   fec->GetBasisType());
+   ParFiniteElementSpace *fes_grad = new ParFiniteElementSpace(pmesh, fec_grad,
+                                                               dim);
+   // Initial gradients.
+   surf_fit_grad = new GridFunction(fes_grad);
+   surf_fit_grad->MakeOwner(fec_grad);
+   for (int d = 0; d < dim; d++)
+   {
+      ParGridFunction surf_fit_grad_comp(fes, surf_fit_grad->GetData()+d*s0.Size());
+      s0.GetDerivative(1, d, surf_fit_grad_comp);
+   }
+   surf_fit_eval_grad = aegrad;
+   surf_fit_eval_grad->SetParMetaInfo(*pmesh, *fes_grad);
+   surf_fit_eval_grad->SetInitialField(*pmesh->GetNodes(), *surf_fit_grad);
+
+   // FE space for Hessians.
+   delete surf_fit_hess;
+   H1_FECollection *fec_hess = new H1_FECollection(fec->GetOrder(), dim,
+                                                   fec->GetBasisType());
+   ParFiniteElementSpace *fes_hess = new ParFiniteElementSpace(pmesh, fec_hess,
+                                                               dim*dim);
+   // Initial Hessians.
+   surf_fit_hess = new GridFunction(fes_hess);
+   surf_fit_hess->MakeOwner(fec_hess);
+   int id = 0;
+   for (int d = 0; d < dim; d++)
+   {
+      for (int idir = 0; idir < dim; idir++)
+      {
+         ParGridFunction surf_fit_grad_comp(fes,
+                                            surf_fit_grad->GetData()+d*s0.Size());
+         ParGridFunction surf_fit_hess_comp(fes,
+                                            surf_fit_hess->GetData()+id*s0.Size());
+         surf_fit_grad_comp.GetDerivative(1, idir, surf_fit_hess_comp);
+         id++;
+      }
+   }
+   surf_fit_eval_hess = aehess;
+   surf_fit_eval_hess->SetParMetaInfo(*pmesh, *fes_hess);
+   surf_fit_eval_hess->SetInitialField(*pmesh->GetNodes(), *surf_fit_hess);
+
+   // Store DOF indices that are marked for fitting. Used to reduce work for
+   // transferring information between source/background and current mesh.
+   surf_fit_marker_dof_index.SetSize(0);
+#ifdef MFEM_USE_GSLIB
+   if (dynamic_cast<InterpolatorFP *>(surf_fit_eval) &&
+       dynamic_cast<InterpolatorFP *>(surf_fit_eval_grad) &&
+       dynamic_cast<InterpolatorFP *>(surf_fit_eval_hess))
+   {
+      for (int i = 0; i < surf_fit_marker->Size(); i++)
+      {
+         if ((*surf_fit_marker)[i] == true)
+         {
+            surf_fit_marker_dof_index.Append(i);
+         }
+      }
+   }
+#endif
+
+   *surf_fit_grad = 0.0;
+   *surf_fit_hess = 0.0;
 }

 void TMOP_Integrator::EnableSurfaceFittingFromSource(
@@ -3022,16 +3112,17 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
   // Setup for level set function
   delete surf_fit_gf;
   surf_fit_gf = new GridFunction(s0);
-   *surf_fit_gf = 0.0;
   surf_fit_marker = &smarker;
   surf_fit_coeff = &coeff;
   surf_fit_eval = &ae;

-   surf_fit_gf_bg = true;
   surf_fit_eval->SetParMetaInfo(*s_bg.ParFESpace()->GetParMesh(),
                                 *s_bg.ParFESpace());
   surf_fit_eval->SetInitialField
   (*s_bg.FESpace()->GetMesh()->GetNodes(), s_bg);
+   GridFunction *nodes = s0.FESpace()->GetMesh()->GetNodes();
+   surf_fit_eval->ComputeAtNewPosition(*nodes, *surf_fit_gf,
+                                       nodes->FESpace()->GetOrdering());

   // Setup for gradient on background mesh
   MFEM_VERIFY(s_bg_grad.ParFESpace()->GetOrdering() ==
@@ -3041,11 +3132,11 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
   delete surf_fit_grad;
   surf_fit_grad = new GridFunction(s0_grad);
   *surf_fit_grad = 0.0;
-   surf_fit_eval_bg_grad = &age;
-   surf_fit_eval_bg_hess = &ahe;
-   surf_fit_eval_bg_grad->SetParMetaInfo(*s_bg_grad.ParFESpace()->GetParMesh(),
-                                         *s_bg_grad.ParFESpace());
-   surf_fit_eval_bg_grad->SetInitialField
+   surf_fit_eval_grad = &age;
+   surf_fit_eval_hess = &ahe;
+   surf_fit_eval_grad->SetParMetaInfo(*s_bg_grad.ParFESpace()->GetParMesh(),
+                                      *s_bg_grad.ParFESpace());
+   surf_fit_eval_grad->SetInitialField
   (*s_bg_grad.FESpace()->GetMesh()->GetNodes(), s_bg_grad);

   // Setup for Hessian on background mesh
@@ -3056,9 +3147,9 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
   delete surf_fit_hess;
   surf_fit_hess = new GridFunction(s0_hess);
   *surf_fit_hess = 0.0;
-   surf_fit_eval_bg_hess->SetParMetaInfo(*s_bg_hess.ParFESpace()->GetParMesh(),
-                                         *s_bg_hess.ParFESpace());
-   surf_fit_eval_bg_hess->SetInitialField
+   surf_fit_eval_hess->SetParMetaInfo(*s_bg_hess.ParFESpace()->GetParMesh(),
+                                      *s_bg_hess.ParFESpace());
+   surf_fit_eval_hess->SetInitialField
   (*s_bg_hess.FESpace()->GetMesh()->GetNodes(), s_bg_hess);

   // Count number of zones that share each of the DOFs
@@ -3863,7 +3954,7 @@ void TMOP_Integrator::AssembleElemVecSurfFit(const FiniteElement &el_x,

   Vector sigma_e(dof_s);
   DenseMatrix surf_fit_grad_e(dof_s, dim);
-   if (surf_fit_gf || surf_fit_gf_bg)
+   if (surf_fit_gf)
   {
      surf_fit_gf->GetSubVector(vdofs, sigma_e);

@@ -3871,7 +3962,7 @@ void TMOP_Integrator::AssembleElemVecSurfFit(const FiniteElement &el_x,
      // The FE coefficients of the gradient go in surf_fit_grad_e.
      Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
      DenseMatrix grad_phys; // This will be (dof x dim, dof).
-      if (surf_fit_gf_bg)
+      if (surf_fit_grad)
      {
         surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
         surf_fit_grad->GetSubVector(dofs, grad_ptr);
@@ -3945,7 +4036,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
   Vector sigma_e(dof_s);
   DenseMatrix surf_fit_grad_e(dof_s, dim);
   DenseMatrix surf_fit_hess_e(dof_s, dim*dim);
-   if (surf_fit_gf || surf_fit_gf_bg)
+   if (surf_fit_gf)
   {
      surf_fit_gf->GetSubVector(vdofs, sigma_e);

@@ -3953,7 +4044,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
      // The FE coefficients of the gradient go in surf_fit_grad_e.
      Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
      DenseMatrix grad_phys; // This will be (dof x dim, dof).
-      if (surf_fit_gf_bg)
+      if (surf_fit_grad)
      {
         surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
         surf_fit_grad->GetSubVector(dofs, grad_ptr);
@@ -3967,7 +4058,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
      // Project the Hessian of sigma in the same space.
      // The FE coefficients of the Hessian go in surf_fit_hess_e.
      Vector hess_ptr(surf_fit_hess_e.GetData(), dof_s*dim*dim);
-      if (surf_fit_gf_bg)
+      if (surf_fit_hess)
      {
         surf_fit_hess->FESpace()->GetElementVDofs(el_id, dofs);
         surf_fit_hess->GetSubVector(dofs, hess_ptr);
@@ -3994,7 +4085,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
      Tpr.SetIntPoint(&ip);
      real_t w = surf_fit_normal * surf_fit_coeff->Eval(Tpr, ip);

-      if (surf_fit_gf || surf_fit_gf_bg)
+      if (surf_fit_gf)
      {
         Vector gg_ptr(surf_fit_hess_s.GetData(), dim * dim);
         surf_fit_hess_e.GetRow(s, gg_ptr);
@@ -4376,6 +4467,130 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
   dx = detv_avg_min / dxscale;
 }

+void TMOP_Integrator::RemapSurfaceFittingLevelSetAtNodes(const Vector &new_x,
+                                                         int new_x_ordering)
+{
+   if (!surf_fit_gf) { return; }
+
+   if (surf_fit_marker_dof_index.Size())
+   {
+      // Interpolate information only at DOFs marked for fitting.
+      const int dim = surf_fit_gf->FESpace()->GetMesh()->Dimension();
+      const int cnt = surf_fit_marker_dof_index.Size();
+      const int total_cnt = new_x.Size()/dim;
+      Vector new_x_sorted(cnt*dim);
+      if (new_x_ordering == 0)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               new_x_sorted(i + d*cnt) = new_x(dof_index + d*total_cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < dim; d++)
+            {
+               new_x_sorted(d + i*dim) = new_x(d + dof_index*dim);
+            }
+         }
+      }
+
+      // Interpolate values of the LS.
+      Vector surf_fit_gf_int, surf_fit_grad_int, surf_fit_hess_int;
+      surf_fit_eval->ComputeAtNewPosition(new_x_sorted, surf_fit_gf_int,
+                                          new_x_ordering);
+      for (int i = 0; i < cnt; i++)
+      {
+         int dof_index = surf_fit_marker_dof_index[i];
+         (*surf_fit_gf)[dof_index] = surf_fit_gf_int(i);
+      }
+
+      // Interpolate gradients of the LS.
+      surf_fit_eval_grad->ComputeAtNewPosition(new_x_sorted, surf_fit_grad_int,
+                                               new_x_ordering);
+      // Assumes surf_fit_grad and surf_fit_gf share the same space
+      const int grad_dim = surf_fit_grad->VectorDim();
+      const int grad_cnt = surf_fit_grad->Size()/grad_dim;
+      if (surf_fit_grad->FESpace()->GetOrdering() == Ordering::byNODES)
+      {
+         for (int d = 0; d < grad_dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               (*surf_fit_grad)[dof_index + d*grad_cnt] =
+                  surf_fit_grad_int(i + d*cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < grad_dim; d++)
+            {
+               (*surf_fit_grad)[dof_index*grad_dim + d] =
+                  surf_fit_grad_int(i*grad_dim + d);
+            }
+         }
+      }
+
+      // Interpolate Hessians of the LS.
+      surf_fit_eval_hess->ComputeAtNewPosition(new_x_sorted, surf_fit_hess_int,
+                                               new_x_ordering);
+      // Assumes surf_fit_hess and surf_fit_gf share the same space
+      const int hess_dim = surf_fit_hess->VectorDim();
+      const int hess_cnt = surf_fit_hess->Size()/hess_dim;
+      if (surf_fit_hess->FESpace()->GetOrdering() == Ordering::byNODES)
+      {
+         for (int d = 0; d < hess_dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               (*surf_fit_hess)[dof_index + d*hess_cnt] =
+                  surf_fit_hess_int(i + d*cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < hess_dim; d++)
+            {
+               (*surf_fit_hess)[dof_index*hess_dim + d] =
+                  surf_fit_hess_int(i*hess_dim + d);
+            }
+         }
+      }
+
+   }
+   else
+   {
+      surf_fit_eval->ComputeAtNewPosition(new_x, *surf_fit_gf, new_x_ordering);
+      if (surf_fit_eval_grad)
+      {
+         surf_fit_eval_grad->ComputeAtNewPosition(new_x, *surf_fit_grad,
+                                                  new_x_ordering);
+      }
+      if (surf_fit_eval_hess)
+      {
+         surf_fit_eval_hess->ComputeAtNewPosition(new_x, *surf_fit_hess,
+                                                  new_x_ordering);
+      }
+   }
+}
+
 void TMOP_Integrator::
 UpdateAfterMeshPositionChange(const Vector &x_new,
                              const FiniteElementSpace &x_fes)
@@ -4406,112 +4621,11 @@ UpdateAfterMeshPositionChange(const Vector &x_new,
      adapt_lim_eval->ComputeAtNewPosition(x_new, *adapt_lim_gf, ordering);
   }

-   // Update surf_fit_gf if surface fitting is enabled.
+   // Update surf_fit_gf (and optionally its gradients) if surface
+   // fitting is enabled.
   if (surf_fit_gf)
   {
-      if (surf_fit_gf_bg)
-      {
-         // Interpolate information for only DOFs marked for fitting.
-         const int dim = surf_fit_gf->FESpace()->GetMesh()->Dimension();
-         const int cnt = surf_fit_marker_dof_index.Size();
-         const int total_cnt = x_new.Size()/dim;
-         Vector new_x_sorted(cnt*dim);
-         if (ordering == 0)
-         {
-            for (int d = 0; d < dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  new_x_sorted(i + d*cnt) = x_new(dof_index + d*total_cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < dim; d++)
-               {
-                  new_x_sorted(d + i*dim) = x_new(d + dof_index*dim);
-               }
-            }
-         }
-
-         Vector surf_fit_gf_int, surf_fit_grad_int, surf_fit_hess_int;
-         surf_fit_eval->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_gf_int, ordering);
-         for (int i = 0; i < cnt; i++)
-         {
-            int dof_index = surf_fit_marker_dof_index[i];
-            (*surf_fit_gf)[dof_index] = surf_fit_gf_int(i);
-         }
-
-         surf_fit_eval_bg_grad->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_grad_int, ordering);
-         // Assumes surf_fit_grad and surf_fit_gf share the same space
-         const int grad_dim = surf_fit_grad->VectorDim();
-         const int grad_cnt = surf_fit_grad->Size()/grad_dim;
-         if (surf_fit_grad->FESpace()->GetOrdering() == Ordering::byNODES)
-         {
-            for (int d = 0; d < grad_dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  (*surf_fit_grad)[dof_index + d*grad_cnt] =
-                     surf_fit_grad_int(i + d*cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < grad_dim; d++)
-               {
-                  (*surf_fit_grad)[dof_index*dim + d] =
-                     surf_fit_grad_int(i*dim + d);
-               }
-            }
-         }
-
-         surf_fit_eval_bg_hess->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_hess_int, ordering);
-         // Assumes surf_fit_hess and surf_fit_gf share the same space
-         const int hess_dim = surf_fit_hess->VectorDim();
-         const int hess_cnt = surf_fit_hess->Size()/hess_dim;
-         if (surf_fit_hess->FESpace()->GetOrdering() == Ordering::byNODES)
-         {
-            for (int d = 0; d < hess_dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  (*surf_fit_hess)[dof_index + d*hess_cnt] =
-                     surf_fit_hess_int(i + d*cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < hess_dim; d++)
-               {
-                  (*surf_fit_hess)[dof_index*dim + d] =
-                     surf_fit_hess_int(i*dim + d);
-               }
-            }
-         }
-      }
-      else
-      {
-         surf_fit_eval->ComputeAtNewPosition(x_new, *surf_fit_gf, ordering);
-      }
+      RemapSurfaceFittingLevelSetAtNodes(x_new, ordering);
   }
 }

@@ -1784,12 +1784,11 @@ protected:
   // Fitting to given physical positions.
   TMOP_QuadraticLimiter *surf_fit_limiter; // Owned. Created internally.
   const GridFunction *surf_fit_pos;        // Not owned. Positions to fit.
-   real_t surf_fit_normal;
-   bool surf_fit_gf_bg;
-   GridFunction *surf_fit_grad, *surf_fit_hess;
-   AdaptivityEvaluator *surf_fit_eval_bg_grad, *surf_fit_eval_bg_hess;
-   Array<int> surf_fit_dof_count;
-   Array<int> surf_fit_marker_dof_index;
+   real_t surf_fit_normal;                  // Normalization factor.
+   GridFunction *surf_fit_grad, *surf_fit_hess; // Owned. Created internally.
+   AdaptivityEvaluator *surf_fit_eval_grad, *surf_fit_eval_hess; // Not owned.
+   Array<int> surf_fit_dof_count;            // Number of dofs per node.
+   Array<int> surf_fit_marker_dof_index;     // Indices of nodes to fit.

   DiscreteAdaptTC *discr_tc;

@@ -1985,6 +1984,10 @@ protected:
   real_t ComputeUntanglerMaxMuBarrier(const Vector &x,
                                       const FiniteElementSpace &fes);

+   // Remaps the internal surface fitting gridfunction object at provided
+   // locations.
+   void RemapSurfaceFittingLevelSetAtNodes(const Vector &new_x,
+                                           int new_x_ordering);
 public:
   /** @param[in] m    TMOP_QualityMetric for r-adaptivity (not owned).
       @param[in] tc   Target-matrix construction algorithm to use (not owned).
@@ -2000,9 +2003,8 @@ public:
        surf_fit_marker(NULL), surf_fit_coeff(NULL),
        surf_fit_gf(NULL), surf_fit_eval(NULL),
        surf_fit_limiter(NULL), surf_fit_pos(NULL),
-        surf_fit_normal(1.0),
-        surf_fit_gf_bg(false), surf_fit_grad(NULL), surf_fit_hess(NULL),
-        surf_fit_eval_bg_grad(NULL), surf_fit_eval_bg_hess(NULL),
+        surf_fit_normal(1.0), surf_fit_grad(NULL), surf_fit_hess(NULL),
+        surf_fit_eval_grad(NULL), surf_fit_eval_hess(NULL),
        discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
        fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
   { PA.enabled = false; }
@@ -2103,9 +2105,15 @@ public:

 #ifdef MFEM_USE_MPI
   /// Parallel support for surface fitting to the zero level set of a function.
+   /// Here, we add two optional inputs: @a aegrad and @a aehess. When provided,
+   /// the first and second derivative of the input level set are computed on
+   /// the initial mesh, and @a aegrad and @a aehess are used to remap grad_s(x)
+   /// from grad_s0(x0) and hess_s(x) from hess_s0(x0), respectively.
   void EnableSurfaceFitting(const ParGridFunction &s0,
                             const Array<bool> &smarker, Coefficient &coeff,
-                             AdaptivityEvaluator &ae);
+                             AdaptivityEvaluator &ae,
+                             AdaptivityEvaluator *aegrad = NULL,
+                             AdaptivityEvaluator *aehess = NULL);

   /** @brief Fitting of certain DOFs in the current mesh to the zero level set
       of a function defined on another (finer) source mesh.
@@ -429,11 +429,13 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 #endif

   real_t scale = 1.0;
-   real_t avg_surf_fit_err, max_surf_fit_err = 0.0;
-   if (surf_fit_max_threshold > 0.0)
+   bool fitting = IsSurfaceFittingEnabled();
+   real_t init_fit_avg_err, init_fit_max_err = 0.0;
+   if (fitting && surf_fit_converge_error)
   {
-      GetSurfaceFittingError(x_out_loc, avg_surf_fit_err, max_surf_fit_err);
-      if (max_surf_fit_err < surf_fit_max_threshold)
+      GetSurfaceFittingError(x_out_loc, init_fit_avg_err, init_fit_max_err);
+      // Check for convergence
+      if (init_fit_max_err < surf_fit_max_err_limit)
      {
         if (print_options.iterations)
         {
@@ -444,11 +446,12 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
         return scale;
      }
   }
-   if (adapt_inc_count >= max_adapt_inc_count)
+
+   if (surf_fit_adapt_count >= surf_fit_adapt_count_limit)
   {
      if (print_options.iterations)
      {
-         mfem::out << "TMOPNewtonSolver converged "
+         mfem::out << "TMOPNewtonSolver terminated "
                   "based on max number of times surface fitting weight can"
                   "be increased. \n";
      }
@@ -467,7 +470,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
      // reference to detect deteriorations.
      MFEM_VERIFY(min_det_ptr != NULL, " Initial mesh was valid, but"
                  " intermediate mesh is invalid. Contact TMOP Developers.");
-      MFEM_VERIFY(min_detJ_threshold == 0.0,
+      MFEM_VERIFY(min_detJ_limit == 0.0,
                  "This setup is not supported. Contact TMOP Developers.");
      *min_det_ptr = untangle_factor * min_detT_in;
   }
@@ -478,6 +481,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
   bool x_out_ok = false;
   real_t energy_out = 0.0, min_detT_out;
   const real_t norm_in = Norm(r);
+   real_t avg_fit_err, max_fit_err = 0.0;

   const real_t detJ_factor = (solver_type == 1) ? 0.25 : 0.5;
   compute_metric_quantile_flag = false;
@@ -488,6 +492,9 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
   // Perform the line search.
   for (int i = 0; i < 12; i++)
   {
+      avg_fit_err = 0.0;
+      max_fit_err = 0.0;
+
      // Update the mesh and get the L-vector in x_out_loc.
      add(x, -scale, c, x_out);
      if (serial)
@@ -502,7 +509,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,

      // Check the changes in detJ.
      min_detT_out = ComputeMinDet(x_out_loc, *fes);
-      if (untangling == false && min_detT_out <= min_detJ_threshold)
+      if (untangling == false && min_detT_out <= min_detJ_limit)
      {
         // No untangling, and detJ got negative (or small) -- no good.
         if (print_options.iterations)
@@ -529,18 +536,19 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
      // Check the changes in total energy.
      ProcessNewState(x_out);

-      real_t avg_fit_err, max_fit_err = 0.0;
-      if (surf_fit_max_threshold > 0.0)
+      // Ensure sufficient decrease in fitting error if we are trying to
+      // converge based on error.
+      if (fitting && surf_fit_converge_error)
      {
         GetSurfaceFittingError(x_out_loc, avg_fit_err, max_fit_err);
-      }
-      if (surf_fit_max_threshold > 0.0 && max_fit_err >= 1.2*max_surf_fit_err)
-      {
-         if (print_options.iterations)
+         if (max_fit_err >= 1.2*init_fit_max_err)
         {
-            mfem::out << "Scale = " << scale << " Surf fit err increased.\n";
+            if (print_options.iterations)
+            {
+               mfem::out << "Scale = " << scale << " Surf fit err increased.\n";
+            }
+            scale *= 0.5; continue;
         }
-         scale *= 0.5; continue;
      }

      if (serial)
@@ -614,7 +622,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,

   if (x_out_ok == false) { scale = 0.0; }

-   if (surf_fit_scale_factor > 0.0) { update_surf_fit_coeff = true; }
+   if (surf_fit_scale_factor > 0.0) { surf_fit_coeff_update = true; }
   compute_metric_quantile_flag = true;

   return scale;
@@ -657,7 +665,7 @@ void TMOPNewtonSolver::GetSurfaceFittingWeight(Array<real_t> &weights) const
   for (int i = 0; i < integs.Size(); i++)
   {
      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
-      if (ti)
+      if (ti && ti->IsSurfaceFittingEnabled())
      {
         weight = ti->GetSurfaceFittingWeight();
         weights.Append(weight);
@@ -668,8 +676,11 @@ void TMOPNewtonSolver::GetSurfaceFittingWeight(Array<real_t> &weights) const
         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
         for (int j = 0; j < ati.Size(); j++)
         {
-            weight = ati[j]->GetSurfaceFittingWeight();
-            weights.Append(weight);
+            if (ati[j]->IsSurfaceFittingEnabled())
+            {
+               weight = ati[j]->GetSurfaceFittingWeight();
+               weights.Append(weight);
+            }
         }
      }
   }
@@ -716,6 +727,39 @@ void TMOPNewtonSolver::GetSurfaceFittingError(const Vector &x_loc,
   }
 }

+bool TMOPNewtonSolver::IsSurfaceFittingEnabled() const
+{
+   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
+   const Array<NonlinearFormIntegrator*> &integs = *nlf->GetDNFI();
+   TMOP_Integrator *ti  = NULL;
+   TMOPComboIntegrator *co = NULL;
+
+   for (int i = 0; i < integs.Size(); i++)
+   {
+      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+      if (ti)
+      {
+         if (ti->IsSurfaceFittingEnabled())
+         {
+            return true;
+         }
+      }
+      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+      if (co)
+      {
+         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+         for (int j = 0; j < ati.Size(); j++)
+         {
+            if (ati[j]->IsSurfaceFittingEnabled())
+            {
+               return true;
+            }
+         }
+      }
+   }
+   return false;
+}
+
 void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
 {
   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
@@ -801,38 +845,46 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
   // adaptive surface fitting is enabled. The idea is to increase the
   // coefficient if the surface fitting error does not sufficiently
   // decrease between subsequent TMOPNewtonSolver iterations.
-   if (update_surf_fit_coeff)
+   if (surf_fit_coeff_update)
   {
      // Get surface fitting errors.
-      GetSurfaceFittingError(x_loc, surf_fit_err_avg, surf_fit_err_max);
+      GetSurfaceFittingError(x_loc, surf_fit_avg_err, surf_fit_max_err);
      // Get array with surface fitting weights.
-      Array<real_t> weights;
-      GetSurfaceFittingWeight(weights);
+      Array<real_t> fitweights;
+      GetSurfaceFittingWeight(fitweights);

      if (print_options.iterations)
      {
         mfem::out << "Avg/Max surface fitting error: " <<
-                   surf_fit_err_avg << " " <<
-                   surf_fit_err_max << "\n";
+                   surf_fit_avg_err << " " <<
+                   surf_fit_max_err << "\n";
         mfem::out << "Min/Max surface fitting weight: " <<
-                   weights.Min() << " " << weights.Max() << "\n";
+                   fitweights.Min() << " " << fitweights.Max() << "\n";
      }

-      real_t change_surf_fit_err = surf_fit_err_avg_prvs-surf_fit_err_avg;
-      real_t rel_change_surf_fit_err = change_surf_fit_err/surf_fit_err_avg_prvs;
+      real_t change_surf_fit_err = surf_fit_avg_err_prvs-surf_fit_avg_err;
+      real_t rel_change_surf_fit_err = change_surf_fit_err/surf_fit_avg_err_prvs;
+
      // Increase the surface fitting coefficient if the surface fitting error
-      // does not decrease sufficiently.
-      if (rel_change_surf_fit_err < surf_fit_rel_change_threshold)
+      // does not decrease sufficiently. If we are converging based on residual,
+      // also make sure we have not reached the maximum fitting weight and
+      // error threshold.
+      if (rel_change_surf_fit_err < surf_fit_err_rel_change_limit &&
+          (surf_fit_converge_error ||
+           (fitweights.Max() < surf_fit_weight_limit &&
+            surf_fit_max_err > surf_fit_max_err_limit)))
      {
-         UpdateSurfaceFittingWeight(surf_fit_scale_factor);
-         adapt_inc_count += 1;
+         real_t scale_factor = std::min(surf_fit_scale_factor,
+                                        surf_fit_weight_limit/fitweights.Max());
+         UpdateSurfaceFittingWeight(scale_factor);
+         surf_fit_adapt_count += 1;
      }
      else
      {
-         adapt_inc_count = 0;
+         surf_fit_adapt_count = 0;
      }
-      surf_fit_err_avg_prvs = surf_fit_err_avg;
-      update_surf_fit_coeff = false;
+      surf_fit_avg_err_prvs = surf_fit_avg_err;
+      surf_fit_coeff_update = false;
   }
 }

@@ -134,18 +134,20 @@ protected:
   int solver_type;
   bool parallel;

-   // Line search step is rejected if min(detJ) <= min_detJ_threshold.
-   real_t min_detJ_threshold = 0.0;
+   // Line search step is rejected if min(detJ) <= min_detJ_limit.
+   real_t min_detJ_limit = 0.0;

   // Surface fitting variables.
-   mutable real_t surf_fit_err_avg_prvs = 10000.0;
-   mutable real_t surf_fit_err_avg, surf_fit_err_max;
-   mutable bool update_surf_fit_coeff = false;
-   real_t surf_fit_max_threshold = -1.0;
-   real_t surf_fit_rel_change_threshold = 0.001;
+   mutable real_t surf_fit_avg_err_prvs = 10000.0;
+   mutable real_t surf_fit_avg_err, surf_fit_max_err;
+   mutable bool surf_fit_coeff_update = false;
+   real_t surf_fit_max_err_limit = -1.0;
+   real_t surf_fit_err_rel_change_limit = 0.001;
   real_t surf_fit_scale_factor = 0.0;
-   mutable int adapt_inc_count = 0;
-   mutable int max_adapt_inc_count = 10;
+   mutable int surf_fit_adapt_count = 0;
+   mutable int surf_fit_adapt_count_limit = 10;
+   mutable real_t surf_fit_weight_limit = 1e10;
+   bool surf_fit_converge_error = false;

   // Minimum determinant over the whole mesh. Used for mesh untangling.
   real_t *min_det_ptr = nullptr;
@@ -191,6 +193,9 @@ protected:
   void GetSurfaceFittingWeight(Array<real_t> &weights) const;
   ///@}

+   /// Check if surface fitting is enabled.
+   bool IsSurfaceFittingEnabled() const;
+
 public:
 #ifdef MFEM_USE_MPI
   TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule, int type = 0)
@@ -224,38 +229,94 @@ public:
   /// (ii) surface fitting weight.
   virtual void ProcessNewState(const Vector &x) const;

-   /** @name Methods for adaptive surface fitting weight. (Experimental) */
-   /// Enable/Disable adaptive surface fitting weight.
-   /// The weight is modified after each TMOPNewtonSolver iteration as:
-   /// w_{k+1} = w_{k} * @a surf_fit_scale_factor if relative change in
-   /// max surface fitting error < @a surf_fit_rel_change_threshold.
-   /// The solver terminates if the maximum surface fitting error does
-   /// not sufficiently decrease for @a max_adapt_inc_count consecutive
-   /// solver iterations or if the max error falls below @a surf_fit_max_threshold.
-   void EnableAdaptiveSurfaceFitting()
-   {
-      surf_fit_scale_factor = 10.0;
-      surf_fit_rel_change_threshold = 0.001;
-   }
+   /** @name Methods for adaptive surface fitting.
+       \brief These methods control the behavior of the weight and the
+       termination of the solver. (Experimental)
+
+       Adaptive fitting weight: The weight is modified after each
+       TMOPNewtonSolver iteration as:
+       w_{k+1} = w_{k} * \ref surf_fit_scale_factor if the relative
+       change in average fitting error < \ref surf_fit_err_rel_change_limit.
+       When converging based on the residual, we enforce the fitting weight
+       to be at-most \ref surf_fit_weight_limit, and increase it only if the
+       fitting error is below user prescribed threshold
+       (\ref surf_fit_max_err_limit).
+       See \ref SetAdaptiveSurfaceFittingScalingFactor and
+       \ref SetAdaptiveSurfaceFittingRelativeChangeThreshold.
+
+       Note that the solver stops if the maximum surface fitting error
+       does not sufficiently decrease for \ref surf_fit_adapt_count_limit (default 10)
+       consecutive increments of the fitting weight during weight adaptation.
+       This typically occurs when the mesh cannot align with the level-set
+       without degrading element quality.
+       See \ref SetMaxNumberofIncrementsForAdaptiveFitting.
+
+       Convergence criterion: There are two modes, residual- and error-based,
+       which can be toggled using \ref SetSurfaceFittingConvergenceBasedOnError.
+
+       (i) Residual based (default): Stop when the norm of the gradient of the
+       TMOP objective reaches the prescribed tolerance. This method is best used
+       with a reasonable value for \ref surf_fit_weight_limit when the
+       adaptive surface fitting scheme is used. See method
+       \ref SetSurfaceFittingWeightLimit.
+
+       (ii) Error based: Stop when the maximum fitting error
+       reaches the user-prescribed threshold, \ref surf_fit_max_err_limit.
+       In this case, \ref surf_fit_weight_limit is ignored during weight
+       adaptation.
+   */
+   ///@{
   void SetAdaptiveSurfaceFittingScalingFactor(real_t factor)
   {
+      MFEM_VERIFY(factor > 1.0, "Scaling factor must be greater than 1.");
      surf_fit_scale_factor = factor;
   }
   void SetAdaptiveSurfaceFittingRelativeChangeThreshold(real_t threshold)
   {
-      surf_fit_rel_change_threshold = threshold;
+      surf_fit_err_rel_change_limit = threshold;
   }
+   /// Used for stopping based on the number of consecutive failed weight
+   /// adaptation iterations.
+   // TODO: Rename to SetMaxNumberofIncrementsForAdaptiveSurfaceFitting
+   // in future.
   void SetMaxNumberofIncrementsForAdaptiveFitting(int count)
   {
-      max_adapt_inc_count = count;
+      surf_fit_adapt_count_limit = count;
   }
+   /// Used for error-based surface fitting termination.
   void SetTerminationWithMaxSurfaceFittingError(real_t max_error)
   {
-      surf_fit_max_threshold = max_error;
+      surf_fit_max_err_limit = max_error;
+      surf_fit_converge_error = true;
   }
+   /// Could be used with both error-based or residual-based convergence.
+   void SetSurfaceFittingMaxErrorLimit(real_t max_error)
+   {
+      surf_fit_max_err_limit = max_error;
+   }
+   /// Used for residual-based surface fitting termination.
+   void SetSurfaceFittingWeightLimit(real_t weight)
+   {
+      surf_fit_weight_limit = weight;
+   }
+   /// Toggle convergence based on residual or error.
+   void SetSurfaceFittingConvergenceBasedOnError(bool mode)
+   {
+      surf_fit_converge_error = mode;
+      if (surf_fit_converge_error)
+      {
+         MFEM_VERIFY(surf_fit_max_err_limit >= 0,
+                     "Fitting error based convergence requires the user to "
+                     "first set the error threshold."
+                     "See SetTerminationWithMaxSurfaceFittingError");
+      }
+   }
+   ///@}
+
+   /// Set minimum determinant enforced during line-search.
   void SetMinimumDeterminantThreshold(real_t threshold)
   {
-      min_detJ_threshold = threshold;
+      min_detJ_limit = threshold;
   }

   virtual void Mult(const Vector &b, Vector &x) const
@@ -124,7 +124,7 @@ T Array<T>::Sum()
 }

 template <class T>
-int Array<T>::IsSorted()
+int Array<T>::IsSorted() const
 {
   T val_prev = operator[](0), val;
   for (int i = 1; i < size; i++)
@@ -74,10 +74,14 @@ public:
   inline Array(int asize, MemoryType mt)
      : size(asize) { asize > 0 ? data.New(asize, mt) : data.Reset(mt); }

-   /** @brief Creates array using an externally allocated pointer @a data_ to
-       @a asize elements. The data pointer will not be deleted by Array. */
-   inline Array(T *data_, int asize)
-   { data.Wrap(data_, asize, false); size = asize; }
+   /** @brief Creates array using an externally allocated host pointer @a data_
+       to @a asize elements. If @a own_data is true, the array takes ownership
+       of the pointer.
+
+       When @a own_data is true, the pointer @a data_ must be allocated with
+       MemoryType given by MemoryManager::GetHostMemoryType(). */
+   inline Array(T *data_, int asize, bool own_data = false)
+   { data.Wrap(data_, asize, own_data); size = asize; }

   /// Copy constructor: deep copy from @a src
   /** This method supports source arrays using any MemoryType. */
@@ -205,7 +209,14 @@ public:
   inline void Copy(Array &copy) const;

   /// Make this Array a reference to a pointer.
-   inline void MakeRef(T *, int);
+   /** When @a own_data is true, the pointer @a data_ must be allocated with
+       MemoryType given by MemoryManager::GetHostMemoryType(). */
+   inline void MakeRef(T *data_, int size_, bool own_data = false);
+
+   /// Make this Array a reference to a pointer.
+   /** When @a own_data is true, the pointer @a data_ must be allocated with
+       MemoryType given by @a mt. */
+   inline void MakeRef(T *data_, int size, MemoryType mt, bool own_data);

   /// Make this Array a reference to 'master'.
   inline void MakeRef(const Array &master);
@@ -262,7 +273,7 @@ public:
   }

   /// Return 1 if the array is sorted from lowest to highest.  Otherwise return 0.
-   int IsSorted();
+   int IsSorted() const;

   /// Fill the entries of the array with the cumulative sum of the entries.
   void PartialSum();
@@ -868,11 +879,19 @@ inline void Array<T>::Copy(Array &copy) const
 }

 template <class T>
-inline void Array<T>::MakeRef(T *p, int s)
+inline void Array<T>::MakeRef(T *data_, int size_, bool own_data)
 {
   data.Delete();
-   data.Wrap(p, s, false);
-   size = s;
+   data.Wrap(data_, size_, own_data);
+   size = size_;
+}
+
+template <class T>
+inline void Array<T>::MakeRef(T *data_, int size_, MemoryType mt, bool own_data)
+{
+   data.Delete();
+   data.Wrap(data_, size_, mt, own_data);
+   size = size_;
 }

 template <class T>
@@ -288,4 +288,3 @@ void ArraysByName<T>::Load(std::istream &in)
 }

 #endif
-
@@ -275,7 +275,7 @@ void GroupTopology::Save(ostream &os) const
   os << "\ncommunication_groups\n";
   os << "number_of_groups " << NGroups() << "\n\n";

-   os << "# number of entities in each group, followed by group ids in group\n";
+   os << "# number of entities in each group, followed by ranks in group\n";
   for (int group_id = 0; group_id < NGroups(); ++group_id)
   {
      int group_size = GetGroupSize(group_id);
@@ -14,6 +14,9 @@
 #ifdef MFEM_USE_CEED
 #include "../fem/ceed/interface/util.hpp"
 #endif
+#ifdef MFEM_USE_MPI
+#include "../linalg/hypre.hpp"
+#endif

 #include <unordered_map>
 #include <string>
@@ -250,6 +253,10 @@ void Device::Configure(const std::string &device, const int device_id)

   // Only '*this' will call the MemoryManager::Destroy() method.
   destroy_mm = true;
+
+#ifdef MFEM_USE_MPI
+   Hypre::InitDevice();
+#endif
 }

 // static method
@@ -19,6 +19,9 @@
 #include "device.hpp"
 #include "mem_manager.hpp"
 #include "../linalg/dtensor.hpp"
+#ifdef MFEM_USE_MPI
+#include <_hypre_utilities.h>
+#endif

 namespace mfem
 {
@@ -780,6 +783,63 @@ inline void forall_3D_grid(int N, int X, int Y, int Z, int G, lambda &&body)
   ForallWrap<3>(true, N, body, X, Y, Z, G);
 }

+#ifdef MFEM_USE_MPI
+
+// Function mfem::hypre_forall_cpu() similar to mfem::forall, but it always
+// executes on the CPU using sequential or OpenMP-parallel execution based on
+// the hypre build time configuration.
+template<typename lambda>
+inline void hypre_forall_cpu(int N, lambda &&body)
+{
+#ifdef HYPRE_USING_OPENMP
+   #pragma omp parallel for HYPRE_SMP_SCHEDULE
+#endif
+   for (int i = 0; i < N; i++) { body(i); }
+}
+
+// Function mfem::hypre_forall_gpu() similar to mfem::forall, but it always
+// executes on the GPU device that hypre was configured with at build time.
+#if defined(HYPRE_USING_GPU)
+template<typename lambda>
+inline void hypre_forall_gpu(int N, lambda &&body)
+{
+#if defined(HYPRE_USING_CUDA)
+   CuWrap1D(N, body);
+#elif defined(HYPRE_USING_HIP)
+   HipWrap1D(N, body);
+#else
+#error Unknown HYPRE GPU backend!
+#endif
+}
+#endif
+
+// Function mfem::hypre_forall() similar to mfem::forall, but it executes on the
+// device, CPU or GPU, that hypre was configured with at build time (when the
+// HYPRE version is < 2.31.0) or at runtime (when HYPRE was configured with GPU
+// support at build time and HYPRE's version is >= 2.31.0). This selection is
+// generally independent of what device was selected in MFEM's runtime
+// configuration.
+template<typename lambda>
+inline void hypre_forall(int N, lambda &&body)
+{
+#if !defined(HYPRE_USING_GPU)
+   hypre_forall_cpu(N, body);
+#elif MFEM_HYPRE_VERSION < 23100
+   hypre_forall_gpu(N, body);
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   if (!HypreUsingGPU())
+   {
+      hypre_forall_cpu(N, body);
+   }
+   else
+   {
+      hypre_forall_gpu(N, body);
+   }
+#endif
+}
+
+#endif // MFEM_USE_MPI
+
 } // namespace mfem

 #endif // MFEM_FORALL_HPP
@@ -1154,6 +1154,10 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
   //  dest   d  | h2d   d2d   d2d
   //        hd  | h2h   d2d   d2d

+   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
+   MFEM_ASSERT(dst_h_ptr != nullptr, "invalid dst_h_ptr = nullptr");
+   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
+
   const bool dst_on_host =
      (dst_flags & Mem::VALID_HOST) &&
      (!(dst_flags & Mem::VALID_DEVICE) ||
@@ -1229,6 +1233,10 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
 void MemoryManager::CopyToHost_(void *dest_h_ptr, const void *src_h_ptr,
                                size_t bytes, unsigned src_flags)
 {
+   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
+   MFEM_ASSERT(dest_h_ptr != nullptr, "invalid dest_h_ptr = nullptr");
+   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
+
   const bool src_on_host = src_flags & Mem::VALID_HOST;
   if (src_on_host)
   {
@@ -1255,6 +1263,10 @@ void MemoryManager::CopyToHost_(void *dest_h_ptr, const void *src_h_ptr,
 void MemoryManager::CopyFromHost_(void *dest_h_ptr, const void *src_h_ptr,
                                  size_t bytes, unsigned &dest_flags)
 {
+   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
+   MFEM_ASSERT(dest_h_ptr != nullptr, "invalid dest_h_ptr = nullptr");
+   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
+
   const bool dest_on_host = dest_flags & Mem::VALID_HOST;
   if (dest_on_host)
   {
@@ -18,8 +18,14 @@
 #include <cstring> // std::memcpy
 #include <type_traits> // std::is_const
 #include <cstddef> // std::max_align_t
+
 #ifdef MFEM_USE_MPI
-#include <HYPRE_config.h> // HYPRE_USING_GPU
+// Enable internal hypre timing routines
+#define HYPRE_TIMING
+#include <HYPRE_utilities.h> // for HYPRE_GetMemoryLocation() and others
+#if (21400 <= MFEM_HYPRE_VERSION) && (MFEM_HYPRE_VERSION < 21900)
+#include <_hypre_utilities.h> // for HYPRE_MEMORY_HOST and others
+#endif
 #endif

 namespace mfem
@@ -869,6 +875,45 @@ public:
 };


+#ifdef MFEM_USE_MPI
+
+#if MFEM_HYPRE_VERSION < 21400
+#define HYPRE_MEMORY_DEVICE (0)
+#define HYPRE_MEMORY_HOST   (1)
+#endif
+#if MFEM_HYPRE_VERSION < 21900
+typedef int HYPRE_MemoryLocation;
+#endif
+
+/// Return the configured HYPRE_MemoryLocation
+inline HYPRE_MemoryLocation GetHypreMemoryLocation()
+{
+#if !defined(HYPRE_USING_GPU)
+   return HYPRE_MEMORY_HOST;
+#elif MFEM_HYPRE_VERSION < 23100
+   return HYPRE_MEMORY_DEVICE;
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   HYPRE_MemoryLocation loc;
+   HYPRE_GetMemoryLocation(&loc);
+   return loc;
+#endif
+}
+
+/// Return true if HYPRE is configured to use GPU
+inline bool HypreUsingGPU()
+{
+#if !defined(HYPRE_USING_GPU)
+   return false;
+#elif MFEM_HYPRE_VERSION < 23100
+   return true;
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   return GetHypreMemoryLocation() != HYPRE_MEMORY_HOST;
+#endif
+}
+
+#endif // MFEM_USE_MPI
+
+
 // Inline methods

 template <typename T>
@@ -1004,10 +1049,12 @@ inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
         // If the following condition is true then MemoryManager::Exists()
         // should also be true:
         IsDeviceMemory(MemoryManager::GetDeviceMemoryType())
-#else
-         // When HYPRE_USING_GPU is defined we always register the 'base' if
-         // the MemoryManager::Exists():
+#elif MFEM_HYPRE_VERSION < 23100
+         // When HYPRE_USING_GPU is defined and HYPRE < 2.31.0, we always
+         // register the 'base' if the MemoryManager::Exists():
         MemoryManager::Exists()
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+         MemoryManager::Exists() && HypreUsingGPU()
 #endif
      )
      {
@@ -1213,9 +1260,10 @@ template <typename T>
 inline void Memory<T>::CopyFrom(const Memory &src, int size)
 {
   MFEM_VERIFY(src.capacity>=size && capacity>=size, "Incorrect size");
+   if (size <= 0) { return; }
   if (!(flags & Registered) && !(src.flags & Registered))
   {
-      if (h_ptr != src.h_ptr && size != 0)
+      if (h_ptr != src.h_ptr)
      {
         MFEM_ASSERT(h_ptr + size <= src.h_ptr || src.h_ptr + size <= h_ptr,
                     "data overlaps!");
@@ -1233,9 +1281,10 @@ template <typename T>
 inline void Memory<T>::CopyFromHost(const T *src, int size)
 {
   MFEM_VERIFY(capacity>=size, "Incorrect size");
+   if (size <= 0) { return; }
   if (!(flags & Registered))
   {
-      if (h_ptr != src && size != 0)
+      if (h_ptr != src)
      {
         MFEM_ASSERT(h_ptr + size <= src || src + size <= h_ptr,
                     "data overlaps!");
@@ -1252,7 +1301,6 @@ inline void Memory<T>::CopyFromHost(const T *src, int size)
 template <typename T>
 inline void Memory<T>::CopyTo(Memory &dest, int size) const
 {
-   MFEM_VERIFY(capacity>=size, "Incorrect size");
   dest.CopyFrom(*this, size);
 }

@@ -1260,9 +1308,10 @@ template <typename T>
 inline void Memory<T>::CopyToHost(T *dest, int size) const
 {
   MFEM_VERIFY(capacity>=size, "Incorrect size");
+   if (size <= 0) { return; }
   if (!(flags & Registered))
   {
-      if (h_ptr != dest && size != 0)
+      if (h_ptr != dest)
      {
         MFEM_ASSERT(h_ptr + size <= dest || dest + size <= h_ptr,
                     "data overlaps!");
@@ -134,7 +134,7 @@ int socketbuf::open(const char hostname[], int port)
      {
         closesocket(socket_descriptor);
         socket_descriptor = -2;
-         return -1;
+         continue;
      }
 #endif

@@ -148,7 +148,7 @@ int socketbuf::open(const char hostname[], int port)
   }

   freeaddrinfo(res);
-   return 0;
+   return (socket_descriptor < 0) ? -1 : 0;
 }

 int socketbuf::close()
@@ -95,7 +95,7 @@ public:
       not called, it returns the number of possible connections established
       by the used constructor. Otherwise, it is exactly the number of
       established connections before calling Finalize(). */
-   inline int Size_of_connections() const { return I[size]; }
+   inline int Size_of_connections() const { HostReadI(); return I[size]; }

   /** Returns index of the connection between element i of TYPE I and
       element j of TYPE II. If there is no connection between element i
@@ -207,7 +207,12 @@ template <> inline void Swap<Table>(Table &a, Table &b)
 void Transpose (const Table &A, Table &At, int ncols_A_ = -1);
 Table * Transpose (const Table &A);

-///  Transpose an Array<int>
+///  @brief Transpose an Array<int>.
+///
+/// The array @a A represents a table where each row @a i has exactly one
+/// connection to the column (TYPE II) index specified by @a A[i].
+///
+/// @note The column (TYPE II) indices in each row of @a At will be sorted.
 void Transpose(const Array<int> &A, Table &At, int ncols_A_ = -1);

 ///  C = A * B  (as boolean matrices)
@@ -400,6 +400,9 @@ inline double StopWatch::SystTime()

 StopWatch::StopWatch() : M(new internal::StopWatch) { }

+StopWatch::StopWatch(const StopWatch &sw)
+   : M(new internal::StopWatch(*(sw.M))) { }
+
 void StopWatch::Clear()
 {
   M->Clear();
--- a/Show More
+++ b/Show More