Merge pull request #4330 from mfem/CurlDim-bugfix

GridFunction::CurlDim() nullptr fix
Merge branch 'master' into CurlDim-bugfix
2024-06-05 11:49:58 +01:00 · 2024-06-05 11:49:08 +01:00 · 2024-06-01 14:03:22 -07:00 · 2024-05-31 17:53:18 -07:00 · 2024-05-31 10:15:56 -07:00 · 2024-05-31 09:29:36 -07:00
152 changed files with 5988 additions and 1512 deletions
@@ -33,6 +33,7 @@ env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
  HYPRE_TOP_DIR: hypre-2.19.0
  METIS_ARCHIVE: metis-4.0.3.tar.gz
+  METIS_ARCHIVE_MAC: metis-4.0.3-mac.tgz
  METIS_TOP_DIR: metis-4.0.3
  MFEM_TOP_DIR: mfem

@@ -52,6 +53,7 @@ jobs:
        mpi: [seq, par]
        build-system: [make, cmake]
        hypre-target: [int32]
+        precision: [fp64]
        exclude:
          - os: ubuntu-latest
            build-system: cmake
@@ -75,6 +77,8 @@ jobs:
          - os: ubuntu-latest
            target: dbg
            config-opts: 'CPPFLAGS+=-Og'
+          - os: macos-latest
+            codecov: NO
          - os: windows-latest
            codecov: NO
          - os: windows-latest
@@ -87,6 +91,7 @@ jobs:
            mpi: par
            build-system: cmake
            hypre-target: int32
+            precision: fp64
            # This option can be set to pass additional configuration options to
            # the MFEM configuration command.
            # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
@@ -96,7 +101,15 @@ jobs:
            mpi: par
            build-system: make
            hypre-target: int64
-    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+            precision: fp64
+          - os: ubuntu-latest
+            target: opt
+            codecov: NO
+            mpi: par
+            build-system: make
+            hypre-target: int32
+            precision: fp32
+    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}

    runs-on: ${{ matrix.os }}

@@ -126,6 +139,17 @@ jobs:
          # Fetch the complete history for codecov to access commits ID
          fetch-depth: 0

+      - name: Xcode version setup (MacOS)
+        if: matrix.os == 'macos-latest'
+        run: |
+          XCODE_PATH="/Applications/Xcode_15.3.app"
+          echo "> sudo xcode-select -s ${XCODE_PATH}"
+          sudo xcode-select -s ${XCODE_PATH}
+          echo "> g++ -v"
+          g++ -v
+          echo "> clang++ -v"
+          clang++ -v
+
      # Only get MPI if defined for the job.
      # TODO: It would be nice to have only one step, e.g. with a dedicated
      # action, but I (@adrienbernede) don't see how at the moment.
@@ -169,25 +193,27 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5

      - name: get hypre
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
          target: ${{ matrix.hypre-target }}
          build-system: make
+          precision: ${{ matrix.precision }}

      - name: get hypre (Windows)
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
          target: ${{ matrix.hypre-target }}
          build-system: cmake
+          precision: ${{ matrix.precision }}

      # Get Metis through cache, or build it.
      # Install will only run on cache miss.
@@ -197,13 +223,13 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: install metis
        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
+        uses: mfem/github-actions/build-metis@v2.5
        with:
-          archive: ${{ env.METIS_ARCHIVE }}
+          archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
          dir: ${{ env.METIS_TOP_DIR }}

      - name: cache vcpkg (Windows)
@@ -228,7 +254,7 @@ jobs:

      # MFEM build and test
      - name: build
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        env:
          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
        with:
@@ -240,6 +266,7 @@ jobs:
          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
          metis-dir: ${{ env.METIS_TOP_DIR }}
          mfem-dir: ${{ env.MFEM_TOP_DIR }}
+          precision: ${{ matrix.precision }}
          config-options: ${{ matrix.config-opts }}
          library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}

@@ -282,7 +309,7 @@ jobs:
      # Code coverage (process and upload reports)
      - name: codecov
        if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.4
+        uses: mfem/github-actions/upload-coverage@v2.5
        with:
          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
          project_dir: ${{ env.MFEM_TOP_DIR }}
@@ -53,11 +53,11 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.5

      - name: Get Hypre
        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
@@ -68,18 +68,18 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: Install Metis
        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
+        uses: mfem/github-actions/build-metis@v2.5
        with:
          archive: ${{ env.METIS_ARCHIVE }}
          dir: ${{ env.METIS_TOP_DIR }}

      # MFEM build and test
      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        with:
          os: ${{ runner.os }}
          target: opt
@@ -44,7 +44,7 @@ jobs:
          path: mfem

      - name: MFEM Build
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        with:
          os: ${{ runner.os }}
          target: opt
@@ -57,6 +57,8 @@ examples/ex2[0-9]
 examples/ex2[0-9]p
 examples/ex3[0-9]
 examples/ex3[0-9]p
+examples/ex4[0-9]
+examples/ex4[0-9]p

 examples/refined.mesh
 examples/displaced.mesh
@@ -232,7 +234,7 @@ miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
 miniapps/meshing/twist-*.mesh
-miniapps/meshing/mesh-explorer.mesh
+miniapps/meshing/mesh-explorer.mesh*
 miniapps/meshing/partitioning.txt
 miniapps/meshing/mesh-explorer-visit*
 miniapps/meshing/mesh-explorer-paraview/
@@ -369,7 +371,7 @@ miniapps/dpg/ParaView
 miniapps/spde/generate_random_field
 miniapps/spde/ParaView

-miniapps/tribol/ContactPatchTest
+miniapps/tribol/contact-patch-test

 # Unit test binary and outputs
 tests/unit/output_meshes
@@ -13,6 +13,9 @@
 # at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
 # LLNL-specific!

+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'

 # The pipeline is divided into stages. Usually, jobs in a given stage wait for
 # the preceding stages to complete before to start. However, we sometimes use
@@ -9,6 +9,10 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
 # We define the following GitLab pipeline variables:
 variables:

@@ -35,9 +35,8 @@ variables:
    - when: on_success

 # Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen
-# to speed-up the allocation.  However this would not be scalable to
-# multiple builds.
+# pre-allocation the same way slurm does.  We use the pci queue on lassen
+# to speed-up the allocation.
 .build_and_test_on_lassen:
  extends: [.on_lassen]
  stage: build_and_test
@@ -45,5 +44,5 @@ variables:
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 45 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
  needs: [setup]
@@ -52,4 +52,4 @@ variables:
    - echo ${JOBID}
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -14,14 +14,14 @@ stages:
  - build_and_test
  - report

-opt_mpi_cuda_xl_16_1_1_12:
+opt_mpi_cuda_gcc:
  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70"
  extends: .build_and_test_on_lassen

-opt_mpi_cuda_hypre_cuda_xl:
+opt_mpi_cuda_hypre_cuda_gcc:
  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
  extends: .build_and_test_on_lassen

 # Jobs report
@@ -32,11 +32,11 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}

 # run
 if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  lalloc 1 -q pci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
  exit 1
@@ -8,82 +8,112 @@
                               https://mfem.org


-Version 4.6.1 (development)
+Version 4.7.1 (development)
 ===========================

+- Added an MFEM example for the eikonal equation. This new solver is based on
+  the proximal Galerkin method introduced by Keith and Surowiec.
+
+- API change: in class GridFunction, 'fec' was renamed to 'fec_owned'.
+
+
+Version 4.7, released on May 7, 2024
+====================================
+
+- Added support for single precision (with corresponding hypre build). The MFEM
+  floating point type was generalized from `double` to `real_t`. For details see
+  https://github.com/orgs/mfem/discussions/4207.
+
 Meshing improvements
 --------------------
+- Added the capability to partition (big) serial meshes in serial code, see the
+  new classes MeshPartitioner and MeshPart. This capability is also exposed as a
+  menu option in the mesh-explorer miniapp in miniapps/meshing.
+
+- Added named attribute sets and basic supporting methods to the Mesh class as a
+  convenient means of referring to sets of domain or boundary attribute numbers.
+  See the new Example 39/39p and data/compass.mesh.
+
 - Introduced formulas for refinement of patches in NURBS meshes. Refinement by
  arbitrary integer factors is also enabled, e.g. in the mesh-explorer miniapp.
  NURBS coarsening and knot removal are also introduced.

-Discretization improvements
---------------------------
- Introduced support for higher order non conformal Nedelec elements on
-  simplices in ParMesh.
-
- Introduced support for internal boundary elements in nonconformal adapted
-  meshes.
-
- Added functionality for construction of cut-surface and cut-volume
-  IntegrationRules through a moment-fitting approach. The cut is specified by
-  the zero level set of a Coefficient. See fem/intrules_cut.hpp and Example 38.
-
- Added a new nonlinear integrator, `HyperbolicFormIntegrator`. This implements
-  both element-wise weak divergence and face-wise numerical flux for a general
-  system of hyperbolic conservation laws. To use this integrator for a specific
-  flux function, users can define a derived class of `FluxFunction`. Currently,
-  advection, Burgers', shallow-water, Euler equations (see, Example 18) are
-  available.
-
-GPU support
----------------------------
- Added support for full assembly on simplices.
-
- Added functionality for BilinearFormIntegrators to use kernels that work for both
-  tensor and unstructured elements.
-
- Added partial assembly for linear elasticity. Does not use sum factorization for now.
-
- Added partial assembly and GPU support for the DG diffusion integrator.
-
-New and updated examples and miniapps
-------------------------------------
- Added a new block solver in miniapp/solvers for the Darcy problem.
-  The new solver is based on a Bramble-Pasciak preconditioning. User can
-  use and implement their own preconditioner for the mass matrix.
-
- Added miniapp to demonstrate new elasticity integrator and unstructured element GPU support,
-  and a block diagonal preconditioner using low order refinement. Allows comparison with
-  currently existing legacy mode integrator. See miniapps/solvers/lor_elast.
-
- Added a new mortar contact patch test miniapp using the Tribol interface
-  physics library (see https://github.com/LLNL/Tribol).  See miniapps/tribol.
-
-Miscellaneous
-------------
- Added support for single and double precision, with corresponding hypre build.
-  Generalized the floating point type from `double` to `real_t`. For more
-  details see https://github.com/orgs/mfem/discussions/4207.
+- Added support for internal boundary elements in nonconforming meshes.

 - The ReadCubit Genesis mesh importer has been rewritten to improve readability.

+Discretization improvements
+---------------------------
+- Added a new nonlinear integrator, `HyperbolicFormIntegrator` that implements
+  both element-wise weak divergence and face-wise numerical flux for a general
+  system of hyperbolic conservation laws. To use the integrator for a specific
+  flux function, users can define a derived class of `FluxFunction`. Currently,
+  advection, Burgers, shallow-water and Euler equations (see Example 18/18p) are
+  available.
+
+- Added a capability to construct cut-surface and cut-volume IntegrationRules
+  through a moment-fitting approach. The cut is specified by the zero level set
+  of a Coefficient. See fem/intrules_cut.hpp and the new Example 38.
+
+- Introduced support for high-order nonconforming Nedelec elements on simplices.
+
+GPU computing
+-------------
+- Added partial assembly and GPU support for the DG diffusion integrator.
+
+- Efficient GPU-accelerated LOR assembly is now supported on surface meshes.
+
+- Added functionality to automatically configure hypre's compute policy to match
+  MFEM's compute policy when hypre is built with GPU support. Requires version
+  hypre-2.31.0 or later.
+
+- Added support for full assembly on simplices.
+
+- Added partial assembly for linear elasticity (no sum factorization for now).
+
+- Added functionality for BilinearFormIntegrators to use kernels that work for
+  both tensor and unstructured elements.
+
+- The RAJA backend will use `seq_exec` for serial loop execution when RAJA
+  v2023.06.00 and beyond is detected as `loop_exec` is deprecated.
+
+- API change: The macro MFEM_HYPRE_FORALL (from hypre.hpp) which was intended
+  for internal use, has been removed and replaced by the function template
+  mfem::hypre_forall in general/forall.hpp.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new miniapp illustrating elastic contact based on the Tribol library,
+  (https://github.com/LLNL/Tribol). See miniapps/tribol.
+
+- Added a miniapp to demonstrate low order refined (LOR) block preconditioning
+  for linear elasticity on GPUs. See miniapps/solvers/lor_elast.
+
+- Added a new block solver in miniapp/solvers for the Darcy problem. The new
+  solver is based on a Bramble-Pasciak preconditioning. User can use and
+  implement their own preconditioner for the mass matrix.
+
+- Added a small miniapp for printing the shape functions of a KnotVector. See
+  miniapps/nurbs/nurbs_printfunc.cpp.
+
+- Added two new example codes: 38 and 39/39p described above. Substantially
+  updated Example 18/18p.
+
+Miscellaneous
+-------------
 - Updated the Doxygen documentation style, which now requires Doxygen version
  1.9.8 or later. See the doc/ directory.

- Improved thread safety for global variables in the library, for example
-  IntegrationRules IntRules, RefinedIntRules, GeometryRefiner
-  GlobGeometryRefiner, and FiniteElement::dof2quad_array.
+- Improved thread safety for global variables in the library, e.g. for IntRules,
+  RefinedIntRules, GlobGeometryRefiner, and FiniteElement::dof2quad_array.

 - PETSc integration now generally requires PETSc version 3.21 or later, though
  depending on the functionality older versions may still work.

- RAJA backend will use seq_exec for serial loop execution when RAJA
-  v2023.06.00 and beyond is detected as loop_exec is deprecated.
+- Various other simplifications, extensions, and bugfixes in the code.
+
+- Added GSLIB-based gather-scatter operator.

- Adding named attribute sets and basic supporting methods to the Mesh class as
-  a convenient means of referring to sets of domain or boundary attribute
-  numbers. Also adding related serial and parallel examples which illustrate.

 Version 4.6, released on September 27, 2023
 ===========================================
@@ -104,7 +134,6 @@ Meshing improvements
  * The edge to knot map for NURBS meshes can be determined automatically. It is
    no longer needed to specify this in the NURBS mesh.
  * Added curve interpolation method for NURBS.
-  * Added new small miniapp for printing of shape functions of a KnotVector
  * See miniapps/nurbs for example meshes and miniapps.

 Discretization improvements
@@ -151,8 +180,6 @@ Linear and nonlinear solvers

 - Added HIP support to the PETSc and SUNDIALS interfaces.

- Efficient GPU-accelerated LOR assembly now supports surface meshes.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new H(div) solver miniapp demonstrating the use of a matrix-free
@@ -58,7 +58,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.6.1)
+set(${PROJECT_NAME}_VERSION 4.7.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -75,6 +75,8 @@ and miniapps. See https://glvis.org and https://mfem.org/building.

 Quick start with GNU make
 =========================
+See also: https://mfem.org/building
+
 Serial build:
   make serial -j 4

@@ -83,6 +85,7 @@ Parallel build:
   (build METIS 4 in ../metis-4.0 relative to mfem/)
   (build hypre in ../hypre relative to mfem/)
   make parallel -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   make cuda -j 4
@@ -116,6 +119,7 @@ Parallel build:
   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
   make -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   (this build requires CMake 3.8 or newer)
@@ -612,9 +616,13 @@ The specific libraries and their options are:
            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
            HYPRE >= 2.22.1   (HYPRE built with CUDA)
            HYPRE >= 2.23.0   (HYPRE built with HIP)
+            HYPRE >= 2.31.0   (runtime selectable HYPRE execution on CPU/GPU)

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
-  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
+  MFEM_USE_METIS_5 = YES (default is to use METIS 4). For building instructions,
+  see the following:
+    - METIS 4.0.3: https://mfem.org/building/#parallel-mpi-version-of-mfem
+    - METIS 5.1.0: https://mfem.org/building/#parallel-build-using-metis-5
  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
  Options: METIS_OPT, METIS_LIB.
  Versions: METIS 4.0.3 or 5.1.0.
@@ -120,6 +120,15 @@ constexpr real_t operator""_r(unsigned long long v)

 // Check dependencies:

+// Define MFEM_MPI_REAL_T to be the appropriate MPI real type
+#ifdef MFEM_USE_MPI
+#ifdef MFEM_USE_SINGLE
+#define MFEM_MPI_REAL_T MPI_FLOAT
+#elif defined MFEM_USE_DOUBLE
+#define MFEM_MPI_REAL_T MPI_DOUBLE
+#endif
+#endif
+
 // Options that require MPI
 #ifndef MFEM_USE_MPI
 #ifdef MFEM_USE_SUPERLU
@@ -388,7 +388,7 @@ GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LINK_LIB_DIR) -L$(GINKGO_LINK_LIB_DIR)\
 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
 AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx
+AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt

 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -110,4 +110,4 @@ config-mk:

 clean:
 	rm -f $(CONFIG_HPP) $(CONFIG_MK) sample-runs-build.log
-	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out
+	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out *.dSYM
@@ -92,4 +92,5 @@ vertices
 -0.70710678 -0.70710678
 0 -1
 0.70710678 -0.70710678
+
 mfem_mesh_end
@@ -48,7 +48,7 @@ PROJECT_NAME           = MFEM
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.6.1
+PROJECT_NUMBER         = v4.7.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -110,9 +110,13 @@ namespace mfem {
 * - <a class="el" href="ex35p_8cpp_source.html">Example 35p</a>: parallel multi-domain damped harmonic oscillators
 * - <a class="el" href="ex36_8cpp_source.html">Example 36</a>: Proximal Galerkin FEM for the obstacle problem
 * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
- * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: Topology optimization
+ * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: topology optimization
 * - <a class="el" href="ex37p_8cpp_source.html">Example 37p</a>: parallel topology optimization
 * - <a class="el" href="ex38_8cpp_source.html">Example 38</a>: cut-surface and cut-volume integration
+ * - <a class="el" href="ex39_8cpp_source.html">Example 39</a>: named mesh attributes
+ * - <a class="el" href="ex39p_8cpp_source.html">Example 39p</a>: parallel named mesh attributes
+ * - <a class="el" href="ex40_8cpp_source.html">Example 40</a>: eikonal equation
+ * - <a class="el" href="ex40p_8cpp_source.html">Example 40p</a>: parallel eikonal equation
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -214,6 +218,8 @@ namespace mfem {
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
+ * - <a class="el" href="contact-patch-test_8cpp_source.html">Contact</a>: mortar contact patch test for elasticity
+ * - <a class="el" href="multidomain_8cpp_source.html">Multidomain miniapp</a>: Multidomain and Submesh demonstration miniapp
 * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
 * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
 * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
@@ -46,7 +46,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
            DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
        })
        // Update the color scheme when the tab is made visible again.
-        // It is possible that the appearance was changed in another tab 
+        // It is possible that the appearance was changed in another tab
        // while this tab was in the background.
        document.addEventListener("visibilitychange", visibilityState => {
            if (document.visibilityState === 'visible') {
@@ -97,7 +97,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
     * @returns `true` for dark-mode, `false` for light-mode user preference
     */
    static get userPreference() {
-        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) || 
+        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) ||
        (DoxygenAwesomeDarkModeToggle.systemPreference && !localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey))
    }

@@ -45,6 +45,7 @@ list(APPEND ALL_EXE_SRCS
  ex37.cpp
  ex38.cpp
  ex39.cpp
+  ex40.cpp
  )

 if (MFEM_USE_MPI)
@@ -87,6 +88,7 @@ if (MFEM_USE_MPI)
    ex36p.cpp
    ex37p.cpp
    ex39p.cpp
+    ex40p.cpp
  )
 endif()

@@ -146,10 +148,10 @@ if (MFEM_ENABLE_TESTING)
  # Add CUDA/HIP tests.
  set(DEVICE_EXAMPLES
      # serial examples with device support:
-      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
+      ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
      # parallel examples with device support:
-      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p
-      ex34p ex35p)
+      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p ex22p ex24p ex25p
+      ex26p ex34p ex35p)
  set(MFEM_TEST_DEVICE)
  if (MFEM_USE_CUDA)
    set(MFEM_TEST_DEVICE "cuda")
@@ -159,6 +161,11 @@ if (MFEM_ENABLE_TESTING)
  if (MFEM_TEST_DEVICE)
    foreach(TEST_NAME ${DEVICE_EXAMPLES})
      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+      if (${TEST_NAME} MATCHES "ex14p")
+        list(APPEND THIS_TEST_OPTIONS "-rs" "2" "-rp" "0" "-pa")
+      elseif (${TEST_NAME} MATCHES "ex14")
+        list(APPEND THIS_TEST_OPTIONS "-r" "2" "-pa")
+      endif()
      if (NOT (${TEST_NAME} MATCHES ".*p$"))
        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
          COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
@@ -646,10 +646,7 @@ real_t HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 real_t HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   real_t loc_energy = 0.5*M.InnerProduct(v, v);
-   real_t energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPITypeMap<real_t>::mpi_type,
-                 MPI_SUM, fespace.GetComm());
+   real_t energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -39,8 +39,8 @@ private:
   // Base Nonlinear Form
   std::unique_ptr<NonlinearForm> nonlinearForm;
   // element-wise inverse mass matrix
-   std::vector<DenseMatrix> invmass; // local scalar inverse mass.
-   std::vector<DenseMatrix> weakdiv; // local weakdivergence. Trial space is ByDim.
+   std::vector<DenseMatrix> invmass; // local scalar inverse mass
+   std::vector<DenseMatrix> weakdiv; // local weak divergence (trial space ByDim)
   // global maximum characteristic speed. Updated by form integrators
   mutable real_t max_char_speed;
   // auxiliary variable used in Mult
@@ -169,9 +169,9 @@ void DGHyperbolicConservationLaws::Mult(const Vector &x, Vector &y) const
 {
   // 0. Reset wavespeed computation before operator application.
   formIntegrator->ResetMaxCharSpeed();
-   // 1. Apply Nonlinear form to obtain an axiliary result
+   // 1. Apply Nonlinear form to obtain an auxiliary result
   //         z = - <F̂(u_h,n), [[v]]>_e
-   //    If weak-divergencee is not preassembled, we also have weak-divergence
+   //    If weak-divergence is not preassembled, we also have weak-divergence
   //         z = - <F̂(u_h,n), [[v]]>_e + (F(u_h), ∇v)
   nonlinearForm->Mult(x, z);
   if (!weakdiv.empty()) // if weak divergence is pre-assembled
@@ -0,0 +1,374 @@
+//                                MFEM Example 40
+//
+// Compile with: make ex40
+//
+// Sample runs: ex40 -step 10 -gr 2.0
+//              ex40 -step 10 -gr 2.0 -o 3 -r 1
+//              ex40 -step 10 -gr 2.0 -r 4 -m ../data/l-shape.mesh
+//              ex40 -step 10 -gr 2.0 -r 2 -m ../data/fichera.mesh
+//
+// Description: This example code demonstrates how to use MFEM to solve the
+//              eikonal equation,
+//
+//                      |∇𝑢| = 1 in Ω,  𝑢 = g on ∂Ω.
+//
+//              The solution of this problem coincides with the unique optimum of
+//              the nonlinear program
+//
+//                   maximize ∫_Ω 𝑢 d𝑥 subject to |∇𝑢| ≤ 1,   𝑢 = g on Ω,      (⋆)
+//
+//              which is the foundation for method implemented below.
+//
+//              Following the proximal Galerkin methodology [1] (see also Example
+//              36), we construct a Legendre function for the unit ball
+//              𝐵₁ := {𝑥 ∈ Rⁿ | |𝑥| < 1}. Our choice is the Hellinger entropy,
+//
+//                    h(𝑥) = −( 1 − |𝑥|² )^{1/2},
+//
+//              although other choices are possible, each leading to a slightly
+//              different algorithm. We then adaptively regularize the optimization
+//              problem (⋆) with the Bregman divergence of the Hellinger entropy,
+//
+//                 maximize  ∫_Ω 𝑢 d𝑥 - αₖ⁻¹ Dₕ(∇𝑢,∇𝑢ₖ₋₁)  subject to  𝑢 = g on Ω.
+//
+//              This results in a sequence of functions ( 𝜓ₖ , 𝑢ₖ ),
+//
+//                      𝑢ₖ → 𝑢,    𝜓ₖ/|𝜓ₖ| → ∇𝑢    as k → \infty,
+//
+//              defined by the nonlinear saddle-point problems
+//
+//               Find 𝜓ₖ ∈ H(div,Ω) and 𝑢ₖ ∈ L²(Ω) such that
+//               ( Zₖ(𝜓ₖ) , τ ) + ( 𝑢ₖ , ∇⋅τ ) = ⟨ g , τ⋅n ⟩         ∀ τ ∈ H(div,Ω)
+//               ( ∇⋅𝜓ₖ , v )                 = ( ∇⋅𝜓ₖ₋₁ - 1 , v )   ∀ v ∈ L²(Ω)
+//
+//              where Zₖ(𝜓) := ∇h⁻¹(αₖ 𝜓) = 𝜓 / ( αₖ⁻² + |𝜓|² )^{1/2} and step size
+//              αₖ > 0. These saddle-point problems are solved using a damped Newton's
+//              method. This example assumes that g = 0 and allows the step size to
+//              grow geometrically, αₖ = α₀rᵏ, where r ≥ 1 is the growth rate.
+//
+//              [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//                  preserving finite element method for pointwise bound constraints.
+//                  arXiv:2307.12444 [math.NA]
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ZCoefficient : public VectorCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   ZCoefficient(int vdim, GridFunction &psi_, real_t alpha_ = 1.0)
+      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+class DZCoefficient : public MatrixCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   DZCoefficient(int height, GridFunction &psi_, real_t alpha_ = 1.0)
+      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int max_it = 5;
+   int ref_levels = 3;
+   real_t alpha = 1.0;
+   real_t growth_rate = 1.0;
+   real_t newton_scaling = 0.9;
+   real_t tichonov = 1e-1;
+   real_t tol = 1e-4;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Initial size alpha");
+   args.AddOption(&growth_rate, "-gr", "--growth-rate",
+                  "Growth rate of the step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+
+   MFEM_ASSERT(mesh.bdr_attributes.Size(),
+               "This example does not currently support meshes"
+               " without boundary attributes."
+              )
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   RT_FECollection RTfec(order, dim);
+   FiniteElementSpace RTfes(&mesh, &RTfec);
+
+   L2_FECollection L2fec(order, dim);
+   FiniteElementSpace L2fes(&mesh, &L2fec);
+
+   cout << "Number of H(div) dofs: "
+        << RTfes.GetTrueVSize() << endl;
+   cout << "Number of L² dofs: "
+        << L2fes.GetTrueVSize() << endl;
+
+   // 5. Define the offsets for the block matrices
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = RTfes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   offsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   // 6. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   GridFunction u_gf, delta_psi_gf;
+   delta_psi_gf.MakeRef(&RTfes,x,offsets[0]);
+   u_gf.MakeRef(&L2fes,x,offsets[1]);
+
+   GridFunction psi_old_gf(&RTfes);
+   GridFunction psi_gf(&RTfes);
+   GridFunction u_old_gf(&L2fes);
+
+   // 7. Define initial guesses for the solution variables.
+   delta_psi_gf = 0.0;
+   psi_gf = 0.0;
+   u_gf = 0.0;
+   psi_old_gf = psi_gf;
+   u_old_gf = u_gf;
+
+   // 8. Prepare for glvis output.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 9. Coefficients to be used later.
+   ConstantCoefficient neg_one(-1.0);
+   ConstantCoefficient zero(0.0);
+   ConstantCoefficient tichonov_cf(tichonov);
+   ConstantCoefficient neg_tichonov_cf(-1.0*tichonov);
+   ZCoefficient Z(sdim, psi_gf, alpha);
+   DZCoefficient DZ(sdim, psi_gf, alpha);
+   ScalarVectorProductCoefficient neg_Z(-1.0, Z);
+   DivergenceGridFunctionCoefficient div_psi_cf(&psi_gf);
+   DivergenceGridFunctionCoefficient div_psi_old_cf(&psi_old_gf);
+   SumCoefficient psi_old_minus_psi(div_psi_old_cf, div_psi_cf, 1.0, -1.0);
+
+   // 10. Assemble constant matrices/vectors to avoid reassembly in the loop.
+   LinearForm b0, b1;
+   b0.MakeRef(&RTfes,rhs.GetBlock(0),0);
+   b1.MakeRef(&L2fes,rhs.GetBlock(1),0);
+
+   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_Z));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+
+   BilinearForm a00(&RTfes);
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(DZ));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(tichonov_cf));
+
+   MixedBilinearForm a10(&RTfes,&L2fes);
+   a10.AddDomainIntegrator(new VectorFEDivergenceIntegrator());
+   a10.Assemble();
+   a10.Finalize();
+   SparseMatrix &A10 = a10.SpMat();
+   SparseMatrix *A01 = Transpose(A10);
+
+   BilinearForm a11(&L2fes);
+   a11.AddDomainIntegrator(new MassIntegrator(neg_tichonov_cf));
+   a11.Assemble();
+   a11.Finalize();
+   SparseMatrix &A11 = a11.SpMat();
+
+   // 11. Iterate.
+   int k;
+   int total_iterations = 0;
+   real_t increment_u = 0.1;
+   GridFunction u_tmp(&L2fes);
+   for (k = 0; k < max_it; k++)
+   {
+      u_tmp = u_old_gf;
+      Z.SetAlpha(alpha);
+      DZ.SetAlpha(alpha);
+
+      mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+
+      int j;
+      for ( j = 0; j < 5; j++)
+      {
+         total_iterations++;
+
+         b0.Assemble();
+         b1.Assemble();
+
+         a00.Assemble(false);
+         a00.Finalize(false);
+         SparseMatrix &A00 = a00.SpMat();
+
+         // Construct Schur-complement preconditioner
+         Vector A00_diag(a00.Height());
+         A00.GetDiag(A00_diag);
+         A00_diag.Reciprocal();
+         SparseMatrix *S = Mult_AtDA(*A01, A00_diag);
+
+         BlockDiagonalPreconditioner prec(offsets);
+         prec.SetDiagonalBlock(0,new DSmoother(A00));
+#ifndef MFEM_USE_SUITESPARSE
+         prec.SetDiagonalBlock(1,new GSSmoother(*S));
+#else
+         prec.SetDiagonalBlock(1,new UMFPackSolver(*S));
+#endif
+         prec.owns_blocks = 1;
+
+         BlockOperator A(offsets);
+         A.SetBlock(0,0,&A00);
+         A.SetBlock(1,0,&A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,&A11);
+
+         GMRES(A,prec,rhs,x,0,2000,500,1e-12,0.0);
+         delete S;
+
+         u_tmp -= u_gf;
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         // Damped Newton update
+         psi_gf.Add(newton_scaling, delta_psi_gf);
+         a00.Update();
+
+         if (visualization)
+         {
+            sol_sock << "solution\n" << mesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+         }
+
+         mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      mfem::out << "Number of Newton iterations = " << j+1 << endl;
+      mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      alpha *= max(growth_rate, 1_r);
+
+   }
+
+   mfem::out << "\n Outer iterations: " << k+1
+             << "\n Total iterations: " << total_iterations
+             << "\n Total dofs:       " << RTfes.GetTrueVSize() + L2fes.GetTrueVSize()
+             << endl;
+
+   delete A01;
+   return 0;
+}
+
+void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
+                        const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(vdim);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   V = psi_vals;
+   V *= phi;
+}
+
+void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                         const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(height);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   K = 0.0;
+   for (int i = 0; i < height; i++)
+   {
+      K(i,i) = phi;
+      for (int j = 0; j < height; j++)
+      {
+         K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
+      }
+   }
+}
@@ -0,0 +1,436 @@
+//                       MFEM Example 40 - Parallel Version
+//
+// Compile with: make ex40p
+//
+// Sample runs: mpirun -np 4 ex40p -step 10 -gr 2.0
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -o 3 -r 1
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -r 4 -m ../data/l-shape.mesh
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -r 2 -m ../data/fichera.mesh
+//
+// Description: This example code demonstrates how to use MFEM to solve the
+//              eikonal equation,
+//
+//                      |∇𝑢| = 1 in Ω,  𝑢 = g on ∂Ω.
+//
+//              The solution of this problem coincides with the unique optimum of
+//              the nonlinear program
+//
+//                   maximize ∫_Ω 𝑢 d𝑥 subject to |∇𝑢| ≤ 1,   𝑢 = g on Ω,      (⋆)
+//
+//              which is the foundation for method implemented below.
+//
+//              Following the proximal Galerkin methodology [1] (see also Example
+//              36), we construct a Legendre function for the unit ball
+//              𝐵₁ := {𝑥 ∈ Rⁿ | |𝑥| < 1}. Our choice is the Hellinger entropy,
+//
+//                    h(𝑥) = −( 1 − |𝑥|² )^{1/2},
+//
+//              although other choices are possible, each leading to a slightly
+//              different algorithm. We then adaptively regularize the optimization
+//              problem (⋆) with the Bregman divergence of the Hellinger entropy,
+//
+//                 maximize  ∫_Ω 𝑢 d𝑥 - αₖ⁻¹ Dₕ(∇𝑢,∇𝑢ₖ₋₁)  subject to  𝑢 = g on Ω.
+//
+//              This results in a sequence of functions ( 𝜓ₖ , 𝑢ₖ ),
+//
+//                      𝑢ₖ → 𝑢,    𝜓ₖ/|𝜓ₖ| → ∇𝑢    as k → \infty,
+//
+//              defined by the nonlinear saddle-point problems
+//
+//               Find 𝜓ₖ ∈ H(div,Ω) and 𝑢ₖ ∈ L²(Ω) such that
+//               ( Zₖ(𝜓ₖ) , τ ) + ( 𝑢ₖ , ∇⋅τ ) = ⟨ g , τ⋅n ⟩         ∀ τ ∈ H(div,Ω)
+//               ( ∇⋅𝜓ₖ , v )                 = ( ∇⋅𝜓ₖ₋₁ - 1 , v )   ∀ v ∈ L²(Ω)
+//
+//              where Zₖ(𝜓) := ∇h⁻¹(αₖ 𝜓) = 𝜓 / ( αₖ⁻² + |𝜓|² )^{1/2} and step size
+//              αₖ > 0. These saddle-point problems are solved using a damped Newton's
+//              method. This example assumes that g = 0 and allows the step size to
+//              grow geometrically, αₖ = α₀rᵏ, where r ≥ 1 is the growth rate.
+//
+//              [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//                  preserving finite element method for pointwise bound constraints.
+//                  arXiv:2307.12444 [math.NA]
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ZCoefficient : public VectorCoefficient
+{
+protected:
+   ParGridFunction *psi;
+   real_t alpha;
+
+public:
+   ZCoefficient(int vdim, ParGridFunction &psi_, real_t alpha_ = 1.0)
+      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+class DZCoefficient : public MatrixCoefficient
+{
+protected:
+   ParGridFunction *psi;
+   real_t alpha;
+
+public:
+   DZCoefficient(int height, ParGridFunction &psi_, real_t alpha_ = 1.0)
+      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int max_it = 5;
+   int ref_levels = 3;
+   real_t alpha = 1.0;
+   real_t growth_rate = 1.0;
+   real_t newton_scaling = 0.9;
+   real_t tichonov = 1e-1;
+   real_t tol = 1e-4;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Initial size alpha");
+   args.AddOption(&growth_rate, "-gr", "--growth-rate",
+                  "Growth rate of the step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2. Read the mesh from the mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+
+   MFEM_ASSERT(mesh.bdr_attributes.Size(),
+               "This example does not currently support meshes"
+               " without boundary attributes."
+              )
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   RT_FECollection RTfec(order, dim);
+   ParFiniteElementSpace RTfes(&pmesh, &RTfec);
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2fes(&pmesh, &L2fec);
+
+   int num_dofs_RT = RTfes.GlobalTrueVSize();
+   int num_dofs_L2 = L2fes.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of H(div) dofs: "
+           << num_dofs_RT << endl;
+      cout << "Number of L² dofs: "
+           << num_dofs_L2 << endl;
+   }
+
+   // 5. Define the offsets for the block matrices
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = RTfes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   offsets.PartialSum();
+
+   Array<int> toffsets(3);
+   toffsets[0] = 0;
+   toffsets[1] = RTfes.GetTrueVSize();
+   toffsets[2] = L2fes.GetTrueVSize();
+   toffsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   BlockVector tx(toffsets), trhs(toffsets);
+   tx = 0.0; trhs = 0.0;
+
+   // 6. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   ParGridFunction u_gf, delta_psi_gf;
+   delta_psi_gf.MakeRef(&RTfes,x,offsets[0]);
+   u_gf.MakeRef(&L2fes,x,offsets[1]);
+
+   ParGridFunction psi_old_gf(&RTfes);
+   ParGridFunction psi_gf(&RTfes);
+   ParGridFunction u_old_gf(&L2fes);
+
+   // 7. Define initial guesses for the solution variables.
+   delta_psi_gf = 0.0;
+   psi_gf = 0.0;
+   u_gf = 0.0;
+   psi_old_gf = psi_gf;
+   u_old_gf = u_gf;
+
+   // 8. Prepare for glvis output.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 9. Coefficients to be used later.
+   ConstantCoefficient neg_one(-1.0);
+   ConstantCoefficient zero(0.0);
+   ConstantCoefficient tichonov_cf(tichonov);
+   ConstantCoefficient neg_tichonov_cf(-1.0*tichonov);
+   ZCoefficient Z(sdim, psi_gf, alpha);
+   DZCoefficient DZ(sdim, psi_gf, alpha);
+   ScalarVectorProductCoefficient neg_Z(-1.0, Z);
+   DivergenceGridFunctionCoefficient div_psi_cf(&psi_gf);
+   DivergenceGridFunctionCoefficient div_psi_old_cf(&psi_old_gf);
+   SumCoefficient psi_old_minus_psi(div_psi_old_cf, div_psi_cf, 1.0, -1.0);
+
+   // 10. Assemble constant matrices/vectors to avoid reassembly in the loop.
+   ParLinearForm b0, b1;
+   b0.MakeRef(&RTfes,rhs.GetBlock(0),0);
+   b1.MakeRef(&L2fes,rhs.GetBlock(1),0);
+
+   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_Z));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+
+   ParBilinearForm a00(&RTfes);
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(DZ));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(tichonov_cf));
+
+   ParMixedBilinearForm a10(&RTfes,&L2fes);
+   a10.AddDomainIntegrator(new VectorFEDivergenceIntegrator());
+   a10.Assemble();
+   a10.Finalize();
+   HypreParMatrix *A10 = a10.ParallelAssemble();
+
+   HypreParMatrix *A01 = A10->Transpose();
+
+   ParBilinearForm a11(&L2fes);
+   a11.AddDomainIntegrator(new MassIntegrator(neg_tichonov_cf));
+   a11.Assemble();
+   a11.Finalize();
+   HypreParMatrix *A11 = a11.ParallelAssemble();
+
+   // 11. Iterate.
+   int k;
+   int total_iterations = 0;
+   real_t increment_u = 0.1;
+   ParGridFunction u_tmp(&L2fes);
+   for (k = 0; k < max_it; k++)
+   {
+      u_tmp = u_old_gf;
+      Z.SetAlpha(alpha);
+      DZ.SetAlpha(alpha);
+
+      if (myid == 0)
+      {
+         mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+      }
+
+      int j;
+      for ( j = 0; j < 5; j++)
+      {
+         total_iterations++;
+
+         b0.Assemble();
+         b0.ParallelAssemble(trhs.GetBlock(0));
+
+         b1.Assemble();
+         b1.ParallelAssemble(trhs.GetBlock(1));
+
+         a00.Assemble(false);
+         a00.Finalize(false);
+         HypreParMatrix *A00 = a00.ParallelAssemble();
+
+         // Construct Schur-complement preconditioner
+         HypreParVector A00_diag(MPI_COMM_WORLD, A00->GetGlobalNumRows(),
+                                 A00->GetRowStarts());
+         A00->GetDiag(A00_diag);
+         HypreParMatrix S_tmp(*A01);
+         S_tmp.InvScaleRows(A00_diag);
+         HypreParMatrix *S = ParMult(A10, &S_tmp, true);
+
+         BlockDiagonalPreconditioner prec(toffsets);
+         HypreBoomerAMG P00(*A00);
+         P00.SetPrintLevel(0);
+         HypreBoomerAMG P11(*S);
+         P11.SetPrintLevel(0);
+         prec.SetDiagonalBlock(0,&P00);
+         prec.SetDiagonalBlock(1,&P11);
+
+         BlockOperator A(toffsets);
+         A.SetBlock(0,0,A00);
+         A.SetBlock(1,0,A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,A11);
+
+         GMRESSolver gmres(MPI_COMM_WORLD);
+         gmres.SetPrintLevel(-1);
+         gmres.SetRelTol(1e-8);
+         gmres.SetMaxIter(2000);
+         gmres.SetKDim(500);
+         gmres.SetOperator(A);
+         gmres.SetPreconditioner(prec);
+         gmres.Mult(trhs,tx);
+         delete S;
+         delete A00;
+
+         delta_psi_gf.SetFromTrueDofs(tx.GetBlock(0));
+         u_gf.SetFromTrueDofs(tx.GetBlock(1));
+
+         u_tmp -= u_gf;
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         // Damped Newton update
+         psi_gf.Add(newton_scaling, delta_psi_gf);
+         a00.Update();
+
+         if (visualization)
+         {
+            sol_sock << "parallel " << num_procs << " " << myid << "\n";
+            sol_sock << "solution\n" << pmesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+         }
+
+         if (myid == 0)
+         {
+            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+         }
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      if (myid == 0)
+      {
+         mfem::out << "Number of Newton iterations = " << j+1 << endl;
+         mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+      }
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      alpha *= max(growth_rate, 1_r);
+
+   }
+
+   // 12. Print stats.
+   if (myid == 0)
+   {
+      mfem::out << "\n Outer iterations: " << k+1
+                << "\n Total iterations: " << total_iterations
+                << "\n Total dofs:       " << RTfes.GetTrueVSize() + L2fes.GetTrueVSize()
+                << endl;
+   }
+
+   // 13. Free the used memory.
+   delete A01;
+   delete A10;
+   delete A11;
+   return 0;
+}
+
+void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
+                        const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(vdim);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   V = psi_vals;
+   V *= phi;
+}
+
+void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                         const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(height);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   K = 0.0;
+   for (int i = 0; i < height; i++)
+   {
+      K(i,i) = phi;
+      for (int j = 0; j < height; j++)
+      {
+         K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
+      }
+   }
+}
@@ -23,14 +23,14 @@ MFEM_LIB_FILE = mfem_is_not_built

 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
   ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-   ex31 ex33 ex34 ex36 ex37 ex38 ex39
+   ex31 ex33 ex34 ex36 ex37 ex38 ex39 ex40
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
   ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-   ex37p ex39p
-SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
-PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-   ex24p ex25p ex26p ex34p ex35p
+   ex37p ex39p ex40p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
+PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
+   ex22p ex24p ex25p ex26p ex34p ex35p

 ifeq ($(MFEM_USE_LAPACK),YES)
   SEQ_EXAMPLES += ex38
@@ -138,6 +138,14 @@ ex10-test-seq: ex10
 	@$(call mfem-test,$<,, Serial example,-tf 5)
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-tf 5)
+ex14-test-seq-cuda: ex14
+	@$(call mfem-test,$<,, Serial CUDA example,-r 2 -pa -d cuda)
+ex14p-test-par-cuda: ex14p
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-rs 2 -rp 0 -pa -d cuda)
+ex14-test-seq-hip: ex14
+	@$(call mfem-test,$<,, Serial HIP example,-r 2 -pa -d hip)
+ex14p-test-par-hip: ex14p
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-rs 2 -rp 0 -pa -d hip)
 ex15-test-seq: ex15
 	@$(call mfem-test,$<,, Serial example,-e 1)
 ex15p-test-par: ex15p
@@ -1,3 +1,14 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
 #include <algorithm>
 #include <assert.h>
 #include <cstdlib>
@@ -709,10 +709,7 @@ real_t HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 real_t HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   real_t loc_energy = 0.5*M.InnerProduct(v, v);
-   real_t energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
-                 fespace.GetComm());
+   real_t energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -66,7 +66,6 @@ int main(int argc, char *argv[])
 {
   // 1. Initialize MPI (required by PUMI) and HYPRE.
   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
   int myid = Mpi::WorldRank();
   Hypre::Init();

@@ -80,8 +80,6 @@ int main(int argc, char *argv[])
 {
   // 1. Initialize MPI (required by PUMI) and HYPRE.
   Mpi::Init(argc, argv);
-   int num_proc = Mpi::WorldSize();
-   int myId = Mpi::WorldRank();
   Hypre::Init();

   // 2. Parse command-line options.
@@ -856,10 +856,7 @@ double HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 double HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   double loc_energy = 0.5*M.InnerProduct(v, v);
-   double energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPI_DOUBLE, MPI_SUM,
-                 fespace.GetComm());
+   double energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -340,9 +340,9 @@ public:
       $ M^{-1} $ (currently returns NULL) */
   virtual MatrixInverse *Inverse() const;

-   /** @brief  Finalizes the matrix initialization if the ::AssemblyLevel is
+   /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
       AssemblyLevel::LEGACY.
-       THe matrix that gets finalized is different if you are using static
+       The matrix that gets finalized is different if you are using static
       condensation or hybridization.*/
   virtual void Finalize(int skip_zeros = 1);

@@ -643,7 +643,7 @@ public:
   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
                       DiagonalPolicy dpolicy = DIAG_ONE);

-   /** @brief  Eliminate the given @a vdofs, storing the eliminated part
+   /** @brief Eliminate the given @a vdofs, storing the eliminated part
       internally in $ M_e $.

       This method works in conjunction with EliminateVDofsInRHS() and allows
@@ -826,7 +826,7 @@ public:
       $ M^{-1} $ (currently unimplemented and returns NULL)*/
   virtual MatrixInverse *Inverse() const;

-   /** @brief  Finalizes the matrix initialization if the ::AssemblyLevel is
+   /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
       AssemblyLevel::LEGACY.*/
   virtual void Finalize(int skip_zeros = 1);

@@ -1741,7 +1741,7 @@ public:
   { vector_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \cdot  \nabla \cdot u, \nabla \cdot v)$ in 2D
+/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \cdot \nabla u, \nabla \cdot v)$ in 2D
    or 3D and where $\hat{V}$ is a vector coefficient, $u$ is in $H^1$ and $v$ is in $H(div)$. */
 class MixedGradDivIntegrator : public MixedScalarVectorIntegrator
 {
@@ -1780,7 +1780,7 @@ public:
   { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \nabla \cdot u, \nabla \cdot v)$ in 2D
+/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \nabla \cdot u, \nabla v)$ in 2D
    or 3D and where $\hat{V}$ is a vector coefficient, $u$ is in $H(div)$ and $v$ is in $H^1$. */
 class MixedDivGradIntegrator : public MixedScalarVectorIntegrator
 {
@@ -1820,7 +1820,7 @@ public:
   { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} u, \nabla \cdot v)$ in 2D or 3D
+/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} u, \nabla v)$ in 2D or 3D
    and where $\hat{V}$ is a vector coefficient, $u$ is in $H^1$ or $L_2$ and $v$ is in $H^1$. */
 class MixedScalarWeakDivergenceIntegrator : public MixedScalarVectorIntegrator
 {
@@ -807,6 +807,7 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)

   QuadratureSpaceBase &qspace = *qf.GetSpace();
   const int ne = qspace.GetNE();
+   qf.HostWrite();
   DenseMatrix values;
   DenseSymmetricMatrix matrix;
   for (int iel = 0; iel < ne; ++iel)
@@ -818,7 +819,7 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
      {
         const IntegrationPoint &ip = ir[iq];
         T.SetIntPoint(&ip);
-         matrix.UseExternalData(&values(0, iq), vdim);
+         matrix.UseExternalData(&values(0, iq), height);
         Eval(matrix, T, ip);
      }
   }
@@ -828,13 +829,12 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
 void SymmetricMatrixCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
-   mat.SetSize(height);
-   Eval(mat, T, ip);
+   Eval(mat_aux, T, ip);
   for (int j = 0; j < width; ++j)
   {
      for (int i = 0; i < height; ++ i)
      {
-         K(i, j) = mat(i, j);
+         K(i, j) = mat_aux(i, j);
      }
   }
 }
@@ -924,6 +924,75 @@ void MatrixArrayCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
   }
 }

+MatrixArrayVectorCoefficient::MatrixArrayVectorCoefficient (int dim)
+   : MatrixCoefficient (dim)
+{
+   Coeff.SetSize(height);
+   ownCoeff.SetSize(height);
+   for (int i = 0; i < height; i++)
+   {
+      Coeff[i] = NULL;
+      ownCoeff[i] = true;
+   }
+}
+
+void MatrixArrayVectorCoefficient::SetTime(real_t t)
+{
+   for (int i=0; i < height; i++)
+   {
+      if (Coeff[i]) { Coeff[i]->SetTime(t); }
+   }
+   this->MatrixCoefficient::SetTime(t);
+}
+
+void MatrixArrayVectorCoefficient::Set(int i, VectorCoefficient * c, bool own)
+{
+   MFEM_ASSERT(i < height && i >= 0, "Row "
+               << i << " does not exist. " <<
+               "Matrix height = " << height << ".");
+   if (ownCoeff[i]) { delete Coeff[i]; }
+   Coeff[i] = c;
+   ownCoeff[i] = own;
+}
+
+MatrixArrayVectorCoefficient::~MatrixArrayVectorCoefficient ()
+{
+   for (int i=0; i < height; i++)
+   {
+      if (ownCoeff[i]) { delete Coeff[i]; }
+   }
+}
+
+void MatrixArrayVectorCoefficient::Eval(int i, Vector &V,
+                                        ElementTransformation &T,
+                                        const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(i < height && i >= 0, "Row "
+               << i << " does not exist. " <<
+               "Matrix height = " << height << ".");
+   if (Coeff[i])
+   {
+      Coeff[i] -> Eval(V, T, ip);
+   }
+   else
+   {
+      V = 0.0;
+   }
+}
+
+void MatrixArrayVectorCoefficient::Eval(DenseMatrix &K,
+                                        ElementTransformation &T,
+                                        const IntegrationPoint &ip)
+{
+   K.SetSize(height, width);
+   Vector V(width);
+   for (int i = 0; i < height; i++)
+   {
+      this->Eval(i, V, T, ip);
+      K.SetRow(i, V);
+   }
+}
+
 void MatrixRestrictedCoefficient::SetTime(real_t t)
 {
   if (c) { c->SetTime(t); }
@@ -1041,6 +1110,27 @@ real_t DeterminantCoefficient::Eval(ElementTransformation &T,
   return ma.Det();
 }

+TraceCoefficient::TraceCoefficient(MatrixCoefficient &A)
+   : a(&A), ma(A.GetHeight(), A.GetWidth())
+{
+   MFEM_ASSERT(A.GetHeight() == A.GetWidth(),
+               "TraceCoefficient:  "
+               "Argument must be a square matrix.");
+}
+
+void TraceCoefficient::SetTime(real_t t)
+{
+   if (a) { a->SetTime(t); }
+   this->Coefficient::SetTime(t);
+}
+
+real_t TraceCoefficient::Eval(ElementTransformation &T,
+                              const IntegrationPoint &ip)
+{
+   a->Eval(ma, T, ip);
+   return ma.Trace();
+}
+
 VectorSumCoefficient::VectorSumCoefficient(int dim)
   : VectorCoefficient(dim),
     ACoef(NULL), BCoef(NULL),
@@ -1326,6 +1416,30 @@ void InverseMatrixCoefficient::Eval(DenseMatrix &M,
   M.Invert();
 }

+ExponentialMatrixCoefficient::ExponentialMatrixCoefficient(MatrixCoefficient &A)
+   : MatrixCoefficient(A.GetHeight(), A.GetWidth()), a(&A)
+{
+   MFEM_ASSERT(A.GetHeight() == A.GetWidth() && A.GetHeight() == 2,
+               "ExponentialMatrixCoefficient:  "
+               << "Argument must be a square 2x2 matrix."
+               << "  Height = " << A.GetHeight()
+               << ", Width = " << A.GetWidth());
+}
+
+void ExponentialMatrixCoefficient::SetTime(real_t t)
+{
+   if (a) { a->SetTime(t); }
+   this->MatrixCoefficient::SetTime(t);
+}
+
+void ExponentialMatrixCoefficient::Eval(DenseMatrix &M,
+                                        ElementTransformation &T,
+                                        const IntegrationPoint &ip)
+{
+   a->Eval(M, T, ip);
+   M.Exponential();
+}
+
 OuterProductCoefficient::OuterProductCoefficient(VectorCoefficient &A,
                                                 VectorCoefficient &B)
   : MatrixCoefficient(A.GetVDim(), B.GetVDim()), a(&A), b(&B),
@@ -1334,6 +1334,46 @@ public:
   virtual ~MatrixArrayCoefficient();
 };

+/** @brief Matrix coefficient defined row-wise by an array of vector
+    coefficients. Rows that are not set will evaluate to zero. The
+    matrix coefficient is stored as an array indexing the rows of
+    the matrix. */
+class MatrixArrayVectorCoefficient : public MatrixCoefficient
+{
+private:
+   Array<VectorCoefficient *> Coeff;
+   Array<bool> ownCoeff;
+
+public:
+   /** @brief Construct a coefficient matrix of dimensions @a dim * @a dim. The
+       actual coefficients still need to be added with Set(). */
+   explicit MatrixArrayVectorCoefficient (int dim);
+
+   /// Set the time for internally stored coefficients
+   void SetTime(real_t t) override;
+
+   /// Get the vector coefficient located at the i-th row of the matrix
+   VectorCoefficient* GetCoeff (int i) { return Coeff[i]; }
+
+   /** @brief Set the coefficient located at the i-th row of the matrix.
+       By this will take ownership of the Coefficient passed in, but this
+       can be overridden with the @a own parameter. */
+   void Set(int i, VectorCoefficient * c, bool own=true);
+
+   using MatrixCoefficient::Eval;
+
+   /// Evaluate coefficient located at the i-th row of the matrix using integration
+   /// point @a ip.
+   void Eval(int i, Vector &V, ElementTransformation &T,
+             const IntegrationPoint &ip);
+
+   /// Evaluate the matrix coefficient @a ip.
+   void Eval(DenseMatrix &K, ElementTransformation &T,
+             const IntegrationPoint &ip) override;
+
+   virtual ~MatrixArrayVectorCoefficient();
+};
+

 /** @brief Derived matrix coefficient that has the value of the parent matrix
    coefficient where it is active and is zero otherwise. */
@@ -1426,12 +1466,13 @@ public:
 class SymmetricMatrixCoefficient : public MatrixCoefficient
 {
 protected:
+
   /// Internal matrix used when evaluating this coefficient as a DenseMatrix.
-   DenseSymmetricMatrix mat;
+   mutable DenseSymmetricMatrix mat_aux;
 public:
   /// Construct a dim x dim matrix coefficient.
   explicit SymmetricMatrixCoefficient(int dimension)
-      : MatrixCoefficient(dimension, true) { }
+      : MatrixCoefficient(dimension, true), mat_aux(height) { }

   /// Get the size of the matrix.
   int GetSize() const { return height; }
@@ -1464,8 +1505,9 @@ public:
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip);

-   /// Return a reference to the constant matrix.
-   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
+   /// @deprecated Return a reference to the internal matrix used when evaluating this coefficient as a DenseMatrix.
+   MFEM_DEPRECATED const DenseSymmetricMatrix& GetMatrix() { return mat_aux; }

   virtual ~SymmetricMatrixCoefficient() { }
 };
@@ -1485,6 +1527,10 @@ public:
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseSymmetricMatrix &M, ElementTransformation &T,
                     const IntegrationPoint &ip) { M = mat; }
+
+   /// Return a reference to the constant matrix.
+   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
 };


@@ -1761,6 +1807,31 @@ public:
                       const IntegrationPoint &ip);
 };

+/// Scalar coefficient defined as the trace of a matrix coefficient
+class TraceCoefficient : public Coefficient
+{
+private:
+   MatrixCoefficient * a;
+
+   mutable DenseMatrix ma;
+
+public:
+   /// Construct with the matrix.
+   TraceCoefficient(MatrixCoefficient &A);
+
+   /// Set the time for internally stored coefficients
+   void SetTime(real_t t);
+
+   /// Reset the matrix coefficient
+   void SetACoef(MatrixCoefficient &A) { a = &A; }
+   /// Return the matrix coefficient
+   MatrixCoefficient * GetACoef() const { return a; }
+
+   /// Evaluate the trace coefficient at @a ip.
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
 /// Vector coefficient defined as the linear combination of two vectors
 class VectorSumCoefficient : public VectorCoefficient
 {
@@ -2112,7 +2183,7 @@ public:
                     const IntegrationPoint &ip);
 };

-/// Matrix coefficient defined as the transpose a matrix coefficient
+/// Matrix coefficient defined as the transpose of a matrix coefficient
 class TransposeMatrixCoefficient : public MatrixCoefficient
 {
 private:
@@ -2135,7 +2206,7 @@ public:
                     const IntegrationPoint &ip);
 };

-/// Matrix coefficient defined as the inverse a matrix coefficient.
+/// Matrix coefficient defined as the inverse of a matrix coefficient.
 class InverseMatrixCoefficient : public MatrixCoefficient
 {
 private:
@@ -2158,6 +2229,29 @@ public:
                     const IntegrationPoint &ip);
 };

+/// Matrix coefficient defined as the exponential of a matrix coefficient.
+class ExponentialMatrixCoefficient : public MatrixCoefficient
+{
+private:
+   MatrixCoefficient * a;
+
+public:
+   /// Construct the matrix coefficient.  Result is $ \exp(A) $.
+   ExponentialMatrixCoefficient(MatrixCoefficient &A);
+
+   /// Set the time for internally stored coefficients
+   void SetTime(real_t t);
+
+   /// Reset the matrix coefficient
+   void SetACoef(MatrixCoefficient &A) { a = &A; }
+   /// Return the matrix coefficient
+   MatrixCoefficient * GetACoef() const { return a; }
+
+   /// Evaluate the matrix coefficient at @a ip.
+   virtual void Eval(DenseMatrix &M, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+};
+
 /// Matrix coefficient defined as the outer product of two vector coefficients.
 class OuterProductCoefficient : public MatrixCoefficient
 {
@@ -1243,25 +1243,16 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
         HypreParMatrix * Ah;
         A_i.Get(Ah);
         hypre_ParCSRMatrix *Aih = *Ah;
-#if !defined(HYPRE_USING_GPU)
-         ess_tdof_list.HostRead();
-         for (int k = 0; k < n; k++)
-         {
-            const int j = ess_tdof_list[k];
-            Aih->diag->data[Aih->diag->i[j]] = 0.0;
-         }
-#else
         Ah->HypreReadWrite();
         const int *d_ess_tdof_list =
-            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
-         const int *d_diag_i = Aih->diag->i;
+            ess_tdof_list.GetMemory().Read(GetHypreMemoryClass(), n);
+         HYPRE_Int *d_diag_i = Aih->diag->i;
         real_t *d_diag_data = Aih->diag->data;
-         MFEM_GPU_FORALL(k, n,
+         mfem::hypre_forall(n, [=] MFEM_HOST_DEVICE (int k)
         {
            const int j = d_ess_tdof_list[k];
            d_diag_data[d_diag_i[j]] = 0.0;
         });
-#endif
      }
      else
      {
@@ -1,8 +1,18 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
 #include "convergence.hpp"

 using namespace std;

-
 namespace mfem
 {

@@ -101,7 +101,7 @@ void DGMassInverse::SetRelTol(const real_t rel_tol_) { rel_tol = rel_tol_; }

 void DGMassInverse::SetAbsTol(const real_t abs_tol_) { abs_tol = abs_tol_; }

-void DGMassInverse::SetMaxIter(const real_t max_iter_) { max_iter = max_iter_; }
+void DGMassInverse::SetMaxIter(const int max_iter_) { max_iter = max_iter_; }

 void DGMassInverse::Update()
 {
@@ -137,7 +137,7 @@ void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const

   const real_t RELTOL = rel_tol;
   const real_t ABSTOL = abs_tol;
-   const real_t MAXIT = max_iter;
+   const int MAXIT = max_iter;
   const bool IT_MODE = iterative_mode;
   const bool CHANGE_BASIS = (d2q != nullptr);

@@ -96,7 +96,7 @@ public:
   /// Set the absolute tolerance.
   void SetAbsTol(const real_t abs_tol_);
   /// Set the maximum number of iterations.
-   void SetMaxIter(const real_t max_iter_);
+   void SetMaxIter(const int max_iter_);
   /// Recompute operator and preconditioner (when coefficient or mesh changes).
   void Update();

@@ -316,7 +316,7 @@ public:
   int GetDim() const { return dim; }

   /** @brief Returns the vector dimension for vector-valued finite elements,
-       which is also the dimension of the interpolation operatrion. */
+       which is also the dimension of the interpolation operation. */
   int GetRangeDim() const { return vdim; }

   /// Returns the dimension of the curl for vector-valued finite elements.
@@ -39,7 +39,7 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
   UseDevice(true);

   fes = new FiniteElementSpace;
-   fec = fes->Load(m, input);
+   fec_owned = fes->Load(m, input);

   skip_comment_lines(input, '#');
   istream::int_type next_char = input.peek();
@@ -81,10 +81,10 @@ GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)
   int vdim, ordering;

   fes = gf_array[0]->FESpace();
-   fec = FiniteElementCollection::New(fes->FEColl()->Name());
+   fec_owned = FiniteElementCollection::New(fes->FEColl()->Name());
   vdim = fes->GetVDim();
   ordering = fes->GetOrdering();
-   fes = new FiniteElementSpace(m, fec, vdim, ordering);
+   fes = new FiniteElementSpace(m, fec_owned, vdim, ordering);
   SetSize(fes->GetVSize());

   if (m->NURBSext)
@@ -153,11 +153,11 @@ GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)

 void GridFunction::Destroy()
 {
-   if (fec)
+   if (fec_owned)
   {
      delete fes;
-      delete fec;
-      fec = NULL;
+      delete fec_owned;
+      fec_owned = NULL;
   }
 }

@@ -325,10 +325,9 @@ int GridFunction::VectorDim() const
   const FiniteElement *fe;
   if (!fes->GetNE())
   {
-      const FiniteElementCollection *fe_coll = fes->FEColl();
      static const Geometry::Type geoms[3] =
      { Geometry::SEGMENT, Geometry::TRIANGLE, Geometry::TETRAHEDRON };
-      fe = fe_coll->
+      fe = fes->FEColl()->
           FiniteElementForGeometry(geoms[fes->GetMesh()->Dimension()-1]);
   }
   else
@@ -350,7 +349,8 @@ int GridFunction::CurlDim() const
   {
      static const Geometry::Type geoms[3] =
      { Geometry::SEGMENT, Geometry::TRIANGLE, Geometry::TETRAHEDRON };
-      fe = fec->FiniteElementForGeometry(geoms[fes->GetMesh()->Dimension()-1]);
+      fe = fes->FEColl()->
+           FiniteElementForGeometry(geoms[fes->GetMesh()->Dimension()-1]);
   }
   else
   {
@@ -1321,9 +1321,9 @@ void GridFunction::ProjectVectorFieldOn(GridFunction &vec_field, int comp)
   }
 }

-void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
-                                                      GridFunction &der,
-                                                      Array<int> &zones_per_dof)
+void GridFunction::AccumulateAndCountDerivativeValues(
+   int comp, int der_comp, GridFunction &der,
+   Array<int> &zones_per_dof) const
 {
   FiniteElementSpace * der_fes = der.FESpace();
   ElementTransformation * transf;
@@ -1374,7 +1374,8 @@ void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
   }
 }

-void GridFunction::GetDerivative(int comp, int der_comp, GridFunction &der)
+void GridFunction::GetDerivative(int comp, int der_comp,
+                                 GridFunction &der) const
 {
   Array<int> overlap;
   AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
@@ -2061,41 +2062,37 @@ void GridFunction::AccumulateAndCountBdrValues(
   Coefficient *coeff[], VectorCoefficient *vcoeff, const Array<int> &attr,
   Array<int> &values_counter)
 {
-   int i, j, fdof, d, ind, vdim;
-   real_t val;
-   const FiniteElement *fe;
-   ElementTransformation *transf;
   Array<int> vdofs;
   Vector vc;

   values_counter.SetSize(Size());
   values_counter = 0;

-   vdim = fes->GetVDim();
-
+   const int vdim = fes->GetVDim();
   HostReadWrite();

-   for (i = 0; i < fes->GetNBE(); i++)
+   for (int i = 0; i < fes->GetNBE(); i++)
   {
      if (attr[fes->GetBdrAttribute(i) - 1] == 0) { continue; }

-      fe = fes->GetBE(i);
-      fdof = fe->GetDof();
-      transf = fes->GetBdrElementTransformation(i);
+      const FiniteElement *fe = fes->GetBE(i);
+      const int fdof = fe->GetDof();
+      ElementTransformation *transf = fes->GetBdrElementTransformation(i);
      const IntegrationRule &ir = fe->GetNodes();
      fes->GetBdrElementVDofs(i, vdofs);

-      for (j = 0; j < fdof; j++)
+      for (int j = 0; j < fdof; j++)
      {
         const IntegrationPoint &ip = ir.IntPoint(j);
         transf->SetIntPoint(&ip);
         if (vcoeff) { vcoeff->Eval(vc, *transf, ip); }
-         for (d = 0; d < vdim; d++)
+         for (int d = 0; d < vdim; d++)
         {
            if (!vcoeff && !coeff[d]) { continue; }

-            val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
-            if ( (ind = vdofs[fdof*d+j]) < 0 )
+            real_t val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
+            int ind = vdofs[fdof*d+j];
+            if ( ind < 0 )
            {
               val = -val, ind = -1-ind;
            }
@@ -2117,10 +2114,11 @@ void GridFunction::AccumulateAndCountBdrValues(
   // iff A_ij != 0. It is sufficient to resolve just the first level of
   // dependency, since A is a projection matrix: A^n = A due to cR.cP = I.
   // Cases like these arise in 3D when boundary edges are constrained by
-   // (depend on) internal faces/elements. We use the virtual method
-   // GetBoundaryClosure from NCMesh to resolve the dependencies.
-
-   if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3)
+   // (depend on) internal faces/elements, or for internal boundaries in 2 or
+   // 3D. We use the virtual method GetBoundaryClosure from NCMesh to resolve
+   // the dependencies.
+   if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 ||
+                                fes->GetMesh()->Dimension() == 3))
   {
      Vector vals;
      Mesh *mesh = fes->GetMesh();
@@ -2128,26 +2126,19 @@ void GridFunction::AccumulateAndCountBdrValues(
      Array<int> bdr_edges, bdr_vertices, bdr_faces;
      ncmesh->GetBoundaryClosure(attr, bdr_vertices, bdr_edges, bdr_faces);

-      for (i = 0; i < bdr_edges.Size(); i++)
+      auto mark_dofs = [&](ElementTransformation &transf, const FiniteElement &fe)
      {
-         int edge = bdr_edges[i];
-         fes->GetEdgeVDofs(edge, vdofs);
-         if (vdofs.Size() == 0) { continue; }
-
-         transf = mesh->GetEdgeTransformation(edge);
-         transf->Attribute = -1; // TODO: set the boundary attribute
-         fe = fes->GetEdgeElement(edge);
         if (!vcoeff)
         {
-            vals.SetSize(fe->GetDof());
-            for (d = 0; d < vdim; d++)
+            vals.SetSize(fe.GetDof());
+            for (int d = 0; d < vdim; d++)
            {
               if (!coeff[d]) { continue; }

-               fe->Project(*coeff[d], *transf, vals);
+               fe.Project(*coeff[d], transf, vals);
               for (int k = 0; k < vals.Size(); k++)
               {
-                  ind = vdofs[d*vals.Size()+k];
+                  const int ind = vdofs[d*vals.Size()+k];
                  if (++values_counter[ind] == 1)
                  {
                     (*this)(ind) = vals(k);
@@ -2161,11 +2152,11 @@ void GridFunction::AccumulateAndCountBdrValues(
         }
         else // vcoeff != NULL
         {
-            vals.SetSize(vdim*fe->GetDof());
-            fe->Project(*vcoeff, *transf, vals);
+            vals.SetSize(vdim*fe.GetDof());
+            fe.Project(*vcoeff, transf, vals);
            for (int k = 0; k < vals.Size(); k++)
            {
-               ind = vdofs[k];
+               const int ind = vdofs[k];
               if (++values_counter[ind] == 1)
               {
                  (*this)(ind) = vals(k);
@@ -2176,6 +2167,26 @@ void GridFunction::AccumulateAndCountBdrValues(
               }
            }
         }
+      };
+
+      for (auto edge : bdr_edges)
+      {
+         fes->GetEdgeVDofs(edge, vdofs);
+         if (vdofs.Size() == 0) { continue; }
+
+         ElementTransformation *transf = mesh->GetEdgeTransformation(edge);
+         const FiniteElement *fe = fes->GetEdgeElement(edge);
+         mark_dofs(*transf, *fe);
+      }
+
+      for (auto face : bdr_faces)
+      {
+         fes->GetFaceVDofs(face, vdofs);
+         if (vdofs.Size() == 0) { continue; }
+
+         ElementTransformation *transf = mesh->GetFaceTransformation(face);
+         const FiniteElement *fe = fes->GetFaceElement(face);
+         mark_dofs(*transf, *fe);
      }
   }
 }
@@ -2228,26 +2239,37 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
      accumulate_dofs(dofs, lvec, *this, values_counter);
   }

-   if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3)
+   if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 ||
+                                fes->GetMesh()->Dimension() == 3))
   {
      Mesh *mesh = fes->GetMesh();
      NCMesh *ncmesh = mesh->ncmesh;
      Array<int> bdr_edges, bdr_vertices, bdr_faces;
      ncmesh->GetBoundaryClosure(bdr_attr, bdr_vertices, bdr_edges, bdr_faces);

-      for (int i = 0; i < bdr_edges.Size(); i++)
+      for (auto edge : bdr_edges)
      {
-         int edge = bdr_edges[i];
         fes->GetEdgeDofs(edge, dofs);
         if (dofs.Size() == 0) { continue; }

         T = mesh->GetEdgeTransformation(edge);
-         T->Attribute = -1; // TODO: set the boundary attribute
         fe = fes->GetEdgeElement(edge);
         lvec.SetSize(fe->GetDof());
         fe->Project(vcoeff, *T, lvec);
         accumulate_dofs(dofs, lvec, *this, values_counter);
      }
+
+      for (auto face : bdr_faces)
+      {
+         fes->GetFaceDofs(face, dofs);
+         if (dofs.Size() == 0) { continue; }
+
+         T = mesh->GetFaceTransformation(face);
+         fe = fes->GetFaceElement(face);
+         lvec.SetSize(fe->GetDof());
+         fe->Project(vcoeff, *T, lvec);
+         accumulate_dofs(dofs, lvec, *this, values_counter);
+      }
   }
 }

@@ -3904,7 +3926,7 @@ void GridFunction::LegacyNCReorder()
      mesh->GetEdgeVertices(i, ev);
      if (old_vertex[ev[0]] > old_vertex[ev[1]])
      {
-         const int *ind = fec->DofOrderForOrientation(Geometry::SEGMENT, -1);
+         const int *ind = fes->FEColl()->DofOrderForOrientation(Geometry::SEGMENT, -1);

         fes->GetEdgeInteriorDofs(i, dofs);
         for (int k = 0; k < dofs.Size(); k++)
@@ -30,14 +30,14 @@ namespace mfem
 class GridFunction : public Vector
 {
 protected:
-   /// FE space on which the grid function lives. Owned if #fec is not NULL.
+   /// FE space on which the grid function lives. Owned if #fec_owned is not NULL.
   FiniteElementSpace *fes;

   /** @brief Used when the grid function is read from a file. It can also be
       set explicitly, see MakeOwner().

       If not NULL, this pointer is owned by the GridFunction. */
-   FiniteElementCollection *fec;
+   FiniteElementCollection *fec_owned;

   long fes_sequence; // see FiniteElementSpace::sequence, Mesh::sequence

@@ -72,16 +72,16 @@ protected:

 public:

-   GridFunction() { fes = NULL; fec = NULL; fes_sequence = 0; UseDevice(true); }
+   GridFunction() { fes = NULL; fec_owned = NULL; fes_sequence = 0; UseDevice(true); }

   /// Copy constructor. The internal true-dof vector #t_vec is not copied.
   GridFunction(const GridFunction &orig)
-      : Vector(orig), fes(orig.fes), fec(NULL), fes_sequence(orig.fes_sequence)
+      : Vector(orig), fes(orig.fes), fec_owned(NULL), fes_sequence(orig.fes_sequence)
   { UseDevice(true); }

   /// Construct a GridFunction associated with the FiniteElementSpace @a *f.
   GridFunction(FiniteElementSpace *f) : Vector(f->GetVSize())
-   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+   { fes = f; fec_owned = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }

   /// Construct a GridFunction using previously allocated array @a data.
   /** The GridFunction does not assume ownership of @a data which is assumed to
@@ -91,13 +91,13 @@ public:
    */
   GridFunction(FiniteElementSpace *f, real_t *data)
      : Vector(data, f->GetVSize())
-   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+   { fes = f; fec_owned = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }

   /** @brief Construct a GridFunction using previously allocated Vector @a base
       starting at the given offset, @a base_offset. */
   GridFunction(FiniteElementSpace *f, Vector &base, int base_offset = 0)
      : Vector(base, base_offset, f->GetVSize())
-   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+   { fes = f; fec_owned = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }

   /// Construct a GridFunction on the given Mesh, using the data from @a input.
   /** The content of @a input should be in the format created by the method
@@ -116,12 +116,12 @@ public:
   GridFunction &operator=(const GridFunction &rhs)
   { return operator=((const Vector &)rhs); }

-   /// Make the GridFunction the owner of #fec and #fes.
-   /** If the new FiniteElementCollection, @a fec_, is NULL, ownership of #fec
+   /// Make the GridFunction the owner of #fec_owned and #fes.
+   /** If the new FiniteElementCollection, @a fec_, is NULL, ownership of #fec_owned
       and #fes is taken away. */
-   void MakeOwner(FiniteElementCollection *fec_) { fec = fec_; }
+   void MakeOwner(FiniteElementCollection *fec_) { fec_owned = fec_; }

-   FiniteElementCollection *OwnFEC() { return fec; }
+   FiniteElementCollection *OwnFEC() { return fec_owned; }

   int VectorDim() const;
   int CurlDim() const;
@@ -321,7 +321,7 @@ public:
       @param[out] der       The resulting derivative (scalar function). The
                             FiniteElementSpace of this function must be set
                             before the call. */
-   void GetDerivative(int comp, int der_comp, GridFunction &der);
+   void GetDerivative(int comp, int der_comp, GridFunction &der) const;

   real_t GetDivergence(ElementTransformation &tr) const;

@@ -443,7 +443,7 @@ protected:
       GetDerivative() method; see its documentation. */
   void AccumulateAndCountDerivativeValues(int comp, int der_comp,
                                           GridFunction &der,
-                                           Array<int> &zones_per_dof);
+                                           Array<int> &zones_per_dof) const;

   void AccumulateAndCountBdrValues(Coefficient *coeff[],
                                    VectorCoefficient *vcoeff,
@@ -1352,6 +1352,85 @@ void OversetFindPointsGSLIB::Interpolate(const Vector &point_pos,
   Interpolate(field_in, field_out);
 }

+GSOPGSLIB::GSOPGSLIB(Array<long long> &ids)
+{
+   gsl_comm = new gslib::comm;
+   cr       = new gslib::crystal;
+#ifdef MFEM_USE_MPI
+   int initialized;
+   MPI_Initialized(&initialized);
+   if (!initialized) { MPI_Init(NULL, NULL); }
+   MPI_Comm comm = MPI_COMM_WORLD;
+   comm_init(gsl_comm, comm);
+#else
+   comm_init(gsl_comm, 0);
+#endif
+   crystal_init(cr, gsl_comm);
+   UpdateIdentifiers(ids);
+}
+
+#ifdef MFEM_USE_MPI
+GSOPGSLIB::GSOPGSLIB(MPI_Comm comm_, Array<long long> &ids)
+   : cr(NULL), gsl_comm(NULL)
+{
+   gsl_comm = new gslib::comm;
+   cr      = new gslib::crystal;
+   comm_init(gsl_comm, comm_);
+   crystal_init(cr, gsl_comm);
+   UpdateIdentifiers(ids);
+}
+#endif
+
+GSOPGSLIB::~GSOPGSLIB()
+{
+   crystal_free(cr);
+   gslib_gs_free(gsl_data);
+   comm_free(gsl_comm);
+   delete gsl_comm;
+   delete cr;
+}
+
+void GSOPGSLIB::UpdateIdentifiers(const Array<long long> &ids)
+{
+   long long minval = ids.Min();
+#ifdef MFEM_USE_MPI
+   MPI_Allreduce(MPI_IN_PLACE, &minval, 1, MPI_LONG_LONG_INT,
+                 MPI_MIN, gsl_comm->c);
+#endif
+   MFEM_VERIFY(minval >= 0, "Unique identifier cannot be negative.");
+   if (gsl_data != NULL) { gslib_gs_free(gsl_data); }
+   num_ids = ids.Size();
+   gsl_data = gslib_gs_setup(ids.GetData(),
+                             ids.Size(),
+                             gsl_comm, 0,
+                             gslib::gs_crystal_router, 0);
+}
+
+void GSOPGSLIB::GS(Vector &senddata, GSOp op)
+{
+   MFEM_VERIFY(senddata.Size() == num_ids,
+               "Incompatible setup and GOP operation.");
+   if (op == GSOp::ADD)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_add,0,gsl_data,0);
+   }
+   else if (op == GSOp::MUL)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_mul,0,gsl_data,0);
+   }
+   else if (op == GSOp::MAX)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_max,0,gsl_data,0);
+   }
+   else if (op == GSOp::MIN)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_min,0,gsl_data,0);
+   }
+   else
+   {
+      MFEM_ABORT("Invalid GSOp operation.");
+   }
+}

 } // namespace mfem

@@ -23,13 +23,16 @@ struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
 struct crystal;
+struct gs_data;
 }

 namespace mfem
 {

 /** \brief FindPointsGSLIB can robustly evaluate a GridFunction on an arbitrary
- *  collection of points. There are three key functions in FindPointsGSLIB:
+ *  collection of points.
+ *
+ *  There are three key functions in FindPointsGSLIB:
 *
 *  1. Setup - constructs the internal data structures of gslib.
 *
@@ -226,6 +229,7 @@ public:

 /** \brief OversetFindPointsGSLIB enables use of findpts for arbitrary number of
    overlapping grids.
+
    The parameters in this class are the same as FindPointsGSLIB with the
    difference of additional inputs required to account for more than 1 mesh. */
 class OversetFindPointsGSLIB : public FindPointsGSLIB
@@ -290,6 +294,63 @@ public:
   using FindPointsGSLIB::Interpolate;
 };

+/** \brief  Class for gather-scatter (gs) operations on Vectors based on
+    corresponding global identifiers.
+
+    This functionality is useful for gs-ops on DOF values across processor
+    boundary, where the global identifier would be the corresponding true DOF
+    index. Operations currently supported are min, max, sum, and multiplication.
+    Note: identifier 0 does not participate in the gather-scatter operation and
+    a given identifier can be included multiple times on a given rank.
+    For example, consider a vector, v:
+    - v = [0.3, 0.4, 0.25, 0.7] on rank1,
+    - v = [0.6, 0.1] on rank 2,
+    - v = [-0.2, 0.3, 0.7, 0.] on rank 3.
+
+    Consider a corresponding Array<int>, a:
+    - a = [1, 2, 3, 1] on rank 1,
+    - a = [3, 2] on rank 2,
+    - a = [1, 2, 0, 3] on rank 3.
+
+    A gather-scatter "minimum" operation, done as follows:
+    GSOPGSLIB gs = GSOPGSLIB(MPI_COMM_WORLD, a);
+    gs.GS(v, GSOp::MIN);
+    would return into v:
+    - v = [-0.2, 0.1, 0., -0.2] on rank 1,
+    - v = [0., 0.1] on rank 2,
+    - v = [-0.2, 0.1, 0.7, 0.] on rank 3,
+    where the values have been compared across all processors based on the
+    integer identifier. */
+class GSOPGSLIB
+{
+protected:
+   struct gslib::crystal *cr;               // gslib's internal data
+   struct gslib::comm *gsl_comm;            // gslib's internal data
+   struct gslib::gs_data *gsl_data = NULL;
+   int num_ids;
+
+public:
+   GSOPGSLIB(Array<long long> &ids);
+
+#ifdef MFEM_USE_MPI
+   GSOPGSLIB(MPI_Comm comm_, Array<long long> &ids);
+#endif
+
+   virtual ~GSOPGSLIB();
+
+   /// Supported operation types. See class description.
+   enum GSOp {ADD, MUL, MIN, MAX};
+
+   /// Update the identifiers used for the gather-scatter operator.
+   /// Same @a ids get grouped together and id == 0 does not participate.
+   /// See class description.
+   void UpdateIdentifiers(const Array<long long> &ids);
+
+   /// Gather-Scatter operation on senddata. Must match length of unique
+   /// identifiers used in the constructor. See class description.
+   void GS(Vector &senddata, GSOp op);
+};
+
 } // namespace mfem

 #endif // MFEM_USE_GSLIB
@@ -18,7 +18,6 @@
 namespace mfem
 {

-
 void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
                                                     ElementTransformation &Tr,
                                                     const Vector &elfun,
@@ -29,7 +28,7 @@ void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
   const int dof = el.GetDof();

 #ifdef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration

   // shape function value at an integration point
   Vector shape(dof);
@@ -62,7 +61,7 @@ void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
      ir = &IntRules.Get(Tr.GetGeometryType(), order);
   }

-   // loop over interation points
+   // loop over integration points
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
@@ -92,7 +91,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
   const int dof2 = el2.GetDof();

 #ifdef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration

   // shape function value at an integration point - first elem
   Vector shape1(dof1);
@@ -122,7 +121,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
   DenseMatrix elvect2_mat(elvect.GetData() + dof1 * num_equations, dof2,
                           num_equations);

-   // obtain integration rule. If integration is rule is given, then use it.
+   // Obtain integration rule. If integration is rule is given, then use it.
   // Otherwise, get (2*p + IntOrderOffset) order integration rule
   const IntegrationRule *ir = IntRule;
   if (!ir)
@@ -149,7 +148,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
      if (nor.Size() == 1)  // if 1D, use 1 or -1.
      {
         // This assume the 1D integration point is in (0,1). This may not work
-         // if this chages.
+         // if this changes.
         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
      }
      else
@@ -18,43 +18,36 @@
 namespace mfem
 {

-//                     MFEM Hyperbolic Conservation Laws
+// This file contains general hyperbolic conservation element/face form
+// integrators. HyperbolicFormIntegrator and RiemannSolver are defined.
 //
-// Description:
+// HyperbolicFormIntegrator is a NonlinearFormIntegrator that implements
+// element weak divergence and interface flux
 //
-//    This file contains general hyperbolic conservation element/face form
-//    integrators.
+//     ∫_T F(u):∇v,   -∫_e F̂(u)⋅[[v]]
 //
-//    HyperbolicFormIntegrator and RiemannSolver are defined.
-//    HyperbolicFormIntegrator is a NonlinearFormIntegrator that implements
-//    element weak divergence and interface flux
+// Here, T is an element, e is an edge, and [[⋅]] is jump. This form integrator
+// is coupled with RiemannSolver that implements the numerical flux F̂. For
+// RiemannSolver, the Rusanov flux, also known as local Lax-Friedrichs flux, is
+// provided.
 //
-//       ∫_T F(u):∇v,   -∫_e F̂(u)⋅[[v]]
+// To implement a specific hyperbolic conservation laws, users can create
+// derived classes from FluxFunction with overloaded ComputeFlux. One can
+// optionally overload ComputeFluxDotN to avoid creating dense matrix when
+// computing normal flux. Several example equations are also defined including:
+// advection, Burgers', shallow water, and Euler equations. Users can control
+// the quadrature rule by either providing the integration rule, or integration
+// order offset. Integration will use 2*p + IntOrderOffset order quadrature
+// rule.
 //
-//    Here, T is an element, e is an edge, and [[⋅]] is jump. This form
-//    integrator is coupled with RiemannSolver that implements the numerical
-//    flux F̂. For RiemannSolver, the Rusanov flux, also known as local
-//    Lax-Friedrichs flux, is provided.
-//
-//    To implement a specific hyperbolic conservation laws, users can create
-//    derived classes from FluxFunction with overloaded ComputeFlux. One can
-//    optionally overload ComputeFluxDotN to avoid creating dense matrix when
-//    computing normal flux. Several example equations are also defined
-//    including: advection, Burgers', shallow water, and Euler equations. Users
-//    can control the quadrature rule by either providing the integration rule,
-//    or integration order offset. Integration will use 2*p + IntOrderOffset
-//    order quadrature rule.
-//
-//    At each call of HyperbolicFormIntegrator::AssembleElementVector
-//    HyperbolicFormIntegrator::AssembleFaceVector, the maximum characteristic
-//    speed will be updated. This will not be reinitialized automatically.
-//    To reinitialize, use HyperbolicFormIntegrator::ResetMaxCharSpeed. See,
-//    ex18.hpp.
-//
-//    Note: To avoid communication overhead, we update the maximum
-//    characteristic speed within each process. Use a proper MPI routine to
-//    gather the information.
+// At each call of HyperbolicFormIntegrator::AssembleElementVector
+// HyperbolicFormIntegrator::AssembleFaceVector, the maximum characteristic
+// speed will be updated. This will not be reinitialized automatically. To
+// reinitialize, use HyperbolicFormIntegrator::ResetMaxCharSpeed. See, ex18.hpp.
 //
+// Note: To avoid communication overhead, we update the maximum characteristic
+// speed within each MPI process only. Use the appropriate MPI routine to gather
+// the information.

 /**
 * @brief Abstract class for hyperbolic flux for a system of hyperbolic
@@ -88,7 +81,7 @@ public:
   virtual real_t ComputeFlux(const Vector &state, ElementTransformation &Tr,
                              DenseMatrix &flux) const = 0;
   /**
-    * @brief Compute normal flux. Optionally overloadded in the
+    * @brief Compute normal flux. Optionally overloaded in the
    * derived class to avoid creating full dense matrix for flux.
    *
    * @param[in] state state at the current integration point
@@ -168,13 +161,13 @@ protected:
 class HyperbolicFormIntegrator : public NonlinearFormIntegrator
 {
 private:
-   // The maximum characterstic speed, updated during element/face vector assembly
+   // The maximum characteristic speed, updated during element/face vector assembly
   real_t max_char_speed;
   const RiemannSolver &rsolver;   // Numerical flux that maps F(u±,x) to hat(F)
   const FluxFunction &fluxFunction;
   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
 #ifndef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration
   Vector shape;              // shape function value at an integration point
   Vector state;              // state value at an integration point
   DenseMatrix flux;          // flux value at an integration point
@@ -307,7 +307,7 @@ static void PADGDiffusionSetupFaceInfo2D(const int nf, const Mesh &mesh,
   }
 }

-// Assigns to perm the permuation:
+// Assigns to perm the permutation:
 //    perm[0] <- normal component
 //    perm[1] <- first tangential component
 //    perm[2] <- second tangential component
@@ -563,7 +563,7 @@ void DiffusionIntegrator::AssemblePatchMatrix_fullQuadrature(
   cdofs.SetSize(maxw[0], maxw[1], maxw[2]);

   // Compute sparsity of the sparse matrix
-   smati = new int[ndof+1];
+   smati = Memory<int>(ndof+1);
   smati[0] = 0;

   for (int dof_j=0; dof_j<ndof; ++dof_j)
@@ -586,8 +586,8 @@ void DiffusionIntegrator::AssemblePatchMatrix_fullQuadrature(
      nnz += ndd;
   }

-   smatj = new int[nnz];
-   smata = new real_t[nnz];
+   smatj = Memory<int>(nnz);
+   smata = Memory<real_t>(nnz);

   for (int i=0; i<nnz; ++i)
   {
@@ -973,7 +973,7 @@ void DiffusionIntegrator::AssemblePatchMatrix_reducedQuadrature(
   cdofs.SetSize(maxw[0], maxw[1], maxw[2]);

   // Compute sparsity of the sparse matrix
-   smati = new int[ndof+1];
+   smati = Memory<int>(ndof+1);
   smati[0] = 0;

   for (int dof_j=0; dof_j<ndof; ++dof_j)
@@ -996,8 +996,8 @@ void DiffusionIntegrator::AssemblePatchMatrix_reducedQuadrature(
      nnz += ndd;
   }

-   smatj = new int[nnz];
-   smata = new real_t[nnz];
+   smatj = Memory<int>(nnz);
+   smata = Memory<real_t>(nnz);

   for (int i=0; i<nnz; ++i)
   {
@@ -157,7 +157,7 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
   static constexpr int aSize = aUpper-aLower;
   static constexpr bool isComponent = (i_block >= 0);

-   //Assuming all elements are the same
+   // Assuming all elements are the same
   const auto &ir = QVec.GetIntRule(0);
   const QuadratureInterpolator *E_To_Q_Map = fespace.GetQuadratureInterpolator(
                                                 ir);
@@ -180,7 +180,7 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
         auto invJ = inv(make_tensor<d, d>(
         [&](int i, int j) { return J(p, i, j, e); }));
         tensor<real_t, aSize, d> gradx;
-         //load grad(x) into gradx
+         // load grad(x) into gradx
         if (isComponent)
         {
            for (int i = 0; i < d; i++)
@@ -198,11 +198,11 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
               }
            }
         }
-         //compute divergence
+         // compute divergence
         real_t div = 0.;
         for (int i = aLower; i < aUpper; i++)
         {
-            //take size of gradx into account
+            // take size of gradx into account
            const int iIndex = isComponent ? 0 : i;
            div += gradx(iIndex,i);
         }
@@ -211,11 +211,11 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
         {
            for (int q = qLower; q < qUpper; q++)
            {
-               //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-               //this contraction could be made slightly cheaper using Voigt
-               //notation, but repeated entries are summed for simplicity.
+               // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+               // this contraction could be made slightly cheaper using Voigt
+               // notation, but repeated entries are summed for simplicity.
               real_t contraction = 0.;
-               //not sure how to combine cases
+               // not sure how to combine cases
               if (isComponent)
               {
                  for (int a = 0; a < d; a++)
@@ -276,7 +276,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
                                   const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
-   //Assuming all elements are the same
+   // Assuming all elements are the same
   const auto &ir = QVec.GetIntRule(0);
   static constexpr int d = dim;
   const int numPoints = ir.GetNPoints();
@@ -299,9 +299,9 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
            {
               for (int q = 0; q < d; q++)
               {
-                  //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-                  //this contraction could be made slightly cheaper using Voigt
-                  //notation, but repeated entries are summed for simplicity.
+                  // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                  // this contraction could be made slightly cheaper using Voigt
+                  // notation, but repeated entries are summed for simplicity.
                  real_t contraction = 0.;
                  for (int a = 0; a < d; a++)
                  {
@@ -321,7 +321,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
      }
   });

-   //Reduce quadrature function to an E-Vector
+   // Reduce quadrature function to an E-Vector
   const auto QRead = Reshape(QVec.Read(), numPoints, d, d, d, numEls);
   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
@@ -348,7 +348,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
   });
 }

-//Templated implementation of ElasticityAssembleEA.
+// Templated implementation of ElasticityAssembleEA.
 template<int dim>
 void ElasticityAssembleEA_(const int i_block,
                           const int j_block,
@@ -360,7 +360,7 @@ void ElasticityAssembleEA_(const int i_block,
                           const DofToQuad &maps,
                           Vector &emat)
 {
-   //Assuming all elements are the same
+   // Assuming all elements are the same
   static constexpr int d = dim;
   const int numPoints = ir.GetNPoints();
   const int numEls = lambda.Size()/numPoints;
@@ -386,7 +386,7 @@ void ElasticityAssembleEA_(const int i_block,
               {
                  for (int m = 0; m < d; m++)
                  {
-                     //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                     // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
                     real_t contraction = 0.;
                     for (int a = 0; a < d; a++)
                     {
@@ -101,7 +101,7 @@ void MomentFittingIntRules::InitVolume(int order, Coefficient& levelset,
         }
      }

-      // assamble the matrix
+      // assemble the matrix
      DenseMatrix Mat(nBasisVolume, ir.GetNPoints());
      for (int ip = 0; ip < ir.GetNPoints(); ip++)
      {
@@ -118,7 +118,7 @@ void MomentFittingIntRules::InitVolume(int order, Coefficient& levelset,
         Mat.SetCol(ip, shape);
      }

-      // compute the svd for the matrix
+      // compute the SVD for the matrix
      VolumeSVD = new DenseMatrixSVD(Mat, 'A', 'A');
      VolumeSVD->Eval(Mat);
   }
@@ -1239,7 +1239,7 @@ void MomentFittingIntRules::OrthoBasis2D(const IntegrationPoint& ip,

   shape.SetSize(nBasis, 2);

-   // evaluate basis inthe point
+   // evaluate basis in the point
   DenseMatrix preshape(nBasis, 2);
   DivFreeBasis2D(ip, shape);

@@ -1597,6 +1597,6 @@ void MomentFittingIntRules::GetSurfaceWeights(ElementTransformation& Tr,
   }
 }

-#endif //MFEM_USE_LAPACK
+#endif // MFEM_USE_LAPACK

 }
@@ -291,7 +291,7 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   const auto ltdof_ldof = HypreRead(R->GetMemoryJ());

   // Go from E-vector format directly to T-vector format
-   MFEM_HYPRE_FORALL(i, ntdofs,
+   mfem::hypre_forall(ntdofs, [=] MFEM_HOST_DEVICE (int i)
   {
      const int j = d_offsets[ltdof_ldof[i]];
      for (int c = 0; c < sdim; ++c)
@@ -269,13 +269,13 @@ void BatchedLOR_H1::Assemble3D()
               real_t vx[8], vy[8], vz[8];
               LORVertexCoordinates3D<ORDER>(X, iel_ho, kx, ky, kz, vx, vy, vz);

-               //MFEM_UNROLL(2)
+               // MFEM_UNROLL(2)
               for (int iqz=0; iqz<2; ++iqz)
               {
-                  //MFEM_UNROLL(2)
+                  // MFEM_UNROLL(2)
                  for (int iqy=0; iqy<2; ++iqy)
                  {
-                     //MFEM_UNROLL(2)
+                     // MFEM_UNROLL(2)
                     for (int iqx=0; iqx<2; ++iqx)
                     {
                        const real_t x = iqx;
@@ -307,21 +307,21 @@ void BatchedLOR_H1::Assemble3D()
                  }
               }

-               //MFEM_UNROLL(2)
+               // MFEM_UNROLL(2)
               for (int iqx=0; iqx<2; ++iqx)
               {
-                  //MFEM_UNROLL(2)
+                  // MFEM_UNROLL(2)
                  for (int jz=0; jz<2; ++jz)
                  {
                     // Note loop starts at iz=jz here, taking advantage of
                     // symmetries.
-                     //MFEM_UNROLL(2)
+                     // MFEM_UNROLL(2)
                     for (int iz=jz; iz<2; ++iz)
                     {
-                        //MFEM_UNROLL(2)
+                        // MFEM_UNROLL(2)
                        for (int iqy=0; iqy<2; ++iqy)
                        {
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int iqz=0; iqz<2; ++iqz)
                           {
                              const real_t mq = const_mq ? MQ(0,0,0,0) : MQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
@@ -356,10 +356,10 @@ void BatchedLOR_H1::Assemble3D()
                              real_t wdetJ = Q(6,iqz,iqy,iqx);
                              mass_A(iqy,iz,jz,iqx) += mq*wdetJ*biz*bjz;
                           }
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int jy=0; jy<2; ++jy)
                           {
-                              //MFEM_UNROLL(2)
+                              // MFEM_UNROLL(2)
                              for (int iy=0; iy<2; ++iy)
                              {
                                 const real_t biy = (iy == iqy) ? 1.0 : 0.0;
@@ -382,16 +382,16 @@ void BatchedLOR_H1::Assemble3D()
                              }
                           }
                        }
-                        //MFEM_UNROLL(2)
+                        // MFEM_UNROLL(2)
                        for (int jy=0; jy<2; ++jy)
                        {
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int jx=0; jx<2; ++jx)
                           {
-                              //MFEM_UNROLL(2)
+                              // MFEM_UNROLL(2)
                              for (int iy=0; iy<2; ++iy)
                              {
-                                 //MFEM_UNROLL(2)
+                                 // MFEM_UNROLL(2)
                                 for (int ix=0; ix<2; ++ix)
                                 {
                                    const real_t bix = (ix == iqx) ? 1.0 : 0.0;
@@ -431,7 +431,7 @@ void BatchedLOR_H1::Assemble3D()
               // Assemble the local matrix into the macro-element sparse matrix
               // in a format similar to coordinate format. The (I,J) arrays
               // are implicit (not stored explicitly).
-               //MFEM_UNROLL(8)
+               // MFEM_UNROLL(8)
               for (int ii_loc=0; ii_loc<nv; ++ii_loc)
               {
                  const int ix = ii_loc%2;
@@ -368,6 +368,72 @@ const
   y.Add(a, Ytmp);
 }

+real_t ParBilinearForm::ParInnerProduct(const ParGridFunction &x,
+                                        const ParGridFunction &y) const
+{
+   MFEM_ASSERT(mat != NULL, "local matrix must be assembled");
+
+   real_t loc = InnerProduct(x, y);
+   real_t glob = 0.;
+
+   MPI_Allreduce(&loc, &glob, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                 pfes->GetComm());
+
+   return glob;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(const ParGridFunction &x,
+                                         const ParGridFunction &y) const
+{
+   MFEM_ASSERT(x.ParFESpace() == pfes, "the parallel spaces must match");
+   MFEM_ASSERT(y.ParFESpace() == pfes, "the parallel spaces must match");
+
+   HypreParVector *x_p = x.ParallelProject();
+   HypreParVector *y_p = y.ParallelProject();
+
+   real_t res = TrueInnerProduct(*x_p, *y_p);
+
+   delete x_p;
+   delete y_p;
+
+   return res;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(HypreParVector &x,
+                                         HypreParVector &y) const
+{
+   MFEM_VERIFY(p_mat.Ptr() != NULL, "parallel matrix must be assembled");
+
+   if (p_mat->GetType() != Operator::Hypre_ParCSR)
+   {
+      return TrueInnerProduct((const Vector&)x, (const Vector&)y);
+   }
+
+   HypreParVector *Ax = new HypreParVector(pfes);
+   HypreParMatrix *A = p_mat.As<HypreParMatrix>();
+
+   A->Mult(x, *Ax);
+
+   real_t res = mfem::InnerProduct(y, *Ax);
+
+   delete Ax;
+
+   return res;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(const Vector &x,
+                                         const Vector &y) const
+{
+   MFEM_VERIFY(p_mat.Ptr() != NULL, "parallel matrix must be assembled");
+
+   Vector Ax(pfes->GetTrueVSize());
+   p_mat->Mult(x, Ax);
+
+   real_t res = mfem::InnerProduct(pfes->GetComm(), y, Ax);
+
+   return res;
+}
+
 void ParBilinearForm::FormLinearSystem(
   const Array<int> &ess_tdof_list, Vector &x, Vector &b,
   OperatorHandle &A, Vector &X, Vector &B, int copy_interior)
@@ -173,6 +173,37 @@ public:
       vectors on the true dofs. */
   void TrueAddMult(const Vector &x, Vector &y, const real_t a = 1.0) const;

+   /// Compute $ y^T M x $
+   /** @warning The calculation is performed on local dofs, assuming that
+       the local vectors are consistent with the prolongations of the true
+       vectors (see ParGridFunction::Distribute()). If this is not the case,
+       use TrueInnerProduct(const ParGridFunction &, const ParGridFunction &)
+       instead.
+       @note It is assumed that the local matrix is assembled and it has
+       not been replaced by the parallel matrix through FormSystemMatrix().
+       @see TrueInnerProduct(const ParGridFunction&, const ParGridFunction&) */
+   real_t ParInnerProduct(const ParGridFunction &x,
+                          const ParGridFunction &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (grid function version)
+   /** @note The ParGridFunction%s are restricted to the true-vectors for
+       for calculation.
+       @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix().
+       @see ParInnerProduct(const ParGridFunction&, const ParGridFunction&) */
+   real_t TrueInnerProduct(const ParGridFunction &x,
+                           const ParGridFunction &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (Hypre vector version)
+   /** @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix(). */
+   real_t TrueInnerProduct(HypreParVector &x, HypreParVector &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (true-vector version)
+   /** @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix(). */
+   real_t TrueInnerProduct(const Vector &x, const Vector &y) const;
+
   /// Return the parallel FE space associated with the ParBilinearForm.
   ParFiniteElementSpace *ParFESpace() const { return pfes; }

@@ -861,17 +861,17 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
         }
      }

-      HYPRE_Int *i_diag = new HYPRE_Int[ldof+1];
-      HYPRE_Int *j_diag = new HYPRE_Int[ltdof];
-      real_t    *d_diag = new real_t[ltdof];
+      HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
+      real_t    *d_diag = Memory<real_t>(ltdof);
      int diag_counter;

-      HYPRE_Int *i_offd = new HYPRE_Int[ldof+1];
-      HYPRE_Int *j_offd = new HYPRE_Int[nnz_offd];
-      real_t    *d_offd = new real_t[nnz_offd];
+      HYPRE_Int *i_offd = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_offd = Memory<HYPRE_Int>(nnz_offd);
+      real_t    *d_offd = Memory<real_t>(nnz_offd);
      int offd_counter;

-      HYPRE_BigInt *cmap   = new HYPRE_BigInt[ldof-ltdof];
+      HYPRE_BigInt *cmap   = Memory<HYPRE_BigInt>(ldof-ltdof);

      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
      HYPRE_BigInt *row_starts = GetDofOffsets();
@@ -39,9 +39,10 @@ ParGridFunction::ParGridFunction(ParMesh *pmesh, const GridFunction *gf,
 {
   const FiniteElementSpace *glob_fes = gf->FESpace();
   // duplicate the FiniteElementCollection from 'gf'
-   fec = FiniteElementCollection::New(glob_fes->FEColl()->Name());
+   fec_owned = FiniteElementCollection::New(glob_fes->FEColl()->Name());
   // create a local ParFiniteElementSpace from the global one:
-   fes = pfes = new ParFiniteElementSpace(pmesh, glob_fes, partitioning, fec);
+   fes = pfes = new ParFiniteElementSpace(pmesh, glob_fes, partitioning,
+                                          fec_owned);
   SetSize(pfes->GetVSize());

   if (partitioning)
@@ -81,7 +82,7 @@ ParGridFunction::ParGridFunction(ParMesh *pmesh, std::istream &input)
   : GridFunction(pmesh, input)
 {
   // Convert the FiniteElementSpace, fes, to a ParFiniteElementSpace:
-   pfes = new ParFiniteElementSpace(pmesh, fec, fes->GetVDim(),
+   pfes = new ParFiniteElementSpace(pmesh, fec_owned, fes->GetVDim(),
                                    fes->GetOrdering());
   delete fes;
   fes = pfes;
@@ -249,6 +250,8 @@ void ParGridFunction::ExchangeFaceNbrData()
   auto send_data_ptr = mpi_gpu_aware ? send_data.Read() : send_data.HostRead();
   auto face_nbr_data_ptr = mpi_gpu_aware ? face_nbr_data.Write() :
                            face_nbr_data.HostWrite();
+   // Wait for the kernel to be done since it updates what's sent and it may be async
+   if (mpi_gpu_aware) { MFEM_STREAM_SYNC; }
   for (int fn = 0; fn < num_face_nbrs; fn++)
   {
      int nbr_rank = pmesh->GetFaceNbrRank(fn);
@@ -518,7 +521,7 @@ void ParGridFunction::CountElementsPerVDof(Array<int> &elem_per_vdof) const
 }

 void ParGridFunction::GetDerivative(int comp, int der_comp,
-                                    ParGridFunction &der)
+                                    ParGridFunction &der) const
 {
   Array<int> overlap;
   AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
@@ -713,10 +716,10 @@ void ParGridFunction::ProjectBdrCoefficient(
         }
      }
   }
+   gcomm.Bcast<int>(values_counter.HostReadWrite());
   for (int i = 0; i < values_counter.Size(); i++)
   {
-      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
-                  bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
                  "internal error");
   }
 #endif
@@ -753,10 +756,10 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
 #ifdef MFEM_DEBUG
   Array<int> ess_vdofs_marker;
   pfes->GetEssentialVDofs(bdr_attr, ess_vdofs_marker);
+   gcomm.Bcast<int>(values_counter.HostReadWrite());
   for (int i = 0; i < values_counter.Size(); i++)
   {
-      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
-                  bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
                  "internal error: " << pfes->GetLocalTDofNumber(i) << ' ' << bool(
                     values_counter[i]));
   }
@@ -231,7 +231,7 @@ public:
   void CountElementsPerVDof(Array<int> &elem_per_vdof) const override;

   /// Parallel version of GridFunction::GetDerivative(); see its documentation.
-   void GetDerivative(int comp, int der_comp, ParGridFunction &der);
+   void GetDerivative(int comp, int der_comp, ParGridFunction &der) const;

   /** Sets the output vector @a dof_vals to the values of the degrees of
       freedom of element @a el. If @a el is greater than or equal to the number
@@ -262,7 +262,6 @@ public:
                              const Array<int> &attr) override
   { ProjectBdrCoefficient(coeff, NULL, attr); }

-   // Only the values in the master are guaranteed to be correct!
   void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                     const Array<int> &bdr_attr) override;

@@ -2949,6 +2949,15 @@ void TMOP_Integrator::EnableSurfaceFitting(const GridFunction &s0,
   MFEM_VERIFY(surf_fit_pos == NULL,
               "Using both fitting approaches is not supported.");

+   const int dim = s0.FESpace()->GetMesh()->Dimension();
+   Mesh *mesh = s0.FESpace()->GetMesh();
+   MFEM_VERIFY(mesh->GetNodes()->Size() == dim*s0.Size(),
+               "Mesh and level-set polynomial order must be the same.");
+   const H1_FECollection *fec = dynamic_cast<const H1_FECollection *>
+                                (s0.FESpace()->FEColl());
+   MFEM_VERIFY(fec, "Only H1_FECollection is supported for the surface fitting "
+               "grid function.");
+
   delete surf_fit_gf;
   surf_fit_gf = new GridFunction(s0);
   surf_fit_gf->CountElementsPerVDof(surf_fit_dof_count);
@@ -2987,12 +2996,24 @@ void TMOP_Integrator::EnableSurfaceFitting(const GridFunction &pos,
 void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
                                           const Array<bool> &smarker,
                                           Coefficient &coeff,
-                                           AdaptivityEvaluator &ae)
+                                           AdaptivityEvaluator &ae,
+                                           AdaptivityEvaluator *aegrad,
+                                           AdaptivityEvaluator *aehess)
 {
   // To have both we must duplicate the markers.
   MFEM_VERIFY(surf_fit_pos == NULL,
               "Using both fitting approaches is not supported.");

+   const int dim = s0.FESpace()->GetMesh()->Dimension();
+   ParMesh *pmesh = s0.ParFESpace()->GetParMesh();
+   MFEM_VERIFY(pmesh->GetNodes()->Size() == dim*s0.Size(),
+               "Mesh and level-set polynomial order must be the same.");
+   const H1_FECollection *fec = dynamic_cast<const H1_FECollection *>
+                                (s0.FESpace()->FEColl());
+   MFEM_VERIFY(fec, "Only H1_FECollection is supported for the surface fitting "
+               "grid function.");
+
+
   delete surf_fit_gf;
   surf_fit_gf = new GridFunction(s0);
   s0.CountElementsPerVDof(surf_fit_dof_count);
@@ -3000,11 +3021,80 @@ void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
   surf_fit_coeff = &coeff;
   surf_fit_eval = &ae;

-   surf_fit_eval->SetParMetaInfo(*s0.ParFESpace()->GetParMesh(),
-                                 *s0.ParFESpace());
+   surf_fit_eval->SetParMetaInfo(*pmesh, *s0.ParFESpace());
   surf_fit_eval->SetInitialField
   (*surf_fit_gf->FESpace()->GetMesh()->GetNodes(), *surf_fit_gf);
-   surf_fit_gf_bg = false;
+
+   if (!aegrad) { return; }
+
+   MFEM_VERIFY(aehess, "AdaptivityEvaluator for Hessians must be provided too.");
+
+   ParFiniteElementSpace *fes = s0.ParFESpace();
+
+   // FE space for gradients.
+   delete surf_fit_grad;
+   H1_FECollection *fec_grad = new H1_FECollection(fec->GetOrder(), dim,
+                                                   fec->GetBasisType());
+   ParFiniteElementSpace *fes_grad = new ParFiniteElementSpace(pmesh, fec_grad,
+                                                               dim);
+   // Initial gradients.
+   surf_fit_grad = new GridFunction(fes_grad);
+   surf_fit_grad->MakeOwner(fec_grad);
+   for (int d = 0; d < dim; d++)
+   {
+      ParGridFunction surf_fit_grad_comp(fes, surf_fit_grad->GetData()+d*s0.Size());
+      s0.GetDerivative(1, d, surf_fit_grad_comp);
+   }
+   surf_fit_eval_grad = aegrad;
+   surf_fit_eval_grad->SetParMetaInfo(*pmesh, *fes_grad);
+   surf_fit_eval_grad->SetInitialField(*pmesh->GetNodes(), *surf_fit_grad);
+
+   // FE space for Hessians.
+   delete surf_fit_hess;
+   H1_FECollection *fec_hess = new H1_FECollection(fec->GetOrder(), dim,
+                                                   fec->GetBasisType());
+   ParFiniteElementSpace *fes_hess = new ParFiniteElementSpace(pmesh, fec_hess,
+                                                               dim*dim);
+   // Initial Hessians.
+   surf_fit_hess = new GridFunction(fes_hess);
+   surf_fit_hess->MakeOwner(fec_hess);
+   int id = 0;
+   for (int d = 0; d < dim; d++)
+   {
+      for (int idir = 0; idir < dim; idir++)
+      {
+         ParGridFunction surf_fit_grad_comp(fes,
+                                            surf_fit_grad->GetData()+d*s0.Size());
+         ParGridFunction surf_fit_hess_comp(fes,
+                                            surf_fit_hess->GetData()+id*s0.Size());
+         surf_fit_grad_comp.GetDerivative(1, idir, surf_fit_hess_comp);
+         id++;
+      }
+   }
+   surf_fit_eval_hess = aehess;
+   surf_fit_eval_hess->SetParMetaInfo(*pmesh, *fes_hess);
+   surf_fit_eval_hess->SetInitialField(*pmesh->GetNodes(), *surf_fit_hess);
+
+   // Store DOF indices that are marked for fitting. Used to reduce work for
+   // transferring information between source/background and current mesh.
+   surf_fit_marker_dof_index.SetSize(0);
+#ifdef MFEM_USE_GSLIB
+   if (dynamic_cast<InterpolatorFP *>(surf_fit_eval) &&
+       dynamic_cast<InterpolatorFP *>(surf_fit_eval_grad) &&
+       dynamic_cast<InterpolatorFP *>(surf_fit_eval_hess))
+   {
+      for (int i = 0; i < surf_fit_marker->Size(); i++)
+      {
+         if ((*surf_fit_marker)[i] == true)
+         {
+            surf_fit_marker_dof_index.Append(i);
+         }
+      }
+   }
+#endif
+
+   *surf_fit_grad = 0.0;
+   *surf_fit_hess = 0.0;
 }

 void TMOP_Integrator::EnableSurfaceFittingFromSource(
@@ -3022,16 +3112,17 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
   // Setup for level set function
   delete surf_fit_gf;
   surf_fit_gf = new GridFunction(s0);
-   *surf_fit_gf = 0.0;
   surf_fit_marker = &smarker;
   surf_fit_coeff = &coeff;
   surf_fit_eval = &ae;

-   surf_fit_gf_bg = true;
   surf_fit_eval->SetParMetaInfo(*s_bg.ParFESpace()->GetParMesh(),
                                 *s_bg.ParFESpace());
   surf_fit_eval->SetInitialField
   (*s_bg.FESpace()->GetMesh()->GetNodes(), s_bg);
+   GridFunction *nodes = s0.FESpace()->GetMesh()->GetNodes();
+   surf_fit_eval->ComputeAtNewPosition(*nodes, *surf_fit_gf,
+                                       nodes->FESpace()->GetOrdering());

   // Setup for gradient on background mesh
   MFEM_VERIFY(s_bg_grad.ParFESpace()->GetOrdering() ==
@@ -3041,11 +3132,11 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
   delete surf_fit_grad;
   surf_fit_grad = new GridFunction(s0_grad);
   *surf_fit_grad = 0.0;
-   surf_fit_eval_bg_grad = &age;
-   surf_fit_eval_bg_hess = &ahe;
-   surf_fit_eval_bg_grad->SetParMetaInfo(*s_bg_grad.ParFESpace()->GetParMesh(),
-                                         *s_bg_grad.ParFESpace());
-   surf_fit_eval_bg_grad->SetInitialField
+   surf_fit_eval_grad = &age;
+   surf_fit_eval_hess = &ahe;
+   surf_fit_eval_grad->SetParMetaInfo(*s_bg_grad.ParFESpace()->GetParMesh(),
+                                      *s_bg_grad.ParFESpace());
+   surf_fit_eval_grad->SetInitialField
   (*s_bg_grad.FESpace()->GetMesh()->GetNodes(), s_bg_grad);

   // Setup for Hessian on background mesh
@@ -3056,9 +3147,9 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
   delete surf_fit_hess;
   surf_fit_hess = new GridFunction(s0_hess);
   *surf_fit_hess = 0.0;
-   surf_fit_eval_bg_hess->SetParMetaInfo(*s_bg_hess.ParFESpace()->GetParMesh(),
-                                         *s_bg_hess.ParFESpace());
-   surf_fit_eval_bg_hess->SetInitialField
+   surf_fit_eval_hess->SetParMetaInfo(*s_bg_hess.ParFESpace()->GetParMesh(),
+                                      *s_bg_hess.ParFESpace());
+   surf_fit_eval_hess->SetInitialField
   (*s_bg_hess.FESpace()->GetMesh()->GetNodes(), s_bg_hess);

   // Count number of zones that share each of the DOFs
@@ -3863,7 +3954,7 @@ void TMOP_Integrator::AssembleElemVecSurfFit(const FiniteElement &el_x,

   Vector sigma_e(dof_s);
   DenseMatrix surf_fit_grad_e(dof_s, dim);
-   if (surf_fit_gf || surf_fit_gf_bg)
+   if (surf_fit_gf)
   {
      surf_fit_gf->GetSubVector(vdofs, sigma_e);

@@ -3871,7 +3962,7 @@ void TMOP_Integrator::AssembleElemVecSurfFit(const FiniteElement &el_x,
      // The FE coefficients of the gradient go in surf_fit_grad_e.
      Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
      DenseMatrix grad_phys; // This will be (dof x dim, dof).
-      if (surf_fit_gf_bg)
+      if (surf_fit_grad)
      {
         surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
         surf_fit_grad->GetSubVector(dofs, grad_ptr);
@@ -3945,7 +4036,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
   Vector sigma_e(dof_s);
   DenseMatrix surf_fit_grad_e(dof_s, dim);
   DenseMatrix surf_fit_hess_e(dof_s, dim*dim);
-   if (surf_fit_gf || surf_fit_gf_bg)
+   if (surf_fit_gf)
   {
      surf_fit_gf->GetSubVector(vdofs, sigma_e);

@@ -3953,7 +4044,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
      // The FE coefficients of the gradient go in surf_fit_grad_e.
      Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
      DenseMatrix grad_phys; // This will be (dof x dim, dof).
-      if (surf_fit_gf_bg)
+      if (surf_fit_grad)
      {
         surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
         surf_fit_grad->GetSubVector(dofs, grad_ptr);
@@ -3967,7 +4058,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
      // Project the Hessian of sigma in the same space.
      // The FE coefficients of the Hessian go in surf_fit_hess_e.
      Vector hess_ptr(surf_fit_hess_e.GetData(), dof_s*dim*dim);
-      if (surf_fit_gf_bg)
+      if (surf_fit_hess)
      {
         surf_fit_hess->FESpace()->GetElementVDofs(el_id, dofs);
         surf_fit_hess->GetSubVector(dofs, hess_ptr);
@@ -3994,7 +4085,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
      Tpr.SetIntPoint(&ip);
      real_t w = surf_fit_normal * surf_fit_coeff->Eval(Tpr, ip);

-      if (surf_fit_gf || surf_fit_gf_bg)
+      if (surf_fit_gf)
      {
         Vector gg_ptr(surf_fit_hess_s.GetData(), dim * dim);
         surf_fit_hess_e.GetRow(s, gg_ptr);
@@ -4376,6 +4467,130 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
   dx = detv_avg_min / dxscale;
 }

+void TMOP_Integrator::RemapSurfaceFittingLevelSetAtNodes(const Vector &new_x,
+                                                         int new_x_ordering)
+{
+   if (!surf_fit_gf) { return; }
+
+   if (surf_fit_marker_dof_index.Size())
+   {
+      // Interpolate information only at DOFs marked for fitting.
+      const int dim = surf_fit_gf->FESpace()->GetMesh()->Dimension();
+      const int cnt = surf_fit_marker_dof_index.Size();
+      const int total_cnt = new_x.Size()/dim;
+      Vector new_x_sorted(cnt*dim);
+      if (new_x_ordering == 0)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               new_x_sorted(i + d*cnt) = new_x(dof_index + d*total_cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < dim; d++)
+            {
+               new_x_sorted(d + i*dim) = new_x(d + dof_index*dim);
+            }
+         }
+      }
+
+      // Interpolate values of the LS.
+      Vector surf_fit_gf_int, surf_fit_grad_int, surf_fit_hess_int;
+      surf_fit_eval->ComputeAtNewPosition(new_x_sorted, surf_fit_gf_int,
+                                          new_x_ordering);
+      for (int i = 0; i < cnt; i++)
+      {
+         int dof_index = surf_fit_marker_dof_index[i];
+         (*surf_fit_gf)[dof_index] = surf_fit_gf_int(i);
+      }
+
+      // Interpolate gradients of the LS.
+      surf_fit_eval_grad->ComputeAtNewPosition(new_x_sorted, surf_fit_grad_int,
+                                               new_x_ordering);
+      // Assumes surf_fit_grad and surf_fit_gf share the same space
+      const int grad_dim = surf_fit_grad->VectorDim();
+      const int grad_cnt = surf_fit_grad->Size()/grad_dim;
+      if (surf_fit_grad->FESpace()->GetOrdering() == Ordering::byNODES)
+      {
+         for (int d = 0; d < grad_dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               (*surf_fit_grad)[dof_index + d*grad_cnt] =
+                  surf_fit_grad_int(i + d*cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < grad_dim; d++)
+            {
+               (*surf_fit_grad)[dof_index*grad_dim + d] =
+                  surf_fit_grad_int(i*grad_dim + d);
+            }
+         }
+      }
+
+      // Interpolate Hessians of the LS.
+      surf_fit_eval_hess->ComputeAtNewPosition(new_x_sorted, surf_fit_hess_int,
+                                               new_x_ordering);
+      // Assumes surf_fit_hess and surf_fit_gf share the same space
+      const int hess_dim = surf_fit_hess->VectorDim();
+      const int hess_cnt = surf_fit_hess->Size()/hess_dim;
+      if (surf_fit_hess->FESpace()->GetOrdering() == Ordering::byNODES)
+      {
+         for (int d = 0; d < hess_dim; d++)
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               (*surf_fit_hess)[dof_index + d*hess_cnt] =
+                  surf_fit_hess_int(i + d*cnt);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            for (int d = 0; d < hess_dim; d++)
+            {
+               (*surf_fit_hess)[dof_index*hess_dim + d] =
+                  surf_fit_hess_int(i*hess_dim + d);
+            }
+         }
+      }
+
+   }
+   else
+   {
+      surf_fit_eval->ComputeAtNewPosition(new_x, *surf_fit_gf, new_x_ordering);
+      if (surf_fit_eval_grad)
+      {
+         surf_fit_eval_grad->ComputeAtNewPosition(new_x, *surf_fit_grad,
+                                                  new_x_ordering);
+      }
+      if (surf_fit_eval_hess)
+      {
+         surf_fit_eval_hess->ComputeAtNewPosition(new_x, *surf_fit_hess,
+                                                  new_x_ordering);
+      }
+   }
+}
+
 void TMOP_Integrator::
 UpdateAfterMeshPositionChange(const Vector &x_new,
                              const FiniteElementSpace &x_fes)
@@ -4406,112 +4621,11 @@ UpdateAfterMeshPositionChange(const Vector &x_new,
      adapt_lim_eval->ComputeAtNewPosition(x_new, *adapt_lim_gf, ordering);
   }

-   // Update surf_fit_gf if surface fitting is enabled.
+   // Update surf_fit_gf (and optionally its gradients) if surface
+   // fitting is enabled.
   if (surf_fit_gf)
   {
-      if (surf_fit_gf_bg)
-      {
-         // Interpolate information for only DOFs marked for fitting.
-         const int dim = surf_fit_gf->FESpace()->GetMesh()->Dimension();
-         const int cnt = surf_fit_marker_dof_index.Size();
-         const int total_cnt = x_new.Size()/dim;
-         Vector new_x_sorted(cnt*dim);
-         if (ordering == 0)
-         {
-            for (int d = 0; d < dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  new_x_sorted(i + d*cnt) = x_new(dof_index + d*total_cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < dim; d++)
-               {
-                  new_x_sorted(d + i*dim) = x_new(d + dof_index*dim);
-               }
-            }
-         }
-
-         Vector surf_fit_gf_int, surf_fit_grad_int, surf_fit_hess_int;
-         surf_fit_eval->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_gf_int, ordering);
-         for (int i = 0; i < cnt; i++)
-         {
-            int dof_index = surf_fit_marker_dof_index[i];
-            (*surf_fit_gf)[dof_index] = surf_fit_gf_int(i);
-         }
-
-         surf_fit_eval_bg_grad->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_grad_int, ordering);
-         // Assumes surf_fit_grad and surf_fit_gf share the same space
-         const int grad_dim = surf_fit_grad->VectorDim();
-         const int grad_cnt = surf_fit_grad->Size()/grad_dim;
-         if (surf_fit_grad->FESpace()->GetOrdering() == Ordering::byNODES)
-         {
-            for (int d = 0; d < grad_dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  (*surf_fit_grad)[dof_index + d*grad_cnt] =
-                     surf_fit_grad_int(i + d*cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < grad_dim; d++)
-               {
-                  (*surf_fit_grad)[dof_index*dim + d] =
-                     surf_fit_grad_int(i*dim + d);
-               }
-            }
-         }
-
-         surf_fit_eval_bg_hess->ComputeAtNewPosition(
-            new_x_sorted, surf_fit_hess_int, ordering);
-         // Assumes surf_fit_hess and surf_fit_gf share the same space
-         const int hess_dim = surf_fit_hess->VectorDim();
-         const int hess_cnt = surf_fit_hess->Size()/hess_dim;
-         if (surf_fit_hess->FESpace()->GetOrdering() == Ordering::byNODES)
-         {
-            for (int d = 0; d < hess_dim; d++)
-            {
-               for (int i = 0; i < cnt; i++)
-               {
-                  int dof_index = surf_fit_marker_dof_index[i];
-                  (*surf_fit_hess)[dof_index + d*hess_cnt] =
-                     surf_fit_hess_int(i + d*cnt);
-               }
-            }
-         }
-         else
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               for (int d = 0; d < hess_dim; d++)
-               {
-                  (*surf_fit_hess)[dof_index*dim + d] =
-                     surf_fit_hess_int(i*dim + d);
-               }
-            }
-         }
-      }
-      else
-      {
-         surf_fit_eval->ComputeAtNewPosition(x_new, *surf_fit_gf, ordering);
-      }
+      RemapSurfaceFittingLevelSetAtNodes(x_new, ordering);
   }
 }

@@ -1784,12 +1784,11 @@ protected:
   // Fitting to given physical positions.
   TMOP_QuadraticLimiter *surf_fit_limiter; // Owned. Created internally.
   const GridFunction *surf_fit_pos;        // Not owned. Positions to fit.
-   real_t surf_fit_normal;
-   bool surf_fit_gf_bg;
-   GridFunction *surf_fit_grad, *surf_fit_hess;
-   AdaptivityEvaluator *surf_fit_eval_bg_grad, *surf_fit_eval_bg_hess;
-   Array<int> surf_fit_dof_count;
-   Array<int> surf_fit_marker_dof_index;
+   real_t surf_fit_normal;                  // Normalization factor.
+   GridFunction *surf_fit_grad, *surf_fit_hess; // Owned. Created internally.
+   AdaptivityEvaluator *surf_fit_eval_grad, *surf_fit_eval_hess; // Not owned.
+   Array<int> surf_fit_dof_count;            // Number of dofs per node.
+   Array<int> surf_fit_marker_dof_index;     // Indices of nodes to fit.

   DiscreteAdaptTC *discr_tc;

@@ -1985,6 +1984,10 @@ protected:
   real_t ComputeUntanglerMaxMuBarrier(const Vector &x,
                                       const FiniteElementSpace &fes);

+   // Remaps the internal surface fitting gridfunction object at provided
+   // locations.
+   void RemapSurfaceFittingLevelSetAtNodes(const Vector &new_x,
+                                           int new_x_ordering);
 public:
   /** @param[in] m    TMOP_QualityMetric for r-adaptivity (not owned).
       @param[in] tc   Target-matrix construction algorithm to use (not owned).
@@ -2000,9 +2003,8 @@ public:
        surf_fit_marker(NULL), surf_fit_coeff(NULL),
        surf_fit_gf(NULL), surf_fit_eval(NULL),
        surf_fit_limiter(NULL), surf_fit_pos(NULL),
-        surf_fit_normal(1.0),
-        surf_fit_gf_bg(false), surf_fit_grad(NULL), surf_fit_hess(NULL),
-        surf_fit_eval_bg_grad(NULL), surf_fit_eval_bg_hess(NULL),
+        surf_fit_normal(1.0), surf_fit_grad(NULL), surf_fit_hess(NULL),
+        surf_fit_eval_grad(NULL), surf_fit_eval_hess(NULL),
        discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
        fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
   { PA.enabled = false; }
@@ -2103,9 +2105,15 @@ public:

 #ifdef MFEM_USE_MPI
   /// Parallel support for surface fitting to the zero level set of a function.
+   /// Here, we add two optional inputs: @a aegrad and @a aehess. When provided,
+   /// the first and second derivative of the input level set are computed on
+   /// the initial mesh, and @a aegrad and @a aehess are used to remap grad_s(x)
+   /// from grad_s0(x0) and hess_s(x) from hess_s0(x0), respectively.
   void EnableSurfaceFitting(const ParGridFunction &s0,
                             const Array<bool> &smarker, Coefficient &coeff,
-                             AdaptivityEvaluator &ae);
+                             AdaptivityEvaluator &ae,
+                             AdaptivityEvaluator *aegrad = NULL,
+                             AdaptivityEvaluator *aehess = NULL);

   /** @brief Fitting of certain DOFs in the current mesh to the zero level set
       of a function defined on another (finer) source mesh.
@@ -429,11 +429,13 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 #endif

   real_t scale = 1.0;
-   real_t avg_surf_fit_err, max_surf_fit_err = 0.0;
-   if (surf_fit_max_threshold > 0.0)
+   bool fitting = IsSurfaceFittingEnabled();
+   real_t init_fit_avg_err, init_fit_max_err = 0.0;
+   if (fitting && surf_fit_converge_error)
   {
-      GetSurfaceFittingError(x_out_loc, avg_surf_fit_err, max_surf_fit_err);
-      if (max_surf_fit_err < surf_fit_max_threshold)
+      GetSurfaceFittingError(x_out_loc, init_fit_avg_err, init_fit_max_err);
+      // Check for convergence
+      if (init_fit_max_err < surf_fit_max_err_limit)
      {
         if (print_options.iterations)
         {
@@ -444,11 +446,12 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
         return scale;
      }
   }
-   if (adapt_inc_count >= max_adapt_inc_count)
+
+   if (surf_fit_adapt_count >= surf_fit_adapt_count_limit)
   {
      if (print_options.iterations)
      {
-         mfem::out << "TMOPNewtonSolver converged "
+         mfem::out << "TMOPNewtonSolver terminated "
                   "based on max number of times surface fitting weight can"
                   "be increased. \n";
      }
@@ -467,7 +470,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
      // reference to detect deteriorations.
      MFEM_VERIFY(min_det_ptr != NULL, " Initial mesh was valid, but"
                  " intermediate mesh is invalid. Contact TMOP Developers.");
-      MFEM_VERIFY(min_detJ_threshold == 0.0,
+      MFEM_VERIFY(min_detJ_limit == 0.0,
                  "This setup is not supported. Contact TMOP Developers.");
      *min_det_ptr = untangle_factor * min_detT_in;
   }
@@ -478,6 +481,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
   bool x_out_ok = false;
   real_t energy_out = 0.0, min_detT_out;
   const real_t norm_in = Norm(r);
+   real_t avg_fit_err, max_fit_err = 0.0;

   const real_t detJ_factor = (solver_type == 1) ? 0.25 : 0.5;
   compute_metric_quantile_flag = false;
@@ -488,6 +492,9 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
   // Perform the line search.
   for (int i = 0; i < 12; i++)
   {
+      avg_fit_err = 0.0;
+      max_fit_err = 0.0;
+
      // Update the mesh and get the L-vector in x_out_loc.
      add(x, -scale, c, x_out);
      if (serial)
@@ -502,7 +509,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,

      // Check the changes in detJ.
      min_detT_out = ComputeMinDet(x_out_loc, *fes);
-      if (untangling == false && min_detT_out <= min_detJ_threshold)
+      if (untangling == false && min_detT_out <= min_detJ_limit)
      {
         // No untangling, and detJ got negative (or small) -- no good.
         if (print_options.iterations)
@@ -529,18 +536,19 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
      // Check the changes in total energy.
      ProcessNewState(x_out);

-      real_t avg_fit_err, max_fit_err = 0.0;
-      if (surf_fit_max_threshold > 0.0)
+      // Ensure sufficient decrease in fitting error if we are trying to
+      // converge based on error.
+      if (fitting && surf_fit_converge_error)
      {
         GetSurfaceFittingError(x_out_loc, avg_fit_err, max_fit_err);
-      }
-      if (surf_fit_max_threshold > 0.0 && max_fit_err >= 1.2*max_surf_fit_err)
-      {
-         if (print_options.iterations)
+         if (max_fit_err >= 1.2*init_fit_max_err)
         {
-            mfem::out << "Scale = " << scale << " Surf fit err increased.\n";
+            if (print_options.iterations)
+            {
+               mfem::out << "Scale = " << scale << " Surf fit err increased.\n";
+            }
+            scale *= 0.5; continue;
         }
-         scale *= 0.5; continue;
      }

      if (serial)
@@ -614,7 +622,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,

   if (x_out_ok == false) { scale = 0.0; }

-   if (surf_fit_scale_factor > 0.0) { update_surf_fit_coeff = true; }
+   if (surf_fit_scale_factor > 0.0) { surf_fit_coeff_update = true; }
   compute_metric_quantile_flag = true;

   return scale;
@@ -657,7 +665,7 @@ void TMOPNewtonSolver::GetSurfaceFittingWeight(Array<real_t> &weights) const
   for (int i = 0; i < integs.Size(); i++)
   {
      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
-      if (ti)
+      if (ti && ti->IsSurfaceFittingEnabled())
      {
         weight = ti->GetSurfaceFittingWeight();
         weights.Append(weight);
@@ -668,8 +676,11 @@ void TMOPNewtonSolver::GetSurfaceFittingWeight(Array<real_t> &weights) const
         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
         for (int j = 0; j < ati.Size(); j++)
         {
-            weight = ati[j]->GetSurfaceFittingWeight();
-            weights.Append(weight);
+            if (ati[j]->IsSurfaceFittingEnabled())
+            {
+               weight = ati[j]->GetSurfaceFittingWeight();
+               weights.Append(weight);
+            }
         }
      }
   }
@@ -716,6 +727,39 @@ void TMOPNewtonSolver::GetSurfaceFittingError(const Vector &x_loc,
   }
 }

+bool TMOPNewtonSolver::IsSurfaceFittingEnabled() const
+{
+   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
+   const Array<NonlinearFormIntegrator*> &integs = *nlf->GetDNFI();
+   TMOP_Integrator *ti  = NULL;
+   TMOPComboIntegrator *co = NULL;
+
+   for (int i = 0; i < integs.Size(); i++)
+   {
+      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+      if (ti)
+      {
+         if (ti->IsSurfaceFittingEnabled())
+         {
+            return true;
+         }
+      }
+      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+      if (co)
+      {
+         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+         for (int j = 0; j < ati.Size(); j++)
+         {
+            if (ati[j]->IsSurfaceFittingEnabled())
+            {
+               return true;
+            }
+         }
+      }
+   }
+   return false;
+}
+
 void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
 {
   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
@@ -801,38 +845,46 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
   // adaptive surface fitting is enabled. The idea is to increase the
   // coefficient if the surface fitting error does not sufficiently
   // decrease between subsequent TMOPNewtonSolver iterations.
-   if (update_surf_fit_coeff)
+   if (surf_fit_coeff_update)
   {
      // Get surface fitting errors.
-      GetSurfaceFittingError(x_loc, surf_fit_err_avg, surf_fit_err_max);
+      GetSurfaceFittingError(x_loc, surf_fit_avg_err, surf_fit_max_err);
      // Get array with surface fitting weights.
-      Array<real_t> weights;
-      GetSurfaceFittingWeight(weights);
+      Array<real_t> fitweights;
+      GetSurfaceFittingWeight(fitweights);

      if (print_options.iterations)
      {
         mfem::out << "Avg/Max surface fitting error: " <<
-                   surf_fit_err_avg << " " <<
-                   surf_fit_err_max << "\n";
+                   surf_fit_avg_err << " " <<
+                   surf_fit_max_err << "\n";
         mfem::out << "Min/Max surface fitting weight: " <<
-                   weights.Min() << " " << weights.Max() << "\n";
+                   fitweights.Min() << " " << fitweights.Max() << "\n";
      }

-      real_t change_surf_fit_err = surf_fit_err_avg_prvs-surf_fit_err_avg;
-      real_t rel_change_surf_fit_err = change_surf_fit_err/surf_fit_err_avg_prvs;
+      real_t change_surf_fit_err = surf_fit_avg_err_prvs-surf_fit_avg_err;
+      real_t rel_change_surf_fit_err = change_surf_fit_err/surf_fit_avg_err_prvs;
+
      // Increase the surface fitting coefficient if the surface fitting error
-      // does not decrease sufficiently.
-      if (rel_change_surf_fit_err < surf_fit_rel_change_threshold)
+      // does not decrease sufficiently. If we are converging based on residual,
+      // also make sure we have not reached the maximum fitting weight and
+      // error threshold.
+      if (rel_change_surf_fit_err < surf_fit_err_rel_change_limit &&
+          (surf_fit_converge_error ||
+           (fitweights.Max() < surf_fit_weight_limit &&
+            surf_fit_max_err > surf_fit_max_err_limit)))
      {
-         UpdateSurfaceFittingWeight(surf_fit_scale_factor);
-         adapt_inc_count += 1;
+         real_t scale_factor = std::min(surf_fit_scale_factor,
+                                        surf_fit_weight_limit/fitweights.Max());
+         UpdateSurfaceFittingWeight(scale_factor);
+         surf_fit_adapt_count += 1;
      }
      else
      {
-         adapt_inc_count = 0;
+         surf_fit_adapt_count = 0;
      }
-      surf_fit_err_avg_prvs = surf_fit_err_avg;
-      update_surf_fit_coeff = false;
+      surf_fit_avg_err_prvs = surf_fit_avg_err;
+      surf_fit_coeff_update = false;
   }
 }

@@ -134,18 +134,20 @@ protected:
   int solver_type;
   bool parallel;

-   // Line search step is rejected if min(detJ) <= min_detJ_threshold.
-   real_t min_detJ_threshold = 0.0;
+   // Line search step is rejected if min(detJ) <= min_detJ_limit.
+   real_t min_detJ_limit = 0.0;

   // Surface fitting variables.
-   mutable real_t surf_fit_err_avg_prvs = 10000.0;
-   mutable real_t surf_fit_err_avg, surf_fit_err_max;
-   mutable bool update_surf_fit_coeff = false;
-   real_t surf_fit_max_threshold = -1.0;
-   real_t surf_fit_rel_change_threshold = 0.001;
+   mutable real_t surf_fit_avg_err_prvs = 10000.0;
+   mutable real_t surf_fit_avg_err, surf_fit_max_err;
+   mutable bool surf_fit_coeff_update = false;
+   real_t surf_fit_max_err_limit = -1.0;
+   real_t surf_fit_err_rel_change_limit = 0.001;
   real_t surf_fit_scale_factor = 0.0;
-   mutable int adapt_inc_count = 0;
-   mutable int max_adapt_inc_count = 10;
+   mutable int surf_fit_adapt_count = 0;
+   mutable int surf_fit_adapt_count_limit = 10;
+   mutable real_t surf_fit_weight_limit = 1e10;
+   bool surf_fit_converge_error = false;

   // Minimum determinant over the whole mesh. Used for mesh untangling.
   real_t *min_det_ptr = nullptr;
@@ -191,6 +193,9 @@ protected:
   void GetSurfaceFittingWeight(Array<real_t> &weights) const;
   ///@}

+   /// Check if surface fitting is enabled.
+   bool IsSurfaceFittingEnabled() const;
+
 public:
 #ifdef MFEM_USE_MPI
   TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule, int type = 0)
@@ -224,38 +229,94 @@ public:
   /// (ii) surface fitting weight.
   virtual void ProcessNewState(const Vector &x) const;

-   /** @name Methods for adaptive surface fitting weight. (Experimental) */
-   /// Enable/Disable adaptive surface fitting weight.
-   /// The weight is modified after each TMOPNewtonSolver iteration as:
-   /// w_{k+1} = w_{k} * @a surf_fit_scale_factor if relative change in
-   /// max surface fitting error < @a surf_fit_rel_change_threshold.
-   /// The solver terminates if the maximum surface fitting error does
-   /// not sufficiently decrease for @a max_adapt_inc_count consecutive
-   /// solver iterations or if the max error falls below @a surf_fit_max_threshold.
-   void EnableAdaptiveSurfaceFitting()
-   {
-      surf_fit_scale_factor = 10.0;
-      surf_fit_rel_change_threshold = 0.001;
-   }
+   /** @name Methods for adaptive surface fitting.
+       \brief These methods control the behavior of the weight and the
+       termination of the solver. (Experimental)
+
+       Adaptive fitting weight: The weight is modified after each
+       TMOPNewtonSolver iteration as:
+       w_{k+1} = w_{k} * \ref surf_fit_scale_factor if the relative
+       change in average fitting error < \ref surf_fit_err_rel_change_limit.
+       When converging based on the residual, we enforce the fitting weight
+       to be at-most \ref surf_fit_weight_limit, and increase it only if the
+       fitting error is below user prescribed threshold
+       (\ref surf_fit_max_err_limit).
+       See \ref SetAdaptiveSurfaceFittingScalingFactor and
+       \ref SetAdaptiveSurfaceFittingRelativeChangeThreshold.
+
+       Note that the solver stops if the maximum surface fitting error
+       does not sufficiently decrease for \ref surf_fit_adapt_count_limit (default 10)
+       consecutive increments of the fitting weight during weight adaptation.
+       This typically occurs when the mesh cannot align with the level-set
+       without degrading element quality.
+       See \ref SetMaxNumberofIncrementsForAdaptiveFitting.
+
+       Convergence criterion: There are two modes, residual- and error-based,
+       which can be toggled using \ref SetSurfaceFittingConvergenceBasedOnError.
+
+       (i) Residual based (default): Stop when the norm of the gradient of the
+       TMOP objective reaches the prescribed tolerance. This method is best used
+       with a reasonable value for \ref surf_fit_weight_limit when the
+       adaptive surface fitting scheme is used. See method
+       \ref SetSurfaceFittingWeightLimit.
+
+       (ii) Error based: Stop when the maximum fitting error
+       reaches the user-prescribed threshold, \ref surf_fit_max_err_limit.
+       In this case, \ref surf_fit_weight_limit is ignored during weight
+       adaptation.
+   */
+   ///@{
   void SetAdaptiveSurfaceFittingScalingFactor(real_t factor)
   {
+      MFEM_VERIFY(factor > 1.0, "Scaling factor must be greater than 1.");
      surf_fit_scale_factor = factor;
   }
   void SetAdaptiveSurfaceFittingRelativeChangeThreshold(real_t threshold)
   {
-      surf_fit_rel_change_threshold = threshold;
+      surf_fit_err_rel_change_limit = threshold;
   }
+   /// Used for stopping based on the number of consecutive failed weight
+   /// adaptation iterations.
+   // TODO: Rename to SetMaxNumberofIncrementsForAdaptiveSurfaceFitting
+   // in future.
   void SetMaxNumberofIncrementsForAdaptiveFitting(int count)
   {
-      max_adapt_inc_count = count;
+      surf_fit_adapt_count_limit = count;
   }
+   /// Used for error-based surface fitting termination.
   void SetTerminationWithMaxSurfaceFittingError(real_t max_error)
   {
-      surf_fit_max_threshold = max_error;
+      surf_fit_max_err_limit = max_error;
+      surf_fit_converge_error = true;
   }
+   /// Could be used with both error-based or residual-based convergence.
+   void SetSurfaceFittingMaxErrorLimit(real_t max_error)
+   {
+      surf_fit_max_err_limit = max_error;
+   }
+   /// Used for residual-based surface fitting termination.
+   void SetSurfaceFittingWeightLimit(real_t weight)
+   {
+      surf_fit_weight_limit = weight;
+   }
+   /// Toggle convergence based on residual or error.
+   void SetSurfaceFittingConvergenceBasedOnError(bool mode)
+   {
+      surf_fit_converge_error = mode;
+      if (surf_fit_converge_error)
+      {
+         MFEM_VERIFY(surf_fit_max_err_limit >= 0,
+                     "Fitting error based convergence requires the user to "
+                     "first set the error threshold."
+                     "See SetTerminationWithMaxSurfaceFittingError");
+      }
+   }
+   ///@}
+
+   /// Set minimum determinant enforced during line-search.
   void SetMinimumDeterminantThreshold(real_t threshold)
   {
-      min_detJ_threshold = threshold;
+      min_detJ_limit = threshold;
   }

   virtual void Mult(const Vector &b, Vector &x) const
@@ -124,7 +124,7 @@ T Array<T>::Sum()
 }

 template <class T>
-int Array<T>::IsSorted()
+int Array<T>::IsSorted() const
 {
   T val_prev = operator[](0), val;
   for (int i = 1; i < size; i++)
@@ -74,10 +74,14 @@ public:
   inline Array(int asize, MemoryType mt)
      : size(asize) { asize > 0 ? data.New(asize, mt) : data.Reset(mt); }

-   /** @brief Creates array using an externally allocated pointer @a data_ to
-       @a asize elements. The data pointer will not be deleted by Array. */
-   inline Array(T *data_, int asize)
-   { data.Wrap(data_, asize, false); size = asize; }
+   /** @brief Creates array using an externally allocated host pointer @a data_
+       to @a asize elements. If @a own_data is true, the array takes ownership
+       of the pointer.
+
+       When @a own_data is true, the pointer @a data_ must be allocated with
+       MemoryType given by MemoryManager::GetHostMemoryType(). */
+   inline Array(T *data_, int asize, bool own_data = false)
+   { data.Wrap(data_, asize, own_data); size = asize; }

   /// Copy constructor: deep copy from @a src
   /** This method supports source arrays using any MemoryType. */
@@ -205,7 +209,14 @@ public:
   inline void Copy(Array &copy) const;

   /// Make this Array a reference to a pointer.
-   inline void MakeRef(T *, int);
+   /** When @a own_data is true, the pointer @a data_ must be allocated with
+       MemoryType given by MemoryManager::GetHostMemoryType(). */
+   inline void MakeRef(T *data_, int size_, bool own_data = false);
+
+   /// Make this Array a reference to a pointer.
+   /** When @a own_data is true, the pointer @a data_ must be allocated with
+       MemoryType given by @a mt. */
+   inline void MakeRef(T *data_, int size, MemoryType mt, bool own_data);

   /// Make this Array a reference to 'master'.
   inline void MakeRef(const Array &master);
@@ -262,7 +273,7 @@ public:
   }

   /// Return 1 if the array is sorted from lowest to highest.  Otherwise return 0.
-   int IsSorted();
+   int IsSorted() const;

   /// Fill the entries of the array with the cumulative sum of the entries.
   void PartialSum();
@@ -868,11 +879,19 @@ inline void Array<T>::Copy(Array &copy) const
 }

 template <class T>
-inline void Array<T>::MakeRef(T *p, int s)
+inline void Array<T>::MakeRef(T *data_, int size_, bool own_data)
 {
   data.Delete();
-   data.Wrap(p, s, false);
-   size = s;
+   data.Wrap(data_, size_, own_data);
+   size = size_;
+}
+
+template <class T>
+inline void Array<T>::MakeRef(T *data_, int size_, MemoryType mt, bool own_data)
+{
+   data.Delete();
+   data.Wrap(data_, size_, mt, own_data);
+   size = size_;
 }

 template <class T>
@@ -288,4 +288,3 @@ void ArraysByName<T>::Load(std::istream &in)
 }

 #endif
-
@@ -275,7 +275,7 @@ void GroupTopology::Save(ostream &os) const
   os << "\ncommunication_groups\n";
   os << "number_of_groups " << NGroups() << "\n\n";

-   os << "# number of entities in each group, followed by group ids in group\n";
+   os << "# number of entities in each group, followed by ranks in group\n";
   for (int group_id = 0; group_id < NGroups(); ++group_id)
   {
      int group_size = GetGroupSize(group_id);
@@ -14,6 +14,9 @@
 #ifdef MFEM_USE_CEED
 #include "../fem/ceed/interface/util.hpp"
 #endif
+#ifdef MFEM_USE_MPI
+#include "../linalg/hypre.hpp"
+#endif

 #include <unordered_map>
 #include <string>
@@ -250,6 +253,10 @@ void Device::Configure(const std::string &device, const int device_id)

   // Only '*this' will call the MemoryManager::Destroy() method.
   destroy_mm = true;
+
+#ifdef MFEM_USE_MPI
+   Hypre::InitDevice();
+#endif
 }

 // static method
@@ -19,6 +19,9 @@
 #include "device.hpp"
 #include "mem_manager.hpp"
 #include "../linalg/dtensor.hpp"
+#ifdef MFEM_USE_MPI
+#include <_hypre_utilities.h>
+#endif

 namespace mfem
 {
@@ -780,6 +783,63 @@ inline void forall_3D_grid(int N, int X, int Y, int Z, int G, lambda &&body)
   ForallWrap<3>(true, N, body, X, Y, Z, G);
 }

+#ifdef MFEM_USE_MPI
+
+// Function mfem::hypre_forall_cpu() similar to mfem::forall, but it always
+// executes on the CPU using sequential or OpenMP-parallel execution based on
+// the hypre build time configuration.
+template<typename lambda>
+inline void hypre_forall_cpu(int N, lambda &&body)
+{
+#ifdef HYPRE_USING_OPENMP
+   #pragma omp parallel for HYPRE_SMP_SCHEDULE
+#endif
+   for (int i = 0; i < N; i++) { body(i); }
+}
+
+// Function mfem::hypre_forall_gpu() similar to mfem::forall, but it always
+// executes on the GPU device that hypre was configured with at build time.
+#if defined(HYPRE_USING_GPU)
+template<typename lambda>
+inline void hypre_forall_gpu(int N, lambda &&body)
+{
+#if defined(HYPRE_USING_CUDA)
+   CuWrap1D(N, body);
+#elif defined(HYPRE_USING_HIP)
+   HipWrap1D(N, body);
+#else
+#error Unknown HYPRE GPU backend!
+#endif
+}
+#endif
+
+// Function mfem::hypre_forall() similar to mfem::forall, but it executes on the
+// device, CPU or GPU, that hypre was configured with at build time (when the
+// HYPRE version is < 2.31.0) or at runtime (when HYPRE was configured with GPU
+// support at build time and HYPRE's version is >= 2.31.0). This selection is
+// generally independent of what device was selected in MFEM's runtime
+// configuration.
+template<typename lambda>
+inline void hypre_forall(int N, lambda &&body)
+{
+#if !defined(HYPRE_USING_GPU)
+   hypre_forall_cpu(N, body);
+#elif MFEM_HYPRE_VERSION < 23100
+   hypre_forall_gpu(N, body);
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   if (!HypreUsingGPU())
+   {
+      hypre_forall_cpu(N, body);
+   }
+   else
+   {
+      hypre_forall_gpu(N, body);
+   }
+#endif
+}
+
+#endif // MFEM_USE_MPI
+
 } // namespace mfem

 #endif // MFEM_FORALL_HPP
@@ -51,7 +51,7 @@ int isockstream::establish()
 {
   // char myname[129];
   char   myname[] = "localhost";
-   int    sfd;
+   int    sfd = -1;
   struct addrinfo hints, *res, *rp;

   memset(&hints, 0, sizeof(hints));
@@ -1154,6 +1154,10 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
   //  dest   d  | h2d   d2d   d2d
   //        hd  | h2h   d2d   d2d

+   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
+   MFEM_ASSERT(dst_h_ptr != nullptr, "invalid dst_h_ptr = nullptr");
+   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
+
   const bool dst_on_host =
      (dst_flags & Mem::VALID_HOST) &&
      (!(dst_flags & Mem::VALID_DEVICE) ||
@@ -1229,6 +1233,10 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
 void MemoryManager::CopyToHost_(void *dest_h_ptr, const void *src_h_ptr,
                                size_t bytes, unsigned src_flags)
 {
+   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
+   MFEM_ASSERT(dest_h_ptr != nullptr, "invalid dest_h_ptr = nullptr");
+   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
+
   const bool src_on_host = src_flags & Mem::VALID_HOST;
   if (src_on_host)
   {
@@ -1255,6 +1263,10 @@ void MemoryManager::CopyToHost_(void *dest_h_ptr, const void *src_h_ptr,
 void MemoryManager::CopyFromHost_(void *dest_h_ptr, const void *src_h_ptr,
                                  size_t bytes, unsigned &dest_flags)
 {
+   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
+   MFEM_ASSERT(dest_h_ptr != nullptr, "invalid dest_h_ptr = nullptr");
+   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
+
   const bool dest_on_host = dest_flags & Mem::VALID_HOST;
   if (dest_on_host)
   {
@@ -18,8 +18,14 @@
 #include <cstring> // std::memcpy
 #include <type_traits> // std::is_const
 #include <cstddef> // std::max_align_t
+
 #ifdef MFEM_USE_MPI
-#include <HYPRE_config.h> // HYPRE_USING_GPU
+// Enable internal hypre timing routines
+#define HYPRE_TIMING
+#include <HYPRE_utilities.h> // for HYPRE_GetMemoryLocation() and others
+#if (21400 <= MFEM_HYPRE_VERSION) && (MFEM_HYPRE_VERSION < 21900)
+#include <_hypre_utilities.h> // for HYPRE_MEMORY_HOST and others
+#endif
 #endif

 namespace mfem
@@ -869,6 +875,45 @@ public:
 };


+#ifdef MFEM_USE_MPI
+
+#if MFEM_HYPRE_VERSION < 21400
+#define HYPRE_MEMORY_DEVICE (0)
+#define HYPRE_MEMORY_HOST   (1)
+#endif
+#if MFEM_HYPRE_VERSION < 21900
+typedef int HYPRE_MemoryLocation;
+#endif
+
+/// Return the configured HYPRE_MemoryLocation
+inline HYPRE_MemoryLocation GetHypreMemoryLocation()
+{
+#if !defined(HYPRE_USING_GPU)
+   return HYPRE_MEMORY_HOST;
+#elif MFEM_HYPRE_VERSION < 23100
+   return HYPRE_MEMORY_DEVICE;
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   HYPRE_MemoryLocation loc;
+   HYPRE_GetMemoryLocation(&loc);
+   return loc;
+#endif
+}
+
+/// Return true if HYPRE is configured to use GPU
+inline bool HypreUsingGPU()
+{
+#if !defined(HYPRE_USING_GPU)
+   return false;
+#elif MFEM_HYPRE_VERSION < 23100
+   return true;
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   return GetHypreMemoryLocation() != HYPRE_MEMORY_HOST;
+#endif
+}
+
+#endif // MFEM_USE_MPI
+
+
 // Inline methods

 template <typename T>
@@ -1004,10 +1049,12 @@ inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
         // If the following condition is true then MemoryManager::Exists()
         // should also be true:
         IsDeviceMemory(MemoryManager::GetDeviceMemoryType())
-#else
-         // When HYPRE_USING_GPU is defined we always register the 'base' if
-         // the MemoryManager::Exists():
+#elif MFEM_HYPRE_VERSION < 23100
+         // When HYPRE_USING_GPU is defined and HYPRE < 2.31.0, we always
+         // register the 'base' if the MemoryManager::Exists():
         MemoryManager::Exists()
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+         MemoryManager::Exists() && HypreUsingGPU()
 #endif
      )
      {
@@ -1213,9 +1260,10 @@ template <typename T>
 inline void Memory<T>::CopyFrom(const Memory &src, int size)
 {
   MFEM_VERIFY(src.capacity>=size && capacity>=size, "Incorrect size");
+   if (size <= 0) { return; }
   if (!(flags & Registered) && !(src.flags & Registered))
   {
-      if (h_ptr != src.h_ptr && size != 0)
+      if (h_ptr != src.h_ptr)
      {
         MFEM_ASSERT(h_ptr + size <= src.h_ptr || src.h_ptr + size <= h_ptr,
                     "data overlaps!");
@@ -1233,9 +1281,10 @@ template <typename T>
 inline void Memory<T>::CopyFromHost(const T *src, int size)
 {
   MFEM_VERIFY(capacity>=size, "Incorrect size");
+   if (size <= 0) { return; }
   if (!(flags & Registered))
   {
-      if (h_ptr != src && size != 0)
+      if (h_ptr != src)
      {
         MFEM_ASSERT(h_ptr + size <= src || src + size <= h_ptr,
                     "data overlaps!");
@@ -1252,7 +1301,6 @@ inline void Memory<T>::CopyFromHost(const T *src, int size)
 template <typename T>
 inline void Memory<T>::CopyTo(Memory &dest, int size) const
 {
-   MFEM_VERIFY(capacity>=size, "Incorrect size");
   dest.CopyFrom(*this, size);
 }

@@ -1260,9 +1308,10 @@ template <typename T>
 inline void Memory<T>::CopyToHost(T *dest, int size) const
 {
   MFEM_VERIFY(capacity>=size, "Incorrect size");
+   if (size <= 0) { return; }
   if (!(flags & Registered))
   {
-      if (h_ptr != dest && size != 0)
+      if (h_ptr != dest)
      {
         MFEM_ASSERT(h_ptr + size <= dest || dest + size <= h_ptr,
                     "data overlaps!");
@@ -134,7 +134,7 @@ int socketbuf::open(const char hostname[], int port)
      {
         closesocket(socket_descriptor);
         socket_descriptor = -2;
-         return -1;
+         continue;
      }
 #endif

@@ -148,7 +148,7 @@ int socketbuf::open(const char hostname[], int port)
   }

   freeaddrinfo(res);
-   return 0;
+   return (socket_descriptor < 0) ? -1 : 0;
 }

 int socketbuf::close()
@@ -207,7 +207,12 @@ template <> inline void Swap<Table>(Table &a, Table &b)
 void Transpose (const Table &A, Table &At, int ncols_A_ = -1);
 Table * Transpose (const Table &A);

-///  Transpose an Array<int>
+///  @brief Transpose an Array<int>.
+///
+/// The array @a A represents a table where each row @a i has exactly one
+/// connection to the column (TYPE II) index specified by @a A[i].
+///
+/// @note The column (TYPE II) indices in each row of @a At will be sorted.
 void Transpose(const Array<int> &A, Table &At, int ncols_A_ = -1);

 ///  C = A * B  (as boolean matrices)
@@ -400,6 +400,9 @@ inline double StopWatch::SystTime()

 StopWatch::StopWatch() : M(new internal::StopWatch) { }

+StopWatch::StopWatch(const StopWatch &sw)
+   : M(new internal::StopWatch(*(sw.M))) { }
+
 void StopWatch::Clear()
 {
   M->Clear();
@@ -40,6 +40,7 @@ private:
 public:
   /// Creates a new (stopped) StopWatch object.
   StopWatch();
+   StopWatch(const StopWatch &);

   /// Clear the elapsed time on the stopwatch and restart it if it's running.
   void Clear();
@@ -23,6 +23,10 @@
 #include "amgxsolver.hpp"
 #ifdef MFEM_USE_AMGX

+#ifdef MFEM_USE_MPI
+#include "../general/communication.hpp"
+#endif
+
 namespace mfem
 {

@@ -81,7 +81,7 @@ void BlockOperator::Mult (const Vector & x, Vector & y) const
      tmp.SetSize(row_offsets[iRow+1] - row_offsets[iRow]);
      for (int jCol=0; jCol < nColBlocks; ++jCol)
      {
-         if (op(iRow,jCol))
+         if (op(iRow,jCol) && coef(iRow,jCol) != 0.)
         {
            op(iRow,jCol)->Mult(xblock.GetBlock(jCol), tmp);
            yblock.GetBlock(iRow).Add(coef(iRow,jCol), tmp);
@@ -112,7 +112,7 @@ void BlockOperator::MultTranspose (const Vector & x, Vector & y) const
      tmp.SetSize(col_offsets[iRow+1] - col_offsets[iRow]);
      for (int jCol=0; jCol < nRowBlocks; ++jCol)
      {
-         if (op(jCol,iRow))
+         if (op(jCol,iRow) && coef(jCol,iRow) != 0.)
         {
            op(jCol,iRow)->MultTranspose(xblock.GetBlock(jCol), tmp);
            yblock.GetBlock(iRow).Add(coef(jCol,iRow), tmp);
@@ -1,3 +1,14 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
 #include "cpardiso.hpp"
 #include "hypre.hpp"
 #include <algorithm>
@@ -532,6 +532,69 @@ MatrixInverse *DenseMatrix::Inverse() const
   return new DenseMatrixInverse(*this);
 }

+void DenseMatrix::Exponential()
+{
+   MFEM_ASSERT(Height() == Width() && Height() <= 2,
+               "The matrix must be square and "
+               << "of size less than or equal to 2."
+               << "  Height() = " << Height()
+               << ", Width() = " << Width());
+
+   switch (Height())
+   {
+      case 1:
+      {
+         data[0] = std::exp(data[0]);
+         break;
+      }
+      case 2:
+      {
+         /// Formulas from Corollary 2.4 of doi:10.1109/9.233156
+         /// Note typo in the paper, in the prefactor in the equation under (i).
+         const real_t a = data[0];
+         const real_t b = data[1];
+         const real_t c = data[2];
+         const real_t d = data[3];
+         const real_t e = (a - d)*(a - d) + 4*b*c;
+         const real_t f = std::exp((a + d)/2.0);
+         const real_t g = std::sqrt(std::abs(e)) / 2.0;
+
+         if (e == 0)
+         {
+            data[0] = 1.0 + (a - d)/2.0;
+            data[3] = 1.0 - (a - d)/2.0;
+         }
+         else if (e > 0)
+         {
+            data[0] = std::cosh(g) + (a - d)/2 * std::sinh(g) / g;
+            data[1] = b * std::sinh(g) / g;
+            data[2] = c * std::sinh(g) / g;
+            data[3] = std::cosh(g) - (a - d)/2 * std::sinh(g) / g;
+         }
+         else
+         {
+            data[0] = std::cos(g) + (a - d)/2 * std::sin(g) / g;
+            data[1] = b * std::sin(g) / g;
+            data[2] = c * std::sin(g) / g;
+            data[3] = std::cos(g) - (a - d)/2 * std::sin(g) / g;
+         }
+         for (int i = 0; i < 4; i++)
+         {
+            data[i] *= f;
+         }
+         break;
+      }
+      case 3:
+      {
+         MFEM_ABORT("3x3 matrices are not currently supported");
+      }
+      default:
+      {
+         MFEM_ABORT("Only 1x1 and 2x2 matrices are currently supported");
+      }
+   }
+}
+
 real_t DenseMatrix::Det() const
 {
   MFEM_ASSERT(Height() == Width() && Height() > 0,
@@ -3217,6 +3280,93 @@ void MultAtB(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &AtB)
 #endif
 }

+void AddMultAtB(const DenseMatrix &A, const DenseMatrix &B,
+                DenseMatrix &AtB)
+{
+   MFEM_ASSERT(AtB.Height() == A.Width() && AtB.Width() == B.Width() &&
+               A.Height() == B.Height(), "incompatible dimensions");
+
+#ifdef MFEM_USE_LAPACK
+   static char transa = 'T', transb = 'N';
+   static real_t alpha = 1.0, beta = 1.0;
+   int m = A.Width(), n = B.Width(), k = A.Height();
+
+#ifdef MFEM_USE_SINGLE
+   sgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
+#elif defined MFEM_USE_DOUBLE
+   dgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
+#endif
+          B.Data(), &k, &beta, AtB.Data(), &m);
+#else
+   const int ah = A.Height();
+   const int aw = A.Width();
+   const int bw = B.Width();
+   const real_t *ad = A.Data();
+   const real_t *bd = B.Data();
+   real_t *cd = AtB.Data();
+
+   for (int j = 0; j < bw; j++)
+   {
+      const real_t *ap = ad;
+      for (int i = 0; i < aw; i++)
+      {
+         real_t d = 0.0;
+         for (int k = 0; k < ah; k++)
+         {
+            d += ap[k] * bd[k];
+         }
+         *(cd++) += d;
+         ap += ah;
+      }
+      bd += ah;
+   }
+#endif
+}
+
+void AddMult_a_AtB(real_t a, const DenseMatrix &A, const DenseMatrix &B,
+                   DenseMatrix &AtB)
+{
+   MFEM_ASSERT(AtB.Height() == A.Width() && AtB.Width() == B.Width() &&
+               A.Height() == B.Height(), "incompatible dimensions");
+
+#ifdef MFEM_USE_LAPACK
+   static char transa = 'T', transb = 'N';
+   real_t alpha = a;
+   static real_t beta = 1.0;
+   int m = A.Width(), n = B.Width(), k = A.Height();
+
+#ifdef MFEM_USE_SINGLE
+   sgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
+#elif defined MFEM_USE_DOUBLE
+   dgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
+#endif
+          B.Data(), &k, &beta, AtB.Data(), &m);
+#else
+   const int ah = A.Height();
+   const int aw = A.Width();
+   const int bw = B.Width();
+   const real_t *ad = A.Data();
+   const real_t *bd = B.Data();
+   real_t *cd = AtB.Data();
+
+   for (int j = 0; j < bw; j++)
+   {
+      const real_t *ap = ad;
+      for (int i = 0; i < aw; i++)
+      {
+         real_t d = 0.0;
+         for (int k = 0; k < ah; k++)
+         {
+            d += ap[k] * bd[k];
+         }
+         *(cd++) += a * d;
+         ap += ah;
+      }
+      bd += ah;
+   }
+#endif
+}
+
 void AddMult_a_AAt(real_t a, const DenseMatrix &A, DenseMatrix &AAt)
 {
   real_t d;
@@ -207,6 +207,10 @@ public:
   /// Replaces the current matrix with its square root inverse
   void SquareRootInverse();

+   /// Replaces the current matrix with its exponential
+   /// (currently only supports 2x2 matrices)
+   void Exponential();
+
   /// Calculates the determinant of the matrix
   /// (optimized for 2x2, 3x3, and 4x4 matrices)
   real_t Det() const;
@@ -580,6 +584,13 @@ void AddMult_a_ABt(real_t a, const DenseMatrix &A, const DenseMatrix &B,
 /// Multiply the transpose of a matrix A with a matrix B:   At*B
 void MultAtB(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &AtB);

+/// AtB += A^t * B
+void AddMultAtB(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &AtB);
+
+/// AtB += a * A^t * B
+void AddMult_a_AtB(real_t a, const DenseMatrix &A, const DenseMatrix &B,
+                   DenseMatrix &AtB);
+
 /// AAt += a * A * A^t
 void AddMult_a_AAt(real_t a, const DenseMatrix &A, DenseMatrix &AAt);

@@ -16,17 +16,19 @@

 #ifdef MFEM_USE_MPI

+#include "../general/globals.hpp"
+#include "sparsemat.hpp"
+#include "hypre_parcsr.hpp"
 #include <mpi.h>

 // Enable internal hypre timing routines
 #define HYPRE_TIMING

 // hypre header files
-#include "seq_mv.h"
-#include "_hypre_parcsr_mv.h"
-#include "_hypre_parcsr_ls.h"
-#include "temp_multivector.h"
-#include "../general/globals.hpp"
+#include <seq_mv.h>
+#include <temp_multivector.h>
+#include <_hypre_parcsr_mv.h>
+#include <_hypre_parcsr_ls.h>

 #ifdef HYPRE_COMPLEX
 #error "MFEM does not work with HYPRE's complex numbers support"
@@ -51,22 +53,6 @@
 #error "MFEM_USE_HIP=YES is required when HYPRE is built with HIP!"
 #endif

-// MFEM_HYPRE_FORALL is a macro similar to mfem::forall, but it executes on the
-// device that hypre was configured with (no matter what device was selected
-// in MFEM's runtime configuration).
-#if defined(HYPRE_USING_CUDA)
-#define MFEM_HYPRE_FORALL(i, N,...) CuWrap1D(N, [=] MFEM_DEVICE      \
-                                       (int i) {__VA_ARGS__})
-#elif defined(HYPRE_USING_HIP)
-#define MFEM_HYPRE_FORALL(i, N,...) HipWrap1D(N, [=] MFEM_DEVICE     \
-                                        (int i) {__VA_ARGS__})
-#else
-#define MFEM_HYPRE_FORALL(i, N,...) for (int i = 0; i < N; i++) { __VA_ARGS__ }
-#endif
-
-#include "sparsemat.hpp"
-#include "hypre_parcsr.hpp"
-
 namespace mfem
 {

@@ -87,6 +73,22 @@ public:
   /// Calling HYPRE_Finalize() manually is not compatible with this class.
   static void Init() { Instance(); }

+   /// @brief Configure HYPRE's compute and memory policy.
+   ///
+   /// By default HYPRE will be configured with the same policy as MFEM unless
+   /// `Hypre::configure_runtime_policy_from_mfem` is false, in which case
+   /// HYPRE's default will be used; if HYPRE is built for the GPU and the
+   /// aforementioned variable is false then HYPRE will use the GPU even if MFEM
+   /// is not.
+   ///
+   /// This function is no-op if HYPRE is built without GPU support or the HYPRE
+   /// version is less than 2.31.0.
+   ///
+   /// This function is NOT called by Init(). Instead it is called by
+   /// Device::Configure() (when MFEM_USE_MPI=YES) after the MFEM device
+   /// configuration is complete.
+   static void InitDevice();
+
   /// @brief Finalize hypre (called automatically at program exit if
   /// Hypre::Init() has been called).
   ///
@@ -94,6 +96,13 @@ public:
   /// called manually to more precisely control when hypre is finalized.
   static void Finalize();

+   /// @brief Use MFEM's device policy to configure HYPRE's device policy, true
+   /// by default. This variable is used by InitDevice().
+   ///
+   /// This value is not used if HYPRE is build without GPU support or the HYPRE
+   /// version is less than 2.31.0.
+   static bool configure_runtime_policy_from_mfem;
+
 private:
   /// Calls HYPRE_Init() when the singleton is constructed.
   Hypre();
@@ -142,15 +151,28 @@ inline int to_int(HYPRE_Int i)


 /// The MemoryClass used by Hypre objects.
-inline constexpr MemoryClass GetHypreMemoryClass()
+inline MemoryClass GetHypreMemoryClass()
 {
 #if !defined(HYPRE_USING_GPU)
   return MemoryClass::HOST;
-#elif defined(HYPRE_USING_UNIFIED_MEMORY)
+#elif MFEM_HYPRE_VERSION < 23100
+#if defined(HYPRE_USING_UNIFIED_MEMORY)
   return MemoryClass::MANAGED;
 #else
   return MemoryClass::DEVICE;
 #endif
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   if (GetHypreMemoryLocation() == HYPRE_MEMORY_HOST)
+   {
+      return MemoryClass::HOST;
+   }
+   // Return the actual memory location, see hypre_GetActualMemLocation():
+#if defined(HYPRE_USING_UNIFIED_MEMORY)
+   return MemoryClass::MANAGED;
+#else
+   return MemoryClass::DEVICE;
+#endif
+#endif
 }

 /// The MemoryType used by MFEM when allocating arrays for Hypre objects.
@@ -158,13 +180,27 @@ inline MemoryType GetHypreMemoryType()
 {
 #if !defined(HYPRE_USING_GPU)
   return Device::GetHostMemoryType();
-#elif defined(HYPRE_USING_UNIFIED_MEMORY)
+#elif MFEM_HYPRE_VERSION < 23100
+#if defined(HYPRE_USING_UNIFIED_MEMORY)
   return MemoryType::MANAGED;
 #else
   return MemoryType::DEVICE;
 #endif
+#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   if (GetHypreMemoryLocation() == HYPRE_MEMORY_HOST)
+   {
+      return Device::GetHostMemoryType();
+   }
+   // Return the actual memory location, see hypre_GetActualMemLocation():
+#if defined(HYPRE_USING_UNIFIED_MEMORY)
+   return MemoryType::MANAGED;
+#else
+   return MemoryType::DEVICE;
+#endif
+#endif
 }

+
 /// Wrapper for hypre's parallel vector class
 class HypreParVector : public Vector
 {
@@ -328,10 +364,10 @@ public:
   HYPRE_Int Randomize(HYPRE_Int seed);

   /// Prints the locally owned rows in parallel
-   void Print(const char *fname) const;
+   void Print(const std::string &fname) const;

   /// Reads a HypreParVector from files saved with HypreParVector::Print
-   void Read(MPI_Comm comm, const char *fname);
+   void Read(MPI_Comm comm, const std::string &fname);

   /// Calls hypre's destroy function
   ~HypreParVector();
@@ -883,12 +919,14 @@ public:
   const Memory<HYPRE_Int> &GetDiagMemoryJ() const { return mem_diag.J; }
   const Memory<real_t> &GetDiagMemoryData() const { return mem_diag.data; }

-   /// Prints the locally owned rows in parallel
-   void Print(const char *fname, HYPRE_Int offi = 0, HYPRE_Int offj = 0) const;
+   /// @brief Prints the locally owned rows in parallel. The resulting files can
+   /// be read with Read_IJMatrix().
+   void Print(const std::string &fname, HYPRE_Int offi = 0,
+              HYPRE_Int offj = 0) const;
   /// Reads the matrix from a file
-   void Read(MPI_Comm comm, const char *fname);
+   void Read(MPI_Comm comm, const std::string &fname);
   /// Read a matrix saved as a HYPRE_IJMatrix
-   void Read_IJMatrix(MPI_Comm comm, const char *fname);
+   void Read_IJMatrix(MPI_Comm comm, const std::string &fname);

   /// Print information about the hypre_ParCSRCommPkg of the HypreParMatrix.
   void PrintCommPkg(std::ostream &out = mfem::out) const;
@@ -1037,29 +1075,40 @@ protected:
   bool A_is_symmetric;

 public:
-   /** Hypre smoother types:
-       0    = Jacobi
-       1    = l1-scaled Jacobi
-       2    = l1-scaled block Gauss-Seidel/SSOR
-       4    = truncated l1-scaled block Gauss-Seidel/SSOR
-       5    = lumped Jacobi
-       6    = Gauss-Seidel
-       10   = On-processor forward solve for matrix w/ triangular structure
-       16   = Chebyshev
-       1001 = Taubin polynomial smoother
-       1002 = FIR polynomial smoother. */
-   enum Type { Jacobi = 0, l1Jacobi = 1, l1GS = 2, l1GStr = 4, lumpedJacobi = 5,
-               GS = 6, OPFS = 10, Chebyshev = 16, Taubin = 1001, FIR = 1002
-             };
+   /// HYPRE smoother types
+   enum Type
+   {
+      Jacobi = 0,       ///< Jacobi
+      l1Jacobi = 1,     ///< l1-scaled Jacobi
+      l1GS = 2,         ///< l1-scaled block Gauss-Seidel/SSOR
+      l1GStr = 4,       ///< truncated l1-scaled block Gauss-Seidel/SSOR
+      lumpedJacobi = 5, ///< lumped Jacobi
+      GS = 6,           ///< Gauss-Seidel
+      OPFS = 10,        /**< On-processor forward solve for matrix w/ triangular
+                             structure */
+      Chebyshev = 16,   ///< Chebyshev
+      Taubin = 1001,    ///< Taubin polynomial smoother
+      FIR = 1002        ///< FIR polynomial smoother
+   };
+
+   /// @deprecated Use DefaultType() instead
 #if !defined(HYPRE_USING_GPU)
-   static constexpr Type default_type = l1GS;
+   MFEM_DEPRECATED static constexpr Type default_type = l1GS;
 #else
-   static constexpr Type default_type = l1Jacobi;
+   MFEM_DEPRECATED static constexpr Type default_type = l1Jacobi;
 #endif

+   /** @brief Default value for the smoother type used by the constructors:
+       Type::l1GS when HYPRE is running on CPU and Type::l1Jacobi when HYPRE is
+       running on GPU. */
+   static Type DefaultType()
+   {
+      return HypreUsingGPU() ? l1Jacobi : l1GS;
+   }
+
   HypreSmoother();

-   HypreSmoother(const HypreParMatrix &A_, int type = default_type,
+   HypreSmoother(const HypreParMatrix &A_, int type = DefaultType(),
                 int relax_times = 1, real_t relax_weight = 1.0,
                 real_t omega = 1.0, int poly_order = 2,
                 real_t poly_fraction = .3, int eig_est_cg_iter = 10);
@@ -1474,13 +1523,54 @@ public:

   virtual void SetOperator(const Operator &op);

-   void SetParams(real_t threshold, int max_levels);
+   /// Set the threshold and levels parameters
+   /** The accuracy and cost of ParaSails are parametrized by the real
+    * @a thresh and integer @a nlevels parameters (0<=thresh<=1,  0<=nlevels).
+    * Lower values of @a thresh and higher values of @a nlevels lead to
+    * more accurate, but more expensive preconditioners. More accurate
+    * preconditioners are also more expensive per iteration. The default
+    * values are @a thresh = 0.1 and @a nlevels = 1.
+    */
+   void SetParams(real_t thresh, int nlevels);
+
+   /// Set the filter parameter
+   /** The filter parameter is used to drop small nonzeros in the preconditioner,
+    * to reduce the cost of applying the preconditioner. Values from 0.055
+    * to 0.1 are recommended. The default value is 0.1.
+    */
   void SetFilter(real_t filter);
-   void SetLoadBal(real_t loadbal);
-   void SetReuse(int reuse);
-   void SetLogging(int logging);
+
+   /// Set symmetry parameter
+   /** The recognized options are:
+    *  0 = nonsymmetric and/or indefinite problem, and nonsymmetric preconditioner
+    *  1 = SPD problem, and SPD (factored) preconditioner
+    *  2 = nonsymmetric, definite problem, and SPD (factored) preconditioner
+    */
   void SetSymmetry(int sym);

+   /// Set the load balance parameter
+   /** A zero value indicates that no load balance is attempted; a value
+    * of unity indicates that perfect load balance will be attempted. The
+    * recommended value is 0.9 to balance the overhead of data exchanges
+    * for load balancing. No load balancing is needed if the preconditioner
+    * is very sparse and fast to construct. The default value is 0.
+    */
+   void SetLoadBal(real_t loadbal);
+
+   /// Set the pattern reuse parameter
+   /** A nonzero value indicates that the pattern of the preconditioner
+    * should be reused for subsequent constructions of the proconditioner.
+    * A zero value inicates that the peconditioner should be constructed
+    * from scratch. The default value is 0.
+    */
+   void SetReuse(int reuse);
+
+   /// Set the logging parameter
+   /** A nonzero value prints statistics of the setup procedure to stdout.
+    * The default value of this parameter is 1.
+    */
+   void SetLogging(int logging);
+
   /// The typecast to HYPRE_Solver returns the internal sai_precond
   virtual operator HYPRE_Solver() const { return sai_precond; }

@@ -16,11 +16,10 @@

 #ifdef MFEM_USE_MPI

-// Enable internal hypre timing routines
-#define HYPRE_TIMING
-
 #include "../general/mem_manager.hpp"
-#include "_hypre_parcsr_mv.h"
+
+// hypre header files
+#include <_hypre_parcsr_mv.h>

 // Older hypre versions do not define HYPRE_BigInt and HYPRE_MPI_BIG_INT, so we
 // define them here for backward compatibility.
@@ -46,10 +45,10 @@ typedef HYPRE_Int HYPRE_BigInt;
 #else // MFEM_HYPRE_VERSION >= 21400

 #define mfem_hypre_TAlloc(type, size) \
-   hypre_TAlloc(type, size, HYPRE_MEMORY_DEVICE)
+   hypre_TAlloc(type, size, mfem::GetHypreMemoryLocation())
 #define mfem_hypre_CTAlloc(type, size) \
-   hypre_CTAlloc(type, size, HYPRE_MEMORY_DEVICE)
-#define mfem_hypre_TFree(ptr) hypre_TFree(ptr, HYPRE_MEMORY_DEVICE)
+   hypre_CTAlloc(type, size, mfem::GetHypreMemoryLocation())
+#define mfem_hypre_TFree(ptr) hypre_TFree(ptr, mfem::GetHypreMemoryLocation())

 #define mfem_hypre_TAlloc_host(type, size) \
   hypre_TAlloc(type, size, HYPRE_MEMORY_HOST)
@@ -309,18 +309,51 @@ public:


 /// Base abstract class for first order time dependent operators.
-/** Operator of the form: (x,t) -> f(x,t), where k = f(x,t) generally solves the
-    algebraic equation F(x,k,t) = G(x,t). The functions F and G represent the
-    _implicit_ and _explicit_ parts of the operator, respectively. For explicit
-    operators, F(x,k,t) = k, so f(x,t) = G(x,t). */
+/** Operator of the form: (u,t) -> k(u,t), where k generally solves the
+    algebraic equation F(u,k,t) = G(u,t). The functions F and G represent the
+    _implicit_ and _explicit_ parts of the operator, respectively.
+
+    A common use for this class is representing a differential algebraic
+    equation of the form $ F(y,\frac{dy}{dt},t) = G(y,t) $.
+
+    For example, consider an ordinary differential equation of the form
+    $ M \frac{dy}{dt} = g(y,t) $. There are various ways of expressing this ODE
+    as a TimeDependentOperator depending on the choices for F and G. Here are
+    some common choices:
+
+      1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t),
+      2. F(u,k,t) = M k and G(u,t) = g(u,t),
+      3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0.
+
+    Note that depending on the ODE solver, some of the above choices may be
+    preferable to the others.
+*/
 class TimeDependentOperator : public Operator
 {
 public:
+   /// Enum used to describe the form of the time-dependent operator.
+   /** The type should be set by classes derived from TimeDependentOperator to
+       describe the form, in terms of the functions F and G, used by the
+       specific derived class. This information can be queried by classes or
+       functions (like time stepping algorithms) to make choices about the
+       algorithm to use, or to ensure that the TimeDependentOperator uses the
+       form expected by the class/function.
+
+       For example, assume that a derived class is implementing the ODE
+       $M \frac{dy}{dt} = g(y,t)$ and chooses to define $F(u,k,t) = M k$ and
+       $G(u,t) = g(u,t)$. Then it cannot use type EXPLICIT, unless $M = I$, or
+       type HOMOGENEOUS, unless $g(u,t) = 0$. If, on the other hand, the derived
+       class chooses to define $F(u,k,t) = k$ and $G(u,t) = M^{-1} g(y,t)$, then
+       the natural choice is to set the type to EXPLICIT, even though setting it
+       to IMPLICIT is also not wrong -- doing so will simply fail to inform
+       methods that query this information that it uses a more specific
+       implementation, EXPLICIT, that may allow the use of algorithms that
+       support only the EXPLICIT type. */
   enum Type
   {
-      EXPLICIT,   ///< This type assumes F(x,k,t) = k, i.e. k = f(x,t) = G(x,t).
+      EXPLICIT,   ///< This type assumes F(u,k,t) = k.
      IMPLICIT,   ///< This is the most general type, no assumptions on F and G.
-      HOMOGENEOUS ///< This type assumes that G(x,t) = 0.
+      HOMOGENEOUS ///< This type assumes that G(u,t) = 0.
   };

   /// Evaluation mode. See SetEvalMode() for details.
@@ -328,29 +361,30 @@ public:
   {
      /** Normal evaluation. */
      NORMAL,
-      /** Assuming additive split, f(x,t) = f1(x,t) + f2(x,t), evaluate the
-          first term, f1. */
+      /** Assuming additive split, k(u,t) = k1(u,t) + k2(u,t), evaluate the
+          first term, k1. */
      ADDITIVE_TERM_1,
-      /** Assuming additive split, f(x,t) = f1(x,t) + f2(x,t), evaluate the
-          second term, f2. */
+      /** Assuming additive split, k(u,t) = k1(u,t) + k2(u,t), evaluate the
+          second term, k2. */
      ADDITIVE_TERM_2
   };

 protected:
   real_t t;  ///< Current time.
-   Type type; ///< Describes the form of the TimeDependentOperator.
+   Type type; /**< @brief Describes the form of the TimeDependentOperator, see
+                   the documentation of #Type. */
   EvalMode eval_mode; ///< Current evaluation mode.

 public:
-   /** @brief Construct a "square" TimeDependentOperator y = f(x,t), where x and
-       y have the same dimension @a n. */
+   /** @brief Construct a "square" TimeDependentOperator (u,t) -> k(u,t), where
+       u and k have the same dimension @a n. */
   explicit TimeDependentOperator(int n = 0, real_t t_ = 0.0,
                                  Type type_ = EXPLICIT)
      : Operator(n) { t = t_; type = type_; eval_mode = NORMAL; }

-   /** @brief Construct a TimeDependentOperator y = f(x,t), where x and y have
-       dimensions @a w and @a h, respectively. */
-   TimeDependentOperator(int h, int w, real_t t_ = 0.0, Type type_ = EXPLICIT)
+   /** @brief Construct a TimeDependentOperator (u,t) -> k(u,t), where u and k
+       have dimensions @a w and @a h, respectively. */
+   TimeDependentOperator(int h, int w, double t_ = 0.0, Type type_ = EXPLICIT)
      : Operator(h, w) { t = t_; type = type_; eval_mode = NORMAL; }

   /// Read the currently set time.
@@ -373,7 +407,7 @@ public:
   /** The evaluation mode is a switch that allows time-stepping methods to
       request evaluation of separate components/terms of the time-dependent
       operator. For example, IMEX methods typically assume additive split of
-       the operator: f(x,t) = f1(x,t) + f2(x,t) and they rely on the ability to
+       the operator: k(u,t) = k1(u,t) + k2(u,t) and they rely on the ability to
       evaluate the two terms separately.

       Generally, setting the evaluation mode should affect the behavior of all
@@ -384,62 +418,104 @@ public:
   { eval_mode = new_eval_mode; }

   /** @brief Perform the action of the explicit part of the operator, G:
-       @a y = G(@a x, t) where t is the current time.
+       @a v = G(@a u, t) where t is the current time.

       Presently, this method is used by some PETSc ODE solvers, for more
       details, see the PETSc Manual. */
-   virtual void ExplicitMult(const Vector &x, Vector &y) const;
+   virtual void ExplicitMult(const Vector &u, Vector &v) const;

   /** @brief Perform the action of the implicit part of the operator, F:
-       @a y = F(@a x, @a k, t) where t is the current time.
+       @a v = F(@a u, @a k, t) where t is the current time.

       Presently, this method is used by some PETSc ODE solvers, for more
       details, see the PETSc Manual.*/
-   virtual void ImplicitMult(const Vector &x, const Vector &k, Vector &y) const;
+   virtual void ImplicitMult(const Vector &u, const Vector &k, Vector &v) const;

-   /** @brief Perform the action of the operator: @a y = k = f(@a x, t), where
-       k solves the algebraic equation F(@a x, k, t) = G(@a x, t) and t is the
-       current time. */
-   virtual void Mult(const Vector &x, Vector &y) const;
+   /** @brief Perform the action of the operator (u,t) -> k(u,t) where t is the
+       current time set by SetTime() and @a k satisfies
+       F(@a u, @a k, t) = G(@a u, t).

-   /** @brief Solve the equation: @a k = f(@a x + @a dt @a k, t), for the
-       unknown @a k at the current time t.
+       For solving an ordinary differential equation of the form
+       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined in
+       various ways, e.g.:

-       For general F and G, the equation for @a k becomes:
-       F(@a x + @a dt @a k, @a k, t) = G(@a x + @a dt @a k, t).
+         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
+         2. F(u,k,t) = M k and G(u,t) = g(u,t)
+         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0.

-       The input vector @a x corresponds to time index (or cycle) n, while the
-       currently set time, #t, and the result vector @a k correspond to time
-       index n+1. The time step @a dt corresponds to the time interval between
-       cycles n and n+1.
+       Regardless of the choice of F and G, this function should always compute
+       @a k = inv(M) g(@a u, t). */
+   virtual void Mult(const Vector &u, Vector &v) const override;

-       This method allows for the abstract implementation of some time
-       integration methods, including diagonal implicit Runge-Kutta (DIRK)
-       methods and the backward Euler method in particular.
+   /** @brief Solve for the unknown @a k, at the current time t, the following
+       equation:
+       F(@a u + @a gamma @a k, @a k, t) = G(@a u + @a gamma @a k, t).
+
+       For solving an ordinary differential equation of the form
+       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined in
+       various ways, e.g.:
+
+         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
+         2. F(u,k,t) = M k and G(u,t) = g(u,t)
+         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0
+
+       Regardless of the choice of F and G, this function should solve for @a k
+       in M @a k = g(@a u + @a gamma @a k, t).
+
+       To see how @a k can be useful, consider the backward Euler method defined
+       by $ y(t + \Delta t) = y(t) + \Delta t k_0 $ where
+       $ M k_0 = g \big( y(t) + \Delta t k_0, t + \Delta t \big) $. A backward
+       Euler integrator can use @a k from this function for $k_0$, with the call
+       using @a u set to $ y(t) $, @a gamma set to $ \Delta t$, and time set to
+       $t + \Delta t$. See class BackwardEulerSolver.
+
+       Generalizing further, consider a diagonally implicit Runge-Kutta (DIRK)
+       method defined by
+       $ y(t + \Delta t) = y(t) + \Delta t \sum_{i=1}^s b_i k_i $ where
+       $ M k_i = g \big( y(t) + \Delta t \sum_{j=1}^i a_{ij} k_j,
+                         t + c_i \Delta t \big) $.
+       A DIRK integrator can use @a k from this function, with @a u set to
+       $ y(t) + \Delta t \sum_{j=1}^{i-1} a_{ij} k_j $ and @a gamma set to
+       $ a_{ii} \Delta t $, for $ k_i $. For example, see class SDIRK33Solver.

       If not re-implemented, this method simply generates an error. */
-   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);
+   virtual void ImplicitSolve(const real_t gamma, const Vector &u, Vector &k);

-   /** @brief Return an Operator representing (dF/dk @a shift + dF/dx) at the
-       given @a x, @a k, and the currently set time.
+   /** @brief Return an Operator representing (dF/dk @a shift + dF/du) at the
+       given @a u, @a k, and the currently set time.

       Presently, this method is used by some PETSc ODE solvers, for more
       details, see the PETSc Manual. */
-   virtual Operator& GetImplicitGradient(const Vector &x, const Vector &k,
+   virtual Operator& GetImplicitGradient(const Vector &u, const Vector &k,
                                         real_t shift) const;

-   /** @brief Return an Operator representing dG/dx at the given point @a x and
+   /** @brief Return an Operator representing dG/du at the given point @a u and
       the currently set time.

       Presently, this method is used by some PETSc ODE solvers, for more
       details, see the PETSc Manual. */
-   virtual Operator& GetExplicitGradient(const Vector &x) const;
+   virtual Operator& GetExplicitGradient(const Vector &u) const;

-   /** @brief Setup the ODE linear system $ A(x,t) = (I - gamma J) $ or
-       $ A = (M - gamma J) $, where $ J(x,t) = \frac{df}{dt(x,t)} $.
+   /** @brief Setup a linear system as needed by some SUNDIALS ODE solvers.

-       @param[in]  x     The state at which $A(x,t)$ should be evaluated.
-       @param[in]  fx    The current value of the ODE rhs function, $f(x,t)$.
+       For solving an ordinary differential equation of the form
+       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined as one
+       of the following:
+
+         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
+         2. F(u,k,t) = M k and G(u,t) = g(u,t)
+         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0
+
+       This function performs setup to solve $ A x = b $ where A is either
+
+         1. A(@a y,t) = I - @a gamma inv(M) J(@a y,t)
+         2. A(@a y,t) = M - @a gamma J(@a y,t)
+         3. A(@a y,t) = M - @a gamma J(@a y,t)
+
+       with J = dg/dy (or a reasonable approximation thereof).
+
+       @param[in]  y     The state at which A(@a y,t) should be evaluated.
+       @param[in]  v     The value of inv(M) g(y,t) for 1 or g(y,t) for 2 & 3.
       @param[in]  jok   Flag indicating if the Jacobian should be updated.
       @param[out] jcur  Flag to signal if the Jacobian was updated.
       @param[in]  gamma The scaled time step value.
@@ -448,10 +524,10 @@ public:

       Presently, this method is used by SUNDIALS ODE solvers, for more
       details, see the SUNDIALS User Guides. */
-   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
+   virtual int SUNImplicitSetup(const Vector &y, const Vector &v,
                                int jok, int *jcur, real_t gamma);

-   /** @brief Solve the ODE linear system $ A x = b $ as setup by
+   /** @brief Solve the ODE linear system A @a x = @a b, where A is defined by
       the method SUNImplicitSetup().

       @param[in]      b   The linear system right-hand side.
@@ -464,7 +540,8 @@ public:
       details, see the SUNDIALS User Guides. */
   virtual int SUNImplicitSolve(const Vector &b, Vector &x, real_t tol);

-   /** @brief Setup the mass matrix in the ODE system $ M y' = f(y,t) $ .
+   /** @brief Setup the mass matrix in the ODE system
+       $ M \frac{dy}{dt} = g(y,t) $ .

       If not re-implemented, this method simply generates an error.

@@ -472,8 +549,8 @@ public:
       details, see the ARKode User Guide. */
   virtual int SUNMassSetup();

-   /** @brief Solve the mass matrix linear system $ M x = b $
-       as setup by the method SUNMassSetup().
+   /** @brief Solve the mass matrix linear system  M @a x = @a b, where M is
+       defined by the method SUNMassSetup().

       @param[in]      b   The linear system right-hand side.
       @param[in,out]  x   On input, the initial guess. On output, the solution.
@@ -485,7 +562,8 @@ public:
       details, see the ARKode User Guide. */
   virtual int SUNMassSolve(const Vector &b, Vector &x, real_t tol);

-   /** @brief Compute the mass matrix-vector product $ v = M x $ .
+   /** @brief Compute the mass matrix-vector product @a v = M @a x, where M is
+       defined by the method SUNMassSetup().

       @param[in]   x The vector to multiply.
       @param[out]  v The result of the matrix-vector product.
@@ -1188,6 +1188,8 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const

   final_norm = std::max(rel_tol*beta, abs_tol);

+   converged = false;
+
   if (beta <= final_norm)
   {
      final_norm = beta;
@@ -1303,8 +1305,6 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
      MFEM_ASSERT(IsFinite(beta), "beta = " << beta);
      if (beta <= final_norm)
      {
-         final_norm = beta;
-         final_iter = j;
         converged = true;

         break;
@@ -1317,7 +1317,9 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
      if (v[i]) { delete v[i]; }
      if (z[i]) { delete z[i]; }
   }
-   converged = false;
+
+   final_norm = beta;
+   final_iter = converged ? j : max_iter;

   // Note: j is off by one when we arrive here
   if (!print_options.iterations && print_options.first_and_last)
@@ -1328,7 +1330,7 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
   }
   if (print_options.summary || (print_options.warnings && !converged))
   {
-      mfem::out << "FGMRES: Number of iterations: " << j-1 << '\n';
+      mfem::out << "FGMRES: Number of iterations: " << final_iter << '\n';
   }
   if (print_options.warnings && !converged)
   {
@@ -58,6 +58,20 @@ DenseSymmetricMatrix &DenseSymmetricMatrix::operator=(real_t c)
   return *this;
 }

+DenseSymmetricMatrix &DenseSymmetricMatrix::operator=(const DenseSymmetricMatrix
+                                                      &m)
+{
+   SetSize(m.height);
+
+   const int hw = m.GetStoredSize();
+   for (int i = 0; i < hw; i++)
+   {
+      data[i] = m.data[i];
+   }
+
+   return *this;
+}
+
 real_t &DenseSymmetricMatrix::Elem(int i, int j)
 {
   return (*this)(i,j);
@@ -89,11 +103,6 @@ MatrixInverse *DenseSymmetricMatrix::Inverse() const
   return nullptr;
 }

-void DenseSymmetricMatrix::Print (std::ostream & os, int width_) const
-{
-   mfem_error("DenseSymmetricMatrix::Print() not implemented!");
-}
-
 DenseSymmetricMatrix::~DenseSymmetricMatrix()
 {
   data.Delete();
@@ -102,6 +102,9 @@ public:

   DenseSymmetricMatrix &operator*=(real_t c);

+   /// Sets the matrix size and elements equal to those of m
+   DenseSymmetricMatrix &operator=(const DenseSymmetricMatrix &m);
+
   std::size_t MemoryUsage() const { return data.Capacity() * sizeof(real_t); }

   /// Shortcut for mfem::Read( GetMemory(), TotalSize(), on_dev).
@@ -134,9 +137,6 @@ public:
   /// Returns a pointer to (an approximation) of the matrix inverse.
   virtual MatrixInverse *Inverse() const;

-   /// Prints matrix to stream out.
-   virtual void Print (std::ostream & out = mfem::out, int width_ = 4) const;
-
   /// Destroys the symmetric matrix.
   virtual ~DenseSymmetricMatrix();
 };
@@ -716,13 +716,7 @@ inline real_t InnerProduct(MPI_Comm comm, const Vector &x, const Vector &y)
 {
   real_t loc_prod = x * y;
   real_t glb_prod;
-#ifdef MFEM_USE_SINGLE
-   MPI_Allreduce(&loc_prod, &glb_prod, 1, MPI_FLOAT, MPI_SUM, comm);
-#elif defined MFEM_USE_DOUBLE
-   MPI_Allreduce(&loc_prod, &glb_prod, 1, MPI_DOUBLE, MPI_SUM, comm);
-#else
-   MFEM_ABORT("Floating point type undefined");
-#endif
+   MPI_Allreduce(&loc_prod, &glb_prod, 1, MFEM_MPI_REAL_T, MPI_SUM, comm);
   return glb_prod;
 }
 #endif
@@ -10,7 +10,7 @@
 # CONTRIBUTING.md for details.

 # The current MFEM version as an integer, see also `CMakeLists.txt`.
-MFEM_VERSION = 40601
+MFEM_VERSION = 40701
 MFEM_VERSION_STRING = $(shell printf "%06d" $(MFEM_VERSION) | \
  sed -e 's/^0*\(.*.\)\(..\)\(..\)$$/\1.\2.\3/' -e 's/\.0/./g' -e 's/\.0$$//')

@@ -205,6 +205,14 @@ MFEM_SHARED_BUILD = $(MFEM_SHARED)
 override static = $(if $(MFEM_STATIC:YES=),,YES)
 override shared = $(if $(MFEM_SHARED:YES=),,YES)

+# Error for package integrations that currently don't support single precision
+ifeq ($(MFEM_USE_SINGLE),YES)
+   PKGS_NO_SINGLE = SUNDIALS SUITESPARSE SUPERLU STRUMPACK GINKGO AMGX SLEPC\
+	 PUMI GSLIB ALGOIM CEED MOONOLITH TRIBOL
+   $(foreach pkg,$(PKGS_NO_SINGLE),$(if $(MFEM_USE_$(pkg):NO=),\
+     $(error Package $(pkg) is NOT supported with single precision)))
+endif
+
 # The default value of CXXFLAGS is based on the value of MFEM_DEBUG
 ifeq ($(MFEM_DEBUG),YES)
   CXXFLAGS ?= $(DEBUG_FLAGS)
@@ -20,9 +20,10 @@
 #include "../general/tic_toc.hpp"
 #include "../general/gecko.hpp"
 #include "../general/kdtree.hpp"
+#include "../general/sets.hpp"
 #include "../fem/quadinterpolator.hpp"

-#include <iostream>
+// headers already included by mesh.hpp: <iostream>, <array>, <map>, <memory>
 #include <sstream>
 #include <fstream>
 #include <limits>
@@ -1338,7 +1339,7 @@ Mesh::FaceInformation::operator Mesh::FaceInfo() const
   return res;
 }

-std::ostream& operator<<(std::ostream& os, const Mesh::FaceInformation& info)
+std::ostream &operator<<(std::ostream &os, const Mesh::FaceInformation& info)
 {
   os << "face topology=";
   switch (info.topology)
@@ -6209,6 +6210,12 @@ const FiniteElementSpace *Mesh::GetNodalFESpace() const

 void Mesh::SetCurvature(int order, bool discont, int space_dim, int ordering)
 {
+   if (order <= 0)
+   {
+      delete Nodes;
+      Nodes = nullptr;
+      return;
+   }
   space_dim = (space_dim == -1) ? spaceDim : space_dim;
   FiniteElementCollection* nfec;
   if (discont)
@@ -11419,7 +11426,8 @@ void Mesh::Printer(std::ostream &os, std::string section_delimiter,

   if (!section_delimiter.empty())
   {
-      os << section_delimiter << endl; // only with formats v1.2 and above
+      os << '\n'
+         << section_delimiter << endl; // only with formats v1.2 and above
   }
 }

@@ -13299,6 +13307,879 @@ void Mesh::GetGeometricParametersFromJacobian(const DenseMatrix &J,
 }


+MeshPart::EntityHelper::EntityHelper(
+   int dim_, const Array<int> (&entity_to_vertex_)[Geometry::NumGeom])
+   : dim(dim_),
+     entity_to_vertex(entity_to_vertex_)
+{
+   int geom_offset = 0;
+   for (int g = Geometry::DimStart[dim]; g < Geometry::DimStart[dim+1]; g++)
+   {
+      geom_offsets[g] = geom_offset;
+      geom_offset += entity_to_vertex[g].Size()/Geometry::NumVerts[g];
+   }
+   geom_offsets[Geometry::DimStart[dim+1]] = geom_offset;
+   num_entities = geom_offset;
+}
+
+MeshPart::Entity MeshPart::EntityHelper::FindEntity(int bytype_entity_id)
+{
+   // Find the 'geom' that corresponds to 'bytype_entity_id'
+   int geom = Geometry::DimStart[dim];
+   while (geom_offsets[geom+1] <= bytype_entity_id) { geom++; }
+   MFEM_ASSERT(geom < Geometry::NumGeom, "internal error");
+   MFEM_ASSERT(Geometry::Dimension[geom] == dim, "internal error");
+   const int nv = Geometry::NumVerts[geom];
+   const int geom_elem_id = bytype_entity_id - geom_offsets[geom];
+   const int *v = &entity_to_vertex[geom][nv*geom_elem_id];
+   return { geom, nv, v };
+}
+
+void MeshPart::Print(std::ostream &os) const
+{
+   os << "MFEM mesh v1.2\n";
+
+   // optional
+   os <<
+      "\n#\n# MFEM Geometry Types (see mesh/geom.hpp):\n#\n"
+      "# POINT       = 0\n"
+      "# SEGMENT     = 1\n"
+      "# TRIANGLE    = 2\n"
+      "# SQUARE      = 3\n"
+      "# TETRAHEDRON = 4\n"
+      "# CUBE        = 5\n"
+      "# PRISM       = 6\n"
+      "# PYRAMID     = 7\n"
+      "#\n";
+
+   const int dim = dimension;
+   os << "\ndimension\n" << dim;
+
+   os << "\n\nelements\n" << num_elements << '\n';
+   {
+      const bool have_element_map = (element_map.Size() == num_elements);
+      MFEM_ASSERT(have_element_map || element_map.Size() == 0,
+                  "invalid MeshPart state");
+      EntityHelper elem_helper(dim, entity_to_vertex);
+      MFEM_ASSERT(elem_helper.num_entities == num_elements,
+                  "invalid MeshPart state");
+      for (int nat_elem_id = 0; nat_elem_id < num_elements; nat_elem_id++)
+      {
+         const int bytype_elem_id = have_element_map ?
+                                    element_map[nat_elem_id] : nat_elem_id;
+         const Entity ent = elem_helper.FindEntity(bytype_elem_id);
+         // Print the element
+         os << attributes[nat_elem_id] << ' ' << ent.geom;
+         for (int i = 0; i < ent.num_verts; i++)
+         {
+            os << ' ' << ent.verts[i];
+         }
+         os << '\n';
+      }
+   }
+
+   os << "\nboundary\n" << num_bdr_elements << '\n';
+   {
+      const bool have_boundary_map = (boundary_map.Size() == num_bdr_elements);
+      MFEM_ASSERT(have_boundary_map || boundary_map.Size() == 0,
+                  "invalid MeshPart state");
+      EntityHelper bdr_helper(dim-1, entity_to_vertex);
+      MFEM_ASSERT(bdr_helper.num_entities == num_bdr_elements,
+                  "invalid MeshPart state");
+      for (int nat_bdr_id = 0; nat_bdr_id < num_bdr_elements; nat_bdr_id++)
+      {
+         const int bytype_bdr_id = have_boundary_map ?
+                                   boundary_map[nat_bdr_id] : nat_bdr_id;
+         const Entity ent = bdr_helper.FindEntity(bytype_bdr_id);
+         // Print the boundary element
+         os << bdr_attributes[nat_bdr_id] << ' ' << ent.geom;
+         for (int i = 0; i < ent.num_verts; i++)
+         {
+            os << ' ' << ent.verts[i];
+         }
+         os << '\n';
+      }
+   }
+
+   os << "\nvertices\n" << num_vertices << '\n';
+   if (!nodes)
+   {
+      const int sdim = space_dimension;
+      os << sdim << '\n';
+      for (int i = 0; i < num_vertices; i++)
+      {
+         os << vertex_coordinates[i*sdim];
+         for (int d = 1; d < sdim; d++)
+         {
+            os << ' ' << vertex_coordinates[i*sdim+d];
+         }
+         os << '\n';
+      }
+   }
+   else
+   {
+      os << "\nnodes\n";
+      nodes->Save(os);
+   }
+
+   os << "\nmfem_serial_mesh_end\n";
+
+   // Start: GroupTopology::Save
+   const int num_groups = my_groups.Size();
+   os << "\ncommunication_groups\n";
+   os << "number_of_groups " << num_groups << "\n\n";
+
+   os << "# number of entities in each group, followed by ranks in group\n";
+   for (int group_id = 0; group_id < num_groups; ++group_id)
+   {
+      const int group_size = my_groups.RowSize(group_id);
+      const int *group_ptr = my_groups.GetRow(group_id);
+      os << group_size;
+      for (int group_member_index = 0; group_member_index < group_size;
+           ++group_member_index)
+      {
+         os << ' ' << group_ptr[group_member_index];
+      }
+      os << '\n';
+   }
+   // End: GroupTopology::Save
+
+   const Table &g2v  = group_shared_entity_to_vertex[Geometry::POINT];
+   const Table &g2ev = group_shared_entity_to_vertex[Geometry::SEGMENT];
+   const Table &g2tv = group_shared_entity_to_vertex[Geometry::TRIANGLE];
+   const Table &g2qv = group_shared_entity_to_vertex[Geometry::SQUARE];
+
+   MFEM_VERIFY(g2v.RowSize(0) == 0, "internal erroor");
+   os << "\ntotal_shared_vertices " << g2v.Size_of_connections() << '\n';
+   if (dimension >= 2)
+   {
+      MFEM_VERIFY(g2ev.RowSize(0) == 0, "internal erroor");
+      os << "total_shared_edges " << g2ev.Size_of_connections()/2 << '\n';
+   }
+   if (dimension >= 3)
+   {
+      MFEM_VERIFY(g2tv.RowSize(0) == 0, "internal erroor");
+      MFEM_VERIFY(g2qv.RowSize(0) == 0, "internal erroor");
+      const int total_shared_faces =
+         g2tv.Size_of_connections()/3 + g2qv.Size_of_connections()/4;
+      os << "total_shared_faces " << total_shared_faces << '\n';
+   }
+   os << "\n# group 0 has no shared entities\n";
+   for (int gr = 1; gr < num_groups; gr++)
+   {
+      {
+         const int  nv = g2v.RowSize(gr);
+         const int *sv = g2v.GetRow(gr);
+         os << "\n# group " << gr << "\nshared_vertices " << nv << '\n';
+         for (int i = 0; i < nv; i++)
+         {
+            os << sv[i] << '\n';
+         }
+      }
+      if (dimension >= 2)
+      {
+         const int  ne = g2ev.RowSize(gr)/2;
+         const int *se = g2ev.GetRow(gr);
+         os << "\nshared_edges " << ne << '\n';
+         for (int i = 0; i < ne; i++)
+         {
+            const int *v = se + 2*i;
+            os << v[0] << ' ' << v[1] << '\n';
+         }
+      }
+      if (dimension >= 3)
+      {
+         const int  nt = g2tv.RowSize(gr)/3;
+         const int *st = g2tv.GetRow(gr);
+         const int  nq = g2qv.RowSize(gr)/4;
+         const int *sq = g2qv.GetRow(gr);
+         os << "\nshared_faces " << nt+nq << '\n';
+         for (int i = 0; i < nt; i++)
+         {
+            os << Geometry::TRIANGLE;
+            const int *v = st + 3*i;
+            for (int j = 0; j < 3; j++) { os << ' ' << v[j]; }
+            os << '\n';
+         }
+         for (int i = 0; i < nq; i++)
+         {
+            os << Geometry::SQUARE;
+            const int *v = sq + 4*i;
+            for (int j = 0; j < 4; j++) { os << ' ' << v[j]; }
+            os << '\n';
+         }
+      }
+   }
+
+   // Write out section end tag for mesh.
+   os << "\nmfem_mesh_end" << endl;
+}
+
+Mesh &MeshPart::GetMesh()
+{
+   if (mesh) { return *mesh; }
+
+   mesh.reset(new Mesh(dimension,
+                       num_vertices,
+                       num_elements,
+                       num_bdr_elements,
+                       space_dimension));
+
+   // Add elements
+   {
+      const bool have_element_map = (element_map.Size() == num_elements);
+      MFEM_ASSERT(have_element_map || element_map.Size() == 0,
+                  "invalid MeshPart state");
+      EntityHelper elem_helper(dimension, entity_to_vertex);
+      MFEM_ASSERT(elem_helper.num_entities == num_elements,
+                  "invalid MeshPart state");
+      const bool have_tet_refine_flags = (tet_refine_flags.Size() > 0);
+      for (int nat_elem_id = 0; nat_elem_id < num_elements; nat_elem_id++)
+      {
+         const int bytype_elem_id = have_element_map ?
+                                    element_map[nat_elem_id] : nat_elem_id;
+         const Entity ent = elem_helper.FindEntity(bytype_elem_id);
+         Element *el = mesh->NewElement(ent.geom);
+         el->SetVertices(ent.verts);
+         el->SetAttribute(attributes[nat_elem_id]);
+         if (ent.geom == Geometry::TETRAHEDRON && have_tet_refine_flags)
+         {
+            constexpr int geom_tet = Geometry::TETRAHEDRON;
+            const int tet_id = (ent.verts - entity_to_vertex[geom_tet])/4;
+            const int ref_flag = tet_refine_flags[tet_id];
+            static_cast<Tetrahedron*>(el)->SetRefinementFlag(ref_flag);
+         }
+         mesh->AddElement(el);
+      }
+   }
+
+   // Add boundary elements
+   {
+      const bool have_boundary_map = (boundary_map.Size() == num_bdr_elements);
+      MFEM_ASSERT(have_boundary_map || boundary_map.Size() == 0,
+                  "invalid MeshPart state");
+      EntityHelper bdr_helper(dimension-1, entity_to_vertex);
+      MFEM_ASSERT(bdr_helper.num_entities == num_bdr_elements,
+                  "invalid MeshPart state");
+      for (int nat_bdr_id = 0; nat_bdr_id < num_bdr_elements; nat_bdr_id++)
+      {
+         const int bytype_bdr_id = have_boundary_map ?
+                                   boundary_map[nat_bdr_id] : nat_bdr_id;
+         const Entity ent = bdr_helper.FindEntity(bytype_bdr_id);
+         Element *bdr = mesh->NewElement(ent.geom);
+         bdr->SetVertices(ent.verts);
+         bdr->SetAttribute(bdr_attributes[nat_bdr_id]);
+         mesh->AddBdrElement(bdr);
+      }
+   }
+
+   // Add vertices
+   if (vertex_coordinates.Size() == space_dimension*num_vertices)
+   {
+      MFEM_ASSERT(!nodes, "invalid MeshPart state");
+      for (int vert_id = 0; vert_id < num_vertices; vert_id++)
+      {
+         mesh->AddVertex(vertex_coordinates + space_dimension*vert_id);
+      }
+   }
+   else
+   {
+      MFEM_ASSERT(vertex_coordinates.Size() == 0, "invalid MeshPart state");
+      for (int vert_id = 0; vert_id < num_vertices; vert_id++)
+      {
+         mesh->AddVertex(0., 0., 0.);
+      }
+      // 'mesh.Nodes' cannot be set here -- they can be set later, if needed
+   }
+
+   mesh->FinalizeTopology(/* generate_bdr: */ false);
+
+   return *mesh;
+}
+
+
+MeshPartitioner::MeshPartitioner(Mesh &mesh_,
+                                 int num_parts_,
+                                 int *partitioning_,
+                                 int part_method)
+   : mesh(mesh_)
+{
+   if (partitioning_)
+   {
+      partitioning.MakeRef(partitioning_, mesh.GetNE(), false);
+   }
+   else
+   {
+      partitioning_ = mesh.GeneratePartitioning(num_parts_, part_method);
+      // Mesh::GeneratePartitioning always uses new[] to allocate the,
+      // partitioning, so we need to tell the memory manager to free it with
+      // delete[] (even if a different host memory type has been selected).
+      const MemoryType mt = MemoryType::HOST;
+      partitioning.MakeRef(partitioning_, mesh.GetNE(), mt, true);
+   }
+
+   Transpose(partitioning, part_to_element, num_parts_);
+   // Note: the element ids in each row of 'part_to_element' are sorted.
+
+   const int dim = mesh.Dimension();
+   if (dim >= 2)
+   {
+      Transpose(mesh.ElementToEdgeTable(), edge_to_element, mesh.GetNEdges());
+   }
+
+   Array<int> boundary_to_part(mesh.GetNBE());
+   // Same logic as in ParMesh::BuildLocalBoundary
+   if (dim >= 3)
+   {
+      for (int i = 0; i < boundary_to_part.Size(); i++)
+      {
+         int face, o, el1, el2;
+         mesh.GetBdrElementFace(i, &face, &o);
+         mesh.GetFaceElements(face, &el1, &el2);
+         boundary_to_part[i] =
+            partitioning[(o % 2 == 0 || el2 < 0) ? el1 : el2];
+      }
+   }
+   else if (dim == 2)
+   {
+      for (int i = 0; i < boundary_to_part.Size(); i++)
+      {
+         int edge = mesh.GetBdrElementFaceIndex(i);
+         int el1 = edge_to_element.GetRow(edge)[0];
+         boundary_to_part[i] = partitioning[el1];
+      }
+   }
+   else if (dim == 1)
+   {
+      for (int i = 0; i < boundary_to_part.Size(); i++)
+      {
+         int vert = mesh.GetBdrElementFaceIndex(i);
+         int el1, el2;
+         mesh.GetFaceElements(vert, &el1, &el2);
+         boundary_to_part[i] = partitioning[el1];
+      }
+   }
+   Transpose(boundary_to_part, part_to_boundary, num_parts_);
+   // Note: the boundary element ids in each row of 'part_to_boundary' are
+   // sorted.
+   boundary_to_part.DeleteAll();
+
+   Table *vert_element = mesh.GetVertexToElementTable(); // we must delete this
+   vertex_to_element.Swap(*vert_element);
+   delete vert_element;
+}
+
+void MeshPartitioner::ExtractPart(int part_id, MeshPart &mesh_part) const
+{
+   const int num_parts = part_to_element.Size();
+
+   MFEM_VERIFY(0 <= part_id && part_id < num_parts,
+               "invalid part_id = " << part_id
+               << ", num_parts = " << num_parts);
+
+   const int dim = mesh.Dimension();
+   const int sdim = mesh.SpaceDimension();
+   const int num_elems = part_to_element.RowSize(part_id);
+   const int *elem_list = part_to_element.GetRow(part_id); // sorted
+   const int num_bdr_elems = part_to_boundary.RowSize(part_id);
+   const int *bdr_elem_list = part_to_boundary.GetRow(part_id); // sorted
+
+   // Initialize 'mesh_part'
+   mesh_part.dimension = dim;
+   mesh_part.space_dimension = sdim;
+   mesh_part.num_vertices = 0;
+   mesh_part.num_elements = num_elems;
+   mesh_part.num_bdr_elements = num_bdr_elems;
+   for (int g = 0; g < Geometry::NumGeom; g++)
+   {
+      mesh_part.entity_to_vertex[g].SetSize(0); // can reuse Array allocation
+   }
+   mesh_part.tet_refine_flags.SetSize(0);
+   mesh_part.element_map.SetSize(0); // 0 or 'num_elements', if needed
+   mesh_part.boundary_map.SetSize(0); // 0 or 'num_bdr_elements', if needed
+   mesh_part.attributes.SetSize(num_elems);
+   mesh_part.bdr_attributes.SetSize(num_bdr_elems);
+   mesh_part.vertex_coordinates.SetSize(0);
+
+   mesh_part.num_parts = num_parts;
+   mesh_part.my_part_id = part_id;
+   mesh_part.my_groups.Clear();
+   for (int g = 0; g < Geometry::NumGeom; g++)
+   {
+      mesh_part.group_shared_entity_to_vertex[g].Clear();
+   }
+   mesh_part.nodes.reset(nullptr);
+   mesh_part.nodal_fes.reset(nullptr);
+   mesh_part.mesh.reset(nullptr);
+
+   // Initialize:
+   // - 'mesh_part.entity_to_vertex' for the elements (boundary elements are
+   //   set later); vertex ids are global at this point - they will be mapped to
+   //   local ids later
+   // - 'mesh_part.attributes'
+   // - 'mesh_part.tet_refine_flags' if needed
+   int geom_marker = 0, num_geom = 0;
+   for (int i = 0; i < num_elems; i++)
+   {
+      const Element *elem = mesh.GetElement(elem_list[i]);
+      const int geom = elem->GetGeometryType();
+      const int nv = Geometry::NumVerts[geom];
+      const int *v = elem->GetVertices();
+      MFEM_VERIFY(numeric_limits<int>::max() - nv >=
+                  mesh_part.entity_to_vertex[geom].Size(),
+                  "overflow in 'entity_to_vertex[geom]', geom: "
+                  << Geometry::Name[geom]);
+      mesh_part.entity_to_vertex[geom].Append(v, nv);
+      mesh_part.attributes[i] = elem->GetAttribute();
+      if (geom == Geometry::TETRAHEDRON)
+      {
+         // Create 'mesh_part.tet_refine_flags' but only if we find at least one
+         // non-zero flag in a tetrahedron.
+         const Tetrahedron *tet = static_cast<const Tetrahedron*>(elem);
+         const int ref_flag = tet->GetRefinementFlag();
+         if (mesh_part.tet_refine_flags.Size() == 0)
+         {
+            if (ref_flag)
+            {
+               // This is the first time we encounter non-zero 'ref_flag'
+               const int num_tets = mesh_part.entity_to_vertex[geom].Size()/nv;
+               mesh_part.tet_refine_flags.SetSize(num_tets, 0);
+               mesh_part.tet_refine_flags.Last() = ref_flag;
+            }
+         }
+         else
+         {
+            mesh_part.tet_refine_flags.Append(ref_flag);
+         }
+      }
+      if ((geom_marker & (1 << geom)) == 0)
+      {
+         geom_marker |= (1 << geom);
+         num_geom++;
+      }
+   }
+   MFEM_ASSERT(mesh_part.tet_refine_flags.Size() == 0 ||
+               mesh_part.tet_refine_flags.Size() ==
+               mesh_part.entity_to_vertex[Geometry::TETRAHEDRON].Size()/4,
+               "internal error");
+   // Initialize 'mesh_part.element_map' if needed
+   if (num_geom > 1)
+   {
+      int offsets[Geometry::NumGeom];
+      int offset = 0;
+      for (int g = Geometry::DimStart[dim]; g < Geometry::DimStart[dim+1]; g++)
+      {
+         offsets[g] = offset;
+         offset += mesh_part.entity_to_vertex[g].Size()/Geometry::NumVerts[g];
+      }
+      mesh_part.element_map.SetSize(num_elems);
+      for (int i = 0; i < num_elems; i++)
+      {
+         const int geom = mesh.GetElementGeometry(elem_list[i]);
+         mesh_part.element_map[i] = offsets[geom]++;
+      }
+   }
+
+   // Initialize:
+   // - 'mesh_part.entity_to_vertex' for the boundary elements; vertex ids are
+   //   global at this point - they will be mapped to local ids later
+   // - 'mesh_part.bdr_attributes'
+   geom_marker = 0; num_geom = 0;
+   for (int i = 0; i < num_bdr_elems; i++)
+   {
+      const Element *bdr_elem = mesh.GetBdrElement(bdr_elem_list[i]);
+      const int geom = bdr_elem->GetGeometryType();
+      const int nv = Geometry::NumVerts[geom];
+      const int *v = bdr_elem->GetVertices();
+      MFEM_VERIFY(numeric_limits<int>::max() - nv >=
+                  mesh_part.entity_to_vertex[geom].Size(),
+                  "overflow in 'entity_to_vertex[geom]', geom: "
+                  << Geometry::Name[geom]);
+      mesh_part.entity_to_vertex[geom].Append(v, nv);
+      mesh_part.bdr_attributes[i] = bdr_elem->GetAttribute();
+      if ((geom_marker & (1 << geom)) == 0)
+      {
+         geom_marker |= (1 << geom);
+         num_geom++;
+      }
+   }
+   // Initialize 'mesh_part.boundary_map' if needed
+   if (num_geom > 1)
+   {
+      int offsets[Geometry::NumGeom];
+      int offset = 0;
+      for (int g = Geometry::DimStart[dim-1]; g < Geometry::DimStart[dim]; g++)
+      {
+         offsets[g] = offset;
+         offset += mesh_part.entity_to_vertex[g].Size()/Geometry::NumVerts[g];
+      }
+      mesh_part.boundary_map.SetSize(num_bdr_elems);
+      for (int i = 0; i < num_bdr_elems; i++)
+      {
+         const int geom = mesh.GetBdrElementGeometry(bdr_elem_list[i]);
+         mesh_part.boundary_map[i] = offsets[geom]++;
+      }
+   }
+
+   // Create the vertex id map, 'vertex_loc_to_glob', which maps local ids to
+   // global ones; the map is sorted, preserving the global ordering.
+   Array<int> vertex_loc_to_glob;
+   {
+      std::unordered_set<int> vertex_set;
+      for (int i = 0; i < num_elems; i++)
+      {
+         const Element *elem = mesh.GetElement(elem_list[i]);
+         const int geom = elem->GetGeometryType();
+         const int nv = Geometry::NumVerts[geom];
+         const int *v = elem->GetVertices();
+         vertex_set.insert(v, v + nv);
+      }
+      vertex_loc_to_glob.SetSize(vertex_set.size());
+      std::copy(vertex_set.begin(), vertex_set.end(), // src
+                vertex_loc_to_glob.begin());          // dest
+   }
+   vertex_loc_to_glob.Sort();
+
+   // Initialize 'mesh_part.num_vertices'
+   mesh_part.num_vertices = vertex_loc_to_glob.Size();
+
+   // Update the vertex ids in the arrays 'mesh_part.entity_to_vertex' from
+   // global to local.
+   for (int g = 0; g < Geometry::NumGeom; g++)
+   {
+      Array<int> &vert_array = mesh_part.entity_to_vertex[g];
+      for (int i = 0; i < vert_array.Size(); i++)
+      {
+         const int glob_id = vert_array[i];
+         const int loc_id = vertex_loc_to_glob.FindSorted(glob_id);
+         MFEM_ASSERT(loc_id >= 0, "internal error: global vertex id not found");
+         vert_array[i] = loc_id;
+      }
+   }
+
+   // Initialize one of 'mesh_part.vertex_coordinates' or 'mesh_part.nodes'
+   if (!mesh.GetNodes())
+   {
+      MFEM_VERIFY(numeric_limits<int>::max()/sdim >= vertex_loc_to_glob.Size(),
+                  "overflow in 'vertex_coordinates', num_vertices = "
+                  << vertex_loc_to_glob.Size() << ", sdim = " << sdim);
+      mesh_part.vertex_coordinates.SetSize(sdim*vertex_loc_to_glob.Size());
+      for (int i = 0; i < vertex_loc_to_glob.Size(); i++)
+      {
+         const real_t *coord = mesh.GetVertex(vertex_loc_to_glob[i]);
+         for (int d = 0; d < sdim; d++)
+         {
+            mesh_part.vertex_coordinates[i*sdim+d] = coord[d];
+         }
+      }
+   }
+   else
+   {
+      const GridFunction &glob_nodes = *mesh.GetNodes();
+      mesh_part.nodal_fes = ExtractFESpace(mesh_part, *glob_nodes.FESpace());
+      // Initialized 'mesh_part.mesh'.
+      // Note: the nodes of 'mesh_part.mesh' are not set.
+
+      mesh_part.nodes = ExtractGridFunction(mesh_part, glob_nodes,
+                                            *mesh_part.nodal_fes);
+
+      // Attach the 'mesh_part.nodes' to the 'mesh_part.mesh'.
+      mesh_part.mesh->NewNodes(*mesh_part.nodes, /* make_owner: */ false);
+      // Note: the vertices of 'mesh_part.mesh' are not set.
+   }
+
+   // Begin constructing the "neighbor" groups, i.e. the groups that contain
+   // 'part_id'.
+   ListOfIntegerSets groups;
+   {
+      // the first group is the local one
+      IntegerSet group;
+      group.Recreate(1, &part_id);
+      groups.Insert(group);
+   }
+
+   // 'shared_faces' : shared face id -> (global_face_id, group_id)
+   // Note: 'shared_faces' will be sorted by 'global_face_id'.
+   Array<Pair<int,int>> shared_faces;
+
+   // Add "neighbor" groups defined by faces
+   // Construct 'shared_faces'.
+   if (dim >= 3)
+   {
+      std::unordered_set<int> face_set;
+      // Construct 'face_set'
+      const Table &elem_to_face = mesh.ElementToFaceTable();
+      for (int loc_elem_id = 0; loc_elem_id < num_elems; loc_elem_id++)
+      {
+         const int glob_elem_id = elem_list[loc_elem_id];
+         const int nfaces = elem_to_face.RowSize(glob_elem_id);
+         const int *faces = elem_to_face.GetRow(glob_elem_id);
+         face_set.insert(faces, faces + nfaces);
+      }
+      // Construct 'shared_faces'; add "neighbor" groups defined by faces.
+      IntegerSet group;
+      for (int glob_face_id : face_set)
+      {
+         int el[2];
+         mesh.GetFaceElements(glob_face_id, &el[0], &el[1]);
+         if (el[1] < 0) { continue; }
+         el[0] = partitioning[el[0]];
+         el[1] = partitioning[el[1]];
+         MFEM_ASSERT(el[0] == part_id || el[1] == part_id, "internal error");
+         if (el[0] != part_id || el[1] != part_id)
+         {
+            group.Recreate(2, el);
+            const int group_id = groups.Insert(group);
+            shared_faces.Append(Pair<int,int>(glob_face_id, group_id));
+         }
+      }
+      shared_faces.Sort(); // sort the shared faces by 'glob_face_id'
+   }
+
+   // 'shared_edges' : shared edge id -> (global_edge_id, group_id)
+   // Note: 'shared_edges' will be sorted by 'global_edge_id'.
+   Array<Pair<int,int>> shared_edges;
+
+   // Add "neighbor" groups defined by edges.
+   // Construct 'shared_edges'.
+   if (dim >= 2)
+   {
+      std::unordered_set<int> edge_set;
+      // Construct 'edge_set'
+      const Table &elem_to_edge = mesh.ElementToEdgeTable();
+      for (int loc_elem_id = 0; loc_elem_id < num_elems; loc_elem_id++)
+      {
+         const int glob_elem_id = elem_list[loc_elem_id];
+         const int nedges = elem_to_edge.RowSize(glob_elem_id);
+         const int *edges = elem_to_edge.GetRow(glob_elem_id);
+         edge_set.insert(edges, edges + nedges);
+      }
+      // Construct 'shared_edges'; add "neighbor" groups defined by edges.
+      IntegerSet group;
+      for (int glob_edge_id : edge_set)
+      {
+         const int nelem = edge_to_element.RowSize(glob_edge_id);
+         const int *elem = edge_to_element.GetRow(glob_edge_id);
+         Array<int> &gr = group; // reference to the 'group' internal Array
+         gr.SetSize(nelem);
+         for (int j = 0; j < nelem; j++)
+         {
+            gr[j] = partitioning[elem[j]];
+         }
+         gr.Sort();
+         gr.Unique();
+         MFEM_ASSERT(gr.FindSorted(part_id) >= 0, "internal error");
+         if (group.Size() > 1)
+         {
+            const int group_id = groups.Insert(group);
+            shared_edges.Append(Pair<int,int>(glob_edge_id, group_id));
+         }
+      }
+      shared_edges.Sort(); // sort the shared edges by 'glob_edge_id'
+   }
+
+   // 'shared_verts' : shared vertex id -> (global_vertex_id, group_id)
+   // Note: 'shared_verts' will be sorted by 'global_vertex_id'.
+   Array<Pair<int,int>> shared_verts;
+
+   // Add "neighbor" groups defined by vertices.
+   // Construct 'shared_verts'.
+   {
+      IntegerSet group;
+      for (int i = 0; i < vertex_loc_to_glob.Size(); i++)
+      {
+         // 'vertex_to_element' maps global vertex ids to global element ids
+         const int glob_vertex_id = vertex_loc_to_glob[i];
+         const int nelem = vertex_to_element.RowSize(glob_vertex_id);
+         const int *elem = vertex_to_element.GetRow(glob_vertex_id);
+         Array<int> &gr = group; // reference to the 'group' internal Array
+         gr.SetSize(nelem);
+         for (int j = 0; j < nelem; j++)
+         {
+            gr[j] = partitioning[elem[j]];
+         }
+         gr.Sort();
+         gr.Unique();
+         MFEM_ASSERT(gr.FindSorted(part_id) >= 0, "internal error");
+         if (group.Size() > 1)
+         {
+            const int group_id = groups.Insert(group);
+            shared_verts.Append(Pair<int,int>(glob_vertex_id, group_id));
+         }
+      }
+   }
+
+   // Done constructing the "neighbor" groups in 'groups'.
+   const int num_groups = groups.Size();
+
+   // Define 'mesh_part.my_groups'
+   groups.AsTable(mesh_part.my_groups);
+
+   // Construct 'mesh_part.group_shared_entity_to_vertex[Geometry::POINT]'
+   Table &group__shared_vertex_to_vertex =
+      mesh_part.group_shared_entity_to_vertex[Geometry::POINT];
+   group__shared_vertex_to_vertex.MakeI(num_groups);
+   for (int sv = 0; sv < shared_verts.Size(); sv++)
+   {
+      const int group_id = shared_verts[sv].two;
+      group__shared_vertex_to_vertex.AddAColumnInRow(group_id);
+   }
+   group__shared_vertex_to_vertex.MakeJ();
+   for (int sv = 0; sv < shared_verts.Size(); sv++)
+   {
+      const int glob_vertex_id = shared_verts[sv].one;
+      const int group_id       = shared_verts[sv].two;
+      const int loc_vertex_id = vertex_loc_to_glob.FindSorted(glob_vertex_id);
+      MFEM_ASSERT(loc_vertex_id >= 0, "internal error");
+      group__shared_vertex_to_vertex.AddConnection(group_id, loc_vertex_id);
+   }
+   group__shared_vertex_to_vertex.ShiftUpI();
+
+   // Construct 'mesh_part.group_shared_entity_to_vertex[Geometry::SEGMENT]'
+   if (dim >= 2)
+   {
+      Table &group__shared_edge_to_vertex =
+         mesh_part.group_shared_entity_to_vertex[Geometry::SEGMENT];
+      group__shared_edge_to_vertex.MakeI(num_groups);
+      for (int se = 0; se < shared_edges.Size(); se++)
+      {
+         const int group_id = shared_edges[se].two;
+         group__shared_edge_to_vertex.AddColumnsInRow(group_id, 2);
+      }
+      group__shared_edge_to_vertex.MakeJ();
+      const Table &edge_to_vertex = *mesh.GetEdgeVertexTable();
+      for (int se = 0; se < shared_edges.Size(); se++)
+      {
+         const int glob_edge_id = shared_edges[se].one;
+         const int group_id     = shared_edges[se].two;
+         const int *v = edge_to_vertex.GetRow(glob_edge_id);
+         for (int i = 0; i < 2; i++)
+         {
+            const int loc_vertex_id = vertex_loc_to_glob.FindSorted(v[i]);
+            MFEM_ASSERT(loc_vertex_id >= 0, "internal error");
+            group__shared_edge_to_vertex.AddConnection(group_id, loc_vertex_id);
+         }
+      }
+      group__shared_edge_to_vertex.ShiftUpI();
+   }
+
+   // Construct 'mesh_part.group_shared_entity_to_vertex[Geometry::TRIANGLE]'
+   // and 'mesh_part.group_shared_entity_to_vertex[Geometry::SQUARE]'.
+   if (dim >= 3)
+   {
+      Table &group__shared_tria_to_vertex =
+         mesh_part.group_shared_entity_to_vertex[Geometry::TRIANGLE];
+      Table &group__shared_quad_to_vertex =
+         mesh_part.group_shared_entity_to_vertex[Geometry::SQUARE];
+      Array<int> vertex_ids;
+      group__shared_tria_to_vertex.MakeI(num_groups);
+      group__shared_quad_to_vertex.MakeI(num_groups);
+      for (int sf = 0; sf < shared_faces.Size(); sf++)
+      {
+         const int glob_face_id = shared_faces[sf].one;
+         const int group_id     = shared_faces[sf].two;
+         const int geom         = mesh.GetFaceGeometry(glob_face_id);
+         mesh_part.group_shared_entity_to_vertex[geom].
+         AddColumnsInRow(group_id, Geometry::NumVerts[geom]);
+      }
+      group__shared_tria_to_vertex.MakeJ();
+      group__shared_quad_to_vertex.MakeJ();
+      for (int sf = 0; sf < shared_faces.Size(); sf++)
+      {
+         const int glob_face_id = shared_faces[sf].one;
+         const int group_id     = shared_faces[sf].two;
+         const int geom         = mesh.GetFaceGeometry(glob_face_id);
+         mesh.GetFaceVertices(glob_face_id, vertex_ids);
+         // Rotate shared triangles that have an adjacent tetrahedron with a
+         // nonzero refinement flag.
+         // See also ParMesh::BuildSharedFaceElems.
+         if (geom == Geometry::TRIANGLE)
+         {
+            int glob_el_id[2];
+            mesh.GetFaceElements(glob_face_id, &glob_el_id[0], &glob_el_id[1]);
+            int side = 0;
+            const Element *el = mesh.GetElement(glob_el_id[0]);
+            const Tetrahedron *tet = nullptr;
+            if (el->GetGeometryType() == Geometry::TETRAHEDRON)
+            {
+               tet = static_cast<const Tetrahedron*>(el);
+            }
+            else
+            {
+               side = 1;
+               el = mesh.GetElement(glob_el_id[1]);
+               if (el->GetGeometryType() == Geometry::TETRAHEDRON)
+               {
+                  tet = static_cast<const Tetrahedron*>(el);
+               }
+            }
+            if (tet && tet->GetRefinementFlag())
+            {
+               // mark the shared face for refinement by reorienting
+               // it according to the refinement flag in the tetrahedron
+               // to which this shared face belongs to.
+               int info[2];
+               mesh.GetFaceInfos(glob_face_id, &info[0], &info[1]);
+               tet->GetMarkedFace(info[side]/64, &vertex_ids[0]);
+            }
+         }
+         for (int i = 0; i < vertex_ids.Size(); i++)
+         {
+            const int glob_id = vertex_ids[i];
+            const int loc_id = vertex_loc_to_glob.FindSorted(glob_id);
+            MFEM_ASSERT(loc_id >= 0, "internal error");
+            vertex_ids[i] = loc_id;
+         }
+         mesh_part.group_shared_entity_to_vertex[geom].
+         AddConnections(group_id, vertex_ids, vertex_ids.Size());
+      }
+      group__shared_tria_to_vertex.ShiftUpI();
+      group__shared_quad_to_vertex.ShiftUpI();
+   }
+}
+
+std::unique_ptr<FiniteElementSpace>
+MeshPartitioner::ExtractFESpace(MeshPart &mesh_part,
+                                const FiniteElementSpace &global_fespace) const
+{
+   mesh_part.GetMesh(); // initialize 'mesh_part.mesh'
+   // Note: the nodes of 'mesh_part.mesh' are not set by GetMesh() unless they
+   // were already constructed, e.g. by ExtractPart().
+
+   return std::unique_ptr<FiniteElementSpace>(
+             new FiniteElementSpace(mesh_part.mesh.get(),
+                                    global_fespace.FEColl(),
+                                    global_fespace.GetVDim(),
+                                    global_fespace.GetOrdering()));
+}
+
+std::unique_ptr<GridFunction>
+MeshPartitioner::ExtractGridFunction(const MeshPart &mesh_part,
+                                     const GridFunction &global_gf,
+                                     FiniteElementSpace &local_fespace) const
+{
+   std::unique_ptr<GridFunction> local_gf(new GridFunction(&local_fespace));
+
+   // Transfer data from 'global_gf' to 'local_gf'.
+   Array<int> gvdofs, lvdofs;
+   Vector loc_vals;
+   const int part_id = mesh_part.my_part_id;
+   const int num_elems = part_to_element.RowSize(part_id);
+   const int *elem_list = part_to_element.GetRow(part_id); // sorted
+   for (int loc_elem_id = 0; loc_elem_id < num_elems; loc_elem_id++)
+   {
+      const int glob_elem_id = elem_list[loc_elem_id];
+      auto glob_dt = global_gf.FESpace()->GetElementVDofs(glob_elem_id, gvdofs);
+      global_gf.GetSubVector(gvdofs, loc_vals);
+      if (glob_dt) { glob_dt->InvTransformPrimal(loc_vals); }
+      auto local_dt = local_fespace.GetElementVDofs(loc_elem_id, lvdofs);
+      if (local_dt) { local_dt->TransformPrimal(loc_vals); }
+      local_gf->SetSubVector(lvdofs, loc_vals);
+   }
+   return local_gf;
+}
+
+
 GeometricFactors::GeometricFactors(const Mesh *mesh, const IntegrationRule &ir,
                                   int flags, MemoryType d_mt)
 {
@@ -30,6 +30,7 @@
 #include <iostream>
 #include <array>
 #include <map>
+#include <memory>

 namespace mfem
 {
@@ -75,8 +76,10 @@ protected:
       visualization purpose in GLVis. */
   mutable int nbInteriorFaces, nbBoundaryFaces;

-   int meshgen; // see MeshGenerator()
-   int mesh_geoms; // sum of (1 << geom) for all geom of all dimensions
+   // see MeshGenerator(); global in parallel
+   int meshgen;
+   // sum of (1 << geom) for all geom of all dimensions; local in parallel
+   int mesh_geoms;

   // Counter for Mesh transformations: refinement, derefinement, rebalancing.
   // Used for checking during Update operations on objects depending on the
@@ -307,11 +310,11 @@ protected:
   void Destroy();         // Delete all owned data.
   void ResetLazyData();

-   Element *ReadElementWithoutAttr(std::istream &);
-   static void PrintElementWithoutAttr(const Element *, std::ostream &);
+   Element *ReadElementWithoutAttr(std::istream &input);
+   static void PrintElementWithoutAttr(const Element *el, std::ostream &os);

-   Element *ReadElement(std::istream &);
-   static void PrintElement(const Element *, std::ostream &);
+   Element *ReadElement(std::istream &input);
+   static void PrintElement(const Element *el, std::ostream &os);

   // Readers for different mesh formats, used in the Load() method.
   // The implementations of these methods are in mesh_readers.cpp.
@@ -558,7 +561,7 @@ protected:
       mfem v1.2 format with the given section_delimiter at the end.
       If @a comments is non-empty, it will be printed after the first line of
       the file, and each line should begin with '#'. */
-   void Printer(std::ostream &out = mfem::out,
+   void Printer(std::ostream &os = mfem::out,
                std::string section_delimiter = "",
                const std::string &comments = "") const;

@@ -854,7 +857,9 @@ public:
       vectors using Mesh::CreatePeriodicVertexMapping.
       @note MFEM requires that each edge of the resulting mesh be uniquely
       identifiable by a pair of distinct vertices. As a consequence, periodic
-       boundaries must be connected by at least three edges. */
+       boundaries must be separated by at least two interior vertices.
+       @note The resulting mesh uses a discontinuous nodal function, see
+       SetCurvature() for further details. */
   static Mesh MakePeriodic(const Mesh &orig_mesh, const std::vector<int> &v2v);

   ///@}
@@ -2124,7 +2129,11 @@ public:
   /// Set the curvature of the mesh nodes using the given polynomial degree.
   /** Creates a nodal GridFunction if one doesn't already exist.

-       @param[in]  order       Polynomial degree of the nodal FE space.
+       @param[in]  order       Polynomial degree of the nodal FE space. If this
+                               value is <= 0 then the method will remove the
+                               nodal GridFunction and the Mesh will use the
+                               vertices array instead; the other arguments are
+                               ignored in this case.
       @param[in]  discont     Whether to use a discontinuous or continuous
                               finite element space (continuous is default).
       @param[in]  space_dim   The space dimension (optional).
@@ -2330,7 +2339,7 @@ public:
                               std::ostream &os, int elem_attr = 0) const;

   void PrintElementsWithPartitioning (int *partitioning,
-                                       std::ostream &out,
+                                       std::ostream &os,
                                       int interior_faces = 0);

   /// Print set of disjoint surfaces:
@@ -2338,13 +2347,13 @@ public:
    * If Aface_face(i,j) != 0, print face j as a boundary
    * element with attribute i+1.
    */
-   void PrintSurfaces(const Table &Aface_face, std::ostream &out) const;
+   void PrintSurfaces(const Table &Aface_face, std::ostream &os) const;

   /// Auxiliary method used by PrintCharacteristics().
   /** It is also used in the `mesh-explorer` miniapp. */
   static void PrintElementsByGeometry(int dim,
                                       const Array<int> &num_elems_by_geom,
-                                       std::ostream &out);
+                                       std::ostream &os);

   /** @brief Compute and print mesh characteristics such as number of vertices,
       number of elements, number of boundary elements, minimal and maximal
@@ -2364,7 +2373,7 @@ public:

 #ifdef MFEM_DEBUG
   /// Output an NCMesh-compatible debug dump.
-   void DebugDump(std::ostream &out) const;
+   void DebugDump(std::ostream &os) const;
 #endif

   /// @}
@@ -2445,7 +2454,334 @@ public:

 /** Overload operator<< for std::ostream and Mesh; valid also for the derived
    class ParMesh */
-std::ostream &operator<<(std::ostream &out, const Mesh &mesh);
+std::ostream &operator<<(std::ostream &os, const Mesh &mesh);
+
+/// @brief Print function for Mesh::FaceInformation.
+std::ostream& operator<<(std::ostream &os, const Mesh::FaceInformation& info);
+
+
+/** @brief Class containing a minimal description of a part (a subset of the
+    elements) of a Mesh and its connectivity to other parts.
+
+    The main purpose of this class is to facilitate the partitioning of serial
+    meshes (in serial, i.e. on one processor) and save the parts in parallel
+    MFEM mesh format.
+
+    Another potential futrure purpose of this class could be to facilitate
+    exchange of MeshParts between MPI ranks for repartitioning purposes. It can
+    also potentially be used to implement parallel mesh I/O functions with
+    partitionings that have number of parts different from the number of MPI
+    tasks.
+
+    @note Parts of NURBS or non-conforming meshes cannot be fully described by
+    this class alone with its current data members. Such extensions may be added
+    in the future.
+*/
+class MeshPart
+{
+protected:
+   struct Entity { int geom; int num_verts; const int *verts; };
+   struct EntityHelper
+   {
+      int dim, num_entities;
+      int geom_offsets[Geometry::NumGeom+1];
+      typedef const Array<int> entity_to_vertex_type[Geometry::NumGeom];
+      entity_to_vertex_type &entity_to_vertex;
+
+      EntityHelper(int dim_,
+                   const Array<int> (&entity_to_vertex_)[Geometry::NumGeom]);
+      Entity FindEntity(int bytype_entity_id);
+   };
+
+public:
+   /// Reference space dimension of the elements
+   int dimension;
+
+   /// Dimension of the physical space into which the MeshPart is embedded.
+   int space_dimension;
+
+   /// Number of vertices
+   int num_vertices;
+
+   /// Number of elements with reference space dimension equal to 'dimension'.
+   int num_elements;
+
+   /** @brief Number of boundary elements with reference space dimension equal
+       to 'dimension'-1. */
+   int num_bdr_elements;
+
+   /**
+      Each 'entity_to_vertex[geom]' describes the entities of Geometry::Type
+      'geom' in terms of their vertices. The number of entities of type 'geom'
+      is:
+
+          num_entities[geom] = size('entity_to_vertex[geom]')/num_vertices[geom]
+
+      The number of all elements, 'num_elements', is:
+
+          'num_elements' = sum_{dim[geom]=='dimension'} num_entities[geom]
+
+      and the number of all boundary elements, 'num_bdr_elements' is:
+
+          'num_bdr_elements' = sum_{dim[geom]=='dimension'-1} num_entities[geom]
+
+      Note that 'entity_to_vertex' does NOT describe all "faces" in the mesh
+      part (i.e. all 'dimension'-1 entities) but only the boundary elements.
+      Also, note that lower dimesional entities ('dimension'-2 and lower) are
+      NOT described by the respective array, i.e. the array will be empty.
+   */
+   Array<int> entity_to_vertex[Geometry::NumGeom];
+
+   /** @brief Store the refinement flags for tetraheral elements. If all tets
+       have zero refinement flags then this array is empty, i.e. has size 0. */
+   Array<int> tet_refine_flags;
+
+   /**
+      Terminology: "by-type" element/boundary ordering: ordered by
+      Geometry::Type and within each Geometry::Type 'geom' ordered as in
+      'entity_to_vertex[geom]'.
+
+      Optional re-ordering of the elements that will be used by (Par)Mesh
+      objects constructed from this MeshPart. This array maps "natural" element
+      ids (used by the Mesh/ParMesh objects) to "by-type" element ids (see
+      above):
+
+          "by-type" element id = element_map["natural" element id]
+
+      The size of the array is either 'num_elements' or 0 when no re-ordering is
+      needed (then "by-type" id == "natural" id).
+   */
+   Array<int> element_map;
+
+   /// Optional re-ordering for the boundary elements, similar to 'element_map'.
+   Array<int> boundary_map;
+
+   /**
+      Element attributes. Ordered using the "natural" element ordering defined
+      by the array 'element_map'. The size of this array is 'num_elements'.
+   */
+   Array<int> attributes;
+
+   /**
+      Boundary element attributes. Ordered using the "natural" boundary element
+      ordering defined by the array 'boundary_map'. The size of this array is
+      'num_bdr_elements'.
+   */
+   Array<int> bdr_attributes;
+
+   /**
+      Optional vertex coordinates. The size of the array is either
+
+          size = 'space_dimension' * 'num_vertices'
+
+      or 0 when the vertex coordinates are not used, i.e. when the MeshPart uses
+      a nodal GridFunction to describe its location in physical space. This
+      array uses Ordering::byVDIM: "X0,Y0,Z0, X1,Y1,Z1, ...".
+   */
+   Array<real_t> vertex_coordinates;
+
+   /**
+      Optional serial Mesh object constructed on demand using the method
+      GetMesh(). One use case for it is when one wants to construct FE spaces
+      and GridFunction%s on the MeshPart for saving or MPI communication.
+   */
+   std::unique_ptr<Mesh> mesh;
+
+   /**
+      Nodal FE space defined on 'mesh' used by the GridFunction 'nodes'. Uses
+      the FE collection from the global nodal FE space.
+   */
+   std::unique_ptr<FiniteElementSpace> nodal_fes;
+
+   /**
+      'nodes': pointer to a GridFunction describing the physical location of the
+      MeshPart. Used for describing high-order and periodic meshes. This
+      GridFunction is defined on the FE space 'nodal_fes' which, in turn, is
+      defined on the Mesh 'mesh'.
+   */
+   std::unique_ptr<GridFunction> nodes;
+
+   /** @name Connectivity to other MeshPart objects */
+   ///@{
+
+   /// Total number of MeshParts
+   int num_parts;
+
+   /** @brief Index of the part described by this MeshPart:
+       0 <= 'my_part_id' < 'num_parts' */
+   int my_part_id;
+
+   /**
+      A group G is a subset of the set { 0, 1, ..., 'num_parts'-1 } for which
+      there is a mesh entity E (of any dimension) in the global mesh such that
+      G is the set of the parts assigned (by the partitioning array) to the
+      elements adjacent to E. The MeshPart describes only the "neighbor" groups,
+      i.e. the groups that contain 'my_part_id'. The Table 'my_groups' defines
+      the "neighbor" groups in terms of their part ids. In other words, it maps
+      "neighbor" group ids to a (sorted) list of part ids. In particular, the
+      number of "neighbor" groups is given by 'my_groups.Size()'. The "local"
+      group { 'my_part_id' } has index 0 in 'my_groups'.
+   */
+   Table my_groups;
+
+   /**
+      Shared entities for this MeshPart are mesh entities of all dimensions less
+      than 'dimension' that are generated by the elements of this MeshPart and
+      at least one other MeshPart.
+
+      The Table 'group_shared_entity_to_vertex[geom]' defines, for each group,
+      the shared entities of Geometry::Type 'geom'. Each row (corresponding to a
+      "neighbor" group, as defined by 'my_groups') in the Table defines the
+      shared entities in a way similar to the arrays 'entity_to_vertex[geom]'.
+      The "local" group (with index 0) does not have any shared entities, so the
+      0-th row in the Table is always empty.
+
+      IMPORTANT: the descriptions of the groups in this MeshPart must match
+      their descriptions in all neighboring MeshParts. This includes the
+      ordering of the shared entities within the group, as well as the vertex
+      ordering of each shared entity.
+   */
+   Table group_shared_entity_to_vertex[Geometry::NumGeom];
+
+   ///@}
+
+   /** @brief Write the MeshPart to a stream using the parallel format
+       "MFEM mesh v1.2". */
+   void Print(std::ostream &os) const;
+
+   /** @brief Construct a serial Mesh object from the MeshPart.
+
+       The nodes of 'mesh' are NOT initialized by this method, however, the
+       nodal FE space and nodal GridFunction can be created and then attached to
+       the 'mesh'. The Mesh is constructed only if 'mesh' is empty, otherwise
+       the method simply returns the object held by 'mesh'.
+   */
+   Mesh &GetMesh();
+};
+
+
+/** @brief Class that allows serial meshes to be partitioned into MeshPart
+    objects, typically one MeshPart at a time, which can then be used to write
+    the local mesh in parallel MFEM mesh format.
+
+    Sample usage of this class: partition a serial mesh and save it in parallel
+    MFEM format:
+    \code
+       // The array 'partitioning' can be obtained e.g. from
+       // mesh->GeneratePartitioning():
+       void usage1(Mesh *mesh, int num_parts, int *partitioning)
+       {
+          MeshPartitioner partitioner(*mesh, num_parts, partitioning);
+          MeshPart mesh_part;
+          for (int i = 0; i < num_parts; i++)
+          {
+             partitioner.ExtractPart(i, mesh_part);
+             ofstream omesh(MakeParFilename("my-mesh.", i));
+             mesh_part.Print(omesh);
+          }
+       }
+    \endcode
+
+    This class can also be used to partition a mesh and GridFunction(s) and save
+    them in parallel:
+    \code
+       // The array 'partitioning' can be obtained e.g. from
+       // mesh->GeneratePartitioning():
+       void usage2(Mesh *mesh, int num_parts, int *partitioning,
+                   GridFunction *gf)
+       {
+          MeshPartitioner partitioner(*mesh, num_parts, partitioning);
+          MeshPart mesh_part;
+          for (int i = 0; i < num_parts; i++)
+          {
+             partitioner.ExtractPart(i, mesh_part);
+             ofstream omesh(MakeParFilename("my-mesh.", i));
+             mesh_part.Print(omesh);
+             auto lfes = partitioner.ExtractFESpace(mesh_part, *gf->FESpace());
+             auto lgf = partitioner.ExtractGridFunction(mesh_part, *gf, *lfes);
+             ofstream ofield(MakeParFilename("my-field.", i));
+             lgf->Save(ofield);
+          }
+       }
+    \endcode
+*/
+class MeshPartitioner
+{
+protected:
+   Mesh &mesh;
+   Array<int> partitioning;
+   Table part_to_element;
+   Table part_to_boundary;
+   Table edge_to_element;
+   Table vertex_to_element;
+
+public:
+   /** @brief Construct a MeshPartitioner.
+
+       @param[in] mesh_         Mesh to be partitioned into MeshPart%s.
+       @param[in] num_parts_    Number of parts to partition the mesh into.
+       @param[in] partitioning_ Partitioning array: for every element in the
+                                mesh gives the partition it belongs to; if NULL,
+                                partitioning will be generated internally by
+                                calling Mesh::GeneratePartitioning().
+       @param[in] part_method   Partitioning method to be used in the call to
+                                Mesh::GeneratePartitioning() when the provided
+                                input partitioning is NULL.
+   */
+   MeshPartitioner(Mesh &mesh_, int num_parts_, int *partitioning_ = NULL,
+                   int part_method = 1);
+
+   /** @brief Construct a MeshPart corresponding to the given @a part_id.
+
+       @param[in]  part_id    Partition index to extract; valid values are in
+                              the range [0, num_parts).
+       @param[out] mesh_part  Output MeshPart object; its contents is
+                              overwritten, while potentially reusing existing
+                              dynamic memory allocations.
+   */
+   void ExtractPart(int part_id, MeshPart &mesh_part) const;
+
+   /** @brief Construct a local version of the given FiniteElementSpace
+       @a global_fespace corresponding to the given @a mesh_part.
+
+       @param[in,out] mesh_part       MeshPart on which to construct the local
+                                      FiniteElementSpace; this object is
+                                      generally modified by this call since it
+                                      calls mesh_part.GetMesh() to ensure the
+                                      local mesh is constructed.
+       @param[in]     global_fespace  The global FiniteElementSpace that should
+                                      be restricted to the @a mesh_part.
+
+       @returns A FiniteElementSpace pointer stored in a unique_ptr. The
+                returned local FiniteElementSpace is built on the Mesh object
+                contained in @a mesh_part (MeshPart::mesh) and it reuses the
+                FiniteElementCollection of the @a global_fespace.
+   */
+   std::unique_ptr<FiniteElementSpace>
+   ExtractFESpace(MeshPart &mesh_part,
+                  const FiniteElementSpace &global_fespace) const;
+
+   /** @brief Construct a local version of the given GridFunction, @a global_gf,
+       corresponding to the given @a mesh_part. The respective data is copied
+       from @a global_gf to the returned local GridFunction.
+
+       @param[in]      mesh_part      MeshPart on which to construct the local
+                                      GridFunction.
+       @param[in]      global_gf      The global GridFunction that should be
+                                      restricted to the @a mesh_part.
+       @param[in,out]  local_fespace  The local FiniteElementSpace corresponding
+                                      to @a mesh_part, e.g. constructed by the
+                                      method ExtractFESpace().
+
+       @returns A GridFunction pointer stored in a unique_ptr. The returned
+                local GridFunction is initialized with data appropriately copied
+                from @a global_gf.
+   */
+   std::unique_ptr<GridFunction>
+   ExtractGridFunction(const MeshPart &mesh_part,
+                       const GridFunction &global_gf,
+                       FiniteElementSpace &local_fespace) const;
+};


 /** @brief Structure for storing mesh geometric factors: coordinates, Jacobians,
@@ -2454,7 +2790,6 @@ std::ostream &operator<<(std::ostream &out, const Mesh &mesh);
    Mesh. See Mesh::GetGeometricFactors(). */
 class GeometricFactors
 {
-
 private:
   void Compute(const GridFunction &nodes,
                MemoryType d_mt = MemoryType::DEFAULT);
@@ -2502,6 +2837,7 @@ public:
   Vector detJ;
 };

+
 /** @brief Structure for storing face geometric factors: coordinates, Jacobians,
    determinants of the Jacobians, and normal vectors. */
 /** Typically objects of this type are constructed and owned by objects of class
@@ -2556,6 +2892,7 @@ public:
   Vector normal;
 };

+
 /// Class used to extrude the nodes of a mesh
 class NodeExtrudeCoefficient : public VectorCoefficient
 {
@@ -2587,9 +2924,6 @@ inline void ShiftRight(int &a, int &b, int &c)
   a = c;  c = b;  b = t;
 }

-/// @brief Print function for Mesh::FaceInformation.
-std::ostream& operator<<(std::ostream& os, const Mesh::FaceInformation& info);
-
 }

 #endif
@@ -1151,15 +1151,24 @@ void Mesh::ReadXML_VTKMesh(std::istream &input, int &curved, int &read_gf,
   }
   if (cells_xml == NULL) { MFEM_ABORT(erstr); }

-   // Read the element attributes, which are stored as CellData named "material"
+   // Read the element attributes, which are stored as CellData named either
+   // "material" or "attribute". We prioritize "material" over "attribute" for
+   // backwards compatibility.
   Array<int> cell_attributes;
+   bool found_attributes = false;
   for (const XMLElement *cell_data_xml = piece->FirstChildElement();
        cell_data_xml != NULL;
        cell_data_xml = cell_data_xml->NextSiblingElement())
   {
-      if (StringCompare(cell_data_xml->Name(), "CellData")
-          && StringCompare(cell_data_xml->Attribute("Scalars"), "material"))
+      const bool is_cell_data =
+         StringCompare(cell_data_xml->Name(), "CellData");
+      const bool is_material =
+         StringCompare(cell_data_xml->Attribute("Scalars"), "material");
+      const bool is_attribute =
+         StringCompare(cell_data_xml->Attribute("Scalars"), "attribute");
+      if (is_cell_data && (is_material || (is_attribute && !found_attributes)))
      {
+         found_attributes = true;
         const XMLElement *data_xml = cell_data_xml->FirstChildElement();
         if (data_xml != NULL && StringCompare(data_xml->Name(), "DataArray"))
         {
@@ -1274,6 +1283,7 @@ void Mesh::ReadVTKMesh(std::istream &input, int &curved, int &read_gf,
   // Read the cell materials
   // bool found_material = false;
   Array<int> cell_attributes;
+   bool found_attributes = false;
   while ((input.good()))
   {
      getline(input, buff);
@@ -1281,8 +1291,10 @@ void Mesh::ReadVTKMesh(std::istream &input, int &curved, int &read_gf,
      {
         break; // We have entered the POINT_DATA block. Quit.
      }
-      else if (buff.rfind("SCALARS material") == 0)
+      else if (buff.rfind("SCALARS material") == 0 ||
+               (buff.rfind("SCALARS attribute") == 0 && !found_attributes))
      {
+         found_attributes = true;
         getline(input, buff); // LOOKUP_TABLE default
         if (buff.rfind("LOOKUP_TABLE default") != 0)
         {
@@ -35,9 +35,9 @@ KnotVector::KnotVector(istream &input)
   GetElements();
 }

-KnotVector::KnotVector(int Order_, int NCP)
+KnotVector::KnotVector(int order, int NCP)
 {
-   Order = Order_;
+   Order = order;
   NumOfControlPoints = NCP;
   knot.SetSize(NumOfControlPoints + Order + 1);
   NumOfElements = 0;
@@ -276,18 +276,18 @@ void KnotVector::PrintFunctions(std::ostream &os, int samples) const

      for (int j = 0; j <samples; j++)
      {
-         x =j*dx;
-         os<< x + e;
+         x = j*dx;
+         os << x + e;

-         CalcShape ( shape, cnt, x);
+         CalcShape(shape, cnt, x);
         for (int d = 0; d < Order+1; d++) { os<<"\t"<<shape[d]; }

-         CalcDShape ( shape, cnt, x);
+         CalcDShape(shape, cnt, x);
         for (int d = 0; d < Order+1; d++) { os<<"\t"<<shape[d]; }

-         CalcD2Shape ( shape, cnt, x);
+         CalcD2Shape(shape, cnt, x);
         for (int d = 0; d < Order+1; d++) { os<<"\t"<<shape[d]; }
-         os<<endl;
+         os << endl;
      }
   }
 }
@@ -378,6 +378,7 @@ void KnotVector::CalcDShape(Vector &grad, int i, real_t xi) const
 }

 // Routine from "The NURBS book" - 2nd ed - Piegl and Tiller
+// Algorithm A2.3 p. 72
 void KnotVector::CalcDnShape(Vector &gradn, int n, int i, real_t xi) const
 {
   int    p = Order, rk, pk, j1, j2,r,j,k;
@@ -522,7 +523,7 @@ void KnotVector::FindMaxima(Array<int> &ks, Vector &xi, Vector &u) const
               }

               arg = (arg1 + arg2)/2;
-               CalcShape ( shape, i, arg);
+               CalcShape(shape, i, arg);
               max = shape[d];
            }

@@ -556,7 +557,7 @@ void KnotVector::FindInterpolant(Array<Vector*> &x)
   A = 0.0;
   for (int i = 0; i < ncp; i++)
   {
-      CalcShape ( shape, i_args[i], xi_args[i]);
+      CalcShape(shape, i_args[i], xi_args[i]);
      for (int p = 0; p < order+1; p++)
      {
         A(i,i_args[i] + p) =  shape[p];
@@ -637,9 +638,9 @@ void KnotVector::Difference(const KnotVector &kv, Vector &diff) const
   }
 }

-void NURBSPatch::init(int dim_)
+void NURBSPatch::init(int dim)
 {
-   Dim = dim_;
+   Dim = dim;
   sd = nd = -1;

   if (kv.Size() == 1)
@@ -751,32 +752,32 @@ NURBSPatch::NURBSPatch(std::istream &input)
   }
 }

-NURBSPatch::NURBSPatch(const KnotVector *kv0, const KnotVector *kv1, int dim_)
+NURBSPatch::NURBSPatch(const KnotVector *kv0, const KnotVector *kv1, int dim)
 {
   kv.SetSize(2);
   kv[0] = new KnotVector(*kv0);
   kv[1] = new KnotVector(*kv1);
-   init(dim_);
+   init(dim);
 }

 NURBSPatch::NURBSPatch(const KnotVector *kv0, const KnotVector *kv1,
-                       const KnotVector *kv2, int dim_)
+                       const KnotVector *kv2, int dim)
 {
   kv.SetSize(3);
   kv[0] = new KnotVector(*kv0);
   kv[1] = new KnotVector(*kv1);
   kv[2] = new KnotVector(*kv2);
-   init(dim_);
+   init(dim);
 }

-NURBSPatch::NURBSPatch(Array<const KnotVector *> &kv_,  int dim_)
+NURBSPatch::NURBSPatch(Array<const KnotVector *> &kvs, int dim)
 {
-   kv.SetSize(kv_.Size());
+   kv.SetSize(kvs.Size());
   for (int i = 0; i < kv.Size(); i++)
   {
-      kv[i] = new KnotVector(*kv_[i]);
+      kv[i] = new KnotVector(*kvs[i]);
   }
-   init(dim_);
+   init(dim);
 }

 NURBSPatch::NURBSPatch(NURBSPatch *parent, int dir, int Order, int NCP)
@@ -859,7 +860,7 @@ void NURBSPatch::Print(std::ostream &os) const

 int NURBSPatch::SetLoopDirection(int dir)
 {
-   if (nj == -1)
+   if (nj == -1)  // 1D case
   {
      if (dir == 0)
      {
@@ -875,7 +876,7 @@ int NURBSPatch::SetLoopDirection(int dir)
         mfem_error();
      }
   }
-   else if (nk == -1)
+   else if (nk == -1)  // 2D case
   {
      if (dir == 0)
      {
@@ -898,7 +899,7 @@ int NURBSPatch::SetLoopDirection(int dir)
         mfem_error();
      }
   }
-   else
+   else  // 3D case
   {
      if (dir == 0)
      {
@@ -992,6 +993,7 @@ void NURBSPatch::GetCoarseningFactors(Array<int> & f) const

 void NURBSPatch::KnotInsert(Array<KnotVector *> &newkv)
 {
+   MFEM_ASSERT(newkv.Size() == kv.Size(), "Invalid input to KnotInsert");
   for (int dir = 0; dir < kv.Size(); dir++)
   {
      KnotInsert(dir, *newkv[dir]);
@@ -1026,6 +1028,7 @@ void NURBSPatch::KnotInsert(int dir, const KnotVector &newkv)

 void NURBSPatch::KnotInsert(Array<Vector *> &newkv)
 {
+   MFEM_ASSERT(newkv.Size() == kv.Size(), "Invalid input to KnotInsert");
   for (int dir = 0; dir < kv.Size(); dir++)
   {
      KnotInsert(dir, *newkv[dir]);
@@ -1143,7 +1146,7 @@ void NURBSPatch::KnotInsert(int dir, const Vector &knot)
            for (int ll = 0; ll < size; ll++)
            {
               newp.slice(ind-1,ll) = alfa*newp.slice(ind-1,ll) +
-                                      (1.0-alfa)*newp.slice(ind, ll);
+                                      (1.0-alfa)*newp.slice(ind,ll);
            }
         }
      }
@@ -1372,6 +1375,8 @@ void NURBSPatch::DegreeElevate(int dir, int t)
      mfem_error("NURBSPatch::DegreeElevate : Incorrect direction!");
   }

+   MFEM_ASSERT(t >= 0, "DegreeElevate cannot decrease the degree.");
+
   int i, j, k, kj, mpi, mul, mh, kind, cind, first, last;
   int r, a, b, oldr, save, s, tr, lbz, rbz, l;
   real_t inv, ua, ub, numer, alf, den, bet, gam;
@@ -1715,8 +1720,10 @@ void NURBSPatch::Get3DRotationMatrix(real_t n[], real_t angle, real_t r,
                                     DenseMatrix &T)
 {
   real_t c, s, c1;
-   real_t l2 = n[0]*n[0] + n[1]*n[1] + n[2]*n[2];
-   real_t l = sqrt(l2);
+   const real_t l2 = n[0]*n[0] + n[1]*n[1] + n[2]*n[2];
+   const real_t l = sqrt(l2);
+
+   MFEM_ASSERT(l2 > 0.0, "3D rotation axis is undefined");

   if (fabs(angle) == (real_t)(M_PI_2))
   {
@@ -1973,7 +1980,7 @@ NURBSExtension::NURBSExtension(std::istream &input, bool spacing)
   // Read topology
   patchTopo = new Mesh;
   patchTopo->LoadPatchTopo(input, edge_to_knot);
-   own_topo = 1;
+   own_topo = true;

   CheckPatches();
   // CheckBdrPatches();
@@ -2172,7 +2179,7 @@ NURBSExtension::NURBSExtension(std::istream &input, bool spacing)
 NURBSExtension::NURBSExtension(NURBSExtension *parent, int newOrder)
 {
   patchTopo = parent->patchTopo;
-   own_topo = 0;
+   own_topo = false;

   parent->edge_to_knot.Copy(edge_to_knot);

@@ -2229,7 +2236,7 @@ NURBSExtension::NURBSExtension(NURBSExtension *parent,
   SetOrderFromOrders();

   patchTopo = parent->patchTopo;
-   own_topo = 0;
+   own_topo = false;

   parent->edge_to_knot.Copy(edge_to_knot);

@@ -2287,8 +2294,8 @@ NURBSExtension::NURBSExtension(Mesh *mesh_array[], int num_pieces)
                 "  parent does not own the patch topology!");
   }
   patchTopo = parent->patchTopo;
-   own_topo = 1;
-   parent->own_topo = 0;
+   own_topo = true;
+   parent->own_topo = false;

   parent->edge_to_knot.Copy(edge_to_knot);

@@ -2442,7 +2449,7 @@ void NURBSExtension::PrintFunctions(const char *basename, int samples) const
   for (int i = 0; i < NumOfKnotVectors; i++)
   {
      std::ostringstream filename;
-      filename << basename<<"_"<<i<<".dat";
+      filename << basename << "_" << i << ".dat";
      os.open(filename.str().c_str());
      knotVectors[i]->PrintFunctions(os,samples);
      os.close();
@@ -2889,7 +2896,6 @@ void NURBSExtension::CheckKVDirection(int p, Array <int> &kvdir)
   // -1: direction is flipped
   //  1: direction is not flipped

-
   for (int i = 0; i < edges.Size(); i++)
   {
      // First side
@@ -3069,7 +3075,7 @@ void NURBSExtension::UpdateUniqueKV()
 bool NURBSExtension::ConsistentKVSets()
 {
   // patchTopo->GetElementEdges is not yet implemented for 1D
-   MFEM_VERIFY(Dimension()>1, "1D not yet implemented.");
+   MFEM_VERIFY(Dimension() > 1, "1D not yet implemented.");

   Array<int> edges, orient, kvdir;
   Vector diff;
@@ -3180,7 +3186,7 @@ const
   }
 }

-void NURBSExtension::GetBdrPatchKnotVectors(int p, Array<KnotVector *> &kv)
+void NURBSExtension::GetBdrPatchKnotVectors(int bp, Array<KnotVector *> &kv)
 {
   Array<int> edges;
   Array<int> orient;
@@ -3189,19 +3195,19 @@ void NURBSExtension::GetBdrPatchKnotVectors(int p, Array<KnotVector *> &kv)

   if (Dimension() == 2)
   {
-      patchTopo->GetBdrElementEdges(p, edges, orient);
+      patchTopo->GetBdrElementEdges(bp, edges, orient);
      kv[0] = KnotVec(edges[0]);
   }
   else if (Dimension() == 3)
   {
-      patchTopo->GetBdrElementEdges(p, edges, orient);
+      patchTopo->GetBdrElementEdges(bp, edges, orient);
      kv[0] = KnotVec(edges[0]);
      kv[1] = KnotVec(edges[1]);
   }
 }

 void NURBSExtension::GetBdrPatchKnotVectors(
-   int p, Array<const KnotVector *> &kv) const
+   int bp, Array<const KnotVector *> &kv) const
 {
   Array<int> edges;
   Array<int> orient;
@@ -3210,12 +3216,12 @@ void NURBSExtension::GetBdrPatchKnotVectors(

   if (Dimension() == 2)
   {
-      patchTopo->GetBdrElementEdges(p, edges, orient);
+      patchTopo->GetBdrElementEdges(bp, edges, orient);
      kv[0] = KnotVec(edges[0]);
   }
   else if (Dimension() == 3)
   {
-      patchTopo->GetBdrElementEdges(p, edges, orient);
+      patchTopo->GetBdrElementEdges(bp, edges, orient);
      kv[0] = KnotVec(edges[0]);
      kv[1] = KnotVec(edges[1]);
   }
@@ -3305,8 +3311,6 @@ void NURBSExtension::GenerateOffsets()
      p_meshOffsets[p]  = meshCounter;
      p_spaceOffsets[p] = spaceCounter;

-
-
      if (dim == 1)
      {
         meshCounter  += KnotVec(0)->GetNE() - 1;
@@ -4748,8 +4752,8 @@ ParNURBSExtension::ParNURBSExtension(MPI_Comm comm, NURBSExtension *parent,
      mfem_error("ParNURBSExtension::ParNURBSExtension :\n"
                 "  parent does not own the patch topology!");
   }
-   own_topo = 1;
-   parent->own_topo = 0;
+   own_topo = true;
+   parent->own_topo = false;

   parent->edge_to_knot.Copy(edge_to_knot);

@@ -4812,7 +4816,7 @@ ParNURBSExtension::ParNURBSExtension(NURBSExtension *parent,

   patchTopo = parent->patchTopo;
   own_topo = parent->own_topo;
-   parent->own_topo = 0;
+   parent->own_topo = false;

   Swap(edge_to_knot, parent->edge_to_knot);

@@ -5057,7 +5061,7 @@ Table *ParNURBSExtension::Get3DGlobalElementDofTable()
   return (new Table(GetGNE(), gel_dof_list));
 }

-void ParNURBSExtension::SetActive(const int *partitioning_,
+void ParNURBSExtension::SetActive(const int *partition,
                                  const Array<bool> &active_bel)
 {
   activeElem.SetSize(GetGNE());
@@ -5065,7 +5069,7 @@ void ParNURBSExtension::SetActive(const int *partitioning_,
   NumOfActiveElems = 0;
   const int MyRank = gtopo.MyRank();
   for (int i = 0; i < GetGNE(); i++)
-      if (partitioning_[i] == MyRank)
+      if (partition[i] == MyRank)
      {
         activeElem[i] = true;
         NumOfActiveElems++;
@@ -5080,7 +5084,7 @@ void ParNURBSExtension::SetActive(const int *partitioning_,
      }
 }

-void ParNURBSExtension::BuildGroups(const int *partitioning_,
+void ParNURBSExtension::BuildGroups(const int *partition,
                                    const Table &elem_dof)
 {
   Table dof_proc;
@@ -5092,7 +5096,7 @@ void ParNURBSExtension::BuildGroups(const int *partitioning_,
   // convert elements to processors
   for (int i = 0; i < dof_proc.Size_of_connections(); i++)
   {
-      dof_proc.GetJ()[i] = partitioning_[dof_proc.GetJ()[i]];
+      dof_proc.GetJ()[i] = partition[dof_proc.GetJ()[i]];
   }

   // the first group is the local one
@@ -256,9 +256,6 @@ ParMesh::ParMesh(MPI_Comm comm, Mesh &mesh, int *partitioning_,
      // build svert_lvert mapping
      BuildSharedVertMapping(nsvert, vert_element, vert_global_local);
      delete vert_element;
-
-      SetMeshGen();
-      meshgen = mesh.meshgen; // copy the global 'meshgen'
   }

   if (mesh.NURBSext)
@@ -1527,6 +1524,7 @@ ParMesh ParMesh::MakeSimplicial(ParMesh &orig_mesh)
 void ParMesh::Finalize(bool refine, bool fix_orientation)
 {
   const int meshgen_save = meshgen; // Mesh::Finalize() may call SetMeshGen()
+   // 'mesh_geoms' is local, so there's no need to save and restore it.

   Mesh::Finalize(refine, fix_orientation);

@@ -4807,7 +4805,7 @@ void ParMesh::Print(std::ostream &os, const std::string &comments) const

   if (NURBSext)
   {
-      Printer(os, comments); // does not print shared boundary
+      Printer(os, "", comments); // does not print shared boundary
      return;
   }

@@ -4935,7 +4933,7 @@ void ParMesh::Print(std::ostream &os, const std::string &comments) const

   if (set_names)
   {
-      os << "mfem_mesh_end\n";
+      os << "\nmfem_mesh_end" << endl;
   }
 }

@@ -5286,7 +5284,7 @@ void ParMesh::PrintAsSerial(std::ostream &os, const std::string &comments) const
   Mesh serialmesh = GetSerialMesh(save_rank);
   if (MyRank == save_rank)
   {
-      serialmesh.Printer(os, comments);
+      serialmesh.Printer(os, "", comments);
   }
   MPI_Barrier(MyComm);
 }
@@ -6325,11 +6323,11 @@ void ParMesh::ParPrint(ostream &os, const std::string &comments) const
   if (Nonconforming())
   {
      // the NC mesh format works both in serial and in parallel
-      Printer(os, comments);
+      Printer(os, "", comments);
      return;
   }

-   // Write out serial mesh.  Tell serial mesh to deliniate the end of it's
+   // Write out serial mesh.  Tell serial mesh to delineate the end of its
   // output with 'mfem_serial_mesh_end' instead of 'mfem_mesh_end', as we will
   // be adding additional parallel mesh information.
   Printer(os, "mfem_serial_mesh_end", comments);
@@ -6346,6 +6344,7 @@ void ParMesh::ParPrint(ostream &os, const std::string &comments) const
   {
      os << "total_shared_faces " << sface_lface.Size() << '\n';
   }
+   os << "\n# group 0 has no shared entities\n";
   for (int gr = 1; gr < GetNGroups(); gr++)
   {
      {
@@ -819,7 +819,7 @@ ParPumiMesh::ParPumiMesh(MPI_Comm comm, apf::Mesh2* apf_mesh,
         apf::Downward verts;
         apf_mesh->getDownward(ent,0,verts);

-         int *v, nv = 0;
+         int *v = nullptr, nv = 0;
         apf::Mesh::Type ftype = apf_mesh->getType(ent);
         if (ftype == apf::Mesh::TRIANGLE)
         {
@@ -890,9 +890,9 @@ GridFunctionPumi::GridFunctionPumi(Mesh* m, apf::Mesh2* PumiM,
 {
   int spDim = m->SpaceDimension();
   // Note: default BasisType for 'fec' is GaussLobatto.
-   fec = new H1_FECollection(mesh_order, m->Dimension());
+   fec_owned = new H1_FECollection(mesh_order, m->Dimension());
   int ordering = Ordering::byVDIM; // x1y1z1/x2y2z2/...
-   fes = new FiniteElementSpace(m, fec, spDim, ordering);
+   fes = new FiniteElementSpace(m, fec_owned, spDim, ordering);
   int data_size = fes->GetVSize();

   // Read PUMI mesh data
@@ -401,6 +401,7 @@ ParSubMesh::ParSubMesh(const ParMesh &parent, SubMesh::From from,

      // Copy boundary attribute numbers into local portion of a parallel
      // grid function
+      parent_bdr_attr_gf.HostReadWrite(); // not modifying all entries
      for (int i=0; i<parent.GetNBE(); i++)
      {
         faceIdx = parent.GetBdrElementFaceIndex(i);
@@ -53,7 +53,7 @@ void Tetrahedron::Init(int ind1, int ind2, int ind3, int ind4, int attr,
 }

 void Tetrahedron::ParseRefinementFlag(int refinement_edges[2], int &type,
-                                      int &flag)
+                                      int &flag) const
 {
   int i, f = refinement_flag;

@@ -134,9 +134,10 @@ void Tetrahedron::CreateRefinementFlag(int refinement_edges[2], int type,
   refinement_flag |= refinement_edges[0];
 }

-void Tetrahedron::GetMarkedFace(const int face, int *fv)
+void Tetrahedron::GetMarkedFace(const int face, int *fv) const
 {
-   int re[2], type, flag, *tv = this->indices;
+   int re[2], type, flag;
+   const int *tv = this->indices;
   ParseRefinementFlag(re, type, flag);
   switch (face)
   {
@@ -58,12 +58,13 @@ public:
   /// Return element's type.
   Type GetType() const override { return Element::TETRAHEDRON; }

-   void  ParseRefinementFlag(int refinement_edges[2], int &type, int &flag);
+   void  ParseRefinementFlag(int refinement_edges[2], int &type,
+                             int &flag) const;
   void CreateRefinementFlag(int refinement_edges[2], int  type, int  flag = 0);

-   void GetMarkedFace(const int face, int *fv);
+   void GetMarkedFace(const int face, int *fv) const;

-   int GetRefinementFlag() { return refinement_flag; }
+   int GetRefinementFlag() const { return refinement_flag; }

   void SetRefinementFlag(int rf) { refinement_flag = rf; }

@@ -797,13 +797,7 @@ void MagneticDiffusionEOperator::buildGrad()

 real_t MagneticDiffusionEOperator::ElectricLosses(ParGridFunction &E_gf) const
 {
-   real_t el = m1->InnerProduct(E_gf,E_gf);
-
-   real_t global_el;
-   MPI_Allreduce(&el, &global_el, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
-                 m2->ParFESpace()->GetComm());
-
-   return el;
+   return m1->ParInnerProduct(E_gf, E_gf);
 }

 // E is the input GF, w is the output GF which is assumed to be an L2 scalar
@@ -307,6 +307,7 @@ int main(int argc, char *argv[])
   }

   // Send the solution by socket to a GLVis server.
+   if (visualization)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
--- a/Show More
+++ b/Show More