Fix test error

More merge fix stuff
Merge remote-tracking branch 'origin/master' into dev-stab-mini
2024-06-12 16:42:22 +02:00 · 2024-06-12 16:42:06 +02:00 · 2024-06-12 16:41:21 +02:00 · 2024-06-12 14:48:42 +02:00 · 2024-06-12 14:46:26 +02:00 · 2024-06-12 14:46:02 +02:00
239 changed files with 18675 additions and 3148 deletions
@@ -33,6 +33,7 @@ env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
  HYPRE_TOP_DIR: hypre-2.19.0
  METIS_ARCHIVE: metis-4.0.3.tar.gz
+  METIS_ARCHIVE_MAC: metis-4.0.3-mac.tgz
  METIS_TOP_DIR: metis-4.0.3
  MFEM_TOP_DIR: mfem

@@ -52,6 +53,7 @@ jobs:
        mpi: [seq, par]
        build-system: [make, cmake]
        hypre-target: [int32]
+        precision: [fp64]
        exclude:
          - os: ubuntu-latest
            build-system: cmake
@@ -75,6 +77,8 @@ jobs:
          - os: ubuntu-latest
            target: dbg
            config-opts: 'CPPFLAGS+=-Og'
+          - os: macos-latest
+            codecov: NO
          - os: windows-latest
            codecov: NO
          - os: windows-latest
@@ -87,6 +91,7 @@ jobs:
            mpi: par
            build-system: cmake
            hypre-target: int32
+            precision: fp64
            # This option can be set to pass additional configuration options to
            # the MFEM configuration command.
            # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
@@ -96,7 +101,15 @@ jobs:
            mpi: par
            build-system: make
            hypre-target: int64
-    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+            precision: fp64
+          - os: ubuntu-latest
+            target: opt
+            codecov: NO
+            mpi: par
+            build-system: make
+            hypre-target: int32
+            precision: fp32
+    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}

    runs-on: ${{ matrix.os }}

@@ -126,6 +139,17 @@ jobs:
          # Fetch the complete history for codecov to access commits ID
          fetch-depth: 0

+      - name: Xcode version setup (MacOS)
+        if: matrix.os == 'macos-latest'
+        run: |
+          XCODE_PATH="/Applications/Xcode_15.3.app"
+          echo "> sudo xcode-select -s ${XCODE_PATH}"
+          sudo xcode-select -s ${XCODE_PATH}
+          echo "> g++ -v"
+          g++ -v
+          echo "> clang++ -v"
+          clang++ -v
+
      # Only get MPI if defined for the job.
      # TODO: It would be nice to have only one step, e.g. with a dedicated
      # action, but I (@adrienbernede) don't see how at the moment.
@@ -169,25 +193,27 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5

      - name: get hypre
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
          target: ${{ matrix.hypre-target }}
          build-system: make
+          precision: ${{ matrix.precision }}

      - name: get hypre (Windows)
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
          target: ${{ matrix.hypre-target }}
          build-system: cmake
+          precision: ${{ matrix.precision }}

      # Get Metis through cache, or build it.
      # Install will only run on cache miss.
@@ -197,13 +223,13 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: install metis
        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
+        uses: mfem/github-actions/build-metis@v2.5
        with:
-          archive: ${{ env.METIS_ARCHIVE }}
+          archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
          dir: ${{ env.METIS_TOP_DIR }}

      - name: cache vcpkg (Windows)
@@ -228,7 +254,7 @@ jobs:

      # MFEM build and test
      - name: build
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        env:
          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
        with:
@@ -240,6 +266,7 @@ jobs:
          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
          metis-dir: ${{ env.METIS_TOP_DIR }}
          mfem-dir: ${{ env.MFEM_TOP_DIR }}
+          precision: ${{ matrix.precision }}
          config-options: ${{ matrix.config-opts }}
          library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}

@@ -282,7 +309,7 @@ jobs:
      # Code coverage (process and upload reports)
      - name: codecov
        if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.4
+        uses: mfem/github-actions/upload-coverage@v2.5
        with:
          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
          project_dir: ${{ env.MFEM_TOP_DIR }}
@@ -53,11 +53,11 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.5

      - name: Get Hypre
        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.4
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
@@ -68,18 +68,18 @@ jobs:
        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: Install Metis
        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
+        uses: mfem/github-actions/build-metis@v2.5
        with:
          archive: ${{ env.METIS_ARCHIVE }}
          dir: ${{ env.METIS_TOP_DIR }}

      # MFEM build and test
      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        with:
          os: ${{ runner.os }}
          target: opt
@@ -44,7 +44,7 @@ jobs:
          path: mfem

      - name: MFEM Build
-        uses: mfem/github-actions/build-mfem@v2.4
+        uses: mfem/github-actions/build-mfem@v2.5
        with:
          os: ${{ runner.os }}
          target: opt
@@ -57,6 +57,8 @@ examples/ex2[0-9]
 examples/ex2[0-9]p
 examples/ex3[0-9]
 examples/ex3[0-9]p
+examples/ex4[0-9]
+examples/ex4[0-9]p

 examples/refined.mesh
 examples/displaced.mesh
@@ -232,7 +234,7 @@ miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
 miniapps/meshing/twist-*.mesh
-miniapps/meshing/mesh-explorer.mesh
+miniapps/meshing/mesh-explorer.mesh*
 miniapps/meshing/partitioning.txt
 miniapps/meshing/mesh-explorer-visit*
 miniapps/meshing/mesh-explorer-paraview/
@@ -270,16 +272,27 @@ miniapps/navier/*_output

 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
+miniapps/nurbs/nurbs_ex3
+miniapps/nurbs/nurbs_ex5
 miniapps/nurbs/nurbs_ex11p
+miniapps/nurbs/nurbs_ex24
+miniapps/nurbs/nurbs_solenoidal
 miniapps/nurbs/nurbs_printfunc
 miniapps/nurbs/nurbs_patch_ex1
 miniapps/nurbs/nurbs_curveint
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
+miniapps/nurbs/sol_?.gf
 miniapps/nurbs/sol.*
 miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
+miniapps/nurbs/Example3*
+miniapps/nurbs/Example5*
+miniapps/nurbs/Solenoidal*
+miniapps/nurbs/ParaView
 miniapps/nurbs/sin-fit.mesh
+miniapps/nurbs/ex5.mesh
+miniapps/nurbs/exsol.mesh
 miniapps/nurbs/CurveInt
 miniapps/nurbs/nurbs_naca_cmesh
 miniapps/nurbs/naca-cmesh.mesh
@@ -369,6 +382,8 @@ miniapps/dpg/ParaView
 miniapps/spde/generate_random_field
 miniapps/spde/ParaView

+miniapps/tribol/contact-patch-test
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -13,6 +13,9 @@
 # at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
 # LLNL-specific!

+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'

 # The pipeline is divided into stages. Usually, jobs in a given stage wait for
 # the preceding stages to complete before to start. However, we sometimes use
@@ -9,6 +9,10 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
 # We define the following GitLab pipeline variables:
 variables:

@@ -35,9 +35,8 @@ variables:
    - when: on_success

 # Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen
-# to speed-up the allocation.  However this would not be scalable to
-# multiple builds.
+# pre-allocation the same way slurm does.  We use the pci queue on lassen
+# to speed-up the allocation.
 .build_and_test_on_lassen:
  extends: [.on_lassen]
  stage: build_and_test
@@ -45,5 +44,5 @@ variables:
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 45 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
  needs: [setup]
@@ -52,4 +52,4 @@ variables:
    - echo ${JOBID}
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -14,14 +14,14 @@ stages:
  - build_and_test
  - report

-opt_mpi_cuda_xl_16_1_1_12:
+opt_mpi_cuda_gcc:
  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70"
  extends: .build_and_test_on_lassen

-opt_mpi_cuda_hypre_cuda_xl:
+opt_mpi_cuda_hypre_cuda_gcc:
  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
  extends: .build_and_test_on_lassen

 # Jobs report
@@ -32,11 +32,11 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}

 # run
 if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  lalloc 1 -q pci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
  exit 1
@@ -8,74 +8,117 @@
                               https://mfem.org


-Version 4.6.1 (development)
+Version 4.7.1 (development)
 ===========================

+- Added NURBS-based H(div) and H(curl) elements in 2D and 3D. Only on single 
+  patch meshes. Only implemented for serial computations.
+
+- Added miniapps to demonstrate the H(div) and H(curl) NURBS elements.
+
+- Added an MFEM example for the eikonal equation. This new solver is based on
+  the proximal Galerkin method introduced by Keith and Surowiec.
+
+- API change: in class GridFunction, 'fec' was renamed to 'fec_owned'.
+
+
+Version 4.7, released on May 7, 2024
+====================================
+
+- Added support for single precision (with corresponding hypre build). The MFEM
+  floating point type was generalized from `double` to `real_t`. For details see
+  https://github.com/orgs/mfem/discussions/4207.
+
 Meshing improvements
 --------------------
+- Added the capability to partition (big) serial meshes in serial code, see the
+  new classes MeshPartitioner and MeshPart. This capability is also exposed as a
+  menu option in the mesh-explorer miniapp in miniapps/meshing.
+
+- Added named attribute sets and basic supporting methods to the Mesh class as a
+  convenient means of referring to sets of domain or boundary attribute numbers.
+  See the new Example 39/39p and data/compass.mesh.
+
 - Introduced formulas for refinement of patches in NURBS meshes. Refinement by
  arbitrary integer factors is also enabled, e.g. in the mesh-explorer miniapp.
  NURBS coarsening and knot removal are also introduced.

-Discretization improvements
---------------------------
- Introduced support for higher order non conformal Nedelec elements on
-  simplices in ParMesh.
- Introduced support for internal boundary elements in nonconformal adapted
-  meshes.
-
- Added functionality for construction of cut-surface and cut-volume
-  IntegrationRules through a moment-fitting approach. The cut is specified by
-  the zero level set of a Coefficient. See fem/intrules_cut.hpp and Example 38.
-
- Added a new nonlinear integrator, `HyperbolicFormIntegrator`. This implements 
-  both element-wise weak divergence and face-wise numerical flux for a general
-  system of hyperbolic conservation laws. To use this integrator for a specific
-  flux function, users can define a derived class of `FluxFunction`. Currently,
-  advection, Burgers', shallow-water, Euler equations (see, Example 18) are
-  available.
-
-GPU support
----------------------------
- Added support for full assembly on simplices.
- Added functionality for BilinearFormIntegrators to use kernels that work for both
-  tensor and unstructured elements.
- Added partial assembly for linear elasticity. Does not use sum factorization for now.
-
-New and updated examples and miniapps
-------------------------------------
- Added a new block solver in miniapp/solvers for the Darcy problem.
-  The new solver is based on a Bramble-Pasciak preconditioning. User can
-  use and implement their own preconditioner for the mass matrix.
-
- Added miniapp to demonstrate new elasticity integrator and unstructured element GPU support,
-  and a block diagonal preconditioner using low order refinement. Allows comparison with
-  currently existing legacy mode integrator. See miniapps/solvers/lor_elast.
-
-Miscellaneous
-------------
- Added support for single and double precision, with corresponding hypre build.
-  Generalized the floating point type from `double` to `real_t`. For more
-  details see https://github.com/orgs/mfem/discussions/4207.
+- Added support for internal boundary elements in nonconforming meshes.

 - The ReadCubit Genesis mesh importer has been rewritten to improve readability.

+Discretization improvements
+---------------------------
+- Added a new nonlinear integrator, `HyperbolicFormIntegrator` that implements
+  both element-wise weak divergence and face-wise numerical flux for a general
+  system of hyperbolic conservation laws. To use the integrator for a specific
+  flux function, users can define a derived class of `FluxFunction`. Currently,
+  advection, Burgers, shallow-water and Euler equations (see Example 18/18p) are
+  available.
+
+- Added a capability to construct cut-surface and cut-volume IntegrationRules
+  through a moment-fitting approach. The cut is specified by the zero level set
+  of a Coefficient. See fem/intrules_cut.hpp and the new Example 38.
+
+- Introduced support for high-order nonconforming Nedelec elements on simplices.
+
+GPU computing
+-------------
+- Added partial assembly and GPU support for the DG diffusion integrator.
+
+- Efficient GPU-accelerated LOR assembly is now supported on surface meshes.
+
+- Added functionality to automatically configure hypre's compute policy to match
+  MFEM's compute policy when hypre is built with GPU support. Requires version
+  hypre-2.31.0 or later.
+
+- Added support for full assembly on simplices.
+
+- Added partial assembly for linear elasticity (no sum factorization for now).
+
+- Added functionality for BilinearFormIntegrators to use kernels that work for
+  both tensor and unstructured elements.
+
+- The RAJA backend will use `seq_exec` for serial loop execution when RAJA
+  v2023.06.00 and beyond is detected as `loop_exec` is deprecated.
+
+- API change: The macro MFEM_HYPRE_FORALL (from hypre.hpp) which was intended
+  for internal use, has been removed and replaced by the function template
+  mfem::hypre_forall in general/forall.hpp.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new miniapp illustrating elastic contact based on the Tribol library,
+  (https://github.com/LLNL/Tribol). See miniapps/tribol.
+
+- Added a miniapp to demonstrate low order refined (LOR) block preconditioning
+  for linear elasticity on GPUs. See miniapps/solvers/lor_elast.
+
+- Added a new block solver in miniapp/solvers for the Darcy problem. The new
+  solver is based on a Bramble-Pasciak preconditioning. User can use and
+  implement their own preconditioner for the mass matrix.
+
+- Added a small miniapp for printing the shape functions of a KnotVector. See
+  miniapps/nurbs/nurbs_printfunc.cpp.
+
+- Added two new example codes: 38 and 39/39p described above. Substantially
+  updated Example 18/18p.
+
+Miscellaneous
+-------------
 - Updated the Doxygen documentation style, which now requires Doxygen version
  1.9.8 or later. See the doc/ directory.

- Improved thread safety for global variables in the library, for example
-  IntegrationRules IntRules, RefinedIntRules, GeometryRefiner
-  GlobGeometryRefiner, and FiniteElement::dof2quad_array.
+- Improved thread safety for global variables in the library, e.g. for IntRules,
+  RefinedIntRules, GlobGeometryRefiner, and FiniteElement::dof2quad_array.

 - PETSc integration now generally requires PETSc version 3.21 or later, though
  depending on the functionality older versions may still work.

- RAJA backend will use seq_exec for serial loop execution when RAJA
-  v2023.06.00 and beyond is detected as loop_exec is deprecated.
+- Various other simplifications, extensions, and bugfixes in the code.
+
+- Added GSLIB-based gather-scatter operator.

- Adding named attribute sets and basic supporting methods to the Mesh class as
-  a convenient means of referring to sets of domain or boundary attribute
-  numbers. Also adding related serial and parallel examples which illustrate.

 Version 4.6, released on September 27, 2023
 ===========================================
@@ -96,7 +139,6 @@ Meshing improvements
  * The edge to knot map for NURBS meshes can be determined automatically. It is
    no longer needed to specify this in the NURBS mesh.
  * Added curve interpolation method for NURBS.
-  * Added new small miniapp for printing of shape functions of a KnotVector
  * See miniapps/nurbs for example meshes and miniapps.

 Discretization improvements
@@ -143,8 +185,6 @@ Linear and nonlinear solvers

 - Added HIP support to the PETSc and SUNDIALS interfaces.

- Efficient GPU-accelerated LOR assembly now supports surface meshes.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new H(div) solver miniapp demonstrating the use of a matrix-free
@@ -58,7 +58,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.6.1)
+set(${PROJECT_NAME}_VERSION 4.7.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -87,10 +87,11 @@ if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
  # Just needed to find the MPI_Fortran libraries to link with
  set(XSDK_ENABLE_Fortran ON)
 endif()
-# SUNDIALS, STRUMPACK, Ginkgo, RAJA and Umpire require C++14:
+# SUNDIALS, STRUMPACK, Ginkgo, Tribol, RAJA and Umpire require C++14:
 if ((MFEM_USE_SUNDIALS OR
     MFEM_USE_STRUMPACK OR
     MFEM_USE_GINKGO OR
+     MFEM_USE_TRIBOL OR
     MFEM_USE_RAJA OR
     MFEM_USE_UMPIRE) AND
    ("${CMAKE_CXX_STANDARD}" LESS "14"))
@@ -503,6 +504,15 @@ if (MFEM_USE_PARELAG)
   find_package(PARELAG REQUIRED)
 endif()

+# Tribol
+if (MFEM_USE_TRIBOL)
+  if (MFEM_USE_MPI)
+    find_package(Tribol REQUIRED tribol redecomp)
+  else()
+    message(FATAL_ERROR " *** Tribol requires that MPI be enabled.")
+  endif()
+endif()
+
 # Enzyme
 if (MFEM_USE_ENZYME)
   find_package(ENZYME REQUIRED)
@@ -548,7 +558,7 @@ set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
    ADIOS2 CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
-    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
+    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -151,7 +151,8 @@ The MFEM source code has the following structure:
  │   ├── solvers
  │   ├── spde
  │   ├── tools
-  │   └── toys
+  │   ├── toys
+  │   └── tribol
  └── tests
      ├── benchmarks
      ├── convergence
@@ -75,6 +75,8 @@ and miniapps. See https://glvis.org and https://mfem.org/building.

 Quick start with GNU make
 =========================
+See also: https://mfem.org/building
+
 Serial build:
   make serial -j 4

@@ -83,6 +85,7 @@ Parallel build:
   (build METIS 4 in ../metis-4.0 relative to mfem/)
   (build hypre in ../hypre relative to mfem/)
   make parallel -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   make cuda -j 4
@@ -116,6 +119,7 @@ Parallel build:
   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
   make -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   (this build requires CMake 3.8 or newer)
@@ -571,6 +575,11 @@ MFEM_USE_PARELAG = YES/NO
   use ParELAG. In fact, ParELAG is dependent on MFEM. Therefore, this option
   currently only concerns the miniapps.

+MFEM_USE_TRIBOL = YES/NO
+   Enables the miniapps that use the Tribol library. MFEM does not currently
+   use Tribol. In fact, Tribol is dependent on MFEM. Therefore, this option
+   currently only concerns the miniapps.
+
 MFEM_USE_ENZYME = YES/NO
   Enables automatic differentiation support through the LLVM plugin Enzyme.
   This requires the compiler to be set to clang (>=14.0.0). We also advise to
@@ -607,9 +616,13 @@ The specific libraries and their options are:
            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
            HYPRE >= 2.22.1   (HYPRE built with CUDA)
            HYPRE >= 2.23.0   (HYPRE built with HIP)
+            HYPRE >= 2.31.0   (runtime selectable HYPRE execution on CPU/GPU)

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
-  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
+  MFEM_USE_METIS_5 = YES (default is to use METIS 4). For building instructions,
+  see the following:
+    - METIS 4.0.3: https://mfem.org/building/#parallel-mpi-version-of-mfem
+    - METIS 5.1.0: https://mfem.org/building/#parallel-build-using-metis-5
  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
  Options: METIS_OPT, METIS_LIB.
  Versions: METIS 4.0.3 or 5.1.0.
@@ -857,6 +870,10 @@ The specific libraries and their options are:
  URL: https://github.com/LLNL/parelag
  Options: PARELAG_DIR, PARELAG_OPT, PARELAG_LIB.

+- Tribol, used when MFEM_USE_TRIBOL = YES.
+  URL: https://github.com/LLNL/Tribol
+  Options: TRIBOL_DIR, TRIBOL_OPT, TRIBOL_LIB.
+
 - Enzyme, used when MFEM_USE_ENZYME = YES. Requires LLVM/Clang >= 14.0.0.
  URL: https://github.com/EnzymeAD/Enzyme
  Options: ENZYME_DIR, ENZYME_OPT, ENZYME_LIB.
@@ -1001,6 +1018,7 @@ MFEM_USE_CALIPER
 MFEM_USE_FMS
 MFEM_USE_BENCHMARK
 MFEM_USE_PARELAG
+MFEM_USE_TRIBOL
 MFEM_USE_ENZYME

 The following options are CMake specific:
@@ -287,3 +287,7 @@ ENDIF()
 IF (DEFINED TPL_ENABLE_PARELAG)
  SET(MFEM_USE_PARELAG ${TPL_ENABLE_PARELAG} CACHE BOOL "Enable ParELAG" FORCE)
 ENDIF()
+
+IF (DEFINED TPL_ENABLE_TRIBOL)
+  SET(MFEM_USE_TRIBOL ${TPL_ENABLE_TRIBOL} CACHE BOOL "Enable Tribol" FORCE)
+ENDIF()
@@ -64,6 +64,7 @@ set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)
 set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
 set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)
+set(MFEM_USE_TRIBOL @MFEM_USE_TRIBOL@)
 set(MFEM_USE_ENZYME @MFEM_USE_ENZYME@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
@@ -18,4 +18,13 @@ include(MfemCmakeUtilities)
 # Note: components are enabled based on the find_package() parameters.
 mfem_find_package(Axom AXOM AXOM_DIR "include" "" "lib" ""
  "Paths to headers required by Axom." "Libraries required by Axom."
-  ADD_COMPONENT Axom "include" axom/config.hpp "lib" axom)
+  ADD_COMPONENT core "include" axom/core.hpp "lib" axom_core
+  ADD_COMPONENT inlet "include" axom/inlet.hpp "lib" axom_inlet
+  ADD_COMPONENT klee "include" axom/klee.hpp "lib" axom_klee
+  ADD_COMPONENT lumberjack "include" axom/lumberjack.hpp "lib" axom_lumberjack
+  ADD_COMPONENT mint "include" axom/mint.hpp "lib" axom_mint
+  ADD_COMPONENT multimat "include" axom/multimat.hpp "lib" axom_multimat
+  ADD_COMPONENT quest "include" axom/quest.hpp "lib" axom_quest
+  ADD_COMPONENT sidre "include" axom/sidre.hpp "lib" axom_sidre
+  ADD_COMPONENT slam "include" axom/slam.hpp "lib" axom_slam
+  ADD_COMPONENT slic "include" axom/slic.hpp "lib" axom_slic)
@@ -36,7 +36,11 @@ include(MfemCmakeUtilities)
 mfem_find_package(Conduit CONDUIT CONDUIT_DIR
  "include;include/conduit" conduit.hpp "lib" conduit
  "Paths to headers required by Conduit." "Libraries required by Conduit."
+  ADD_COMPONENT blueprint
+    "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint
+  ADD_COMPONENT blueprint_mpi
+    "include;include/conduit" conduit_blueprint_mpi.hpp "lib" conduit_blueprint_mpi
  ADD_COMPONENT relay
    "include;include/conduit" conduit_relay.hpp "lib" conduit_relay
-  ADD_COMPONENT blueprint
-    "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint)
+  ADD_COMPONENT relay_mpi
+    "include;include/conduit" conduit_relay_mpi.hpp "lib" conduit_relay_mpi)
@@ -16,12 +16,22 @@
 #   - MUMPS_VERSION

 include(MfemCmakeUtilities)
+
+# Toggle which precision of MUMPS to use depending on the precision of MFEM.
+if (MFEM_USE_DOUBLE)
+  set(_mumps_header dmumps_c.h)
+  set(_mumps_lib dmumps)
+elseif(MFEM_USE_SINGLE)
+  set(_mumps_header smumps_c.h)
+  set(_mumps_lib smumps)
+endif()
+
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
-  "include" dmumps_c.h "lib" dmumps
+  "include" ${_mumps_header} "lib" ${_mumps_lib}
  "Paths to headers required by MUMPS."
  "Libraries required by MUMPS."
-  ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
-  ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)
+  ADD_COMPONENT mumps_common "include" ${_mumps_header} "lib" mumps_common
+  ADD_COMPONENT pord "include" ${_mumps_header} "lib" pord)

 if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
  try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
@@ -0,0 +1,22 @@
+# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - TRIBOL_FOUND
+#   - TRIBOL_LIBRARIES
+#   - TRIBOL_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+# Note: components are enabled based on the find_package() parameters.
+mfem_find_package(Tribol TRIBOL TRIBOL_DIR "include" tribol/config.hpp "lib" tribol
+  "Paths to headers required by Tribol." "Libraries required by Tribol."
+  ADD_COMPONENT redecomp
+    "include" redecomp/redecomp.hpp "lib" redecomp)
@@ -852,8 +852,8 @@ function(mfem_export_mk_files)
      MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED
      MFEM_USE_CALIPER MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2
      MFEM_USE_MKL_CPARDISO MFEM_USE_MKL_PARDISO MFEM_USE_ADFORWARD
-      MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_MOONOLITH
-      MFEM_USE_ALGOIM MFEM_USE_ENZYME)
+      MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG MFEM_USE_TRIBOL
+      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -120,6 +120,15 @@ constexpr real_t operator""_r(unsigned long long v)

 // Check dependencies:

+// Define MFEM_MPI_REAL_T to be the appropriate MPI real type
+#ifdef MFEM_USE_MPI
+#ifdef MFEM_USE_SINGLE
+#define MFEM_MPI_REAL_T MPI_FLOAT
+#elif defined MFEM_USE_DOUBLE
+#define MFEM_MPI_REAL_T MPI_DOUBLE
+#endif
+#endif
+
 // Options that require MPI
 #ifndef MFEM_USE_MPI
 #ifdef MFEM_USE_SUPERLU
@@ -65,6 +65,7 @@ MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
 MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
 MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@
+MFEM_USE_TRIBOL        = @MFEM_USE_TRIBOL@
 MFEM_USE_ENZYME        = @MFEM_USE_ENZYME@

 # Compiler, compile options, and link options
@@ -67,6 +67,7 @@ option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
 option(MFEM_USE_PARELAG "Enable ParELAG" OFF)
+option(MFEM_USE_TRIBOL "Enable Tribol" OFF)
 option(MFEM_USE_ENZYME "Enable Enzyme" OFF)

 # Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
@@ -212,8 +213,15 @@ set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH

 set(AXOM_DIR "${MFEM_DIR}/../axom" CACHE PATH "Path to the Axom library.")
 # May need to add "Boost" as requirement.
-set(Axom_REQUIRED_PACKAGES "Conduit/relay/blueprint" CACHE STRING
-    "Additional packages required by Axom.")
+if (MFEM_USE_SIDRE)
+    if (MFEM_USE_MPI)
+        set(Axom_REQUIRED_PACKAGES "Conduit/blueprint/blueprint_mpi/relay/relay_mpi" CACHE STRING
+            "Additional packages required by Axom.")
+    elseif()
+        set(Axom_REQUIRED_PACKAGES "Conduit/blueprint/relay" CACHE STRING
+            "Additional packages required by Axom.")
+    endif()
+endif()

 set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING
    "Directory where PUMI is installed")
@@ -250,6 +258,10 @@ set(PARELAG_INCLUDE_DIRS "${PARELAG_DIR}/src;${PARELAG_DIR}/build/src" CACHE
 set(PARELAG_LIBRARIES "${PARELAG_DIR}/build/src/libParELAG.a" CACHE STRING
    "The ParELAG library.")

+set(TRIBOL_DIR "${MFEM_DIR}/../tribol" CACHE PATH "Path to Tribol")
+set(Tribol_REQUIRED_PACKAGES "Axom/core/mint/slam/slic" CACHE STRING 
+    "Additional packages required by Tribol")
+
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
@@ -167,8 +167,21 @@ MFEM_USE_ADFORWARD     = NO
 MFEM_USE_CODIPACK      = NO
 MFEM_USE_BENCHMARK     = NO
 MFEM_USE_PARELAG       = NO
+MFEM_USE_TRIBOL        = NO
 MFEM_USE_ENZYME        = NO

+# Process MFEM_PRECISION -> MFEM_USE_SINGLE, MFEM_USE_DOUBLE
+ifneq ($(filter double Double DOUBLE,$(MFEM_PRECISION)),)
+   MFEM_USE_DOUBLE = YES
+   MFEM_USE_SINGLE = NO
+else ifneq ($(filter single Single SINGLE,$(MFEM_PRECISION)),)
+   MFEM_USE_DOUBLE = NO
+   MFEM_USE_SINGLE = YES
+else ifeq ($(MAKECMDGOALS),config)
+   $(error Invalid floating-point precision: \
+     MFEM_PRECISION = $(MFEM_PRECISION))
+endif
+
 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
 ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
@@ -318,13 +331,13 @@ MPI_FORTRAN_LIB = -lmpifort
 # MUMPS library configuration
 MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib \
- -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
+MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib
 ifeq ($(MFEM_USE_SINGLE),YES)
   MUMPS_LIB += -lsmumps
 else
   MUMPS_LIB += -ldmumps
 endif
+MUMPS_LIB += -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)

 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
@@ -375,7 +388,7 @@ GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LINK_LIB_DIR) -L$(GINKGO_LINK_LIB_DIR)\
 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
 AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx
+AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt

 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -576,6 +589,16 @@ PARELAG_DIR = @MFEM_DIR@/../parelag
 PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
 PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG

+# Tribol library configuration
+ifeq ($(MFEM_USE_TRIBOL),YES)
+   BASE_FLAGS = -std=c++14
+endif
+AXOM_DIR = @MFEM_DIR@/../axom
+TRIBOL_DIR = @MFEM_DIR@/../tribol
+TRIBOL_OPT = -I$(TRIBOL_DIR)/include -I$(AXOM_DIR)/include
+TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
+   -laxom_slam -laxom_slic -laxom_core
+
 # Enzyme configuration

 # If you want to enable automatic differentiation at compile time, use the
@@ -110,4 +110,4 @@ config-mk:

 clean:
 	rm -f $(CONFIG_HPP) $(CONFIG_MK) sample-runs-build.log
-	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out
+	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out *.dSYM
@@ -32,7 +32,7 @@ groups_serial=(
 '"examples"
   "Examples:"
   "examples"
-   "ex{,1,2,3}[0-9].cpp"'
+   "ex{,[1-9]}[0-9].cpp"'
 #   "ex1.cpp"'
 '"sundials"
   "SUNDIALS examples:"
@@ -58,6 +58,10 @@ groups_serial=(
   "HiOp examples:"
   "examples/hiop"
   "ex9.cpp"'
+'"moonolith"
+   "Moonolith examples:"
+   "examples/moonolith"
+   "ex1.cpp"'
 '"pumi"
   "PUMI examples:"
   "examples/pumi"
@@ -66,25 +70,38 @@ groups_serial=(
 '"meshing"
   "Meshing miniapps:"
   "miniapps/meshing"
-   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
+   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp mesh-quality.cpp
+    polar-nc.cpp reflector.cpp shaper.cpp trimmer.cpp twist.cpp
    mesh-optimizer.cpp minimal-surface.cpp"'
 '"adjoint"
   "Adjoint miniapps:"
   "miniapps/adjoint"
   "cvsRoberts_ASAi_dns.cpp"'
+'"autodiff"
+   "Autodiff miniapps:"
+   "miniapps/autodiff"
+   "seq_example.cpp seq_test.cpp"' # 'seq_test.cpp' has no sample runs
+'"dpg"
+   "DPG miniapps:"
+   "miniapps/dpg"
+   "{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
 '"gslib"
   "GSLIB miniapps:"
   "miniapps/gslib"
   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp "'
+# todo: miniapps/mtop
 '"nurbs"
   "NURBS miniapps:"
   "miniapps/nurbs"
   "nurbs_ex1.cpp"'
+# todo: add other nurbs miniapps
+# todo: miniapps/solvers (serial)
 '"tools"
   "Tools miniapps:"
   "miniapps/tools"
   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
    lor-transfer.cpp"'
+# todo: add other tools miniapps
 '"toys"
   "Toys miniapps:"
   "miniapps/toys"
@@ -100,7 +117,7 @@ groups_parallel=(
 '"examples"
   "Examples:"
   "examples"
-   "ex{,1,2,3}[0-9]p.cpp"'
+   "ex{,[1-9]}[0-9]p.cpp"'
 #   "ex1p.cpp"'
 '"sundials"
   "SUNDIALS examples:"
@@ -126,6 +143,10 @@ groups_parallel=(
   "HiOp examples:"
   "examples/hiop"
   "ex9p.cpp"'
+'"moonolith"
+   "Moonolith examples:"
+   "examples/moonolith"
+   "ex{1,2}p.cpp"'
 '"pumi"
   "PUMI examples:"
   "examples/pumi"
@@ -138,24 +159,41 @@ groups_parallel=(
 '"meshing"
   "Meshing miniapps:"
   "miniapps/meshing"
-   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp"'
+   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp
+    fit-node-position.cpp"'
 '"electromagnetics"
   "Electromagnetics miniapps:"
   "miniapps/electromagnetics"
   "joule.cpp"'
-#   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
+#   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
 '"adjoint"
   "Adjoint miniapps:"
   "miniapps/adjoint"
   "adjoint_advection_diffusion.cpp"'
+'"autodiff"
+   "Autodiff miniapps:"
+   "miniapps/autodiff"
+   "par_example.cpp"'
+'"dpg"
+   "DPG miniapps:"
+   "miniapps/dpg"
+   "p{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
 '"gslib"
   "GSLIB miniapps:"
   "miniapps/gslib"
   "pfindpts.cpp schwarz_ex1p.cpp"'
+'"hdiv-linear-solver"
+   "H(div) linear solver miniapps:"
+   "miniapps/hdiv-linear-solver"
+   "grad_div.cpp darcy.cpp"'
+# 'miniapps/hooke/hooke.cpp' has no sample runs
+# todo: miniapps/mtop
+# todo: miniapps/multidomain
 '"navier"
   "Navier miniapps:"
   "miniapps/navier"
   "navier_cht.cpp"'
+# todo: add other navier miniapps
 '"nurbs"
   "NURBS miniapps:"
   "miniapps/nurbs"
@@ -164,14 +202,18 @@ groups_parallel=(
   "Shifted miniapps:"
   "miniapps/shifted"
   "distance.cpp"'
+# todo: add other shifted miniapps
 '"solvers"
   "Solvers miniapps:"
   "miniapps/solvers"
   "block-solvers.cpp"'
+# todo: add other solvers miniapps
+# todo: miniapps/spde
 '"tools"
   "Tools miniapps:"
   "miniapps/tools"
-   "convert-cd.cpp get-values.cpp load-dc.cpp"'
+   "convert-dc.cpp get-values.cpp load-dc.cpp"'
+# todo: add other tools miniapps
 '"convergence"
   "Convergence tests:"
   "tests/convergence"
@@ -186,7 +228,7 @@ groups_all=(
 '"examples"
   "Examples:"
   "examples"
-   "ex\"{,1,2,3}[0-9]\"{,p}.cpp"'
+   "ex\"{,[1-9]}[0-9]\"{,p}.cpp"'
 '"sundials"
   "SUNDIALS examples:"
   "examples/sundials"
@@ -215,10 +257,14 @@ groups_all=(
   "HiOp examples:"
   "examples/hiop"
   "ex9.cpp ex9p.cpp"'
+'"moonolith"
+   "Moonolith examples:"
+   "examples/moonolith"
+   "ex1.cpp ex{1,2}p.cpp"'
 '"pumi"
   "PUMI examples:"
   "examples/pumi"
-   "ex1.cpp ex1p.cpp ex2.cpp ex6p.cpp"'
+   "ex1.cpp ex2.cpp ex1p.cpp ex6p.cpp"'
 '"superlu"
   "Superlu examples:"
   "examples/superlu"
@@ -226,43 +272,67 @@ groups_all=(
 '"meshing"
   "Meshing miniapps:"
   "miniapps/meshing"
-   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
-    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp"'
+   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp mesh-quality.cpp
+    polar-nc.cpp reflector.cpp shaper.cpp trimmer.cpp twist.cpp
+    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp
+    fit-node-position.cpp"'
 '"electromagnetics"
   "Electromagnetics miniapps:"
   "miniapps/electromagnetics"
   "joule.cpp"'
-#   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
+#   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
 '"adjoint"
   "Adjoint miniapps:"
   "miniapps/adjoint"
-   "adjoint_advection_diffusion.cpp cvsRoberts_ASAi_dns.cpp"'
+   "cvsRoberts_ASAi_dns.cpp adjoint_advection_diffusion.cpp"'
+'"autodiff"
+   "Autodiff miniapps:"
+   "miniapps/autodiff"
+   "seq_example.cpp seq_test.cpp par_example.cpp"'
+# 'seq_test.cpp' has no sample runs
+'"dpg"
+   "DPG miniapps:"
+   "miniapps/dpg"
+   "{,p}{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
 '"gslib"
   "GSLIB miniapps:"
   "miniapps/gslib"
   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp pfindpts.cpp
    schwarz_ex1p.cpp"'
+'"hdiv-linear-solver"
+   "H(div) linear solver miniapps:"
+   "miniapps/hdiv-linear-solver"
+   "grad_div.cpp darcy.cpp"'
+# 'miniapps/hooke/hooke.cpp' has no sample runs
+# todo: miniapps/mtop
+# todo: miniapps/multidomain
 '"navier"
   "Navier miniapps:"
   "miniapps/navier"
   "navier_cht.cpp"'
+# todo: add other navier miniapps
 '"nurbs"
   "NURBS miniapps:"
   "miniapps/nurbs"
   "nurbs_ex1.cpp nurbs_ex1p.cpp nurbs_ex11p.cpp"'
+# todo: add other nurbs miniapps
 '"shifted"
   "Shifted miniapps:"
   "miniapps/shifted"
   "distance.cpp"'
+# todo: add other shifted miniapps
 '"solvers"
   "Solvers miniapps:"
   "miniapps/solvers"
   "block-solvers.cpp"'
+# todo: add other solvers miniapps
+# todo: miniapps/spde
 '"tools"
   "Tools miniapps:"
   "miniapps/tools"
   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
    lor-transfer.cpp"'
+# todo: add other tools miniapps
 '"toys"
   "Toys miniapps:"
   "miniapps/toys"
@@ -386,7 +456,7 @@ function help_message()
      mfem_config [${mfem_config}]
         Set MFEM configuration options
      make [${make}], mpiexec [${mpiexec}], mpiexec_np [${mpiexec_np}]
-         Their values can also set using the respective uppercase environment
+         Their values can also be set using the respective uppercase environment
         variable
      mfem_build_dir [${mfem_build_dir}]
         Same as '-d': set this variable to something different from <mfem_dir>
@@ -92,4 +92,5 @@ vertices
 -0.70710678 -0.70710678
 0 -1
 0.70710678 -0.70710678
+
 mfem_mesh_end
@@ -18,9 +18,9 @@ elements
 boundary
 4
 1 1 0 1
-1 1 2 3
-1 1 3 0
-1 1 1 2
+2 1 2 3
+3 1 3 0
+4 1 1 2

 edges
 4
@@ -48,7 +48,7 @@ PROJECT_NAME           = MFEM
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.6.1
+PROJECT_NUMBER         = v4.7.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -980,6 +980,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/mtop \
                         @MFEM_SOURCE_DIR@/miniapps/multidomain \
                         @MFEM_SOURCE_DIR@/miniapps/navier \
+                         @MFEM_SOURCE_DIR@/miniapps/stabilized \
                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
                         @MFEM_SOURCE_DIR@/miniapps/parelag \
                         @MFEM_SOURCE_DIR@/miniapps/performance \
@@ -987,6 +988,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/solvers \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
                         @MFEM_SOURCE_DIR@/miniapps/toys \
+                         @MFEM_SOURCE_DIR@/miniapps/tribol \
                         @MFEM_SOURCE_DIR@/miniapps/spde \
                         @MFEM_SOURCE_DIR@/miniapps/dpg \
                         @MFEM_SOURCE_DIR@/miniapps/dpg/util
@@ -1048,7 +1050,8 @@ RECURSIVE              = NO
 EXCLUDE                = @MFEM_SOURCE_DIR@/config/_config.hpp \
                         @MFEM_SOURCE_DIR@/config/get_hypre_version.cpp \
                         @MFEM_SOURCE_DIR@/general/tinyxml2.h \
-                         @MFEM_SOURCE_DIR@/general/tinyxml2.cpp
+                         @MFEM_SOURCE_DIR@/general/tinyxml2.cpp \
+                         @MFEM_SOURCE_DIR@/linalg/lapack.hpp

 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -110,9 +110,13 @@ namespace mfem {
 * - <a class="el" href="ex35p_8cpp_source.html">Example 35p</a>: parallel multi-domain damped harmonic oscillators
 * - <a class="el" href="ex36_8cpp_source.html">Example 36</a>: Proximal Galerkin FEM for the obstacle problem
 * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
- * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: Topology optimization
+ * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: topology optimization
 * - <a class="el" href="ex37p_8cpp_source.html">Example 37p</a>: parallel topology optimization
 * - <a class="el" href="ex38_8cpp_source.html">Example 38</a>: cut-surface and cut-volume integration
+ * - <a class="el" href="ex39_8cpp_source.html">Example 39</a>: named mesh attributes
+ * - <a class="el" href="ex39p_8cpp_source.html">Example 39p</a>: parallel named mesh attributes
+ * - <a class="el" href="ex40_8cpp_source.html">Example 40</a>: eikonal equation
+ * - <a class="el" href="ex40p_8cpp_source.html">Example 40p</a>: parallel eikonal equation
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -178,6 +182,21 @@ namespace mfem {
 *   <a class="el" href="examples_2superlu_2ex1p_8cpp_source.html">1p</a>,
 *   demonstrating the use of MFEM's \link superlu.hpp SuperLU integration\endlink.
 *
+ * <H4>NURBS Examples</H4>
+ * - Variants of Examples
+ *  <a class="el" href="nurbs__ex1_8cpp_source.html">1</a>,
+ *  <a class="el" href="nurbs__ex1p_8cpp_source.html">1p</a>,
+ *  <a class="el" href="nurbs__ex3_8cpp_source.html">3</a>,
+ *  <a class="el" href="nurbs__ex5_8cpp_source.html">5</a>,
+ *  <a class="el" href="nurbs__ex11p_8cpp_source.html">11p</a>, and
+ *  <a class="el" href="nurbs__ex24_8cpp_source.html">24</a>,
+ *   demonstrating howto perform NURBS-based Isogeometric Analysis.
+ * - Variant of Example <a class="el" href="nurbs__patch__ex1_8cpp_source.html">1</a>: demonstrates the use of patch integration
+ * - <a class="el" href="nurbs__solenoidal_8cpp_source.html">NURBS Divergence-free</a>: solve a solenoidal vector projection with NURBS-based H(div) elements
+ * - <a class="el" href="nurbs__curveint_8cpp_source.html">NURBS Interpolation</a>: NURBS interpolation of given geometry
+ * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
+ * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
+ *
 * <H3>Miniapps</H3>
 * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
 * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
@@ -214,6 +233,8 @@ namespace mfem {
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
+ * - <a class="el" href="contact-patch-test_8cpp_source.html">Contact</a>: mortar contact patch test for elasticity
+ * - <a class="el" href="multidomain_8cpp_source.html">Multidomain miniapp</a>: Multidomain and Submesh demonstration miniapp
 * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
 * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
 * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
@@ -46,7 +46,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
            DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
        })
        // Update the color scheme when the tab is made visible again.
-        // It is possible that the appearance was changed in another tab 
+        // It is possible that the appearance was changed in another tab
        // while this tab was in the background.
        document.addEventListener("visibilitychange", visibilityState => {
            if (document.visibilityState === 'visible') {
@@ -97,7 +97,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
     * @returns `true` for dark-mode, `false` for light-mode user preference
     */
    static get userPreference() {
-        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) || 
+        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) ||
        (DoxygenAwesomeDarkModeToggle.systemPreference && !localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey))
    }

@@ -45,6 +45,7 @@ list(APPEND ALL_EXE_SRCS
  ex37.cpp
  ex38.cpp
  ex39.cpp
+  ex40.cpp
  )

 if (MFEM_USE_MPI)
@@ -87,6 +88,7 @@ if (MFEM_USE_MPI)
    ex36p.cpp
    ex37p.cpp
    ex39p.cpp
+    ex40p.cpp
  )
 endif()

@@ -146,10 +148,10 @@ if (MFEM_ENABLE_TESTING)
  # Add CUDA/HIP tests.
  set(DEVICE_EXAMPLES
      # serial examples with device support:
-      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
+      ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
      # parallel examples with device support:
-      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p
-      ex34p ex35p)
+      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p ex22p ex24p ex25p
+      ex26p ex34p ex35p)
  set(MFEM_TEST_DEVICE)
  if (MFEM_USE_CUDA)
    set(MFEM_TEST_DEVICE "cuda")
@@ -159,6 +161,11 @@ if (MFEM_ENABLE_TESTING)
  if (MFEM_TEST_DEVICE)
    foreach(TEST_NAME ${DEVICE_EXAMPLES})
      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+      if (${TEST_NAME} MATCHES "ex14p")
+        list(APPEND THIS_TEST_OPTIONS "-rs" "2" "-rp" "0" "-pa")
+      elseif (${TEST_NAME} MATCHES "ex14")
+        list(APPEND THIS_TEST_OPTIONS "-r" "2" "-pa")
+      endif()
      if (NOT (${TEST_NAME} MATCHES ".*p$"))
        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
          COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
@@ -646,10 +646,7 @@ real_t HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 real_t HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   real_t loc_energy = 0.5*M.InnerProduct(v, v);
-   real_t energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPITypeMap<real_t>::mpi_type,
-                 MPI_SUM, fespace.GetComm());
+   real_t energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -18,6 +18,12 @@
 //               ex14 -m ../data/amr-quad.mesh -r 3
 //               ex14 -m ../data/amr-hex.mesh
 //               ex14 -m ../data/fichera-amr.mesh
+//               ex14 -pa -r 1 -o 3
+//               ex14 -pa -r 1 -o 3 -m ../data/fichera.mesh
+//
+// Device sample runs:
+//               ex14 -pa -r 2 -d cuda -o 3
+//               ex14 -pa -r 2 -d cuda -o 3 -m ../data/fichera.mesh
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -46,7 +52,9 @@ int main(int argc, char *argv[])
   real_t sigma = -1.0;
   real_t kappa = -1.0;
   real_t eta = 0.0;
+   bool pa = false;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -62,9 +70,13 @@ int main(int argc, char *argv[])
                  "One of the three DG penalty parameters, should be positive."
                  " Negative values are replaced with (order+1)^2.");
   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -77,117 +89,129 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
   //    NURBS meshes are projected to second order meshes.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   Mesh mesh(mesh_file);
+   const int dim = mesh.Dimension();

-   // 3. Refine the mesh to increase the resolution. In this example we do
+   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. By default, or if ref_levels < 0,
   //    we choose it to be the largest number that gives a final mesh with no
   //    more than 50,000 elements.
   {
      if (ref_levels < 0)
      {
-         ref_levels = (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+         ref_levels = (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
      }
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh->UniformRefinement();
+         mesh.UniformRefinement();
      }
   }
-   if (mesh->NURBSext)
+   if (mesh.NURBSext)
   {
-      mesh->SetCurvature(max(order, 1));
+      mesh.SetCurvature(max(order, 1));
   }

-   // 4. Define a finite element space on the mesh. Here we use discontinuous
+   // 5. Define a finite element space on the mesh. Here we use discontinuous
   //    finite elements of the specified order >= 0.
-   FiniteElementCollection *fec = new DG_FECollection(order, dim);
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of unknowns: " << fespace->GetVSize() << endl;
+   const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
+   DG_FECollection fec(order, dim, bt);
+   FiniteElementSpace fespace(&mesh, &fec);
+   cout << "Number of unknowns: " << fespace.GetVSize() << endl;

-   // 5. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system.
-   LinearForm *b = new LinearForm(fespace);
+   LinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   ConstantCoefficient zero(0.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->AddBdrFaceIntegrator(
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.AddBdrFaceIntegrator(
      new DGDirichletLFIntegrator(zero, one, sigma, kappa));
-   b->Assemble();
+   b.Assemble();

-   // 6. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
-   GridFunction x(fespace);
+   GridFunction x(&fespace);
   x = 0.0;

-   // 7. Set up the bilinear form a(.,.) on the finite element space
+   // 8. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    domain integrator and the interior and boundary DG face integrators.
   //    Note that boundary conditions are imposed weakly in the form, so there
   //    is no need for dof elimination. After assembly and finalizing we
   //    extract the corresponding sparse matrix A.
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   BilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      MFEM_VERIFY(!pa, "BR2 not yet compatible with partial assembly.");
+      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
-   a->Assemble();
-   a->Finalize();
-   const SparseMatrix &A = a->SpMat();
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.Assemble();
+   a.Finalize();

-#ifndef MFEM_USE_SUITESPARSE
-   // 8. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   // 9. Define a simple symmetric Gauss-Seidel preconditioner and use it to
   //    solve the system Ax=b with PCG in the symmetric case, and GMRES in the
-   //    non-symmetric one.
-   GSSmoother M(A);
-   if (sigma == -1.0)
+   //    non-symmetric one. (Note that tolerances are squared: 1e-12 corresponds
+   //    to a relative tolerance of 1e-6).
+   //
+   //    If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   if (pa)
   {
-      PCG(A, M, *b, x, 1, 500, 1e-12, 0.0);
+      MFEM_VERIFY(sigma == -1.0,
+                  "The case of PA with sigma != -1 is not yet supported.");
+      CG(a, b, x, 1, 500, 1e-12, 0.0);
   }
   else
   {
-      GMRES(A, M, *b, x, 1, 500, 10, 1e-12, 0.0);
-   }
+      const SparseMatrix &A = a.SpMat();
+#ifndef MFEM_USE_SUITESPARSE
+      GSSmoother M(A);
+      if (sigma == -1.0)
+      {
+         PCG(A, M, b, x, 1, 500, 1e-12, 0.0);
+      }
+      else
+      {
+         GMRES(A, M, b, x, 1, 500, 10, 1e-12, 0.0);
+      }
 #else
-   // 8. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(A);
-   umf_solver.Mult(*b, x);
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(A);
+      umf_solver.Mult(b, x);
 #endif
+   }

-   // 9. Save the refined mesh and the solution. This output can be viewed later
-   //    using GLVis: "glvis -m refined.mesh -g sol.gf".
+   // 10. Save the refined mesh and the solution. This output can be viewed
+   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
   ofstream mesh_ofs("refined.mesh");
   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
+   mesh.Print(mesh_ofs);
   ofstream sol_ofs("sol.gf");
   sol_ofs.precision(8);
   x.Save(sol_ofs);

-   // 10. Send the solution by socket to a GLVis server.
+   // 11. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
+      sol_sock << "solution\n" << mesh << x << flush;
   }

-   // 11. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete mesh;
-
   return 0;
 }
@@ -17,6 +17,12 @@
 //               mpirun -np 4 ex14p -m ../data/inline-segment.mesh -rs 5
 //               mpirun -np 4 ex14p -m ../data/amr-quad.mesh -rs 3
 //               mpirun -np 4 ex14p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -o 3
+//               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -m ../data/fichera.mesh -o 3
+//
+// Device sample runs:
+//               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -o 3
+//               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -m ../data/fichera.mesh -o 3
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -38,11 +44,14 @@ using namespace mfem;

 class CustomSolverMonitor : public IterativeSolverMonitor
 {
+private:
+   const ParMesh &pmesh;
+   ParGridFunction &pgf;
 public:
-   CustomSolverMonitor(const ParMesh *m,
-                       ParGridFunction *f) :
-      pmesh(m),
-      pgf(f) {}
+   CustomSolverMonitor(const ParMesh &pmesh_,
+                       ParGridFunction &pgf_) :
+      pmesh(pmesh_),
+      pgf(pgf_) {}

   void MonitorSolution(int i, real_t norm, const Vector &x, bool final)
   {
@@ -50,30 +59,24 @@ public:
      int  visport   = 19916;
      int  num_procs, myid;

-      MPI_Comm_size(pmesh->GetComm(),&num_procs);
-      MPI_Comm_rank(pmesh->GetComm(),&myid);
+      MPI_Comm_size(pmesh.GetComm(), &num_procs);
+      MPI_Comm_rank(pmesh.GetComm(), &myid);

-      pgf->SetFromTrueDofs(x);
+      pgf.SetFromTrueDofs(x);

      socketstream sol_sock(vishost, visport);
      sol_sock << "parallel " << num_procs << " " << myid << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << *pgf
+      sol_sock << "solution\n" << pmesh << pgf
               << "window_title 'Iteration no " << i << "'"
               << "keys rRjlc\n" << flush;
   }
-
-private:
-   const ParMesh *pmesh;
-   ParGridFunction *pgf;
 };

 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI and HYPRE.
   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
   Hypre::Init();

   // 2. Parse command-line options.
@@ -84,7 +87,9 @@ int main(int argc, char *argv[])
   real_t sigma = -1.0;
   real_t kappa = -1.0;
   real_t eta = 0.0;
+   bool pa = false;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -103,13 +108,17 @@ int main(int argc, char *argv[])
                  "One of the three DG penalty parameters, should be positive."
                  " Negative values are replaced with (order+1)^2.");
   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
-      if (myid == 0)
+      if (Mpi::Root())
      {
         args.PrintUsage(cout);
      }
@@ -119,16 +128,19 @@ int main(int argc, char *argv[])
   {
      kappa = (order+1)*(order+1);
   }
-   if (myid == 0)
+   if (Mpi::Root())
   {
      args.PrintOptions(cout);
   }

+   Device device(device_config);
+   if (Mpi::Root()) { device.Print(); }
+
   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral and hexahedral meshes
   //    with the same code. NURBS meshes are projected to second order meshes.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   Mesh mesh(mesh_file);
+   int dim = mesh.Dimension();

   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ser_ref_levels' of uniform refinement. By default,
@@ -137,53 +149,54 @@ int main(int argc, char *argv[])
   {
      if (ser_ref_levels < 0)
      {
-         ser_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+         ser_ref_levels = (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
      }
      for (int l = 0; l < ser_ref_levels; l++)
      {
-         mesh->UniformRefinement();
+         mesh.UniformRefinement();
      }
   }
-   if (mesh->NURBSext)
+   if (mesh.NURBSext)
   {
-      mesh->SetCurvature(max(order, 1));
+      mesh.SetCurvature(max(order, 1));
   }

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
   {
      for (int l = 0; l < par_ref_levels; l++)
      {
-         pmesh->UniformRefinement();
+         pmesh.UniformRefinement();
      }
   }

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use discontinuous finite elements of the specified order >= 0.
-   FiniteElementCollection *fec = new DG_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
-   if (myid == 0)
+   const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
+   DG_FECollection fec(order, dim, bt);
+   ParFiniteElementSpace fespace(&pmesh, &fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   if (Mpi::Root())
   {
      cout << "Number of unknowns: " << size << endl;
   }

   // 7. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system.
-   ParLinearForm *b = new ParLinearForm(fespace);
+   ParLinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   ConstantCoefficient zero(0.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->AddBdrFaceIntegrator(
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.AddBdrFaceIntegrator(
      new DGDirichletLFIntegrator(zero, one, sigma, kappa));
-   b->Assemble();
+   b.Assemble();

   // 8. Define the solution vector x as a parallel finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
-   ParGridFunction x(fespace);
+   ParGridFunction x(&fespace);
   x = 0.0;

   // 9. Set up the bilinear form a(.,.) on the finite element space
@@ -192,42 +205,51 @@ int main(int argc, char *argv[])
   //    Note that boundary conditions are imposed weakly in the form, so there
   //    is no need for dof elimination. After serial and parallel assembly we
   //    extract the corresponding parallel matrix A.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      MFEM_VERIFY(!pa, "BR2 not yet compatible with partial assembly.");
+      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
-   a->Assemble();
-   a->Finalize();
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.Assemble();
+   a.Finalize();

   // 10. Define the parallel (hypre) matrix and vectors representing a(.,.),
   //     b(.) and the finite element approximation.
-   HypreParMatrix *A = a->ParallelAssemble();
-   HypreParVector *B = b->ParallelAssemble();
-   HypreParVector *X = x.ParallelProject();
+   OperatorHandle A;

-   delete a;
-   delete b;
-
-   // 11. Depending on the symmetry of A, define and apply a parallel PCG or
-   //     GMRES solver for AX=B using the BoomerAMG preconditioner from hypre.
-   HypreSolver *amg = new HypreBoomerAMG(*A);
-   if (sigma == -1.0)
+   std::unique_ptr<HypreBoomerAMG> amg;
+   if (pa)
   {
-      HyprePCG pcg(*A);
-      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(500);
-      pcg.SetPrintLevel(2);
-      pcg.SetPreconditioner(*amg);
-      pcg.Mult(*B, *X);
+      A.Reset(&a, false);
   }
   else
   {
-      CustomSolverMonitor monitor(pmesh, &x);
+      A.SetType(Operator::Hypre_ParCSR);
+      a.ParallelAssemble(A);
+      amg.reset(new HypreBoomerAMG(*A.As<HypreParMatrix>()));
+   }
+
+   // 11. Depending on the symmetry of A, define and apply a parallel PCG or
+   //     GMRES solver for AX=B using the BoomerAMG preconditioner from hypre.
+   if (sigma == -1.0)
+   {
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(500);
+      cg.SetPrintLevel(1);
+      cg.SetOperator(*A);
+      if (amg) { cg.SetPreconditioner(*amg); }
+      cg.Mult(b, x);
+   }
+   else
+   {
+      CustomSolverMonitor monitor(pmesh, x);
      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetAbsTol(0.0);
      gmres.SetRelTol(1e-12);
@@ -235,51 +257,37 @@ int main(int argc, char *argv[])
      gmres.SetKDim(10);
      gmres.SetPrintLevel(1);
      gmres.SetOperator(*A);
-      gmres.SetPreconditioner(*amg);
+      if (amg) { gmres.SetPreconditioner(*amg); }
      gmres.SetMonitor(monitor);
-      gmres.Mult(*B, *X);
+      gmres.Mult(b, x);
   }
-   delete amg;

-   // 12. Extract the parallel grid function corresponding to the finite element
-   //     approximation X. This is the local solution on each processor.
-   x = *X;
-
-   // 13. Save the refined mesh and the solution in parallel. This output can
+   // 12. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      mesh_name << "mesh." << setfill('0') << setw(6) << Mpi::WorldRank();
+      sol_name << "sol." << setfill('0') << setw(6) << Mpi::WorldRank();

      ofstream mesh_ofs(mesh_name.str().c_str());
      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
+      pmesh.Print(mesh_ofs);

      ofstream sol_ofs(sol_name.str().c_str());
      sol_ofs.precision(8);
      x.Save(sol_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock << "parallel " << Mpi::WorldSize() << " " << Mpi::WorldRank() << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
+      sol_sock << "solution\n" << pmesh << x << flush;
   }

-   // 15. Free the used memory.
-   delete X;
-   delete B;
-   delete A;
-
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
   return 0;
 }
@@ -39,8 +39,8 @@ private:
   // Base Nonlinear Form
   std::unique_ptr<NonlinearForm> nonlinearForm;
   // element-wise inverse mass matrix
-   std::vector<DenseMatrix> invmass; // local scalar inverse mass.
-   std::vector<DenseMatrix> weakdiv; // local weakdivergence. Trial space is ByDim.
+   std::vector<DenseMatrix> invmass; // local scalar inverse mass
+   std::vector<DenseMatrix> weakdiv; // local weak divergence (trial space ByDim)
   // global maximum characteristic speed. Updated by form integrators
   mutable real_t max_char_speed;
   // auxiliary variable used in Mult
@@ -169,9 +169,9 @@ void DGHyperbolicConservationLaws::Mult(const Vector &x, Vector &y) const
 {
   // 0. Reset wavespeed computation before operator application.
   formIntegrator->ResetMaxCharSpeed();
-   // 1. Apply Nonlinear form to obtain an axiliary result
+   // 1. Apply Nonlinear form to obtain an auxiliary result
   //         z = - <F̂(u_h,n), [[v]]>_e
-   //    If weak-divergencee is not preassembled, we also have weak-divergence
+   //    If weak-divergence is not preassembled, we also have weak-divergence
   //         z = - <F̂(u_h,n), [[v]]>_e + (F(u_h), ∇v)
   nonlinearForm->Mult(x, z);
   if (!weakdiv.empty()) // if weak divergence is pre-assembled
@@ -44,7 +44,7 @@ protected:
   BilinearForm *M;
   BilinearForm *K;

-   SparseMatrix Mmat, Kmat, Kmat0;
+   SparseMatrix Mmat, Kmat;
   SparseMatrix *T; // T = M + dt K
   real_t current_dt;

@@ -83,25 +83,24 @@ WaveOperator::WaveOperator(FiniteElementSpace &f,
   : SecondOrderTimeDependentOperator(f.GetTrueVSize(), (real_t) 0.0),
     fespace(f), M(NULL), K(NULL), T(NULL), current_dt(0.0), z(height)
 {
-   const real_t rel_tol = 1e-8;
-
-   fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
+   // Assemble Laplace matrix
   c2 = new ConstantCoefficient(speed*speed);
-
   K = new BilinearForm(&fespace);
   K->AddDomainIntegrator(new DiffusionIntegrator(*c2));
   K->Assemble();

-   Array<int> dummy;
-   K->FormSystemMatrix(dummy, Kmat0);
-   K->FormSystemMatrix(ess_tdof_list, Kmat);
-
+   // Assemble Mass matrix
   M = new BilinearForm(&fespace);
   M->AddDomainIntegrator(new MassIntegrator());
   M->Assemble();
+
+   // Apply Bcs
+   fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   K->FormSystemMatrix(ess_tdof_list, Kmat);
   M->FormSystemMatrix(ess_tdof_list, Mmat);

+   // Configure preconditioner
+   const real_t rel_tol = 1e-8;
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(rel_tol);
   M_solver.SetAbsTol(0.0);
@@ -110,14 +109,13 @@ WaveOperator::WaveOperator(FiniteElementSpace &f,
   M_solver.SetPreconditioner(M_prec);
   M_solver.SetOperator(Mmat);

+   // Configure solver
   T_solver.iterative_mode = false;
   T_solver.SetRelTol(rel_tol);
   T_solver.SetAbsTol(0.0);
   T_solver.SetMaxIter(100);
   T_solver.SetPrintLevel(0);
   T_solver.SetPreconditioner(T_prec);
-
-   T = NULL;
 }

 void WaveOperator::Mult(const Vector &u, const Vector &du_dt,
@@ -126,9 +124,11 @@ void WaveOperator::Mult(const Vector &u, const Vector &du_dt,
   // Compute:
   //    d2udt2 = M^{-1}*-K(u)
   // for d2udt2
-   Kmat.Mult(u, z);
+   K->FullMult(u, z);
   z.Neg(); // z = -z
+   z.SetSubVector(ess_tdof_list, 0.0);
   M_solver.Mult(z, d2udt2);
+   d2udt2.SetSubVector(ess_tdof_list, 0.0);
 }

 void WaveOperator::ImplicitSolve(const real_t fac0, const real_t fac1,
@@ -142,14 +142,11 @@ void WaveOperator::ImplicitSolve(const real_t fac0, const real_t fac1,
      T = Add(1.0, Mmat, fac0, Kmat);
      T_solver.SetOperator(*T);
   }
-   Kmat0.Mult(u, z);
+   K->FullMult(u, z);
   z.Neg();
-
-   for (int i = 0; i < ess_tdof_list.Size(); i++)
-   {
-      z[ess_tdof_list[i]] = 0.0;
-   }
+   z.SetSubVector(ess_tdof_list, 0.0);
   T_solver.Mult(z, d2udt2);
+   d2udt2.SetSubVector(ess_tdof_list, 0.0);
 }

 void WaveOperator::SetParameters(const Vector &u)
@@ -314,7 +311,6 @@ int main(int argc, char *argv[])
         ess_bdr = 0;
      }
   }
-
   WaveOperator oper(fespace, ess_bdr, speed);

   u_gf.SetFromTrueDofs(u);
@@ -19,8 +19,11 @@
 //    ex33 -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
 //    ex33 -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1
 //
-//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
-//        for all alpha.
+//  Note: The manufactured solution used in this problem is
+//
+//            u = ∏_{i=0}^{dim-1} sin(π x_i) ,
+//
+//        regardless of the value of alpha.
 //
 // Description:
 //
@@ -114,7 +117,8 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
                  "--no-verification",
-                  "Use sinusoidal function (f) for analytic comparison.");
+                  "Use sinusoidal function (f) for manufactured "
+                  "solution test.");
   args.Parse();
   if (!args.Good())
   {
@@ -163,7 +167,7 @@ int main(int argc, char *argv[])
   // 5. Define a finite element space on the mesh.
   H1_FECollection fec(order, dim);
   FiniteElementSpace fespace(&mesh, &fec);
-   cout << "Number of finite element unknowns: "
+   cout << "Number of degrees of freedom: "
        << fespace.GetTrueVSize() << endl;

   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
@@ -379,29 +383,29 @@ int main(int argc, char *argv[])
      FunctionCoefficient sol(solution);
      real_t l2_error = u.ComputeL2Error(sol);

-      string analytic_solution,expected_mesh;
+      string manufactured_solution,expected_mesh;
      switch (dim)
      {
         case 1:
-            analytic_solution = "sin(π x)";
+            manufactured_solution = "sin(π x)";
            expected_mesh = "inline_segment.mesh";
            break;
         case 2:
-            analytic_solution = "sin(π x) sin(π y)";
+            manufactured_solution = "sin(π x) sin(π y)";
            expected_mesh = "inline_quad.mesh";
            break;
         default:
-            analytic_solution = "sin(π x) sin(π y) sin(π z)";
+            manufactured_solution = "sin(π x) sin(π y) sin(π z)";
            expected_mesh = "inline_hex.mesh";
            break;
      }

      mfem::out << "\n" << string(80,'=')
                << "\n\nSolution Verification in "<< dim << "D \n\n"
-                << "Analytic solution : " << analytic_solution << "\n"
-                << "Expected mesh     : " << expected_mesh <<"\n"
-                << "Your mesh         : " << mesh_file << "\n"
-                << "L2 error          : " << l2_error << "\n\n"
+                << "Manufactured solution : " << manufactured_solution << "\n"
+                << "Expected mesh         : " << expected_mesh <<"\n"
+                << "Your mesh             : " << mesh_file << "\n"
+                << "L2 error              : " << l2_error << "\n\n"
                << string(80,'=') << endl;
   }

@@ -131,7 +131,7 @@ void RationalApproximation_AAA(const Vector &val, const Vector &pt,
      }

 #ifdef MFEM_USE_LAPACK
-      DenseMatrixSVD svd(Am,false,true);
+      DenseMatrixSVD svd(Am,'N','A');
      svd.Eval(Am);
      DenseMatrix &v = svd.RightSingularvectors();
      v.GetRow(k,w);
@@ -346,7 +346,7 @@ void ComputePartialFractionApproximation(real_t & alpha,
   }
   else
   {
-      if (abs(alpha - 0.5) > eps && print_warning)
+      if (abs(alpha - 0.5) > eps)
      {
         alpha = 0.5;
      }
@@ -368,6 +368,8 @@ void ComputePartialFractionApproximation(real_t & alpha,


   return;
+#else
+   MFEM_CONTRACT_VAR(print_warning);
 #endif

   Vector x(npoints);
@@ -19,8 +19,11 @@
 //    mpirun -np 4 ex33p -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
 //    mpirun -np 4 ex33p -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1

-//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
-//        for all alpha.
+//  Note: The manufactured solution used in this problem is
+//
+//            u = ∏_{i=0}^{dim-1} sin(π x_i) ,
+//
+//        regardless of the value of alpha.
 //
 // Description:
 //
@@ -120,7 +123,8 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
                  "--no-verification",
-                  "Use sinusoidal function (f) for analytic comparison.");
+                  "Use sinusoidal function (f) for manufactured "
+                  "solution test.");
   args.Parse();
   if (!args.Good())
   {
@@ -180,10 +184,11 @@ int main(int argc, char *argv[])
   // 5. Define a finite element space on the mesh.
   H1_FECollection fec(order, dim);
   ParFiniteElementSpace fespace(&pmesh, &fec);
+   HYPRE_BigInt size = fespace.GlobalTrueVSize();
   if (Mpi::Root())
   {
-      cout << "Number of finite element unknowns: "
-           << fespace.GetTrueVSize() << endl;
+      cout << "Number of degrees of freedom: "
+           << size << endl;
   }

   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
@@ -223,7 +228,7 @@ int main(int argc, char *argv[])
   if (verification)
   {
      // This statement is only relevant for the verification of the code. It
-      // uses a different f such that an analytic solution is known and easy
+      // uses a different f such that an manufactured solution is known and easy
      // to compare with the numerical one. The FPDE becomes:
      // (-Δ)^α u = (2\pi ^2)^α sin(\pi x) sin(\pi y) on [0,1]^2
      // -> u(x,y) = sin(\pi x) sin(\pi y)
@@ -415,29 +420,29 @@ int main(int argc, char *argv[])

      if (Mpi::Root())
      {
-         string analytic_solution,expected_mesh;
+         string manufactured_solution,expected_mesh;
         switch (dim)
         {
            case 1:
-               analytic_solution = "sin(π x)";
+               manufactured_solution = "sin(π x)";
               expected_mesh = "inline_segment.mesh";
               break;
            case 2:
-               analytic_solution = "sin(π x) sin(π y)";
+               manufactured_solution = "sin(π x) sin(π y)";
               expected_mesh = "inline_quad.mesh";
               break;
            default:
-               analytic_solution = "sin(π x) sin(π y) sin(π z)";
+               manufactured_solution = "sin(π x) sin(π y) sin(π z)";
               expected_mesh = "inline_hex.mesh";
               break;
         }

         mfem::out << "\n" << string(80,'=')
                   << "\n\nSolution Verification in "<< dim << "D \n\n"
-                   << "Analytic solution : " << analytic_solution << "\n"
-                   << "Expected mesh     : " << expected_mesh <<"\n"
-                   << "Your mesh         : " << mesh_file << "\n"
-                   << "L2 error          : " << l2_error << "\n\n"
+                   << "Manufactured solution : " << manufactured_solution << "\n"
+                   << "Expected mesh         : " << expected_mesh <<"\n"
+                   << "Your mesh             : " << mesh_file << "\n"
+                   << "L2 error              : " << l2_error << "\n\n"
                   << string(80,'=') << endl;
      }
   }
@@ -199,7 +199,6 @@ public:
      {
         mesh->GetElementTransformation(elem, &Tr);
         MFIRs.GetSurfaceIntegrationRule(Tr, ir);
-         Vector w;
         MFIRs.GetSurfaceWeights(Tr, ir, w);
         SurfaceWeights.SetCol(elem, w);

@@ -0,0 +1,374 @@
+//                                MFEM Example 40
+//
+// Compile with: make ex40
+//
+// Sample runs: ex40 -step 10 -gr 2.0
+//              ex40 -step 10 -gr 2.0 -o 3 -r 1
+//              ex40 -step 10 -gr 2.0 -r 4 -m ../data/l-shape.mesh
+//              ex40 -step 10 -gr 2.0 -r 2 -m ../data/fichera.mesh
+//
+// Description: This example code demonstrates how to use MFEM to solve the
+//              eikonal equation,
+//
+//                      |∇𝑢| = 1 in Ω,  𝑢 = g on ∂Ω.
+//
+//              The solution of this problem coincides with the unique optimum of
+//              the nonlinear program
+//
+//                   maximize ∫_Ω 𝑢 d𝑥 subject to |∇𝑢| ≤ 1,   𝑢 = g on Ω,      (⋆)
+//
+//              which is the foundation for method implemented below.
+//
+//              Following the proximal Galerkin methodology [1] (see also Example
+//              36), we construct a Legendre function for the unit ball
+//              𝐵₁ := {𝑥 ∈ Rⁿ | |𝑥| < 1}. Our choice is the Hellinger entropy,
+//
+//                    h(𝑥) = −( 1 − |𝑥|² )^{1/2},
+//
+//              although other choices are possible, each leading to a slightly
+//              different algorithm. We then adaptively regularize the optimization
+//              problem (⋆) with the Bregman divergence of the Hellinger entropy,
+//
+//                 maximize  ∫_Ω 𝑢 d𝑥 - αₖ⁻¹ Dₕ(∇𝑢,∇𝑢ₖ₋₁)  subject to  𝑢 = g on Ω.
+//
+//              This results in a sequence of functions ( 𝜓ₖ , 𝑢ₖ ),
+//
+//                      𝑢ₖ → 𝑢,    𝜓ₖ/|𝜓ₖ| → ∇𝑢    as k → \infty,
+//
+//              defined by the nonlinear saddle-point problems
+//
+//               Find 𝜓ₖ ∈ H(div,Ω) and 𝑢ₖ ∈ L²(Ω) such that
+//               ( Zₖ(𝜓ₖ) , τ ) + ( 𝑢ₖ , ∇⋅τ ) = ⟨ g , τ⋅n ⟩         ∀ τ ∈ H(div,Ω)
+//               ( ∇⋅𝜓ₖ , v )                 = ( ∇⋅𝜓ₖ₋₁ - 1 , v )   ∀ v ∈ L²(Ω)
+//
+//              where Zₖ(𝜓) := ∇h⁻¹(αₖ 𝜓) = 𝜓 / ( αₖ⁻² + |𝜓|² )^{1/2} and step size
+//              αₖ > 0. These saddle-point problems are solved using a damped Newton's
+//              method. This example assumes that g = 0 and allows the step size to
+//              grow geometrically, αₖ = α₀rᵏ, where r ≥ 1 is the growth rate.
+//
+//              [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//                  preserving finite element method for pointwise bound constraints.
+//                  arXiv:2307.12444 [math.NA]
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ZCoefficient : public VectorCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   ZCoefficient(int vdim, GridFunction &psi_, real_t alpha_ = 1.0)
+      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+class DZCoefficient : public MatrixCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   DZCoefficient(int height, GridFunction &psi_, real_t alpha_ = 1.0)
+      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int max_it = 5;
+   int ref_levels = 3;
+   real_t alpha = 1.0;
+   real_t growth_rate = 1.0;
+   real_t newton_scaling = 0.9;
+   real_t tichonov = 1e-1;
+   real_t tol = 1e-4;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Initial size alpha");
+   args.AddOption(&growth_rate, "-gr", "--growth-rate",
+                  "Growth rate of the step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+
+   MFEM_ASSERT(mesh.bdr_attributes.Size(),
+               "This example does not currently support meshes"
+               " without boundary attributes."
+              )
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   RT_FECollection RTfec(order, dim);
+   FiniteElementSpace RTfes(&mesh, &RTfec);
+
+   L2_FECollection L2fec(order, dim);
+   FiniteElementSpace L2fes(&mesh, &L2fec);
+
+   cout << "Number of H(div) dofs: "
+        << RTfes.GetTrueVSize() << endl;
+   cout << "Number of L² dofs: "
+        << L2fes.GetTrueVSize() << endl;
+
+   // 5. Define the offsets for the block matrices
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = RTfes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   offsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   // 6. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   GridFunction u_gf, delta_psi_gf;
+   delta_psi_gf.MakeRef(&RTfes,x,offsets[0]);
+   u_gf.MakeRef(&L2fes,x,offsets[1]);
+
+   GridFunction psi_old_gf(&RTfes);
+   GridFunction psi_gf(&RTfes);
+   GridFunction u_old_gf(&L2fes);
+
+   // 7. Define initial guesses for the solution variables.
+   delta_psi_gf = 0.0;
+   psi_gf = 0.0;
+   u_gf = 0.0;
+   psi_old_gf = psi_gf;
+   u_old_gf = u_gf;
+
+   // 8. Prepare for glvis output.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 9. Coefficients to be used later.
+   ConstantCoefficient neg_one(-1.0);
+   ConstantCoefficient zero(0.0);
+   ConstantCoefficient tichonov_cf(tichonov);
+   ConstantCoefficient neg_tichonov_cf(-1.0*tichonov);
+   ZCoefficient Z(sdim, psi_gf, alpha);
+   DZCoefficient DZ(sdim, psi_gf, alpha);
+   ScalarVectorProductCoefficient neg_Z(-1.0, Z);
+   DivergenceGridFunctionCoefficient div_psi_cf(&psi_gf);
+   DivergenceGridFunctionCoefficient div_psi_old_cf(&psi_old_gf);
+   SumCoefficient psi_old_minus_psi(div_psi_old_cf, div_psi_cf, 1.0, -1.0);
+
+   // 10. Assemble constant matrices/vectors to avoid reassembly in the loop.
+   LinearForm b0, b1;
+   b0.MakeRef(&RTfes,rhs.GetBlock(0),0);
+   b1.MakeRef(&L2fes,rhs.GetBlock(1),0);
+
+   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_Z));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+
+   BilinearForm a00(&RTfes);
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(DZ));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(tichonov_cf));
+
+   MixedBilinearForm a10(&RTfes,&L2fes);
+   a10.AddDomainIntegrator(new VectorFEDivergenceIntegrator());
+   a10.Assemble();
+   a10.Finalize();
+   SparseMatrix &A10 = a10.SpMat();
+   SparseMatrix *A01 = Transpose(A10);
+
+   BilinearForm a11(&L2fes);
+   a11.AddDomainIntegrator(new MassIntegrator(neg_tichonov_cf));
+   a11.Assemble();
+   a11.Finalize();
+   SparseMatrix &A11 = a11.SpMat();
+
+   // 11. Iterate.
+   int k;
+   int total_iterations = 0;
+   real_t increment_u = 0.1;
+   GridFunction u_tmp(&L2fes);
+   for (k = 0; k < max_it; k++)
+   {
+      u_tmp = u_old_gf;
+      Z.SetAlpha(alpha);
+      DZ.SetAlpha(alpha);
+
+      mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+
+      int j;
+      for ( j = 0; j < 5; j++)
+      {
+         total_iterations++;
+
+         b0.Assemble();
+         b1.Assemble();
+
+         a00.Assemble(false);
+         a00.Finalize(false);
+         SparseMatrix &A00 = a00.SpMat();
+
+         // Construct Schur-complement preconditioner
+         Vector A00_diag(a00.Height());
+         A00.GetDiag(A00_diag);
+         A00_diag.Reciprocal();
+         SparseMatrix *S = Mult_AtDA(*A01, A00_diag);
+
+         BlockDiagonalPreconditioner prec(offsets);
+         prec.SetDiagonalBlock(0,new DSmoother(A00));
+#ifndef MFEM_USE_SUITESPARSE
+         prec.SetDiagonalBlock(1,new GSSmoother(*S));
+#else
+         prec.SetDiagonalBlock(1,new UMFPackSolver(*S));
+#endif
+         prec.owns_blocks = 1;
+
+         BlockOperator A(offsets);
+         A.SetBlock(0,0,&A00);
+         A.SetBlock(1,0,&A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,&A11);
+
+         GMRES(A,prec,rhs,x,0,2000,500,1e-12,0.0);
+         delete S;
+
+         u_tmp -= u_gf;
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         // Damped Newton update
+         psi_gf.Add(newton_scaling, delta_psi_gf);
+         a00.Update();
+
+         if (visualization)
+         {
+            sol_sock << "solution\n" << mesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+         }
+
+         mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      mfem::out << "Number of Newton iterations = " << j+1 << endl;
+      mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      alpha *= max(growth_rate, 1_r);
+
+   }
+
+   mfem::out << "\n Outer iterations: " << k+1
+             << "\n Total iterations: " << total_iterations
+             << "\n Total dofs:       " << RTfes.GetTrueVSize() + L2fes.GetTrueVSize()
+             << endl;
+
+   delete A01;
+   return 0;
+}
+
+void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
+                        const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(vdim);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   V = psi_vals;
+   V *= phi;
+}
+
+void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                         const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(height);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   K = 0.0;
+   for (int i = 0; i < height; i++)
+   {
+      K(i,i) = phi;
+      for (int j = 0; j < height; j++)
+      {
+         K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
+      }
+   }
+}
@@ -0,0 +1,436 @@
+//                       MFEM Example 40 - Parallel Version
+//
+// Compile with: make ex40p
+//
+// Sample runs: mpirun -np 4 ex40p -step 10 -gr 2.0
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -o 3 -r 1
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -r 4 -m ../data/l-shape.mesh
+//              mpirun -np 4 ex40p -step 10 -gr 2.0 -r 2 -m ../data/fichera.mesh
+//
+// Description: This example code demonstrates how to use MFEM to solve the
+//              eikonal equation,
+//
+//                      |∇𝑢| = 1 in Ω,  𝑢 = g on ∂Ω.
+//
+//              The solution of this problem coincides with the unique optimum of
+//              the nonlinear program
+//
+//                   maximize ∫_Ω 𝑢 d𝑥 subject to |∇𝑢| ≤ 1,   𝑢 = g on Ω,      (⋆)
+//
+//              which is the foundation for method implemented below.
+//
+//              Following the proximal Galerkin methodology [1] (see also Example
+//              36), we construct a Legendre function for the unit ball
+//              𝐵₁ := {𝑥 ∈ Rⁿ | |𝑥| < 1}. Our choice is the Hellinger entropy,
+//
+//                    h(𝑥) = −( 1 − |𝑥|² )^{1/2},
+//
+//              although other choices are possible, each leading to a slightly
+//              different algorithm. We then adaptively regularize the optimization
+//              problem (⋆) with the Bregman divergence of the Hellinger entropy,
+//
+//                 maximize  ∫_Ω 𝑢 d𝑥 - αₖ⁻¹ Dₕ(∇𝑢,∇𝑢ₖ₋₁)  subject to  𝑢 = g on Ω.
+//
+//              This results in a sequence of functions ( 𝜓ₖ , 𝑢ₖ ),
+//
+//                      𝑢ₖ → 𝑢,    𝜓ₖ/|𝜓ₖ| → ∇𝑢    as k → \infty,
+//
+//              defined by the nonlinear saddle-point problems
+//
+//               Find 𝜓ₖ ∈ H(div,Ω) and 𝑢ₖ ∈ L²(Ω) such that
+//               ( Zₖ(𝜓ₖ) , τ ) + ( 𝑢ₖ , ∇⋅τ ) = ⟨ g , τ⋅n ⟩         ∀ τ ∈ H(div,Ω)
+//               ( ∇⋅𝜓ₖ , v )                 = ( ∇⋅𝜓ₖ₋₁ - 1 , v )   ∀ v ∈ L²(Ω)
+//
+//              where Zₖ(𝜓) := ∇h⁻¹(αₖ 𝜓) = 𝜓 / ( αₖ⁻² + |𝜓|² )^{1/2} and step size
+//              αₖ > 0. These saddle-point problems are solved using a damped Newton's
+//              method. This example assumes that g = 0 and allows the step size to
+//              grow geometrically, αₖ = α₀rᵏ, where r ≥ 1 is the growth rate.
+//
+//              [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//                  preserving finite element method for pointwise bound constraints.
+//                  arXiv:2307.12444 [math.NA]
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ZCoefficient : public VectorCoefficient
+{
+protected:
+   ParGridFunction *psi;
+   real_t alpha;
+
+public:
+   ZCoefficient(int vdim, ParGridFunction &psi_, real_t alpha_ = 1.0)
+      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+class DZCoefficient : public MatrixCoefficient
+{
+protected:
+   ParGridFunction *psi;
+   real_t alpha;
+
+public:
+   DZCoefficient(int height, ParGridFunction &psi_, real_t alpha_ = 1.0)
+      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   void SetAlpha(real_t alpha_) { alpha = alpha_; }
+};
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int max_it = 5;
+   int ref_levels = 3;
+   real_t alpha = 1.0;
+   real_t growth_rate = 1.0;
+   real_t newton_scaling = 0.9;
+   real_t tichonov = 1e-1;
+   real_t tol = 1e-4;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Initial size alpha");
+   args.AddOption(&growth_rate, "-gr", "--growth-rate",
+                  "Growth rate of the step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2. Read the mesh from the mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+
+   MFEM_ASSERT(mesh.bdr_attributes.Size(),
+               "This example does not currently support meshes"
+               " without boundary attributes."
+              )
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   RT_FECollection RTfec(order, dim);
+   ParFiniteElementSpace RTfes(&pmesh, &RTfec);
+
+   L2_FECollection L2fec(order, dim);
+   ParFiniteElementSpace L2fes(&pmesh, &L2fec);
+
+   int num_dofs_RT = RTfes.GlobalTrueVSize();
+   int num_dofs_L2 = L2fes.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of H(div) dofs: "
+           << num_dofs_RT << endl;
+      cout << "Number of L² dofs: "
+           << num_dofs_L2 << endl;
+   }
+
+   // 5. Define the offsets for the block matrices
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = RTfes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   offsets.PartialSum();
+
+   Array<int> toffsets(3);
+   toffsets[0] = 0;
+   toffsets[1] = RTfes.GetTrueVSize();
+   toffsets[2] = L2fes.GetTrueVSize();
+   toffsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   BlockVector tx(toffsets), trhs(toffsets);
+   tx = 0.0; trhs = 0.0;
+
+   // 6. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   ParGridFunction u_gf, delta_psi_gf;
+   delta_psi_gf.MakeRef(&RTfes,x,offsets[0]);
+   u_gf.MakeRef(&L2fes,x,offsets[1]);
+
+   ParGridFunction psi_old_gf(&RTfes);
+   ParGridFunction psi_gf(&RTfes);
+   ParGridFunction u_old_gf(&L2fes);
+
+   // 7. Define initial guesses for the solution variables.
+   delta_psi_gf = 0.0;
+   psi_gf = 0.0;
+   u_gf = 0.0;
+   psi_old_gf = psi_gf;
+   u_old_gf = u_gf;
+
+   // 8. Prepare for glvis output.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 9. Coefficients to be used later.
+   ConstantCoefficient neg_one(-1.0);
+   ConstantCoefficient zero(0.0);
+   ConstantCoefficient tichonov_cf(tichonov);
+   ConstantCoefficient neg_tichonov_cf(-1.0*tichonov);
+   ZCoefficient Z(sdim, psi_gf, alpha);
+   DZCoefficient DZ(sdim, psi_gf, alpha);
+   ScalarVectorProductCoefficient neg_Z(-1.0, Z);
+   DivergenceGridFunctionCoefficient div_psi_cf(&psi_gf);
+   DivergenceGridFunctionCoefficient div_psi_old_cf(&psi_old_gf);
+   SumCoefficient psi_old_minus_psi(div_psi_old_cf, div_psi_cf, 1.0, -1.0);
+
+   // 10. Assemble constant matrices/vectors to avoid reassembly in the loop.
+   ParLinearForm b0, b1;
+   b0.MakeRef(&RTfes,rhs.GetBlock(0),0);
+   b1.MakeRef(&L2fes,rhs.GetBlock(1),0);
+
+   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_Z));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
+   b1.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+
+   ParBilinearForm a00(&RTfes);
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(DZ));
+   a00.AddDomainIntegrator(new VectorFEMassIntegrator(tichonov_cf));
+
+   ParMixedBilinearForm a10(&RTfes,&L2fes);
+   a10.AddDomainIntegrator(new VectorFEDivergenceIntegrator());
+   a10.Assemble();
+   a10.Finalize();
+   HypreParMatrix *A10 = a10.ParallelAssemble();
+
+   HypreParMatrix *A01 = A10->Transpose();
+
+   ParBilinearForm a11(&L2fes);
+   a11.AddDomainIntegrator(new MassIntegrator(neg_tichonov_cf));
+   a11.Assemble();
+   a11.Finalize();
+   HypreParMatrix *A11 = a11.ParallelAssemble();
+
+   // 11. Iterate.
+   int k;
+   int total_iterations = 0;
+   real_t increment_u = 0.1;
+   ParGridFunction u_tmp(&L2fes);
+   for (k = 0; k < max_it; k++)
+   {
+      u_tmp = u_old_gf;
+      Z.SetAlpha(alpha);
+      DZ.SetAlpha(alpha);
+
+      if (myid == 0)
+      {
+         mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+      }
+
+      int j;
+      for ( j = 0; j < 5; j++)
+      {
+         total_iterations++;
+
+         b0.Assemble();
+         b0.ParallelAssemble(trhs.GetBlock(0));
+
+         b1.Assemble();
+         b1.ParallelAssemble(trhs.GetBlock(1));
+
+         a00.Assemble(false);
+         a00.Finalize(false);
+         HypreParMatrix *A00 = a00.ParallelAssemble();
+
+         // Construct Schur-complement preconditioner
+         HypreParVector A00_diag(MPI_COMM_WORLD, A00->GetGlobalNumRows(),
+                                 A00->GetRowStarts());
+         A00->GetDiag(A00_diag);
+         HypreParMatrix S_tmp(*A01);
+         S_tmp.InvScaleRows(A00_diag);
+         HypreParMatrix *S = ParMult(A10, &S_tmp, true);
+
+         BlockDiagonalPreconditioner prec(toffsets);
+         HypreBoomerAMG P00(*A00);
+         P00.SetPrintLevel(0);
+         HypreBoomerAMG P11(*S);
+         P11.SetPrintLevel(0);
+         prec.SetDiagonalBlock(0,&P00);
+         prec.SetDiagonalBlock(1,&P11);
+
+         BlockOperator A(toffsets);
+         A.SetBlock(0,0,A00);
+         A.SetBlock(1,0,A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,A11);
+
+         GMRESSolver gmres(MPI_COMM_WORLD);
+         gmres.SetPrintLevel(-1);
+         gmres.SetRelTol(1e-8);
+         gmres.SetMaxIter(2000);
+         gmres.SetKDim(500);
+         gmres.SetOperator(A);
+         gmres.SetPreconditioner(prec);
+         gmres.Mult(trhs,tx);
+         delete S;
+         delete A00;
+
+         delta_psi_gf.SetFromTrueDofs(tx.GetBlock(0));
+         u_gf.SetFromTrueDofs(tx.GetBlock(1));
+
+         u_tmp -= u_gf;
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         // Damped Newton update
+         psi_gf.Add(newton_scaling, delta_psi_gf);
+         a00.Update();
+
+         if (visualization)
+         {
+            sol_sock << "parallel " << num_procs << " " << myid << "\n";
+            sol_sock << "solution\n" << pmesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+         }
+
+         if (myid == 0)
+         {
+            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+         }
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      if (myid == 0)
+      {
+         mfem::out << "Number of Newton iterations = " << j+1 << endl;
+         mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+      }
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      alpha *= max(growth_rate, 1_r);
+
+   }
+
+   // 12. Print stats.
+   if (myid == 0)
+   {
+      mfem::out << "\n Outer iterations: " << k+1
+                << "\n Total iterations: " << total_iterations
+                << "\n Total dofs:       " << RTfes.GetTrueVSize() + L2fes.GetTrueVSize()
+                << endl;
+   }
+
+   // 13. Free the used memory.
+   delete A01;
+   delete A10;
+   delete A11;
+   return 0;
+}
+
+void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
+                        const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(vdim);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   V = psi_vals;
+   V *= phi;
+}
+
+void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                         const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(height);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
+
+   K = 0.0;
+   for (int i = 0; i < height; i++)
+   {
+      K(i,i) = phi;
+      for (int j = 0; j < height; j++)
+      {
+         K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
+      }
+   }
+}
@@ -23,14 +23,14 @@ MFEM_LIB_FILE = mfem_is_not_built

 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
   ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-   ex31 ex33 ex34 ex36 ex37 ex38 ex39
+   ex31 ex33 ex34 ex36 ex37 ex38 ex39 ex40
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
   ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-   ex37p ex39p
-SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
-PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-   ex24p ex25p ex26p ex34p ex35p
+   ex37p ex39p ex40p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
+PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
+   ex22p ex24p ex25p ex26p ex34p ex35p

 ifeq ($(MFEM_USE_LAPACK),YES)
   SEQ_EXAMPLES += ex38
@@ -138,6 +138,14 @@ ex10-test-seq: ex10
 	@$(call mfem-test,$<,, Serial example,-tf 5)
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-tf 5)
+ex14-test-seq-cuda: ex14
+	@$(call mfem-test,$<,, Serial CUDA example,-r 2 -pa -d cuda)
+ex14p-test-par-cuda: ex14p
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-rs 2 -rp 0 -pa -d cuda)
+ex14-test-seq-hip: ex14
+	@$(call mfem-test,$<,, Serial HIP example,-r 2 -pa -d hip)
+ex14p-test-par-hip: ex14p
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-rs 2 -rp 0 -pa -d hip)
 ex15-test-seq: ex15
 	@$(call mfem-test,$<,, Serial example,-e 1)
 ex15p-test-par: ex15p
@@ -1,3 +1,14 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
 #include <algorithm>
 #include <assert.h>
 #include <cstdlib>
@@ -709,10 +709,7 @@ real_t HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 real_t HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   real_t loc_energy = 0.5*M.InnerProduct(v, v);
-   real_t energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
-                 fespace.GetComm());
+   real_t energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -66,7 +66,6 @@ int main(int argc, char *argv[])
 {
   // 1. Initialize MPI (required by PUMI) and HYPRE.
   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
   int myid = Mpi::WorldRank();
   Hypre::Init();

@@ -80,8 +80,6 @@ int main(int argc, char *argv[])
 {
   // 1. Initialize MPI (required by PUMI) and HYPRE.
   Mpi::Init(argc, argv);
-   int num_proc = Mpi::WorldSize();
-   int myId = Mpi::WorldRank();
   Hypre::Init();

   // 2. Parse command-line options.
@@ -856,10 +856,7 @@ double HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const

 double HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   double loc_energy = 0.5*M.InnerProduct(v, v);
-   double energy;
-   MPI_Allreduce(&loc_energy, &energy, 1, MPI_DOUBLE, MPI_SUM,
-                 fespace.GetComm());
+   double energy = 0.5*M.ParInnerProduct(v, v);
   return energy;
 }

@@ -18,6 +18,7 @@ set(SRCS
  integ/bilininteg_convection_pa.cpp
  integ/bilininteg_convection_ea.cpp
  integ/bilininteg_curlcurl_pa.cpp
+  integ/bilininteg_dgdiffusion_pa.cpp
  integ/bilininteg_dgtrace_pa.cpp
  integ/bilininteg_dgtrace_ea.cpp
  integ/bilininteg_diffusion_mf.cpp
@@ -117,6 +118,7 @@ set(SRCS
  quadinterpolator.cpp
  quadinterpolator_face.cpp
  restriction.cpp
+  normal_deriv_restriction.cpp
  staticcond.cpp
  tmop.cpp
  tmop/tmop_pa.cpp
@@ -228,6 +230,7 @@ set(HDRS
  quadinterpolator.hpp
  quadinterpolator_face.hpp
  restriction.hpp
+  normal_deriv_restriction.hpp
  fespacehierarchy.hpp
  staticcond.hpp
  tbilinearform.hpp
@@ -340,9 +340,9 @@ public:
       $ M^{-1} $ (currently returns NULL) */
   virtual MatrixInverse *Inverse() const;

-   /** @brief  Finalizes the matrix initialization if the ::AssemblyLevel is
+   /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
       AssemblyLevel::LEGACY.
-       THe matrix that gets finalized is different if you are using static
+       The matrix that gets finalized is different if you are using static
       condensation or hybridization.*/
   virtual void Finalize(int skip_zeros = 1);

@@ -643,7 +643,7 @@ public:
   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
                       DiagonalPolicy dpolicy = DIAG_ONE);

-   /** @brief  Eliminate the given @a vdofs, storing the eliminated part
+   /** @brief Eliminate the given @a vdofs, storing the eliminated part
       internally in $ M_e $.

       This method works in conjunction with EliminateVDofsInRHS() and allows
@@ -826,7 +826,7 @@ public:
       $ M^{-1} $ (currently unimplemented and returns NULL)*/
   virtual MatrixInverse *Inverse() const;

-   /** @brief  Finalizes the matrix initialization if the ::AssemblyLevel is
+   /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
       AssemblyLevel::LEGACY.*/
   virtual void Finalize(int skip_zeros = 1);

@@ -282,6 +282,22 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
      int_face_X.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
      int_face_Y.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
      int_face_Y.UseDevice(true); // ensure 'int_face_Y = 0.0' is done on device
+
+      bool needs_normal_derivs = false;
+      auto &integs = *a->GetFBFI();
+      for (int i = 0; i < integs.Size(); ++i)
+      {
+         if (integs[i]->RequiresFaceNormalDerivatives())
+         {
+            needs_normal_derivs = true;
+            break;
+         }
+      }
+      if (needs_normal_derivs)
+      {
+         int_face_dXdn.SetSize(int_face_restrict_lex->Height());
+         int_face_dYdn.SetSize(int_face_restrict_lex->Height());
+      }
   }

   const bool has_bdr_integs = (a->GetBFBFI()->Size() > 0 ||
@@ -296,6 +312,22 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
      bdr_face_Y.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
      bdr_face_Y.UseDevice(true); // ensure 'faceBoundY = 0.0' is done on device

+      bool needs_normal_derivs = false;
+      auto &integs = *a->GetBFBFI();
+      for (int i = 0; i < integs.Size(); ++i)
+      {
+         if (integs[i]->RequiresFaceNormalDerivatives())
+         {
+            needs_normal_derivs = true;
+            break;
+         }
+      }
+      if (needs_normal_derivs)
+      {
+         bdr_face_dXdn.SetSize(bdr_face_restrict_lex->Height());
+         bdr_face_dYdn.SetSize(bdr_face_restrict_lex->Height());
+      }
+
      const Mesh &mesh = *trial_fes->GetMesh();
      // See LinearFormExtension::Update for explanation of f_to_be logic.
      std::unordered_map<int,int> f_to_be;
@@ -542,8 +574,8 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         localY = 0.0;
         for (int i = 0; i < iSz; ++i)
         {
-            AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes,
-                               false, localY);
+            AddMultWithMarkers(*integrators[i], localX, elem_markers[i],
+                               elem_attributes, false, localY);
         }
         elem_restrict->MultTranspose(localY, y);
      }
@@ -557,15 +589,57 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
   const int iFISz = intFaceIntegrators.Size();
   if (int_face_restrict_lex && iFISz>0)
   {
-      int_face_restrict_lex->Mult(x, int_face_X);
-      if (int_face_X.Size()>0)
+      // When assembling interior face integrators for DG spaces, we need to
+      // exchange the face-neighbor information. This happens inside member
+      // functions of the 'int_face_restrict_lex'. To avoid repeated calls to
+      // ParGridFunction::ExchangeFaceNbrData, if we have a parallel space
+      // with interior face integrators, we create a ParGridFunction that
+      // will be used to cache the face-neighbor data. x_dg should be passed
+      // to any restriction operator that may need to use face-neighbor data.
+      const Vector *x_dg = &x;
+#ifdef MFEM_USE_MPI
+      ParGridFunction x_pgf;
+      if (auto *pfes = dynamic_cast<ParFiniteElementSpace*>(a->FESpace()))
+      {
+         x_pgf.MakeRef(pfes, const_cast<Vector&>(x), 0);
+         x_dg = &x_pgf;
+      }
+#endif
+
+      int_face_restrict_lex->Mult(*x_dg, int_face_X);
+      if (int_face_dXdn.Size() > 0)
+      {
+         int_face_restrict_lex->NormalDerivativeMult(*x_dg, int_face_dXdn);
+      }
+      if (int_face_X.Size() > 0)
      {
         int_face_Y = 0.0;
+
+         // if normal derivatives are needed by at least one integrator...
+         if (int_face_dYdn.Size() > 0)
+         {
+            int_face_dYdn = 0.0;
+         }
+
         for (int i = 0; i < iFISz; ++i)
         {
-            intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
+            if (intFaceIntegrators[i]->RequiresFaceNormalDerivatives())
+            {
+               intFaceIntegrators[i]->AddMultPAFaceNormalDerivatives(
+                  int_face_X, int_face_dXdn,
+                  int_face_Y, int_face_dYdn);
+            }
+            else
+            {
+               intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
+            }
         }
         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         if (int_face_dYdn.Size() > 0)
+         {
+            int_face_restrict_lex->NormalDerivativeAddMultTranspose(
+               int_face_dYdn, y);
+         }
      }
   }

@@ -579,9 +653,19 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
      Array<Array<int>*> &bdr_markers = *a->GetBBFI_Marker();
      Array<Array<int>*> &bdr_face_markers = *a->GetBFBFI_Marker();
      bdr_face_restrict_lex->Mult(x, bdr_face_X);
-      if (bdr_face_X.Size()>0)
+      if (bdr_face_dXdn.Size() > 0)
+      {
+         bdr_face_restrict_lex->NormalDerivativeMult(x, bdr_face_dXdn);
+      }
+      if (bdr_face_X.Size() > 0)
      {
         bdr_face_Y = 0.0;
+
+         // if normal derivatives are needed by at least one integrator...
+         if (bdr_face_dYdn.Size() > 0)
+         {
+            bdr_face_dYdn = 0.0;
+         }
         for (int i = 0; i < n_bdr_integs; ++i)
         {
            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
@@ -589,10 +673,23 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         }
         for (int i = 0; i < n_bdr_face_integs; ++i)
         {
-            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
-                               bdr_attributes, false, bdr_face_Y);
+            if (bdr_face_integs[i]->RequiresFaceNormalDerivatives())
+            {
+               AddMultNormalDerivativesWithMarkers(
+                  *bdr_face_integs[i], bdr_face_X, bdr_face_dXdn,
+                  bdr_face_markers[i], bdr_attributes, bdr_face_Y, bdr_face_dYdn);
+            }
+            else
+            {
+               AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
+                                  bdr_attributes, false, bdr_face_Y);
+            }
         }
         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         if (bdr_face_dYdn.Size() > 0)
+         {
+            bdr_face_restrict_lex->NormalDerivativeAddMultTranspose(bdr_face_dYdn, y);
+         }
      }
   }
 }
@@ -693,6 +790,37 @@ static void AddWithMarkers_(
   });
 }

+void PABilinearFormExtension::AddMultNormalDerivativesWithMarkers(
+   const BilinearFormIntegrator &integ,
+   const Vector &x,
+   const Vector &dxdn,
+   const Array<int> *markers,
+   const Array<int> &attributes,
+   Vector &y,
+   Vector &dydn) const
+{
+   if (markers)
+   {
+      tmp_evec.SetSize(y.Size() + dydn.Size());
+      tmp_evec = 0.0;
+      Vector tmp_y(tmp_evec, 0, y.Size());
+      Vector tmp_dydn(tmp_evec, y.Size(), dydn.Size());
+
+      integ.AddMultPAFaceNormalDerivatives(x, dxdn, tmp_y, tmp_dydn);
+
+      const int ne = attributes.Size();
+      const int nd_1 = x.Size() / ne;
+      const int nd_2 = dxdn.Size() / ne;
+
+      AddWithMarkers_(ne, nd_1, tmp_y, *markers, attributes, y);
+      AddWithMarkers_(ne, nd_2, tmp_dydn, *markers, attributes, dydn);
+   }
+   else
+   {
+      integ.AddMultPAFaceNormalDerivatives(x, dxdn, y, dydn);
+   }
+}
+
 void PABilinearFormExtension::AddMultWithMarkers(
   const BilinearFormIntegrator &integ,
   const Vector &x,
@@ -74,6 +74,8 @@ protected:
   mutable Vector localX, localY;
   mutable Vector int_face_X, int_face_Y;
   mutable Vector bdr_face_X, bdr_face_Y;
+   mutable Vector int_face_dXdn, int_face_dYdn;
+   mutable Vector bdr_face_dXdn, bdr_face_dYdn;
   const Operator *elem_restrict; // Not owned
   const FaceRestriction *int_face_restrict_lex; // Not owned
   const FaceRestriction *bdr_face_restrict_lex; // Not owned
@@ -113,6 +115,23 @@ protected:
                           const Array<int> &attributes,
                           const bool transpose,
                           Vector &y) const;
+
+   /// @brief Performs the same function as AddMultWithMarkers, but takes as
+   /// input and output face normal derivatives.
+   ///
+   /// This is required when the integrator requires face normal derivatives,
+   /// for example, DGDiffusionIntegrator.
+   ///
+   /// This is called when the integrator's member function
+   /// BilinearFormIntegrator::RequiresFaceNormalDerivatives() returns true.
+   void AddMultNormalDerivativesWithMarkers(
+      const BilinearFormIntegrator &integ,
+      const Vector &x,
+      const Vector &dxdn,
+      const Array<int> *markers,
+      const Array<int> &attributes,
+      Vector &y,
+      Vector &dydn) const;
 };

 /// Data and methods for element-assembled bilinear forms
@@ -189,6 +189,12 @@ void BilinearFormIntegrator::AssembleTraceFaceMatrix (int elem,
              " Integrator class.");
 }

+void BilinearFormIntegrator::AddMultPAFaceNormalDerivatives(
+   const Vector &x, const Vector &dxdn, Vector &y, Vector &dydn) const
+{
+   MFEM_ABORT("Not implemented.");
+}
+
 void BilinearFormIntegrator::AssembleElementVector(
   const FiniteElement &el, ElementTransformation &Tr, const Vector &elfun,
   Vector &elvect)
@@ -1533,6 +1539,351 @@ const IntegrationRule &ConvectionIntegrator::GetRule(
   return GetRule(el,el,Trans);
 }

+
+void LaplaceIntegrator::AssembleElementMatrix(const FiniteElement &el,
+                                              ElementTransformation &Trans,
+                                              DenseMatrix &elmat )
+{
+   int nd = el.GetDof();
+   real_t w;
+
+   elmat.SetSize(nd);
+   shape.SetSize(nd);
+   laplace.SetSize(nd);
+
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, Trans);
+
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint (&ip);
+
+      el.CalcPhysShape(Trans, shape);
+      el.CalcPhysLaplacian(Trans, laplace);
+
+      w = Trans.Weight() * ip.weight * alpha;
+      if (Q)
+      {
+         w *= Q -> Eval(Trans, ip);
+      }
+      shape *= w;
+      AddMultVWt(shape, laplace, elmat);
+   }
+}
+
+void LaplaceIntegrator::AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                               const FiniteElement &test_fe,
+                                               ElementTransformation &Trans,
+                                               DenseMatrix &elmat)
+{
+   int tr_nd = trial_fe.GetDof();
+   int te_nd = test_fe.GetDof();
+
+   elmat.SetSize(te_nd, tr_nd);
+   laplace.SetSize(tr_nd);
+   shape.SetSize(te_nd);
+
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
+                                                            Trans);
+
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint (&ip);
+
+      test_fe.CalcPhysShape(Trans, shape);
+      trial_fe.CalcPhysLaplacian(Trans, laplace);
+
+      real_t w = Trans.Weight() * ip.weight * alpha;
+      if (Q)
+      {
+         w *= Q -> Eval(Trans, ip);
+      }
+      AddMult_a_VWt(w, shape, laplace, elmat);
+   }
+}
+
+const IntegrationRule &LaplaceIntegrator::GetRule(
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans)
+{
+   int order = trial_fe.GetOrder() + test_fe.GetOrder();
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
+void LaplaceGradIntegrator::AssembleElementMatrix(const FiniteElement &el,
+                                                  ElementTransformation &Trans,
+                                                  DenseMatrix &elmat )
+{
+   int nd = el.GetDof();
+   dim = el.GetDim();
+
+   elmat.SetSize(nd);
+   dshape.SetSize(nd,dim);
+   adjJ.SetSize(dim);
+   laplace.SetSize(nd);
+   vec2.SetSize(dim);
+   BdFidxT.SetSize(nd);
+
+   Vector vec1;
+
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, Trans);
+
+   Q->Eval(Q_ir, Trans, *ir);
+
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      el.CalcDShape(ip, dshape);
+      el.CalcPhysLaplacian(Trans, laplace);
+
+      Trans.SetIntPoint(&ip);
+      CalcAdjugate(Trans.Jacobian(), adjJ);
+      Q_ir.GetColumnReference(i, vec1);
+      vec1 *= alpha * ip.weight;
+
+      adjJ.Mult(vec1, vec2);
+      dshape.Mult(vec2, BdFidxT);
+
+      AddMultVWt(BdFidxT, laplace, elmat);
+   }
+}
+
+void LaplaceGradIntegrator::AssembleElementMatrix2(const FiniteElement
+                                                   &trial_fe,
+                                                   const FiniteElement &test_fe,
+                                                   ElementTransformation &Trans,
+                                                   DenseMatrix &elmat)
+{
+   dim = trial_fe.GetDim();
+   int tr_nd = trial_fe.GetDof();
+   int te_nd = test_fe.GetDof();
+
+   elmat.SetSize(te_nd, tr_nd);
+   laplace.SetSize(tr_nd);
+   dshape.SetSize(te_nd,dim);
+   adjJ.SetSize(dim);
+   vec2.SetSize(dim);
+   BdFidxT.SetSize(te_nd);
+
+   Vector vec1;
+
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
+                                                            Trans);
+
+   Q->Eval(Q_ir, Trans, *ir);
+
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      test_fe.CalcDShape(ip, dshape);
+      trial_fe.CalcPhysLaplacian(Trans, laplace);
+
+      Trans.SetIntPoint(&ip);
+      CalcAdjugate(Trans.Jacobian(), adjJ);
+      Q_ir.GetColumnReference(i, vec1);
+      vec1 *= alpha * ip.weight;
+
+      adjJ.Mult(vec1, vec2);
+      dshape.Mult(vec2, BdFidxT);
+
+      AddMultVWt(BdFidxT, laplace,elmat);
+   }
+}
+
+const IntegrationRule &LaplaceGradIntegrator::GetRule(
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans)
+{
+   int order = trial_fe.GetOrder() + test_fe.GetOrder();
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
+void LaplaceLaplaceIntegrator::AssembleElementMatrix(const FiniteElement &el,
+                                                     ElementTransformation &Trans,
+                                                     DenseMatrix &elmat )
+{
+   int nd = el.GetDof();
+   real_t w;
+
+   elmat.SetSize(nd);
+   laplace.SetSize(nd);
+
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, Trans);
+
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint (&ip);
+
+      el.CalcPhysLaplacian(Trans, laplace);
+
+      w = Trans.Weight() * ip.weight * alpha;
+      if (Q)
+      {
+         w *= Q -> Eval(Trans, ip);
+      }
+      AddMult_a_VVt(w, laplace, elmat);
+   }
+}
+
+void LaplaceLaplaceIntegrator::AssembleElementMatrix2(const FiniteElement
+                                                      &trial_fe,
+                                                      const FiniteElement &test_fe,
+                                                      ElementTransformation &Trans,
+                                                      DenseMatrix &elmat)
+{
+   int
+   dim = trial_fe.GetDim();
+   int tr_nd = trial_fe.GetDof();
+   int te_nd = test_fe.GetDof();
+   real_t w;
+
+   elmat.SetSize(te_nd, tr_nd);
+   laplace.SetSize(tr_nd);
+   te_laplace.SetSize(te_nd);
+
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
+                                                            Trans);
+
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint (&ip);
+
+      trial_fe.CalcPhysLaplacian(Trans, laplace);
+      test_fe.CalcPhysLaplacian(Trans, te_laplace);
+
+      w = Trans.Weight() * ip.weight * alpha;
+      if (Q)
+      {
+         w *= Q -> Eval(Trans, ip);
+      }
+      AddMult_a_VWt(w, te_laplace, laplace, elmat);
+   }
+}
+
+const IntegrationRule &LaplaceLaplaceIntegrator::GetRule(
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans)
+{
+   int order = trial_fe.GetOrder() + test_fe.GetOrder();
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
+void InverseEstimateIntegrator::AssembleElementMatrix(const FiniteElement &el,
+                                                      ElementTransformation &Trans,
+                                                      DenseMatrix &elmat )
+{
+   elmat = 0.0;
+
+   int nd = el.GetDof();
+   int dim = el.GetDim();
+
+   shape.SetSize(nd);
+   dshape.SetSize(nd,dim);
+   laplace.SetSize(nd);
+
+   lapmat.SetSize(nd,nd);
+   bimat.SetSize(nd,nd);
+   ovec.SetSize(nd);
+
+   real_t w,q;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = Trans.OrderGrad(&el) + Trans.Order() + el.GetOrder();
+      ir = &IntRules.Get(el.GetGeomType(), order);
+   }
+
+   bimat = 0.0;
+   lapmat = 0.0;
+   ovec = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint(&ip);
+      w = Trans.Weight()*ip.weight;
+      if (Q)
+      {
+         q = Q->Eval(Trans, ip);
+      }
+
+      el.CalcPhysDShape(Trans, dshape);
+      AddMult_a_AAt(w*q, dshape, lapmat);
+
+      el.CalcPhysLaplacian(Trans, laplace);
+      AddMult_a_VVt(w*q*q, laplace, bimat);
+
+      el.CalcPhysShape(Trans, shape);
+      ovec.Add(w, shape);
+   }
+
+   // Power method
+   Vector x(nd);
+   x.Randomize(696383532);
+
+   // Correct nullspace + inverse
+   AddMult_a_VVt(1.0, ovec, lapmat);
+   DenseMatrixInverse L_inv(lapmat);
+
+   // DenseMatrix M_i, Q_i;
+   real_t alpha= 0.0, eval_i = 0.0, eval_prev = 0.0;
+
+   // Inverse power method
+   Vector x_tmp(nd);
+   int iter = 0;
+   const real_t rel_tol = 1e-4;
+
+   alpha = ovec*ovec;
+   ovec *= 1.0/sqrt(alpha);
+   do
+   {
+      // Othogonalize
+      alpha = x*ovec;
+      x.Add(-alpha, ovec);
+
+      // MatVec (2x)
+      bimat.Mult(x, x_tmp);
+      L_inv.Mult(x_tmp, x);
+
+      eval_prev = eval_i;
+      eval_i = x.Norml2();
+      x *= 1.0/eval_i;
+      ++iter;
+   }
+   while ((iter < 10000) && (fabs(eval_i - eval_prev)/fabs(eval_i) > rel_tol));
+   MFEM_VERIFY(fabs(eval_i - eval_prev)/fabs(eval_i) <= rel_tol,
+               "Inverse power method did not converge."
+               << "\n\t iter      = " << iter
+               << "\n\t eval_i    = " << eval_i
+               << "\n\t eval_prev = " << eval_prev
+               << "\n\t fabs(eval_i - eval_prev)/fabs(eval_i) = "
+               << fabs(eval_i - eval_prev)/fabs(eval_i));
+   cout<<"evev = "<<eval_i<<" "<<iter<<endl;
+}
+
+const IntegrationRule &InverseEstimateIntegrator::GetRule(
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans)
+{
+   // int order = Trans.OrderGrad(&trial_fe) + Trans.Order() + test_fe.GetOrder() - 2;
+   int order = trial_fe.GetOrder() + test_fe.GetOrder();
+   return IntRules.Get(trial_fe.GetGeomType(), order);
+}
+
+
 void VectorMassIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
  DenseMatrix &elmat )
@@ -3423,7 +3774,7 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   const FiniteElement &el1, const FiniteElement &el2,
   FaceElementTransformations &Trans, DenseMatrix &elmat)
 {
-   int dim, ndof1, ndof2, ndofs;
+   int ndof1, ndof2, ndofs;
   bool kappa_is_nonzero = (kappa != 0.);
   real_t w, wq = 0.0;

@@ -3466,17 +3817,9 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
   {
-      // a simple choice for the integration order; is this OK?
-      int order;
-      if (ndof2)
-      {
-         order = 2*max(el1.GetOrder(), el2.GetOrder());
-      }
-      else
-      {
-         order = 2*el1.GetOrder();
-      }
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      const int order = (ndof2) ? max(el1.GetOrder(),
+                                      el2.GetOrder()) : el1.GetOrder();
+      ir = &GetRule(order, Trans);
   }

   // assemble: < {(Q \nabla u).n},[v] >      --> elmat
@@ -3654,6 +3997,13 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   }
 }

+const IntegrationRule &DGDiffusionIntegrator::GetRule(
+   int order, FaceElementTransformations &T)
+{
+   // order is typically the maximum of the order of the left and right elements
+   // neighboring the given face.
+   return IntRules.Get(T.GetGeometryType(), 2*order);
+}

 // static method
 void DGElasticityIntegrator::AssembleBlock(
@@ -266,6 +266,39 @@ public:
                                    Vector &flux, Vector *d_energy = NULL)
   { return 0.0; }

+   /** @brief For bilinear forms on element faces, specifies if the normal
+              derivatives are needed on the faces or just the face restriction.
+
+       @details if RequiresFaceNormalDerivatives() == true, then
+                AddMultPAFaceNormalDerivatives(...) should be invoked in place
+                of AddMultPA(...) and L2NormalDerivativeFaceRestriction should
+                be used to compute the normal derivatives. This is used for some
+                DG integrators, for example DGDiffusionIntegrator.
+
+       @returns whether normal derivatives appear in the bilinear form.
+   */
+   virtual bool RequiresFaceNormalDerivatives() const { return false; }
+
+   /// Method for partially assembled action.
+   /** @brief For bilinear forms on element faces that depend on the normal
+              derivative on the faces, computes the action of integrator to the
+              face values @a x and reference-normal derivatives @a dxdn and adds
+              the result to @a y and @a dydn.
+
+      @details This method can be called only after the method AssemblePA() has
+               been called.
+
+      @param[in]     x E-vector of face values (provided by
+                       FaceRestriction::Mult)
+      @param[in]     dxdn E-vector of face reference-normal derivatives
+                          (provided by FaceRestriction::NormalDerivativeMult)
+      @param[in,out] y E-vector of face values to add action to.
+      @param[in,out] dydn E-vector of face reference-normal derivative values to
+                          add action to.
+   */
+   virtual void AddMultPAFaceNormalDerivatives(const Vector &x, const Vector &dxdn,
+                                               Vector &y, Vector &dydn) const;
+
   virtual ~BilinearFormIntegrator() { }
 };

@@ -2417,6 +2450,135 @@ public:
                                      DenseMatrix &);
 };

+
+/// $\alpha (Q \Delta u, v)$
+class LaplaceIntegrator : public BilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+   real_t alpha;
+
+private:
+   Vector laplace, shape;
+
+public:
+   LaplaceIntegrator(Coefficient &q, real_t a = 1.0)
+      : Q(&q) { alpha = a; }
+
+   virtual void AssembleElementMatrix(const FiniteElement &,
+                                      ElementTransformation &,
+                                      DenseMatrix &);
+
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe,
+                                         ElementTransformation &Trans);
+};
+
+/// $\alpha (u, Q \Delta v)$
+class TransposeLaplaceIntegrator : public TransposeIntegrator
+{
+public:
+   TransposeLaplaceIntegrator (Coefficient &q, real_t a = 1.0)
+      : TransposeIntegrator(new LaplaceIntegrator(q, a)) { }
+};
+
+/// $\alpha (\Delta u, Q \cdot \nabla v)$
+class LaplaceGradIntegrator : public BilinearFormIntegrator
+{
+protected:
+   VectorCoefficient *Q;
+   real_t alpha;
+   int dim;
+
+private:
+   Vector laplace, vec2, BdFidxT;
+   DenseMatrix dshape, adjJ, Q_ir;
+
+public:
+   LaplaceGradIntegrator(VectorCoefficient &q, real_t a = 1.0)
+      : Q(&q) { alpha = a; }
+
+   virtual void AssembleElementMatrix(const FiniteElement &,
+                                      ElementTransformation &,
+                                      DenseMatrix &);
+
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe,
+                                         ElementTransformation &Trans);
+};
+
+/// $\alpha (Q \cdot \nabla u,  \Delta v)$
+class GradLaplaceIntegrator : public TransposeIntegrator
+{
+public:
+   GradLaplaceIntegrator(VectorCoefficient &q, real_t a = 1.0)
+      : TransposeIntegrator(new LaplaceGradIntegrator(q, a)) { }
+};
+
+/// $\alpha (Q \Delta u, \Delta v)$
+class  LaplaceLaplaceIntegrator : public BilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+   real_t alpha;
+
+private:
+   Vector laplace, te_laplace;
+
+public:
+   LaplaceLaplaceIntegrator(Coefficient &q, real_t a = 1.0)
+      : Q(&q) { alpha = a; }
+
+   virtual void AssembleElementMatrix(const FiniteElement &,
+                                      ElementTransformation &,
+                                      DenseMatrix &);
+
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe,
+                                         ElementTransformation &Trans);
+};
+
+// Alias for @LaplaceLaplaceIntegrator.
+using BiHarmonicIntegrator = LaplaceLaplaceIntegrator;
+
+/// Get the inverse estimate
+class InverseEstimateIntegrator : public BilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+
+private:
+   Vector laplace, shape, ovec;//, vec2, BdFidxT;
+   DenseMatrix dshape, lapmat, bimat;//, adjJ, Q_ir;
+
+public:
+   InverseEstimateIntegrator(Coefficient &q)
+      : Q(&q) { }
+
+   virtual void AssembleElementMatrix(const FiniteElement &,
+                                      ElementTransformation &,
+                                      DenseMatrix &);
+
+   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
+                                         const FiniteElement &test_fe,
+                                         ElementTransformation &Trans);
+};
+
 /** Class for integrating the bilinear form $a(u,v) := (Q u, v)$,
    where $u=(u_1,\dots,u_n)$ and $v=(v_1,\dots,v_n)$, $u_i$ and $v_i$ are defined
    by scalar FE through standard transformation. */
@@ -3229,6 +3391,13 @@ protected:
   Vector shape1, shape2, dshape1dn, dshape2dn, nor, nh, ni;
   DenseMatrix jmat, dshape1, dshape2, mq, adjJ;

+
+   // PA extension
+   Vector pa_data; // (Q, h, dot(n,J)|el0, dot(n,J)|el1)
+   const DofToQuad *maps; ///< Not owned
+   int dim, nf, nq, dofs1D, quad1D;
+   IntegrationRules irs{0, Quadrature1D::GaussLobatto};
+
 public:
   DGDiffusionIntegrator(const real_t s, const real_t k)
      : Q(NULL), MQ(NULL), sigma(s), kappa(k) { }
@@ -3237,10 +3406,26 @@ public:
   DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k)
      : Q(NULL), MQ(&q), sigma(s), kappa(k) { }
   using BilinearFormIntegrator::AssembleFaceMatrix;
-   virtual void AssembleFaceMatrix(const FiniteElement &el1,
-                                   const FiniteElement &el2,
-                                   FaceElementTransformations &Trans,
-                                   DenseMatrix &elmat);
+   void AssembleFaceMatrix(const FiniteElement &el1,
+                           const FiniteElement &el2,
+                           FaceElementTransformations &Trans,
+                           DenseMatrix &elmat) override;
+
+   bool RequiresFaceNormalDerivatives() const override { return true; }
+
+   using BilinearFormIntegrator::AssemblePA;
+
+   void AssemblePAInteriorFaces(const FiniteElementSpace &fes) override;
+
+   void AssemblePABoundaryFaces(const FiniteElementSpace &fes) override;
+
+   void AddMultPAFaceNormalDerivatives(const Vector &x, const Vector &dxdn,
+                                       Vector &y, Vector &dydn) const override;
+
+   const IntegrationRule &GetRule(int order, FaceElementTransformations &T);
+
+private:
+   void SetupPA(const FiniteElementSpace &fes, FaceType type);
 };

 /** Integrator for the "BR2" diffusion stabilization term
@@ -807,6 +807,7 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)

   QuadratureSpaceBase &qspace = *qf.GetSpace();
   const int ne = qspace.GetNE();
+   qf.HostWrite();
   DenseMatrix values;
   DenseSymmetricMatrix matrix;
   for (int iel = 0; iel < ne; ++iel)
@@ -818,7 +819,7 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
      {
         const IntegrationPoint &ip = ir[iq];
         T.SetIntPoint(&ip);
-         matrix.UseExternalData(&values(0, iq), vdim);
+         matrix.UseExternalData(&values(0, iq), height);
         Eval(matrix, T, ip);
      }
   }
@@ -828,13 +829,12 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
 void SymmetricMatrixCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
-   mat.SetSize(height);
-   Eval(mat, T, ip);
+   Eval(mat_aux, T, ip);
   for (int j = 0; j < width; ++j)
   {
      for (int i = 0; i < height; ++ i)
      {
-         K(i, j) = mat(i, j);
+         K(i, j) = mat_aux(i, j);
      }
   }
 }
@@ -989,10 +989,7 @@ void MatrixArrayVectorCoefficient::Eval(DenseMatrix &K,
   for (int i = 0; i < height; i++)
   {
      this->Eval(i, V, T, ip);
-      for (int j = 0; j < width; j++)
-      {
-         K(i,j) = V(j);
-      }
+      K.SetRow(i, V);
   }
 }

@@ -1507,6 +1504,295 @@ void CrossCrossCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
   M *= ((a == NULL ) ? aConst : a->Eval(T, ip) );
 }

+
+InverseEstimateCoefficient::InverseEstimateCoefficient(FiniteElementSpace *f)
+   : fes(f), Q(NULL), ir(NULL)
+{
+   ComputeInverseEstimates();
+}
+
+InverseEstimateCoefficient::InverseEstimateCoefficient(FiniteElementSpace *f,
+                                                       Coefficient &q)
+   : fes(f), Q(&q), ir(NULL)
+{
+   ComputeInverseEstimates();
+}
+
+GridFunction *InverseEstimateCoefficient::GetGridFunction()
+{
+   FiniteElementCollection* fec_ec = new L2_FECollection(0,
+                                                         fes ->GetMesh()->Dimension());
+   FiniteElementSpace *fes_ec = new FiniteElementSpace(fes ->GetMesh(), fec_ec);
+   GridFunction *gf = new GridFunction(fes_ec, elemInvEst.GetData());
+   gf->MakeOwner(fec_ec);
+   return gf;
+}
+
+void InverseEstimateCoefficient::ComputeInverseEstimates()
+{
+   elemInvEst.SetSize(fes -> GetNE());
+   SetIntRule(*fes->GetFE(0));
+   for (int i = 0; i < fes -> GetNE(); i++)
+   {
+      elemInvEst[i] = ElementInverseEstimate(*fes->GetFE(i),
+                                             *fes->GetElementTransformation(i));
+   }
+}
+
+void InverseEstimateCoefficient::SetIntRule(const FiniteElement &el)
+{
+   ir = &IntRules.Get(el.GetGeomType(), 2*el.GetOrder());
+}
+
+real_t InverseEstimateCoefficient::ElementInverseEstimate(
+   const FiniteElement &el,
+   ElementTransformation &Trans)
+{
+   if (el.GetOrder() < 2)
+   {
+      return std::numeric_limits<real_t>::min();
+   }
+
+   int nd = el.GetDof();
+   int dim = el.GetDim();
+
+   shape.SetSize(nd);
+   dshape.SetSize(nd,dim);
+   laplace.SetSize(nd);
+
+   lapmat.SetSize(nd,nd);
+   bimat.SetSize(nd,nd);
+   ovec.SetSize(nd);
+
+   real_t w,q = 1.0;
+
+   bimat = 0.0;
+   lapmat = 0.0;
+   ovec = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint(&ip);
+      w = Trans.Weight()*ip.weight;
+      if (Q)
+      {
+         q = Q->Eval(Trans, ip);
+      }
+
+      el.CalcPhysDShape(Trans, dshape);
+      AddMult_a_AAt(w*q, dshape, lapmat);
+
+      el.CalcPhysLaplacian(Trans, laplace);
+      AddMult_a_VVt(w*q*q, laplace, bimat);
+
+      el.CalcPhysShape(Trans, shape);
+      ovec.Add(w, shape);
+   }
+   ovec *= 1.0/ovec.Norml2();
+
+   // Correct nullspace
+   AddMultVVt(ovec, lapmat);
+
+   // Return largest eigenvalue
+   return bimat.Eigenvalue(lapmat);
+}
+
+ElasticInverseEstimateCoefficient
+::ElasticInverseEstimateCoefficient(FiniteElementSpace *f)
+   : fes(f), Q(NULL), ir(NULL)
+{
+   ComputeInverseEstimates();
+}
+
+ElasticInverseEstimateCoefficient
+::ElasticInverseEstimateCoefficient(FiniteElementSpace *f,
+                                    Coefficient &q)
+   : fes(f), Q(&q), ir(NULL)
+{
+   ComputeInverseEstimates();
+}
+
+GridFunction *ElasticInverseEstimateCoefficient::GetGridFunction()
+{
+   FiniteElementCollection* fec_ec = new L2_FECollection(0,
+                                                         fes ->GetMesh()->Dimension());
+   FiniteElementSpace *fes_ec = new FiniteElementSpace(fes ->GetMesh(), fec_ec);
+   GridFunction *gf = new GridFunction(fes_ec, elemInvEst.GetData());
+   gf->MakeOwner(fec_ec);
+   return gf;
+}
+
+void ElasticInverseEstimateCoefficient::ComputeInverseEstimates()
+{
+   elemInvEst.SetSize(fes -> GetNE());
+   SetIntRule(*fes->GetFE(0));
+   int dim = fes->GetFE(0)->GetDim();
+
+   emat.SetSize(dim,dim);
+   divmat.SetSize(dim,dim);
+   for (int i = 0; i < dim; i++)
+   {
+      for (int j = 0; j < dim; j++)
+      {
+         emat(i,j)= new DenseMatrix();
+         divmat(i,j)= new DenseMatrix();
+      }
+   }
+
+   hmap.SetSize(dim,dim);
+
+   if (dim == 2)
+   {
+      hmap(0,0) = 0;
+      hmap(0,1) =  hmap(1,0) =  1;
+      hmap(1,1) = 2;
+   }
+   else if (dim == 2)
+   {
+      hmap(0,0) = 0;
+      hmap(0,1) = hmap(1,0) = 1;
+      hmap(0,2) = hmap(2,0) = 2;
+      hmap(1,1) = 3;
+      hmap(1,2) = hmap(2,1) = 4;
+      hmap(2,2) = 5;
+   }
+   else
+   {
+      mfem_error("Only implemented for 2D and 3D");
+   }
+
+   for (int i = 0; i < fes -> GetNE(); i++)
+   {
+      elemInvEst[i] = ElementInverseEstimate(*fes->GetFE(i),
+                                             *fes->GetElementTransformation(i));
+   }
+}
+
+void ElasticInverseEstimateCoefficient::SetIntRule(const FiniteElement &el)
+{
+   ir = &IntRules.Get(el.GetGeomType(), 2*el.GetOrder());
+}
+
+real_t ElasticInverseEstimateCoefficient::ElementInverseEstimate(
+   const FiniteElement &el,
+   ElementTransformation &Trans)
+{
+   // if (el.GetDerivType() != (int) FiniteElement::HESS)
+   //  {
+   //     return std::numeric_limits<real_t>::min();
+   //  }
+
+   int nd = el.GetDof();
+   int dim = el.GetDim();
+
+   shape.SetSize(nd);
+   dshape.SetSize(nd,dim);
+   hshape.SetSize(nd,dim*(dim+1)/2);
+
+   for (int i = 0; i < dim; i++)
+   {
+      for (int j = 0; j < dim; j++)
+      {
+         emat(i,j)->SetSize(nd,nd);
+         *emat(i,j) = 0.0;
+         divmat(i,j)->SetSize(nd,nd);
+         *divmat(i,j) = 0.0;
+      }
+   }
+
+   real_t w,q = 1.0;
+   for (int ii = 0; ii < ir->GetNPoints(); ii++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(ii);
+      Trans.SetIntPoint(&ip);
+      w = Trans.Weight()*ip.weight;
+      if (Q)
+      {
+         q = Q->Eval(Trans, ip);
+      }
+
+      el.CalcPhysDShape(Trans, dshape);
+      for (int i = 0; i < dim; i++)
+      {
+         for (int j = 0; j < dim; j++)
+         {
+            AddMult_a_VVt(w*q, Vector(dshape.GetColumn(i),nd), *emat(j,j));
+
+            AddMult_a_VWt(w*q, Vector(dshape.GetColumn(i),nd),
+                          Vector(dshape.GetColumn(j),nd), *emat(j,i));
+
+            AddMult_a_VWt(w*q, Vector(dshape.GetColumn(j),nd),
+                          Vector(dshape.GetColumn(i),nd), *emat(i,j));
+
+            AddMult_a_VVt(w*q, Vector(dshape.GetColumn(j),nd), *emat(i,i));
+         }
+      }
+
+      el.CalcPhysHessian(Trans, hshape);
+      for (int i = 0; i < dim; i++)
+      {
+         for (int j = 0; j < dim; j++)
+         {
+            for (int k = 0; k < dim; k++)
+            {
+               AddMult_a_VWt(w*q*q, Vector(hshape.GetColumn(hmap(i,i)),nd),
+                             Vector(hshape.GetColumn(hmap(k,k)),nd), *divmat(j,j));
+
+               AddMult_a_VWt(w*q*q, Vector(hshape.GetColumn(hmap(i,i)),nd),
+                             Vector(hshape.GetColumn(hmap(k,j)),nd), *divmat(j,k));
+
+               AddMult_a_VWt(w*q*q, Vector(hshape.GetColumn(hmap(i,j)),nd),
+                             Vector(hshape.GetColumn(hmap(k,k)),nd), *divmat(i,j));
+
+               AddMult_a_VWt(w*q*q, Vector(hshape.GetColumn(hmap(i,j)),nd),
+                             Vector(hshape.GetColumn(hmap(k,j)),nd), *divmat(i,k));
+            }
+         }
+      }
+   }
+
+   // Collect matrices
+   emat_tot.SetSize(nd*dim,nd*dim);
+   divmat_tot.SetSize(nd*dim,nd*dim);
+   for (int i = 0; i < dim; i++)
+   {
+      for (int j = 0; j < dim; j++)
+      {
+         emat_tot  .SetSubMatrix(i*nd, j*nd, *emat(i,j));
+         divmat_tot.SetSubMatrix(i*nd, j*nd, *divmat(i,j));
+      }
+   }
+
+   // Correct nullspace
+   DenseMatrix ns;
+   emat_tot.NullSpace(ns, 1e-10);
+   for (int i = 0; i < ns.Width(); i++)
+   {
+      AddMultVVt(Vector(ns.GetColumn(i),nd*dim), emat_tot);
+   }
+
+   // Return largest eigenvalue
+   return divmat_tot.Eigenvalue(emat_tot);
+}
+
+ElasticInverseEstimateCoefficient::~ElasticInverseEstimateCoefficient()
+{
+   for (int i = 0; i < emat.NumRows(); i++)
+   {
+      for (int j = 0; j < emat.NumCols(); j++)
+      {
+         delete emat(i,j);
+      }
+   }
+   for (int i = 0; i < divmat.NumRows(); i++)
+   {
+      for (int j = 0; j < divmat.NumCols(); j++)
+      {
+         delete divmat(i,j);
+      }
+   }
+}
+
 real_t LpNormLoop(real_t p, Coefficient &coeff, Mesh &mesh,
                  const IntegrationRule *irs[])
 {
@@ -1352,7 +1352,7 @@ public:
   /// Set the time for internally stored coefficients
   void SetTime(real_t t) override;

-   /// Get the coefficient located at (i,j) in the matrix.
+   /// Get the vector coefficient located at the i-th row of the matrix
   VectorCoefficient* GetCoeff (int i) { return Coeff[i]; }

   /** @brief Set the coefficient located at the i-th row of the matrix.
@@ -1466,12 +1466,13 @@ public:
 class SymmetricMatrixCoefficient : public MatrixCoefficient
 {
 protected:
+
   /// Internal matrix used when evaluating this coefficient as a DenseMatrix.
-   DenseSymmetricMatrix mat;
+   mutable DenseSymmetricMatrix mat_aux;
 public:
   /// Construct a dim x dim matrix coefficient.
   explicit SymmetricMatrixCoefficient(int dimension)
-      : MatrixCoefficient(dimension, true) { }
+      : MatrixCoefficient(dimension, true), mat_aux(height) { }

   /// Get the size of the matrix.
   int GetSize() const { return height; }
@@ -1504,8 +1505,9 @@ public:
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip);

-   /// Return a reference to the constant matrix.
-   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
+   /// @deprecated Return a reference to the internal matrix used when evaluating this coefficient as a DenseMatrix.
+   MFEM_DEPRECATED const DenseSymmetricMatrix& GetMatrix() { return mat_aux; }

   virtual ~SymmetricMatrixCoefficient() { }
 };
@@ -1525,6 +1527,10 @@ public:
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseSymmetricMatrix &M, ElementTransformation &T,
                     const IntegrationPoint &ip) { M = mat; }
+
+   /// Return a reference to the constant matrix.
+   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
 };


@@ -2322,6 +2328,127 @@ public:
 };
 ///@}

+/** @brief
+*/
+class InverseEstimateCoefficient : public Coefficient
+{
+private:
+   ///
+   Vector elemInvEst;
+   /// FE space on which the grid function lives. Owned if #fec is not NULL.
+   FiniteElementSpace *fes;
+
+   ///
+   const IntegrationRule *ir;
+
+   ///
+   Coefficient *Q;
+   Vector laplace, shape, ovec, evec;
+   DenseMatrix dshape, lapmat, bimat;
+
+   ///
+   void SetIntRule(const FiniteElement &el);
+
+   ///
+   void ComputeInverseEstimates();
+
+   real_t ElementInverseEstimate(const FiniteElement &el,
+                                 ElementTransformation &Trans);
+
+public:
+   ///
+   InverseEstimateCoefficient(FiniteElementSpace *f);
+   InverseEstimateCoefficient(FiniteElementSpace *f, Coefficient &q);
+
+   /// Caller gets owner ship of GridFunction and
+   GridFunction *GetGridFunction();
+
+   /// Reset the scalar factor
+   void SetDiffusion(Coefficient &q)
+   {
+      if (Q != &q)
+      {
+         Q = &q;
+         ComputeInverseEstimates();
+      }
+   }
+   /// Return the scalar factor
+   Coefficient * GetDiffusion() const { return Q; }
+
+   /// Evaluate the coefficient at @a ip.
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip)
+   { return elemInvEst[T.ElementNo]; }
+
+};
+
+class ElasticInverseEstimateCoefficient : public Coefficient
+{
+private:
+   ///
+   Vector elemInvEst;
+   /// FE space on which the grid function lives. Owned if #fec is not NULL.
+   FiniteElementSpace *fes;
+
+   ///
+   const IntegrationRule *ir;
+
+   ///
+   Coefficient *Q;
+   Vector shape, ovec, evec;
+   DenseMatrix dshape, hshape, emat_tot, divmat_tot;
+   Array2D<DenseMatrix*> emat,divmat;
+   Array2D<int> hmap;
+
+   ///
+   void SetIntRule(const FiniteElement &el);
+
+   ///
+   void ComputeInverseEstimates();
+
+   ///
+   real_t ElementInverseEstimate(const FiniteElement &el,
+                                 ElementTransformation &Trans);
+
+public:
+   ///
+   ElasticInverseEstimateCoefficient(FiniteElementSpace *f);
+   ElasticInverseEstimateCoefficient(FiniteElementSpace *f, Coefficient &q);
+
+   /// Caller gets owner ship of GridFunction and
+   GridFunction *GetGridFunction();
+
+   /// Reset the scalar factor
+   void SetDiffusion(Coefficient &q)
+   {
+      if (Q != &q)
+      {
+         Q = &q;
+         ComputeInverseEstimates();
+      }
+   }
+   void SetShearModulus(Coefficient &q) { SetDiffusion(q);}
+
+   /// Return the scalar factor
+   Coefficient * GetDiffusion() const { return Q; }
+   Coefficient * GetModulus() const { return GetDiffusion(); }
+
+   /// Evaluate the coefficient at @a ip.
+   virtual real_t Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip)
+   { return elemInvEst[T.ElementNo]; }
+
+   // Destructor
+   ~ElasticInverseEstimateCoefficient();
+
+};
+
+
+///@}
+
+
+
+
 /** @brief Vector quadrature function coefficient which requires that the
    quadrature rules used for this vector coefficient be the same as those that
    live within the supplied QuadratureFunction. */
@@ -1243,25 +1243,16 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
         HypreParMatrix * Ah;
         A_i.Get(Ah);
         hypre_ParCSRMatrix *Aih = *Ah;
-#if !defined(HYPRE_USING_GPU)
-         ess_tdof_list.HostRead();
-         for (int k = 0; k < n; k++)
-         {
-            const int j = ess_tdof_list[k];
-            Aih->diag->data[Aih->diag->i[j]] = 0.0;
-         }
-#else
         Ah->HypreReadWrite();
         const int *d_ess_tdof_list =
-            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
-         const int *d_diag_i = Aih->diag->i;
+            ess_tdof_list.GetMemory().Read(GetHypreMemoryClass(), n);
+         HYPRE_Int *d_diag_i = Aih->diag->i;
         real_t *d_diag_data = Aih->diag->data;
-         MFEM_GPU_FORALL(k, n,
+         mfem::hypre_forall(n, [=] MFEM_HOST_DEVICE (int k)
         {
            const int j = d_ess_tdof_list[k];
            d_diag_data[d_diag_i[j]] = 0.0;
         });
-#endif
      }
      else
      {
@@ -1,8 +1,18 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
 #include "convergence.hpp"

 using namespace std;

-
 namespace mfem
 {

@@ -101,7 +101,7 @@ void DGMassInverse::SetRelTol(const real_t rel_tol_) { rel_tol = rel_tol_; }

 void DGMassInverse::SetAbsTol(const real_t abs_tol_) { abs_tol = abs_tol_; }

-void DGMassInverse::SetMaxIter(const real_t max_iter_) { max_iter = max_iter_; }
+void DGMassInverse::SetMaxIter(const int max_iter_) { max_iter = max_iter_; }

 void DGMassInverse::Update()
 {
@@ -137,7 +137,7 @@ void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const

   const real_t RELTOL = rel_tol;
   const real_t ABSTOL = abs_tol;
-   const real_t MAXIT = max_iter;
+   const int MAXIT = max_iter;
   const bool IT_MODE = iterative_mode;
   const bool CHANGE_BASIS = (d2q != nullptr);

@@ -96,7 +96,7 @@ public:
   /// Set the absolute tolerance.
   void SetAbsTol(const real_t abs_tol_);
   /// Set the maximum number of iterations.
-   void SetMaxIter(const real_t max_iter_);
+   void SetMaxIter(const int max_iter_);
   /// Recompute operator and preconditioner (when coefficient or mesh changes).
   void Update();

@@ -52,6 +52,15 @@ protected:
   const DenseMatrix &EvalTransAdjugateJ();
   const DenseMatrix &EvalInverseJ();

+   /// @name Tolerance used for point comparisons
+   ///@{
+#ifdef MFEM_USE_DOUBLE
+   static constexpr real_t tol_0 = 1e-15;
+#elif defined(MFEM_USE_SINGLE)
+   static constexpr real_t tol_0 = 1e-7;
+#endif
+   ///@}
+
 public:

   /** This enumeration declares the values stored in
@@ -176,7 +185,7 @@ public:
       returned. This method is not 100 percent reliable for non-linear
       transformations. */
   virtual int TransformBack(const Vector &pt, IntegrationPoint &ip,
-                             const real_t phys_tol = 1e-15) = 0;
+                             const real_t phys_tol = tol_0) = 0;

   virtual ~ElementTransformation() { }
 };
@@ -281,9 +290,15 @@ public:
        rel_qpts_order(-1),
        solver_type(NewtonElementProject),
        max_iter(16),
+#ifdef MFEM_USE_DOUBLE
        ref_tol(1e-15),
        phys_rtol(1e-15),
        ip_tol(1e-8),
+#elif defined(MFEM_USE_SINGLE)
+        ref_tol(1e-7),
+        phys_rtol(1e-7),
+        ip_tol(1e-4),
+#endif
        print_level(-1)
   { }

@@ -449,7 +464,7 @@ public:
       returned. This method is not 100 percent reliable for non-linear
       transformations. */
   virtual int TransformBack(const Vector & v, IntegrationPoint & ip,
-                             const real_t phys_rel_tol = 1e-15)
+                             const real_t phys_rel_tol = tol_0)
   {
      InverseElementTransformation inv_tr(this);
      inv_tr.SetPhysicalRelTol(phys_rel_tol);
@@ -13,6 +13,7 @@
 #define MFEM_FACE_MAP_UTILS_HPP

 #include "../../general/array.hpp"
+#include "../../general/backends.hpp"
 #include <utility> // std::pair
 #include <vector>

@@ -51,6 +52,189 @@ void FillFaceMap(const int n_face_dofs_per_component,
 void GetTensorFaceMap(const int dim, const int order, const int face_id,
                      Array<int> &face_map);

+/// @brief Given a face DOF index in native (counter-clockwise) ordering, return
+/// the corresponding DOF index in lexicographic ordering (for a quadrilateral
+/// element).
+MFEM_HOST_DEVICE
+inline int ToLexOrdering2D(const int face_id, const int size1d, const int i)
+{
+   if (face_id==2 || face_id==3)
+   {
+      return size1d-1-i;
+   }
+   else
+   {
+      return i;
+   }
+}
+
+/// @brief Given a face DOF index on a shared face, ordered lexicographically
+/// relative to element 1, return the corresponding face DOF index ordered
+/// lexicographically relative to element 2.
+MFEM_HOST_DEVICE
+inline int PermuteFace2D(const int face_id1, const int face_id2,
+                         const int orientation, const int size1d,
+                         const int index)
+{
+   int new_index;
+   // Convert from element 1 lex ordering to native ordering
+   if (face_id1 == 2 || face_id1 == 3)
+   {
+      new_index = size1d-1-index;
+   }
+   else
+   {
+      new_index = index;
+   }
+   // Permute based on face orientations
+   if (orientation == 1)
+   {
+      new_index = size1d-1-new_index;
+   }
+   // Covert to element 2 lex ordering
+   return ToLexOrdering2D(face_id2, size1d, new_index);
+}
+
+/// @brief Given a face DOF index in native (counter-clockwise) ordering, return
+/// the corresponding DOF index in lexicographic ordering (for a hexahedral
+/// element).
+MFEM_HOST_DEVICE
+inline int ToLexOrdering3D(const int face_id, const int size1d, const int i,
+                           const int j)
+{
+   if (face_id==2 || face_id==1 || face_id==5)
+   {
+      return i + j*size1d;
+   }
+   else if (face_id==3 || face_id==4)
+   {
+      return (size1d-1-i) + j*size1d;
+   }
+   else // face_id==0
+   {
+      return i + (size1d-1-j)*size1d;
+   }
+}
+
+/// @brief Given the index of a face DOF in lexicographic ordering relative
+/// element 1, permute the index so that it is lexicographically ordered
+/// relative to element 2.
+///
+/// The given face corresponds to local face index @a face_id1 relative to
+/// element 1, and @a face_id2 (with @a orientation) relative to element 2.
+MFEM_HOST_DEVICE
+inline int PermuteFace3D(const int face_id1, const int face_id2,
+                         const int orientation,
+                         const int size1d, const int index)
+{
+   int i=0, j=0, new_i=0, new_j=0;
+   i = index%size1d;
+   j = index/size1d;
+   // Convert from lex ordering
+   if (face_id1==3 || face_id1==4)
+   {
+      i = size1d-1-i;
+   }
+   else if (face_id1==0)
+   {
+      j = size1d-1-j;
+   }
+   // Permute based on face orientations
+   switch (orientation)
+   {
+      case 0:
+         new_i = i;
+         new_j = j;
+         break;
+      case 1:
+         new_i = j;
+         new_j = i;
+         break;
+      case 2:
+         new_i = j;
+         new_j = (size1d-1-i);
+         break;
+      case 3:
+         new_i = (size1d-1-i);
+         new_j = j;
+         break;
+      case 4:
+         new_i = (size1d-1-i);
+         new_j = (size1d-1-j);
+         break;
+      case 5:
+         new_i = (size1d-1-j);
+         new_j = (size1d-1-i);
+         break;
+      case 6:
+         new_i = (size1d-1-j);
+         new_j = i;
+         break;
+      case 7:
+         new_i = i;
+         new_j = (size1d-1-j);
+         break;
+   }
+   return ToLexOrdering3D(face_id2, size1d, new_i, new_j);
+}
+
+/// @brief Given a face DOF (or quadrature) index ordered lexicographically
+/// relative to element 1, return the associated (i, j) coordinates.
+///
+/// The returned coordinates will be relative to element 1 or element 2
+/// according to the value of side (side == 0 corresponds element 1).
+MFEM_HOST_DEVICE
+inline void FaceIdxToVolIdx2D(const int qi, const int nq, const int face_id0,
+                              const int face_id1, const int side, int &i, int &j)
+{
+   // Note: in 2D, a consistently ordered mesh will always have the element 2
+   // face reversed relative to element 1, so orientation is determined entirely
+   // by side. (In 3D, separate orientation information is needed).
+   const int orientation = side;
+
+   const int face_id = (side == 0) ? face_id0 : face_id1;
+   const int edge_idx = (side == 0) ? qi : PermuteFace2D(face_id0, face_id1,
+                                                         orientation, nq, qi);
+
+   const int level = (face_id == 0 || face_id == 3) ? 0 : (nq-1);
+   const bool x_axis = (face_id == 0 || face_id == 2);
+
+   i = x_axis ? edge_idx : level;
+   j = x_axis ? level : edge_idx;
+}
+
+/// @brief Given a face DOF (or quadrature) index ordered lexicographically
+/// relative to element 1, return the associated (i, j, k) coordinates.
+///
+/// The returned coordinates will be relative to element 1 or element 2
+/// according to the value of side (side == 0 corresponds element 1).
+MFEM_HOST_DEVICE
+inline void FaceIdxToVolIdx3D(const int index, const int size1d,
+                              const int face_id0, const int face_id1,
+                              const int side, const int orientation,
+                              int& i, int& j, int& k)
+{
+   MFEM_VERIFY_KERNEL(face_id1 >= 0 || side == 0,
+                      "Accessing second side but face_id1 is not valid.");
+
+   const int face_id = (side == 0) ? face_id0 : face_id1;
+   const int fidx = (side == 0) ? index
+                    : PermuteFace3D(face_id0, face_id1, orientation, size1d, index);
+
+   const bool xy_plane = (face_id == 0 || face_id == 5);
+   const bool yz_plane = (face_id == 2 || face_id == 4);
+
+   const int level = (face_id == 0 || face_id == 1 || face_id == 4)
+                     ? 0 : (size1d-1);
+
+   const int _i = fidx % size1d;
+   const int _j = fidx / size1d;
+
+   k = xy_plane ? level : _j;
+   j = yz_plane ? _i : xy_plane ? _j : level;
+   i = yz_plane ? level : _i;
+}
+
 } // namespace internal

 } // namespace mfem
@@ -221,7 +221,7 @@ void FiniteElement::CalcPhysLaplacian(ElementTransformation &Trans,
   {
      for (int nd = 0; nd < dof; nd++)
      {
-         Laplacian[nd] = hess(nd,0) + hess(nd,4) + hess(nd,5);
+         Laplacian[nd] = hess(nd,0) + hess(nd,3) + hess(nd,5);
      }
   }
   else if (dim == 2)
@@ -259,10 +259,10 @@ void FiniteElement::CalcPhysLinLaplacian(ElementTransformation &Trans,
      scale[1] = 2*Gij(0,1);
      scale[2] = 2*Gij(0,2);

-      scale[3] = 2*Gij(1,2);
-      scale[4] =   Gij(2,2);
+      scale[3] =   Gij(1,1);
+      scale[4] = 2*Gij(1,2);

-      scale[5] =   Gij(1,1);
+      scale[5] =   Gij(2,2);
   }
   else if (dim == 2)
   {
@@ -299,12 +299,12 @@ void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,
      map[2] = 2;

      map[3] = 1;
-      map[4] = 5;
-      map[5] = 3;
+      map[4] = 3;
+      map[5] = 4;

      map[6] = 2;
-      map[7] = 3;
-      map[8] = 4;
+      map[7] = 4;
+      map[8] = 5;
   }
   else if (dim == 2)
   {
@@ -299,7 +299,8 @@ public:
      NONE, ///< No derivatives implemented
      GRAD, ///< Implements CalcDShape methods
      DIV,  ///< Implements CalcDivShape methods
-      CURL  ///< Implements CalcCurlShape methods
+      CURL, ///< Implements CalcCurlShape methods
+      HESS  ///< Implements CalcHessian & CalcDShape methods
   };

   /** @brief Construct FiniteElement with given
@@ -316,7 +317,7 @@ public:
   int GetDim() const { return dim; }

   /** @brief Returns the vector dimension for vector-valued finite elements,
-       which is also the dimension of the interpolation operatrion. */
+       which is also the dimension of the interpolation operation. */
   int GetRangeDim() const { return vdim; }

   /// Returns the dimension of the curl for vector-valued finite elements.
@@ -356,7 +357,7 @@ public:

   /** @brief Returns the FiniteElement::DerivType of the element describing the
       spatial derivative method implemented, one of {NONE, GRAD,
-       DIV, CURL}. */
+       DIV, CURL, HESS}. */
   int GetDerivType() const { return deriv_type; }

   /** @brief Returns the FiniteElement::DerivType of the element describing how
@@ -394,7 +395,32 @@ public:
   /// Get a const reference to the nodes of the element
   const IntegrationRule & GetNodes() const { return Nodes; }

-   // virtual functions for finite elements on vector spaces
+   /** @brief Evaluate the Hessians of all shape functions of a scalar finite
+       element in reference space at the given point @a ip. */
+   /** Each row of the result DenseMatrix @a Hessian contains upper triangular
+       part of the Hessian of one shape function.
+       The order in 2D is {u_xx, u_xy, u_yy}.
+       The size (#dof x (#dim (#dim+1)/2) of @a Hessian must be set in advance.*/
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &Hessian) const;
+
+   /** @brief Evaluate the Hessian of all shape functions of a scalar finite
+       element in physical space at the given point @a ip. */
+   /** The size (#dof, #dim*(#dim+1)/2) of @a Hessian must be set in advance. */
+   void CalcPhysHessian(ElementTransformation &Trans,
+                        DenseMatrix& Hessian) const;
+
+   /** @brief Evaluate the Laplacian of all shape functions of a scalar finite
+       element in physical space at the given point @a ip. */
+   /** The size (#dof) of @a Laplacian must be set in advance. */
+   void CalcPhysLaplacian(ElementTransformation &Trans,
+                          Vector& Laplacian) const;
+
+   /** @brief Evaluate the Laplacian of all shape functions of a scalar finite
+       element in physical space at the given point @a ip. */
+   /** The size (#dof) of @a Laplacian must be set in advance. */
+   void CalcPhysLinLaplacian(ElementTransformation &Trans,
+                             Vector& Laplacian) const;

   /** @brief Evaluate the values of all shape functions of a *vector* finite
       element in reference space at the given point @a ip. */
@@ -454,30 +480,6 @@ public:
   */
   virtual void GetFaceDofs(int face, int **dofs, int *ndofs) const;

-   /** @brief Evaluate the Hessians of all shape functions of a scalar finite
-       element in reference space at the given point @a ip. */
-   /** Each row of the result DenseMatrix @a Hessian contains upper triangular
-       part of the Hessian of one shape function.
-       The order in 2D is {u_xx, u_xy, u_yy}.
-       The size (#dof x (#dim (#dim+1)/2) of @a Hessian must be set in advance.*/
-   virtual void CalcHessian(const IntegrationPoint &ip,
-                            DenseMatrix &Hessian) const;
-
-   /** @brief Evaluate the Hessian of all shape functions of a scalar finite
-       element in reference space at the given point @a ip. */
-   /** The size (#dof, #dim*(#dim+1)/2) of @a Hessian must be set in advance. */
-   virtual void CalcPhysHessian(ElementTransformation &Trans,
-                                DenseMatrix& Hessian) const;
-
-   /** @brief Evaluate the Laplacian of all shape functions of a scalar finite
-       element in reference space at the given point @a ip. */
-   /** The size (#dof) of @a Laplacian must be set in advance. */
-   virtual void CalcPhysLaplacian(ElementTransformation &Trans,
-                                  Vector& Laplacian) const;
-
-   virtual void CalcPhysLinLaplacian(ElementTransformation &Trans,
-                                     Vector& Laplacian) const;
-
   /** @brief Return the local interpolation matrix @a I (Dof x Dof) where the
       fine element is the image of the base geometry under the given
       transformation. */
@@ -349,10 +349,10 @@ void NURBS3DFiniteElement::CalcHessian (const IntegrationPoint &ip,
            d2sum[1] += ( hessian(o,1) = dsx*dsy*sz*weights(o) );
            d2sum[2] += ( hessian(o,2) = dsx*sy*dsz*weights(o) );

-            d2sum[3] += ( hessian(o,3) = sx*dsy*dsz*weights(o) );
+            d2sum[3] += ( hessian(o,3) = sx*d2sy*sz*weights(o) );
+            d2sum[4] += ( hessian(o,4) = sx*dsy*dsz*weights(o) );

-            d2sum[4] += ( hessian(o,4) = sx*sy*d2sz*weights(o) );
-            d2sum[5] += ( hessian(o,5) = sx*d2sy*sz*weights(o) );
+            d2sum[5] += ( hessian(o,5) = sx*sy*d2sz*weights(o) );
         }
      }
   }
@@ -387,19 +387,632 @@ void NURBS3DFiniteElement::CalcHessian (const IntegrationPoint &ip,
                     + u[o]*sum*(2*dsum[0]*dsum[2] - d2sum[2]);

      hessian(o,3) = hessian(o,3)*sum
-                     - du(o,1)*sum*dsum[2]
-                     - du(o,2)*sum*dsum[1]
-                     + u[o]*sum*(2*dsum[1]*dsum[2] - d2sum[3]);
+                     - 2*du(o,1)*sum*dsum[1]
+                     + u[o]*sum*(2*dsum[1]*dsum[1] - d2sum[3]);

      hessian(o,4) = hessian(o,4)*sum
-                     - 2*du(o,2)*sum*dsum[2]
-                     + u[o]*sum*(2*dsum[2]*dsum[2] - d2sum[4]);
+                     - du(o,1)*sum*dsum[2]
+                     - du(o,2)*sum*dsum[1]
+                     + u[o]*sum*(2*dsum[1]*dsum[2] - d2sum[4]);

      hessian(o,5) = hessian(o,5)*sum
-                     - 2*du(o,1)*sum*dsum[1]
-                     + u[o]*sum*(2*dsum[1]*dsum[1] - d2sum[5]);
-
+                     - 2*du(o,2)*sum*dsum[2]
+                     + u[o]*sum*(2*dsum[2]*dsum[2] - d2sum[5]);
   }
 }

+
+void NURBS_HDiv2DFiniteElement::SetOrder() const
+{
+   orders[0] = kv[0]->GetOrder();
+   orders[1] = kv[1]->GetOrder();
+
+   if (kv1[0]) { delete kv1[0]; }
+   if (kv1[1]) { delete kv1[1]; }
+
+   kv1[0] = kv[0]->DegreeElevate(1);
+   kv1[1] = kv[1]->DegreeElevate(1);
+
+   shape_x.SetSize(orders[0]+1);
+   shape_y.SetSize(orders[1]+1);
+
+   dshape_x.SetSize(orders[0]+1);
+   dshape_y.SetSize(orders[1]+1);
+
+   d2shape_x.SetSize(orders[0]+1);
+   d2shape_y.SetSize(orders[1]+1);
+
+   shape1_x.SetSize(orders[0]+2);
+   shape1_y.SetSize(orders[1]+2);
+
+   dshape1_x.SetSize(orders[0]+2);
+   dshape1_y.SetSize(orders[1]+2);
+
+   d2shape1_x.SetSize(orders[0]+2);
+   d2shape1_y.SetSize(orders[1]+2);
+
+   order = max(orders[0]+1, orders[1]+1);
+   dof = (orders[0] + 2)*(orders[1] + 1)
+         + (orders[1] + 1)*(orders[1] + 2);
+   u.SetSize(dof);
+   du.SetSize(dof);
+   weights.SetSize(dof);
+}
+
+void NURBS_HDiv2DFiniteElement::CalcVShape(const IntegrationPoint &ip,
+                                           DenseMatrix &shape) const
+{
+   kv[0]->CalcShape(shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape(shape_y, ijk[1], ip.y);
+
+   kv1[0]->CalcShape(shape1_x, ijk[0], ip.x);
+   kv1[1]->CalcShape(shape1_y, ijk[1], ip.y);
+
+   int o = 0;
+   for (int j = 0; j <= orders[1]; j++)
+   {
+      const real_t sy = shape_y(j);
+      for (int i = 0; i <= orders[0]+1; i++, o++)
+      {
+         shape(o,0) = shape1_x(i)*sy;
+         shape(o,1) = 0.0;
+      }
+   }
+
+   for (int j = 0; j <= orders[1]+1; j++)
+   {
+      const real_t sy1 = shape1_y(j);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         shape(o,0) = 0.0;
+         shape(o,1) = shape_x(i)*sy1;
+      }
+   }
+}
+
+void NURBS_HDiv2DFiniteElement::CalcVShape(ElementTransformation &Trans,
+                                           DenseMatrix &shape) const
+{
+   CalcVShape(Trans.GetIntPoint(), shape);
+   const DenseMatrix & J = Trans.Jacobian();
+   MFEM_ASSERT(J.Width() == 2 && J.Height() == 2,
+               "NURBS_HDiv2DFiniteElement cannot be embedded in "
+               "3 dimensional spaces");
+   for (int i=0; i<dof; i++)
+   {
+      real_t sx = shape(i, 0);
+      real_t sy = shape(i, 1);
+      shape(i, 0) = sx * J(0, 0) + sy * J(0, 1);
+      shape(i, 1) = sx * J(1, 0) + sy * J(1, 1);
+   }
+   shape *= (1.0 / Trans.Weight());
+}
+
+void NURBS_HDiv2DFiniteElement::CalcDivShape(const IntegrationPoint &ip,
+                                             Vector &divshape) const
+{
+   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
+
+   kv1[0]->CalcDShape(dshape1_x, ijk[0], ip.x);
+   kv1[1]->CalcDShape(dshape1_y, ijk[1], ip.y);
+
+   int o = 0;
+   for (int j = 0; j <= orders[1]; j++)
+   {
+      const real_t sy = shape_y(j);
+      for (int i = 0; i <= orders[0]+1; i++, o++)
+      {
+         divshape(o) = dshape1_x(i)*sy;
+      }
+   }
+
+   for (int j = 0; j <= orders[1]+1; j++)
+   {
+      const real_t dsy1 = dshape1_y(j);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         divshape(o) = shape_x(i)*dsy1;
+      }
+   }
+}
+
+NURBS_HDiv2DFiniteElement::~NURBS_HDiv2DFiniteElement()
+{
+   if (kv1[0]) { delete kv1[0]; }
+   if (kv1[1]) { delete kv1[1]; }
+}
+
+
+void NURBS_HDiv3DFiniteElement::SetOrder() const
+{
+   orders[0] = kv[0]->GetOrder();
+   orders[1] = kv[1]->GetOrder();
+   orders[2] = kv[2]->GetOrder();
+
+   if (kv1[0]) { delete kv1[0]; }
+   if (kv1[1]) { delete kv1[1]; }
+   if (kv1[2]) { delete kv1[2]; }
+
+   kv1[0] = kv[0]->DegreeElevate(1);
+   kv1[1] = kv[1]->DegreeElevate(1);
+   kv1[2] = kv[2]->DegreeElevate(1);
+
+   shape_x.SetSize(orders[0]+1);
+   shape_y.SetSize(orders[1]+1);
+   shape_z.SetSize(orders[2]+1);
+
+   dshape_x.SetSize(orders[0]+1);
+   dshape_y.SetSize(orders[1]+1);
+   dshape_z.SetSize(orders[2]+1);
+
+   d2shape_x.SetSize(orders[0]+1);
+   d2shape_y.SetSize(orders[1]+1);
+   d2shape_z.SetSize(orders[2]+1);
+
+   shape1_x.SetSize(orders[0]+2);
+   shape1_y.SetSize(orders[1]+2);
+   shape1_z.SetSize(orders[2]+2);
+
+   dshape1_x.SetSize(orders[0]+2);
+   dshape1_y.SetSize(orders[1]+2);
+   dshape1_z.SetSize(orders[2]+2);
+
+   d2shape1_x.SetSize(orders[0]+2);
+   d2shape1_y.SetSize(orders[1]+2);
+   d2shape1_z.SetSize(orders[2]+2);
+
+   order = max(orders[0]+1, max( orders[1]+1, orders[2]+1));
+   dof = (orders[0] + 2)*(orders[1] + 1)*(orders[2] + 1) +
+         (orders[0] + 1)*(orders[1] + 2)*(orders[2] + 1) +
+         (orders[0] + 1)*(orders[1] + 1)*(orders[2] + 2);
+   u.SetSize(dof);
+   du.SetSize(dof);
+   weights.SetSize(dof);
+}
+
+void NURBS_HDiv3DFiniteElement::CalcVShape(const IntegrationPoint &ip,
+                                           DenseMatrix &shape) const
+{
+   kv[0]->CalcShape(shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape(shape_y, ijk[1], ip.y);
+   kv[2]->CalcShape(shape_z, ijk[2], ip.z);
+
+   kv1[0]->CalcShape(shape1_x, ijk[0], ip.x);
+   kv1[1]->CalcShape(shape1_y, ijk[1], ip.y);
+   kv1[2]->CalcShape(shape1_z, ijk[2], ip.z);
+
+   shape = 0.0;
+   int o = 0;
+   for (int k = 0; k <= orders[2]; k++)
+   {
+      const real_t sz = shape_z(k);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         const real_t sy_sz = shape_y(j)*sz;
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            shape(o,0) = shape1_x(i)*sy_sz;
+         }
+      }
+   }
+
+   for (int  k = 0; k <= orders[2]; k++)
+   {
+      const real_t sz = shape_z(k);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         const real_t sy1_sz = shape1_y(j)*sz;
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            shape(o,1) = shape_x(i)*sy1_sz;
+         }
+      }
+   }
+
+   for (int  k = 0; k <= orders[2]+1; k++)
+   {
+      const real_t sz1 = shape1_z(k);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         const real_t sy_sz1 = shape_y(j)*sz1;
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            shape(o,2) = shape_x(i)*sy_sz1;
+         }
+      }
+   }
+}
+
+void NURBS_HDiv3DFiniteElement::CalcVShape(ElementTransformation &Trans,
+                                           DenseMatrix &shape) const
+{
+   CalcVShape(Trans.GetIntPoint(), shape);
+   const DenseMatrix & J = Trans.Jacobian();
+   MFEM_ASSERT(J.Width() == 3 && J.Height() == 3,
+               "RT_R2D_FiniteElement cannot be embedded in "
+               "3 dimensional spaces");
+   for (int i=0; i<dof; i++)
+   {
+      real_t sx = shape(i, 0);
+      real_t sy = shape(i, 1);
+      real_t sz = shape(i, 2);
+      shape(i, 0) = sx * J(0, 0) + sy * J(0, 1) + sz * J(0, 2);
+      shape(i, 1) = sx * J(1, 0) + sy * J(1, 1) + sz * J(1, 2);
+      shape(i, 2) = sx * J(2, 0) + sy * J(2, 1) + sz * J(2, 2);
+   }
+   shape *= (1.0 / Trans.Weight());
+}
+
+void NURBS_HDiv3DFiniteElement::CalcDivShape(const IntegrationPoint &ip,
+                                             Vector &divshape) const
+{
+   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
+   kv[2]->CalcShape ( shape_z, ijk[2], ip.z);
+
+   kv1[0]->CalcDShape(dshape1_x, ijk[0], ip.x);
+   kv1[1]->CalcDShape(dshape1_y, ijk[1], ip.y);
+   kv1[2]->CalcDShape(dshape1_z, ijk[2], ip.z);
+
+   int o = 0;
+   for (int  k = 0; k <= orders[2]; k++)
+   {
+      const real_t sz = shape_z(k);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         const real_t sy_sz = shape_y(j)*sz;
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            divshape(o) = dshape1_x(i)*sy_sz;
+         }
+      }
+   }
+
+   for (int  k = 0; k <= orders[2]; k++)
+   {
+      const real_t sz = shape_z(k);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         const real_t dy1_sz = dshape1_y(j)*sz;
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            divshape(o) = shape_x(i)*dy1_sz;
+         }
+      }
+   }
+
+   for (int  k = 0; k <= orders[2]+1; k++)
+   {
+      const real_t dz1 = dshape1_z(k);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         const real_t sy_dz1 = shape_y(j)*dz1;
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            divshape(o) = shape_x(i)*sy_dz1;
+         }
+      }
+   }
+}
+
+NURBS_HDiv3DFiniteElement::~NURBS_HDiv3DFiniteElement()
+{
+   if (kv1[0]) { delete kv1[0]; }
+   if (kv1[1]) { delete kv1[1]; }
+   if (kv1[2]) { delete kv1[2]; }
+}
+
+void NURBS_HCurl2DFiniteElement::SetOrder() const
+{
+   orders[0] = kv[0]->GetOrder();
+   orders[1] = kv[1]->GetOrder();
+
+   if (kv1[0]) { delete kv1[0]; }
+   if (kv1[1]) { delete kv1[1]; }
+
+   kv1[0] = kv[0]->DegreeElevate(1);
+   kv1[1] = kv[1]->DegreeElevate(1);
+
+   shape_x.SetSize(orders[0]+1);
+   shape_y.SetSize(orders[1]+1);
+
+   dshape_x.SetSize(orders[0]+1);
+   dshape_y.SetSize(orders[1]+1);
+
+   d2shape_x.SetSize(orders[0]+1);
+   d2shape_y.SetSize(orders[1]+1);
+
+   shape1_x.SetSize(orders[0]+2);
+   shape1_y.SetSize(orders[1]+2);
+
+   dshape1_x.SetSize(orders[0]+2);
+   dshape1_y.SetSize(orders[1]+2);
+
+   d2shape1_x.SetSize(orders[0]+2);
+   d2shape1_y.SetSize(orders[1]+2);
+
+   order = max(orders[0]+1, orders[1]+1);
+   dof = (orders[0] + 1)*(orders[1] + 2)
+         + (orders[1] + 2)*(orders[1] + 1);
+   u.SetSize(dof);
+   du.SetSize(dof);
+   weights.SetSize(dof);
+}
+
+void NURBS_HCurl2DFiniteElement::CalcVShape(const IntegrationPoint &ip,
+                                            DenseMatrix &shape) const
+{
+   kv[0]->CalcShape(shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape(shape_y, ijk[1], ip.y);
+
+   kv1[0]->CalcShape(shape1_x, ijk[0], ip.x);
+   kv1[1]->CalcShape(shape1_y, ijk[1], ip.y);
+
+   int o = 0;
+   for (int j = 0; j <= orders[1]+1; j++)
+   {
+      const real_t sy1 = shape1_y(j);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         shape(o,0) = shape_x(i)*sy1;
+         shape(o,1) = 0.0;
+      }
+   }
+
+   for (int j = 0; j <= orders[1]; j++)
+   {
+      const real_t sy = shape_y(j);
+      for (int i = 0; i <= orders[0]+1; i++, o++)
+      {
+         shape(o,0) = 0.0;
+         shape(o,1) = shape1_x(i)*sy;
+      }
+   }
+}
+
+void NURBS_HCurl2DFiniteElement::CalcVShape(ElementTransformation &Trans,
+                                            DenseMatrix &shape) const
+{
+   CalcVShape(Trans.GetIntPoint(), shape);
+   const DenseMatrix & JI = Trans.InverseJacobian();
+   MFEM_ASSERT(JI.Width() == 2 && JI.Height() == 2,
+               "NURBS_HCurl2DFiniteElement cannot be embedded in "
+               "3 dimensional spaces");
+   for (int i=0; i<dof; i++)
+   {
+      real_t sx = shape(i, 0);
+      real_t sy = shape(i, 1);
+      shape(i, 0) = sx * JI(0, 0) + sy * JI(1, 0);
+      shape(i, 1) = sx * JI(0, 1) + sy * JI(1, 1);
+   }
+}
+
+void NURBS_HCurl2DFiniteElement::CalcCurlShape(const IntegrationPoint &ip,
+                                               DenseMatrix &curl_shape) const
+{
+   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
+
+   kv1[0]->CalcDShape(dshape1_x, ijk[0], ip.x);
+   kv1[1]->CalcDShape(dshape1_y, ijk[1], ip.y);
+
+   int o = 0;
+   for (int j = 0; j <= orders[1]+1; j++)
+   {
+      const real_t dsy1 = dshape1_y(j);
+      for (int i = 0; i <= orders[0]; i++, o++)
+      {
+         curl_shape(o,0) = -shape_x(i)*dsy1;
+      }
+   }
+
+   for (int j = 0; j <= orders[1]; j++)
+   {
+      const real_t sy = shape_y(j);
+      for (int i = 0; i <= orders[0]+1; i++, o++)
+      {
+         curl_shape(o,0) = dshape1_x(i)*sy;
+      }
+   }
+}
+
+NURBS_HCurl2DFiniteElement::~NURBS_HCurl2DFiniteElement()
+{
+   if (kv1[0]) { delete kv1[0]; }
+   if (kv1[1]) { delete kv1[1]; }
+}
+
+
+void NURBS_HCurl3DFiniteElement::SetOrder() const
+{
+   orders[0] = kv[0]->GetOrder();
+   orders[1] = kv[1]->GetOrder();
+   orders[2] = kv[2]->GetOrder();
+
+   if (kv1[0]) { delete kv1[0]; }
+   if (kv1[1]) { delete kv1[1]; }
+   if (kv1[2]) { delete kv1[2]; }
+
+   kv1[0] = kv[0]->DegreeElevate(1);
+   kv1[1] = kv[1]->DegreeElevate(1);
+   kv1[2] = kv[2]->DegreeElevate(1);
+
+   shape_x.SetSize(orders[0]+1);
+   shape_y.SetSize(orders[1]+1);
+   shape_z.SetSize(orders[2]+1);
+
+   dshape_x.SetSize(orders[0]+1);
+   dshape_y.SetSize(orders[1]+1);
+   dshape_z.SetSize(orders[2]+1);
+
+   d2shape_x.SetSize(orders[0]+1);
+   d2shape_y.SetSize(orders[1]+1);
+   d2shape_z.SetSize(orders[2]+1);
+
+   shape1_x.SetSize(orders[0]+2);
+   shape1_y.SetSize(orders[1]+2);
+   shape1_z.SetSize(orders[2]+2);
+
+   dshape1_x.SetSize(orders[0]+2);
+   dshape1_y.SetSize(orders[1]+2);
+   dshape1_z.SetSize(orders[2]+2);
+
+   d2shape1_x.SetSize(orders[0]+2);
+   d2shape1_y.SetSize(orders[1]+2);
+   d2shape1_z.SetSize(orders[2]+2);
+
+   order = max(orders[0]+1, max( orders[1]+1, orders[2]+1));
+   dof = (orders[0] + 1)*(orders[1] + 2)*(orders[2] + 2) +
+         (orders[0] + 2)*(orders[1] + 1)*(orders[2] + 2) +
+         (orders[0] + 2)*(orders[1] + 2)*(orders[2] + 1);
+   u.SetSize(dof);
+   du.SetSize(dof);
+   weights.SetSize(dof);
+}
+
+void NURBS_HCurl3DFiniteElement::CalcVShape(const IntegrationPoint &ip,
+                                            DenseMatrix &shape) const
+{
+   kv[0]->CalcShape(shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape(shape_y, ijk[1], ip.y);
+   kv[2]->CalcShape(shape_z, ijk[2], ip.z);
+
+   kv1[0]->CalcShape(shape1_x, ijk[0], ip.x);
+   kv1[1]->CalcShape(shape1_y, ijk[1], ip.y);
+   kv1[2]->CalcShape(shape1_z, ijk[2], ip.z);
+
+   shape = 0.0;
+   int o = 0;
+   for (int  k = 0; k <= orders[2]+1; k++)
+   {
+      const real_t sz1 = shape1_z(k);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         const real_t sy1_sz1 = shape1_y(j)*sz1;
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            shape(o,0) = shape_x(i)*sy1_sz1;
+         }
+      }
+   }
+
+   for (int  k = 0; k <= orders[2]+1; k++)
+   {
+      const real_t sz1 = shape1_z(k);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         const real_t sy_sz1 = shape_y(j)*sz1;
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            shape(o,1) = shape1_x(i)*sy_sz1;
+         }
+      }
+   }
+
+   for (int  k = 0; k <= orders[2]; k++)
+   {
+      const real_t sz = shape_z(k);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         const real_t sy1_sz = shape1_y(j)*sz;
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            shape(o,2) = shape1_x(i)*sy1_sz;
+         }
+      }
+   }
+}
+
+void NURBS_HCurl3DFiniteElement::CalcVShape(ElementTransformation &Trans,
+                                            DenseMatrix &shape) const
+{
+   CalcVShape(Trans.GetIntPoint(), shape);
+   const DenseMatrix & JI = Trans.InverseJacobian();
+   MFEM_ASSERT(JI.Width() == 3 && JI.Height() == 3,
+               "NURBS_HCurl3DFiniteElement must be in a"
+               "3 dimensional spaces");
+   for (int i=0; i<dof; i++)
+   {
+      real_t sx = shape(i, 0);
+      real_t sy = shape(i, 1);
+      real_t sz = shape(i, 2);
+      shape(i, 0) = sx * JI(0, 0) + sy * JI(1, 0) + sz * JI(2, 0);
+      shape(i, 1) = sx * JI(0, 1) + sy * JI(1, 1) + sz * JI(2, 1);
+      shape(i, 2) = sx * JI(0, 2) + sy * JI(1, 2) + sz * JI(2, 2);
+   }
+}
+
+void NURBS_HCurl3DFiniteElement::CalcCurlShape(const IntegrationPoint &ip,
+                                               DenseMatrix &curl_shape) const
+{
+   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
+   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
+   kv[2]->CalcShape ( shape_z, ijk[2], ip.z);
+
+   kv1[0]->CalcShape(shape1_x, ijk[0], ip.x);
+   kv1[1]->CalcShape(shape1_y, ijk[1], ip.y);
+   kv1[2]->CalcShape(shape1_z, ijk[2], ip.z);
+
+   kv1[0]->CalcDShape(dshape1_x, ijk[0], ip.x);
+   kv1[1]->CalcDShape(dshape1_y, ijk[1], ip.y);
+   kv1[2]->CalcDShape(dshape1_z, ijk[2], ip.z);
+
+   int o = 0;
+   for (int  k = 0; k <= orders[2]+1; k++)
+   {
+      const real_t sz1 = shape1_z(k), dsz1 = dshape1_z(k);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         const real_t sy1_dsz1 = shape1_y(j)*dsz1,
+                      dsy1_sz1 = dshape1_y(j)*sz1;
+         for (int i = 0; i <= orders[0]; i++, o++)
+         {
+            curl_shape(o,0) = 0.0;
+            curl_shape(o,1) =  shape_x(i)*sy1_dsz1;
+            curl_shape(o,2) = -shape_x(i)*dsy1_sz1;
+         }
+      }
+   }
+
+   for (int  k = 0; k <= orders[2]+1; k++)
+   {
+      const real_t sz1 = shape1_z(k), dsz1 = dshape1_z(k);
+      for (int j = 0; j <= orders[1]; j++)
+      {
+         const real_t sy_dsz1 = shape_y(j)*dsz1,
+                      sy_sz1 = shape_y(j)*sz1;
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            curl_shape(o,0) = -shape1_x(i)*sy_dsz1;
+            curl_shape(o,1) = 0.0;
+            curl_shape(o,2) = dshape1_x(i)*sy_sz1;
+         }
+      }
+   }
+
+   for (int  k = 0; k <= orders[2]; k++)
+   {
+      const real_t sz = shape_z(k);
+      for (int j = 0; j <= orders[1]+1; j++)
+      {
+         const real_t sy1_sz = shape1_y(j)*sz,
+                      dsy1_sz = dshape1_y(j)*sz;
+         for (int i = 0; i <= orders[0]+1; i++, o++)
+         {
+            curl_shape(o,0) = shape1_x(i)*dsy1_sz;
+            curl_shape(o,1) = -dshape1_x(i)*sy1_sz;
+            curl_shape(o,2) = 0.0;
+         }
+      }
+   }
+}
+
+NURBS_HCurl3DFiniteElement::~NURBS_HCurl3DFiniteElement()
+{
+   if (kv1[0]) { delete kv1[0]; }
+   if (kv1[1]) { delete kv1[1]; }
+   if (kv1[2]) { delete kv1[2]; }
+}
+
 }
@@ -20,7 +20,7 @@ namespace mfem
 class KnotVector;

 /// An arbitrary order and dimension NURBS element
-class NURBSFiniteElement : public ScalarFiniteElement
+class NURBSFiniteElement
 {
 protected:
   mutable Array <const KnotVector*> kv;
@@ -30,31 +30,34 @@ protected:

 public:
   /** @brief Construct NURBSFiniteElement with given
-       @param D    Reference space dimension
-       @param G    Geometry type (of type Geometry::Type)
-       @param Do   Number of degrees of freedom in the FiniteElement
-       @param O    Order/degree of the FiniteElement
-       @param F    FunctionSpace type of the FiniteElement
+       @param dim    Reference space dimension
    */
-   NURBSFiniteElement(int D, Geometry::Type G, int Do, int O, int F)
-      : ScalarFiniteElement(D, G, Do, O, F)
+   NURBSFiniteElement(int dim)
   {
      ijk = NULL;
      patch = elem = -1;
      kv.SetSize(dim);
-      weights.SetSize(dof);
-      weights = 1.0;
   }

+   /// Resets the patch and element data stored in the element
   void                 Reset      ()         const { patch = elem = -1; }
+   /// Set which IJK in patch should be evaluated
   void                 SetIJK     (const int *IJK) const { ijk = IJK; }
+   /// Get which patch is currently considered
   int                  GetPatch   ()         const { return patch; }
+   /// Set which patch should be evaluated
   void                 SetPatch   (int p)    const { patch = p; }
+   /// Set which elemenet should be evaluated
   int                  GetElement ()         const { return elem; }
+   /// Get which element is currently considered
   void                 SetElement (int e)    const { elem = e; }
+   /// Get the KnotVectors
   Array <const KnotVector*> &KnotVectors()   const { return kv; }
+   /// Get the Weights
   Vector              &Weights    ()         const { return weights; }
-   /// Update the NURBSFiniteElement according to the currently set knot vectors
+   /// Update the polynomial order according to the currently set knotvectors
+   /// Resizes all internal data members to have the correct size
+   /// related to the polynomial order
   virtual void         SetOrder   ()         const { }

   /// Returns the indices (i,j) in 2D or (i,j,k) in 3D of this element in the
@@ -64,7 +67,8 @@ public:


 /// An arbitrary order 1D NURBS element on a segment
-class NURBS1DFiniteElement : public NURBSFiniteElement
+class NURBS1DFiniteElement : public ScalarFiniteElement,
+   public NURBSFiniteElement
 {
 protected:
   mutable Vector shape_x;
@@ -72,7 +76,8 @@ protected:
 public:
   /// Construct the NURBS1DFiniteElement of order @a p
   NURBS1DFiniteElement(int p)
-      : NURBSFiniteElement(1, Geometry::SEGMENT, p + 1, p, FunctionSpace::Qk),
+      : ScalarFiniteElement(1, Geometry::SEGMENT, p + 1, p, FunctionSpace::Qk),
+        NURBSFiniteElement(1),
        shape_x(p + 1) { }

   virtual void SetOrder() const;
@@ -84,7 +89,8 @@ public:
 };

 /// An arbitrary order 2D NURBS element on a square
-class NURBS2DFiniteElement : public NURBSFiniteElement
+class NURBS2DFiniteElement : public ScalarFiniteElement,
+   public NURBSFiniteElement
 {
 protected:
   mutable Vector u, shape_x, shape_y, dshape_x, dshape_y, d2shape_x, d2shape_y;
@@ -93,16 +99,18 @@ protected:
 public:
   /// Construct the NURBS2DFiniteElement of order @a p
   NURBS2DFiniteElement(int p)
-      : NURBSFiniteElement(2, Geometry::SQUARE, (p + 1)*(p + 1), p,
-                           FunctionSpace::Qk),
+      : ScalarFiniteElement(2, Geometry::SQUARE, (p + 1)*(p + 1), p,
+                            FunctionSpace::Qk),
+        NURBSFiniteElement(2),
        u(dof), shape_x(p + 1), shape_y(p + 1), dshape_x(p + 1),
        dshape_y(p + 1), d2shape_x(p + 1), d2shape_y(p + 1), du(dof,2)
   { orders[0] = orders[1] = p; }

   /// Construct the NURBS2DFiniteElement with x-order @a px and y-order @a py
   NURBS2DFiniteElement(int px, int py)
-      : NURBSFiniteElement(2, Geometry::SQUARE, (px + 1)*(py + 1),
-                           std::max(px, py), FunctionSpace::Qk),
+      : ScalarFiniteElement(2, Geometry::SQUARE, (px + 1)*(py + 1),
+                            std::max(px, py), FunctionSpace::Qk),
+        NURBSFiniteElement(2),
        u(dof), shape_x(px + 1), shape_y(py + 1), dshape_x(px + 1),
        dshape_y(py + 1), d2shape_x(px + 1), d2shape_y(py + 1), du(dof,2)
   { orders[0] = px; orders[1] = py; }
@@ -116,7 +124,8 @@ public:
 };

 /// An arbitrary order 3D NURBS element on a cube
-class NURBS3DFiniteElement : public NURBSFiniteElement
+class NURBS3DFiniteElement : public ScalarFiniteElement,
+   public NURBSFiniteElement
 {
 protected:
   mutable Vector u, shape_x, shape_y, shape_z;
@@ -127,8 +136,9 @@ protected:
 public:
   /// Construct the NURBS3DFiniteElement of order @a p
   NURBS3DFiniteElement(int p)
-      : NURBSFiniteElement(3, Geometry::CUBE, (p + 1)*(p + 1)*(p + 1), p,
-                           FunctionSpace::Qk),
+      : ScalarFiniteElement(3, Geometry::CUBE, (p + 1)*(p + 1)*(p + 1), p,
+                            FunctionSpace::Qk),
+        NURBSFiniteElement(3),
        u(dof), shape_x(p + 1), shape_y(p + 1), shape_z(p + 1),
        dshape_x(p + 1), dshape_y(p + 1), dshape_z(p + 1),
        d2shape_x(p + 1), d2shape_y(p + 1), d2shape_z(p + 1), du(dof,3)
@@ -137,8 +147,9 @@ public:
   /// Construct the NURBS3DFiniteElement with x-order @a px and y-order @a py
   /// and z-order @a pz
   NURBS3DFiniteElement(int px, int py, int pz)
-      : NURBSFiniteElement(3, Geometry::CUBE, (px + 1)*(py + 1)*(pz + 1),
-                           std::max(std::max(px,py),pz), FunctionSpace::Qk),
+      : ScalarFiniteElement(3, Geometry::CUBE, (px + 1)*(py + 1)*(pz + 1),
+                            std::max(std::max(px,py),pz), FunctionSpace::Qk),
+        NURBSFiniteElement(2),
        u(dof), shape_x(px + 1), shape_y(py + 1), shape_z(pz + 1),
        dshape_x(px + 1), dshape_y(py + 1), dshape_z(pz + 1),
        d2shape_x(px + 1), d2shape_y(py + 1), d2shape_z(pz + 1), du(dof,3)
@@ -152,6 +163,352 @@ public:
                             DenseMatrix &hessian) const;
 };

+
+/** An arbitrary order H(div)-conforming 2D NURBS element on a square.
+    More details in the following papers:
+
+[1] Annalisa Buffa, Carlo De Falco, Giancarlo Sangalli
+"Isogeometric analysis: stable elements for the 2D Stokes equation."
+International Journal for Numerical Methods in Fluids 65 (11‐12) 1407-1422
+
+[2] John A Evans, Thomas JR Hughes
+"Isogeometric divergence-conforming B-splines for the unsteady Navier–Stokes equations."
+Journal of Computational Physics (241) 141-167
+*/
+class NURBS_HDiv2DFiniteElement : public VectorFiniteElement,
+   public NURBSFiniteElement
+{
+protected:
+   mutable Vector shape_x, shape_y, dshape_x, dshape_y, d2shape_x, d2shape_y;
+   mutable Vector shape1_x, shape1_y, dshape1_x, dshape1_y, d2shape1_x, d2shape1_y;
+   mutable Vector u;
+   mutable DenseMatrix du;
+   mutable Array <const KnotVector*> kv1;
+
+public:
+   /// Construct the NURBS_HDiv2DFiniteElement of order @a p
+   NURBS_HDiv2DFiniteElement(int p)
+      : VectorFiniteElement(2, Geometry::SQUARE, 2*(p + 1)*(p + 2), p,
+                            H_DIV,FunctionSpace::Qk),
+        NURBSFiniteElement(2),
+        shape_x(p + 1), shape_y(p + 1), dshape_x(p + 1),
+        dshape_y(p + 1), d2shape_x(p + 1), d2shape_y(p + 1),
+        shape1_x(p + 2), shape1_y(p + 2), dshape1_x(p + 2),
+        dshape1_y(p + 2), d2shape1_x(p + 2), d2shape1_y(p + 2),
+        u(dof), du(dof,2)
+   {
+      orders[0] = orders[1] = p;
+      kv1.SetSize(dim);
+      kv1[0] = nullptr;
+      kv1[1] = nullptr;
+   }
+
+   /// Construct the NURBS_HDiv2DFiniteElement with x-order @a px and y-order @a py
+   NURBS_HDiv2DFiniteElement(int px, int py)
+      : VectorFiniteElement(2, Geometry::SQUARE,
+                            (px + 2)*(py + 1)+(px + 1)*(py + 2),
+                            std::max(px, py), H_DIV, FunctionSpace::Qk),
+        NURBSFiniteElement(2),
+        shape_x(px + 1), shape_y(py + 1), dshape_x(px + 1),
+        dshape_y(py + 1), d2shape_x(px + 1), d2shape_y(py + 1),
+        shape1_x(px + 2), shape1_y(py + 2), dshape1_x(px + 2),
+        dshape1_y(py + 2), d2shape1_x(px + 2), d2shape1_y(py + 2),
+        u(dof), du(dof,2)
+   {
+      orders[0] = px; orders[1] = py;
+      kv1.SetSize(dim);
+      kv1[0] = nullptr;
+      kv1[1] = nullptr;
+   }
+
+   virtual void SetOrder() const;
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+
+   /** @brief Evaluate the values of all shape functions of a *vector* finite
+       element in physical space at the point described by @a Trans. */
+   /** Each row of the result DenseMatrix @a shape contains the components of
+       one vector shape function. The size (#dof x SDim) of @a shape must be set
+       in advance, where SDim >= #dim is the physical space dimension as
+       described by @a Trans. */
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const;
+
+   /** @brief Evaluate the divergence of all shape functions of a *vector*
+       finite element in reference space at the given point @a ip. */
+   /** The size (#dof) of the result Vector @a divshape must be set in advance.
+    */
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+
+   ~NURBS_HDiv2DFiniteElement();
+};
+
+
+/** An arbitrary order H(div)-conforming 3D NURBS element on a cube
+    More details in the following papers:
+
+    [1] Annalisa Buffa, Carlo De Falco, Giancarlo Sangalli
+    "Isogeometric analysis: stable elements for the 2D Stokes equation."
+    International Journal for Numerical Methods in Fluids 65 (11‐12) 1407-1422
+
+    [2] John A Evans, Thomas JR Hughes
+    "Isogeometric divergence-conforming B-splines for the unsteady
+    Navier–Stokes equations."
+    Journal of Computational Physics (241) 141-167 */
+class NURBS_HDiv3DFiniteElement : public VectorFiniteElement,
+   public NURBSFiniteElement
+{
+protected:
+   mutable Vector shape_x, shape_y, shape_z;
+   mutable Vector dshape_x, dshape_y, dshape_z;
+   mutable Vector d2shape_x, d2shape_y, d2shape_z;
+   mutable Vector shape1_x, shape1_y, shape1_z;
+   mutable Vector dshape1_x, dshape1_y, dshape1_z;
+   mutable Vector d2shape1_x, d2shape1_y, d2shape1_z;
+   mutable Vector u;
+   mutable DenseMatrix du;
+   mutable Array <const KnotVector*> kv1;
+
+public:
+   /// Construct the NURBS_HDiv3DFiniteElement of order @a p
+   NURBS_HDiv3DFiniteElement(int p)
+      : VectorFiniteElement(3, Geometry::CUBE, 3*(p + 1)*(p + 1)*(p + 2),
+                            p, H_DIV,FunctionSpace::Qk),
+        NURBSFiniteElement(3),
+        shape_x(p + 1), shape_y(p + 1), shape_z(p + 1),
+        dshape_x(p + 1), dshape_y(p + 1), dshape_z(p + 1),
+        d2shape_x(p + 1), d2shape_y(p + 1), d2shape_z(p + 1),
+        shape1_x(p + 2), shape1_y(p + 2), shape1_z(p + 2),
+        dshape1_x(p + 2), dshape1_y(p + 2),dshape1_z(p + 2),
+        d2shape1_x(p + 2), d2shape1_y(p + 2), d2shape1_z(p + 2),
+        u(dof), du(dof,3)
+   {
+      orders[0] = orders[1] = orders[2] = p;
+      kv1.SetSize(dim);
+      kv1[0] = nullptr;
+      kv1[1] = nullptr;
+      kv1[2] = nullptr;
+   }
+
+   /// Construct the NURBS_HDiv3DFiniteElement with x-order @a px, y-order @a py and z-order @a pz
+   NURBS_HDiv3DFiniteElement(int px, int py, int pz)
+      : VectorFiniteElement(3, Geometry::CUBE,
+                            (px + 2)*(py + 1)*(pz + 1) +
+                            (px + 1)*(py + 2)*(pz + 1) +
+                            (px + 1)*(py + 1)*(pz + 2),
+                            std::max(px, py), H_DIV, FunctionSpace::Qk),
+        NURBSFiniteElement(3),
+        shape_x(px + 1), shape_y(py + 1), shape_z(pz + 1),
+        dshape_x(px + 1), dshape_y(py + 1), dshape_z(pz + 1),
+        d2shape_x(px + 1), d2shape_y(py + 1), d2shape_z(pz + 1),
+        shape1_x(px + 2), shape1_y(py + 2), shape1_z(pz + 2),
+        dshape1_x(px + 2), dshape1_y(py + 2),dshape1_z(pz + 2),
+        d2shape1_x(px + 2), d2shape1_y(py + 2), d2shape1_z(pz + 2),
+        u(dof), du(dof,3)
+   {
+      orders[0] = px; orders[1] = py; orders[2] = pz;
+      kv1.SetSize(dim);
+      kv1[0] = nullptr;
+      kv1[1] = nullptr;
+      kv1[2] = nullptr;
+   }
+
+   virtual void SetOrder() const;
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+
+   /** @brief Evaluate the values of all shape functions of a *vector* finite
+       element in physical space at the point described by @a Trans. */
+   /** Each row of the result DenseMatrix @a shape contains the components of
+       one vector shape function. The size (#dof x SDim) of @a shape must be set
+       in advance, where SDim >= #dim is the physical space dimension as
+       described by @a Trans. */
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const;
+
+   /** @brief Evaluate the divergence of all shape functions of a *vector*
+       finite element in reference space at the given point @a ip. */
+   /** The size (#dof) of the result Vector @a divshape must be set in advance.
+    */
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+
+   ~NURBS_HDiv3DFiniteElement();
+};
+
+
+/** An arbitrary order H(curl)-conforming 2D NURBS element on a square
+    More details in the following paper:
+
+    [1] Annalisa Buffa, Giancarlo Sangalli, Rafael Vázquez
+    "Isogeometric analysis in electromagnetics: B-splines approximation."
+    Computer Methods in Applied Mechanics and Engineering (199) 1143-1152 */
+class NURBS_HCurl2DFiniteElement : public VectorFiniteElement,
+   public NURBSFiniteElement
+{
+protected:
+   mutable Vector shape_x, shape_y, dshape_x, dshape_y, d2shape_x, d2shape_y;
+   mutable Vector shape1_x, shape1_y, dshape1_x, dshape1_y, d2shape1_x, d2shape1_y;
+   mutable Vector u;
+   mutable DenseMatrix du;
+   mutable Array <const KnotVector*> kv1;
+
+public:
+   /// Construct the NURBS_HCurl2DFiniteElement of order @a p
+   NURBS_HCurl2DFiniteElement(int p)
+      : VectorFiniteElement(2, Geometry::SQUARE, 2*(p + 1)*(p + 2), p,
+                            H_CURL,FunctionSpace::Qk),
+        NURBSFiniteElement(2),
+        shape_x(p + 1), shape_y(p + 1), dshape_x(p + 1),
+        dshape_y(p + 1), d2shape_x(p + 1), d2shape_y(p + 1),
+        shape1_x(p + 2), shape1_y(p + 2), dshape1_x(p + 2),
+        dshape1_y(p + 2), d2shape1_x(p + 2), d2shape1_y(p + 2),
+        u(dof), du(dof,2)
+   {
+      orders[0] = orders[1] = p;
+      kv1.SetSize(dim);
+      kv1[0] = nullptr;
+      kv1[1] = nullptr;
+   }
+
+   /// Construct the NURBS_HCurl2DFiniteElement with x-order @a px and y-order @a py
+   NURBS_HCurl2DFiniteElement(int px, int py)
+      : VectorFiniteElement(2, Geometry::SQUARE,
+                            (px + 1)*(py + 2)+(px + 2)*(py + 1),
+                            std::max(px, py), H_CURL, FunctionSpace::Qk),
+        NURBSFiniteElement(2),
+        shape_x(px + 1), shape_y(py + 1), dshape_x(px + 1),
+        dshape_y(py + 1), d2shape_x(px + 1), d2shape_y(py + 1),
+        shape1_x(px + 2), shape1_y(py + 2), dshape1_x(px + 2),
+        dshape1_y(py + 2), d2shape1_x(px + 2), d2shape1_y(py + 2),
+        u(dof), du(dof,2)
+   {
+      orders[0] = px; orders[1] = py;
+      kv1.SetSize(dim);
+      kv1[0] = nullptr;
+      kv1[1] = nullptr;
+   }
+
+   virtual void SetOrder() const;
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+
+   /** @brief Evaluate the values of all shape functions of a *vector* finite
+       element in physical space at the point described by @a Trans. */
+   /** Each row of the result DenseMatrix @a shape contains the components of
+       one vector shape function. The size (#dof x SDim) of @a shape must be set
+       in advance, where SDim >= #dim is the physical space dimension as
+       described by @a Trans. */
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const;
+
+   /** @brief Evaluate the curl of all shape functions of a *vector* finite
+       element in reference space at the given point @a ip. */
+   /** Each row of the result DenseMatrix @a curl_shape contains the components
+       of the curl of one vector shape function. The size (#dof x CDim) of
+       @a curl_shape must be set in advance, where CDim = 3 for #dim = 3 and
+       CDim = 1 for #dim = 2. */
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+
+   ~NURBS_HCurl2DFiniteElement();
+};
+
+
+/** An arbitrary order H(curl)-conforming 3D NURBS element on a cube
+    More details in the following paper:
+
+    [1] Annalisa Buffa, Giancarlo Sangalli, Rafael Vázquez
+    "Isogeometric analysis in electromagnetics: B-splines approximation."
+    Computer Methods in Applied Mechanics and Engineering (199) 1143-1152 */
+class NURBS_HCurl3DFiniteElement : public VectorFiniteElement,
+   public NURBSFiniteElement
+{
+protected:
+   mutable Vector shape_x, shape_y, shape_z;
+   mutable Vector dshape_x, dshape_y, dshape_z;
+   mutable Vector d2shape_x, d2shape_y, d2shape_z;
+   mutable Vector shape1_x, shape1_y, shape1_z;
+   mutable Vector dshape1_x, dshape1_y, dshape1_z;
+   mutable Vector d2shape1_x, d2shape1_y, d2shape1_z;
+   mutable Vector u;
+   mutable DenseMatrix du;
+   mutable Array <const KnotVector*> kv1;
+
+public:
+   /// Construct the NURBS_HCurl3DFiniteElement of order @a p
+   NURBS_HCurl3DFiniteElement(int p)
+      : VectorFiniteElement(3, Geometry::CUBE, 3*(p + 1)*(p + 2)*(p + 2), p,
+                            H_CURL,FunctionSpace::Qk),
+        NURBSFiniteElement(3),
+        shape_x(p + 1), shape_y(p + 1), shape_z(p + 1),
+        dshape_x(p + 1), dshape_y(p + 1), dshape_z(p + 1),
+        d2shape_x(p + 1), d2shape_y(p + 1), d2shape_z(p + 1),
+        shape1_x(p + 2), shape1_y(p + 2), shape1_z(p + 2),
+        dshape1_x(p + 2), dshape1_y(p + 2),dshape1_z(p + 2),
+        d2shape1_x(p + 2), d2shape1_y(p + 2), d2shape1_z(p + 2),
+        u(dof), du(dof,3)
+   {
+      orders[0] = orders[1] = orders[2] = p;
+      kv1.SetSize(dim);
+      kv1[0] = nullptr;
+      kv1[1] = nullptr;
+      kv1[2] = nullptr;
+   }
+
+   /// Construct the NURBS_HCurl3DFiniteElement with x-order @a px, y-order @a py and z-order @a pz
+   NURBS_HCurl3DFiniteElement(int px, int py, int pz)
+      : VectorFiniteElement(3, Geometry::CUBE,
+                            (px + 1)*(py + 2)*(pz + 2) +
+                            (px + 2)*(py + 1)*(pz + 2) +
+                            (px + 2)*(py + 2)*(pz + 1),
+                            std::max(std::max(px, py), pz), H_CURL, FunctionSpace::Qk),
+        NURBSFiniteElement(3),
+        shape_x(px + 1), shape_y(py + 1), shape_z(pz + 1),
+        dshape_x(px + 1), dshape_y(py + 1), dshape_z(pz + 1),
+        d2shape_x(px + 1), d2shape_y(py + 1), d2shape_z(pz + 1),
+        shape1_x(px + 2), shape1_y(py + 2), shape1_z(pz + 2),
+        dshape1_x(px + 2), dshape1_y(py + 2),dshape1_z(pz + 2),
+        d2shape1_x(px + 2), d2shape1_y(py + 2), d2shape1_z(pz + 2),
+        u(dof), du(dof,3)
+   {
+      orders[0] = px; orders[1] = py; orders[2] = pz;
+      kv1.SetSize(dim);
+      kv1[0] = nullptr;
+      kv1[1] = nullptr;
+      kv1[2] = nullptr;
+   }
+
+   virtual void SetOrder() const;
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+
+   /** @brief Evaluate the values of all shape functions of a *vector* finite
+       element in physical space at the point described by @a Trans. */
+   /** Each row of the result DenseMatrix @a shape contains the components of
+       one vector shape function. The size (#dof x SDim) of @a shape must be set
+       in advance, where SDim >= #dim is the physical space dimension as
+       described by @a Trans. */
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const;
+
+   /** @brief Evaluate the curl of all shape functions of a *vector* finite
+       element in reference space at the given point @a ip. */
+   /** Each row of the result DenseMatrix @a curl_shape contains the components
+       of the curl of one vector shape function. The size (#dof x CDim) of
+       @a curl_shape must be set in advance, where CDim = 3 for #dim = 3 and
+       CDim = 1 for #dim = 2. */
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+
+   ~NURBS_HCurl3DFiniteElement();
+};
+
 } // namespace mfem

 #endif
@@ -344,6 +344,32 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
   {
      fec = new Local_FECollection(name + 6);
   }
+   else if (!strncmp(name, "NURBS_HDiv", 10))
+   {
+      if (name[10] != '\0')
+      {
+         // "NURBS" + "number" --> fixed order nurbs collection
+         fec = new NURBS_HDivFECollection(atoi(name + 10));
+      }
+      else
+      {
+         // "NURBS" --> variable order nurbs collection
+         fec = new NURBS_HDivFECollection();
+      }
+   }
+   else if (!strncmp(name, "NURBS_HCurl", 11))
+   {
+      if (name[11] != '\0')
+      {
+         // "NURBS" + "number" --> fixed order nurbs collection
+         fec = new NURBS_HCurlFECollection(atoi(name + 11));
+      }
+      else
+      {
+         // "NURBS" --> variable order nurbs collection
+         fec = new NURBS_HCurlFECollection();
+      }
+   }
   else if (!strncmp(name, "NURBS", 5))
   {
      if (name[5] != '\0')
@@ -3533,4 +3559,192 @@ FiniteElementCollection *NURBSFECollection::GetTraceCollection() const
   return NULL;
 }

+
+NURBS_HDivFECollection::NURBS_HDivFECollection(int Order, const int dim)
+   : NURBSFECollection((Order == VariableOrder) ? 1 : Order)
+{
+   const int order = (Order == VariableOrder) ? 1 : Order;
+
+   SegmentFE       = new NURBS1DFiniteElement(order);
+   QuadrilateralFE = new NURBS2DFiniteElement(order);
+
+   QuadrilateralVFE  = new NURBS_HDiv2DFiniteElement(order);
+   ParallelepipedVFE = new NURBS_HDiv3DFiniteElement(order);
+
+   if (dim != -1) { SetDim(dim); }
+   SetOrder(Order);
+}
+
+void NURBS_HDivFECollection::SetDim(int dim)
+{
+   if (dim == 2)
+   {
+      sFE = SegmentFE;
+      qFE = QuadrilateralVFE;
+      hFE = nullptr;
+   }
+   else if (dim == 3)
+   {
+      sFE = nullptr;
+      qFE = QuadrilateralFE;
+      hFE = ParallelepipedVFE;
+   }
+   else
+   {
+      mfem::err<<"Dimension = "<<dim<<endl;
+      mfem_error ("NURBS_HDivFECollection: wrong dimension!");
+   }
+}
+
+NURBS_HDivFECollection::~NURBS_HDivFECollection()
+{
+   delete SegmentFE;
+   delete QuadrilateralFE;
+   delete QuadrilateralVFE;
+   delete ParallelepipedVFE;
+}
+
+const FiniteElement *
+NURBS_HDivFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   switch (GeomType)
+   {
+      case Geometry::SEGMENT:     return sFE;
+      case Geometry::SQUARE:      return qFE;
+      case Geometry::CUBE:        return hFE;
+      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
+         mfem_error ("NURBS_HDivFECollection: unknown geometry type.");
+   }
+   return QuadrilateralFE; // Make some compilers happy
+}
+
+void NURBS_HDivFECollection::SetOrder(int Order) const
+{
+   mOrder = Order;
+   if (Order != VariableOrder)
+   {
+      snprintf(name, 16, "NURBS_HDiv%i", Order);
+   }
+   else
+   {
+      snprintf(name, 16, "NURBS_HDiv");
+   }
+}
+
+int NURBS_HDivFECollection::DofForGeometry(Geometry::Type GeomType) const
+{
+   mfem_error("NURBS_HDivFECollection::DofForGeometry");
+   return 0; // Make some compilers happy
+}
+
+const int *NURBS_HDivFECollection::DofOrderForOrientation(
+   Geometry::Type GeomType,
+   int Or) const
+{
+   mfem_error("NURBS_HDivFECollection::DofOrderForOrientation");
+   return NULL;
+}
+
+FiniteElementCollection *NURBS_HDivFECollection::GetTraceCollection() const
+{
+   MFEM_ABORT("NURBS finite elements can not be statically condensed!");
+   return NULL;
+}
+
+NURBS_HCurlFECollection::NURBS_HCurlFECollection(int Order, const int dim)
+   : NURBSFECollection((Order == VariableOrder) ? 1 : Order)
+{
+   const int order = (Order == VariableOrder) ? 1 : Order;
+
+   SegmentFE       = new NURBS1DFiniteElement(order+1);
+   QuadrilateralFE = new NURBS2DFiniteElement(order+1);
+
+   QuadrilateralVFE  = new NURBS_HCurl2DFiniteElement(order);
+   ParallelepipedVFE = new NURBS_HCurl3DFiniteElement(order);
+   if (dim != -1) { SetDim(dim); }
+   SetOrder(Order);
+}
+
+void NURBS_HCurlFECollection::SetDim(int dim)
+{
+   if (dim == 2)
+   {
+      sFE = SegmentFE;
+      qFE = QuadrilateralVFE;
+      hFE = nullptr;
+   }
+   else if (dim == 3)
+   {
+      sFE = nullptr;
+      qFE = QuadrilateralFE;
+      hFE = ParallelepipedVFE;
+   }
+   else
+   {
+      mfem::err<<"Dimension = "<<dim<<endl;
+      mfem_error ("NURBS_HCurlFECollection: wrong dimension!");
+   }
+}
+
+
+
+NURBS_HCurlFECollection::~NURBS_HCurlFECollection()
+{
+   delete SegmentFE;
+   delete QuadrilateralFE;
+   delete QuadrilateralVFE;
+   delete ParallelepipedVFE;
+}
+
+const FiniteElement *
+NURBS_HCurlFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   switch (GeomType)
+   {
+      case Geometry::SEGMENT:     return sFE;
+      case Geometry::SQUARE:      return qFE;
+      case Geometry::CUBE:        return hFE;
+      default:
+         if (error_mode == RETURN_NULL) { return nullptr; }
+         mfem_error ("NURBS_HCurlFECollection: unknown geometry type.");
+   }
+   return QuadrilateralFE; // Make some compilers happy
+}
+
+void NURBS_HCurlFECollection::SetOrder(int Order) const
+{
+   mOrder = Order;
+   if (Order != VariableOrder)
+   {
+      snprintf(name, 16, "NURBS_HCurl%i", Order);
+   }
+   else
+   {
+      snprintf(name, 16, "NURBS_HCurl");
+   }
+}
+
+int NURBS_HCurlFECollection::DofForGeometry(Geometry::Type GeomType) const
+{
+   mfem_error("NURBS_HCurlFECollection::DofForGeometry");
+   return 0; // Make some compilers happy
+}
+
+const int *NURBS_HCurlFECollection::DofOrderForOrientation(
+   Geometry::Type GeomType,
+   int Or) const
+{
+   mfem_error("NURBS_HCurlFECollection::DofOrderForOrientation");
+   return NULL;
+}
+
+FiniteElementCollection *NURBS_HCurlFECollection::GetTraceCollection() const
+{
+   MFEM_ABORT("NURBS finite elements can not be statically condensed!");
+   return NULL;
+}
+
+
+
 }
@@ -680,8 +680,8 @@ public:
 /// Arbitrary order non-uniform rational B-splines (NURBS) finite elements.
 class NURBSFECollection : public FiniteElementCollection
 {
-private:
-   PointFiniteElement   *PointFE;
+protected:
+   PointFiniteElement *PointFE;
   NURBS1DFiniteElement *SegmentFE;
   NURBS2DFiniteElement *QuadrilateralFE;
   NURBS3DFiniteElement *ParallelepipedFE;
@@ -701,13 +701,15 @@ public:
      order, or VariableOrder (default). */
   explicit NURBSFECollection(int Order = VariableOrder);

-   void Reset() const
+   virtual void Reset() const
   {
      SegmentFE->Reset();
      QuadrilateralFE->Reset();
      ParallelepipedFE->Reset();
   }

+   virtual void SetDim(const int dim) {};
+
   /** @brief Get the order of the NURBS collection: either a positive number,
       when using fixed order, or VariableOrder. */
   /** @note Not to be confused with FiniteElementCollection::GetOrder(). */
@@ -715,7 +717,7 @@ public:

   /** @brief Set the order and the name, based on the given @a Order: either a
       positive number for fixed order, or VariableOrder. */
-   void SetOrder(int Order) const;
+   virtual void SetOrder(int Order) const;

   const FiniteElement *
   FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -734,6 +736,109 @@ public:
   virtual ~NURBSFECollection();
 };

+/// Arbitrary order H(div) NURBS finite elements.
+class NURBS_HDivFECollection : public NURBSFECollection
+{
+private:
+
+   NURBS1DFiniteElement *SegmentFE;
+   NURBS2DFiniteElement *QuadrilateralFE;
+
+   NURBS_HDiv2DFiniteElement *QuadrilateralVFE;
+   NURBS_HDiv3DFiniteElement *ParallelepipedVFE;
+
+   FiniteElement *sFE;
+   FiniteElement *qFE;
+   FiniteElement *hFE;
+
+public:
+
+   /** @brief The parameter @a Order must be either a positive number, for fixed
+      order, or VariableOrder (default). */
+   explicit NURBS_HDivFECollection(int Order = VariableOrder, const int vdim = -1);
+
+   virtual void Reset() const override
+   {
+      SegmentFE->Reset();
+      QuadrilateralFE->Reset();
+      QuadrilateralVFE->Reset();
+      ParallelepipedVFE->Reset();
+   }
+
+   virtual void SetDim(const int dim) override;
+
+   /** @brief Set the order and the name, based on the given @a Order: either a
+       positive number for fixed order, or VariableOrder. */
+   virtual void SetOrder(int Order) const override;
+
+   const FiniteElement *
+   FiniteElementForGeometry(Geometry::Type GeomType) const override;
+
+   int DofForGeometry(Geometry::Type GeomType) const override;
+
+   const int *DofOrderForOrientation(Geometry::Type GeomType,
+                                     int Or) const override;
+
+   const char *Name() const override { return name; }
+
+   int GetContType() const override { return CONTINUOUS; }
+
+   FiniteElementCollection *GetTraceCollection() const override;
+
+   virtual ~NURBS_HDivFECollection();
+};
+
+/// Arbitrary order H(curl) NURBS finite elements.
+class NURBS_HCurlFECollection : public NURBSFECollection
+{
+private:
+   NURBS1DFiniteElement *SegmentFE;
+   NURBS2DFiniteElement *QuadrilateralFE;
+
+   NURBS_HCurl2DFiniteElement *QuadrilateralVFE;
+   NURBS_HCurl3DFiniteElement *ParallelepipedVFE;
+
+   FiniteElement *sFE;
+   FiniteElement *qFE;
+   FiniteElement *hFE;
+public:
+
+   /** @brief The parameter @a Order must be either a positive number, for fixed
+      order, or VariableOrder (default). */
+   explicit NURBS_HCurlFECollection(int Order = VariableOrder,
+                                    const int vdim = -1);
+
+   virtual void Reset() const override
+   {
+      SegmentFE->Reset();
+      QuadrilateralFE->Reset();
+      QuadrilateralVFE->Reset();
+      ParallelepipedVFE->Reset();
+   }
+
+   virtual void SetDim(const int dim) override;
+
+   /** @brief Set the order and the name, based on the given @a Order: either a
+       positive number for fixed order, or VariableOrder. */
+   virtual void SetOrder(int Order) const override;
+
+   const FiniteElement *
+   FiniteElementForGeometry(Geometry::Type GeomType) const override;
+
+   int DofForGeometry(Geometry::Type GeomType) const override;
+
+   const int *DofOrderForOrientation(Geometry::Type GeomType,
+                                     int Or) const override;
+
+   const char *Name() const override { return name; }
+
+   int GetContType() const override { return CONTINUOUS; }
+
+   FiniteElementCollection *GetTraceCollection() const override;
+
+   virtual ~NURBS_HCurlFECollection();
+};
+
 /// Piecewise-(bi/tri)linear continuous finite elements.
 class LinearFECollection : public FiniteElementCollection
 {
@@ -1525,6 +1525,67 @@ SparseMatrix *FiniteElementSpace::RefinementMatrix_main(
   return P;
 }

+SparseMatrix *FiniteElementSpace::VariableOrderRefinementMatrix(
+   const int coarse_ndofs, const Table &coarse_elem_dof) const
+{
+   MFEM_VERIFY(mesh->GetLastOperation() == Mesh::REFINE, "");
+
+   Array<int> dofs, coarse_dofs, coarse_vdofs;
+   Vector row;
+
+   Mesh::GeometryList elem_geoms(*mesh);
+
+   SparseMatrix *P = new SparseMatrix(GetVSize(), coarse_ndofs*vdim);
+
+   Array<int> mark(P->Height());
+   mark = 0;
+
+   const CoarseFineTransformations &rtrans = mesh->GetRefinementTransforms();
+   DenseMatrix lP;
+   IsoparametricTransformation isotr;
+   for (int k = 0; k < mesh->GetNE(); k++)
+   {
+      const Embedding &emb = rtrans.embeddings[k];
+      const Geometry::Type geom = mesh->GetElementBaseGeometry(k);
+
+      const FiniteElement *fe = GetFE(k);
+      isotr.SetIdentityTransformation(geom);
+      const int ldof = fe->GetDof();
+      lP.SetSize(ldof, ldof);
+      const DenseTensor &pmats = rtrans.point_matrices[geom];
+      isotr.SetPointMat(pmats(emb.matrix));
+      fe->GetLocalInterpolation(isotr, lP);
+
+      const int fine_ldof = lP.Height();
+
+      elem_dof->GetRow(k, dofs);
+      coarse_elem_dof.GetRow(emb.parent, coarse_dofs);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         coarse_dofs.Copy(coarse_vdofs);
+         DofsToVDofs(vd, coarse_vdofs, coarse_ndofs);
+
+         for (int i = 0; i < fine_ldof; i++)
+         {
+            const int r = DofToVDof(dofs[i], vd);
+            int m = (r >= 0) ? r : (-1 - r);
+
+            if (!mark[m])
+            {
+               lP.GetRow(i, row);
+               P->SetRow(r, coarse_vdofs, row);
+               mark[m] = 1;
+            }
+         }
+      }
+   }
+
+   MFEM_VERIFY(mark.Sum() == P->Height(), "Not all rows of P set.");
+   P->Finalize();
+   return P;
+}
+
 void FiniteElementSpace::GetLocalRefinementMatrices(
   Geometry::Type geom, DenseTensor &localP) const
 {
@@ -1556,15 +1617,20 @@ SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
               "Previous mesh is not coarser.");

   Mesh::GeometryList elem_geoms(*mesh);
-
-   DenseTensor localP[Geometry::NumGeom];
-   for (int i = 0; i < elem_geoms.Size(); i++)
+   if (!IsVariableOrder())
   {
-      GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
+      DenseTensor localP[Geometry::NumGeom];
+      for (int i = 0; i < elem_geoms.Size(); i++)
+      {
+         GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
+      }
+      return RefinementMatrix_main(old_ndofs, *old_elem_dof, old_elem_fos,
+                                   localP);
+   }
+   else
+   {
+      return VariableOrderRefinementMatrix(old_ndofs, *old_elem_dof);
   }
-
-   return RefinementMatrix_main(old_ndofs, *old_elem_dof, old_elem_fos,
-                                localP);
 }

 FiniteElementSpace::RefinementOperator::RefinementOperator(
@@ -1582,9 +1648,12 @@ FiniteElementSpace::RefinementOperator::RefinementOperator(

   Mesh::GeometryList elem_geoms(*fespace->GetMesh());

-   for (int i = 0; i < elem_geoms.Size(); i++)
+   if (!fespace->IsVariableOrder())
   {
-      fespace->GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
+      for (int i = 0; i < elem_geoms.Size(); i++)
+      {
+         fespace->GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
+      }
   }

   ConstructDoFTransArray();
@@ -1597,10 +1666,13 @@ FiniteElementSpace::RefinementOperator::RefinementOperator(
 {
   Mesh::GeometryList elem_geoms(*fespace->GetMesh());

-   for (int i = 0; i < elem_geoms.Size(); i++)
+   if (!fespace->IsVariableOrder())
   {
-      fespace->GetLocalRefinementMatrices(*coarse_fes, elem_geoms[i],
-                                          localP[elem_geoms[i]]);
+      for (int i = 0; i < elem_geoms.Size(); i++)
+      {
+         fespace->GetLocalRefinementMatrices(*coarse_fes, elem_geoms[i],
+                                             localP[elem_geoms[i]]);
+      }
   }

   // Make a copy of the coarse elem_dof Table.
@@ -1676,11 +1748,25 @@ void FiniteElementSpace::RefinementOperator::Mult(const Vector &x,

   Vector subY, subX;

+   DenseMatrix eP;
+   IsoparametricTransformation isotr;
+
   for (int k = 0; k < mesh_ref->GetNE(); k++)
   {
      const Embedding &emb = trans_ref.embeddings[k];
      const Geometry::Type geom = mesh_ref->GetElementBaseGeometry(k);
-      const DenseMatrix &lP = localP[geom](emb.matrix);
+      if (fespace->IsVariableOrder())
+      {
+         const FiniteElement *fe = fespace->GetFE(k);
+         isotr.SetIdentityTransformation(geom);
+         const int ldof = fe->GetDof();
+         eP.SetSize(ldof, ldof);
+         const DenseTensor &pmats = trans_ref.point_matrices[geom];
+         isotr.SetPointMat(pmats(emb.matrix));
+         fe->GetLocalInterpolation(isotr, eP);
+      }
+      const DenseMatrix &lP = (fespace->IsVariableOrder()) ? eP : localP[geom](
+                                 emb.matrix);

      subY.SetSize(lP.Height());

@@ -1745,11 +1831,28 @@ void FiniteElementSpace::RefinementOperator::MultTranspose(const Vector &x,

   Vector subY, subX, subYt;

+   DenseMatrix eP;
+   IsoparametricTransformation isotr;
+   const FiniteElement *fe = nullptr;
+
   for (int k = 0; k < mesh_ref->GetNE(); k++)
   {
      const Embedding &emb = trans_ref.embeddings[k];
      const Geometry::Type geom = mesh_ref->GetElementBaseGeometry(k);
-      const DenseMatrix &lP = localP[geom](emb.matrix);
+
+      if (fespace->IsVariableOrder())
+      {
+         fe = fespace->GetFE(k);
+         isotr.SetIdentityTransformation(geom);
+         const int ldof = fe->GetDof();
+         eP.SetSize(ldof);
+         const DenseTensor &pmats = trans_ref.point_matrices[geom];
+         isotr.SetPointMat(pmats(emb.matrix));
+         fe->GetLocalInterpolation(isotr, eP);
+      }
+
+      const DenseMatrix &lP = (fespace->IsVariableOrder()) ? eP : localP[geom](
+                                 emb.matrix);

      DofTransformation *doftrans = fespace->GetElementDofs(k, f_dofs);
      old_elem_dof->GetRow(emb.parent, c_dofs);
@@ -2108,9 +2211,12 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
   Mesh::GeometryList elem_geoms(*mesh);

   DenseTensor localR[Geometry::NumGeom];
-   for (int i = 0; i < elem_geoms.Size(); i++)
+   if (!IsVariableOrder())
   {
-      GetLocalDerefinementMatrices(elem_geoms[i], localR[elem_geoms[i]]);
+      for (int i = 0; i < elem_geoms.Size(); i++)
+      {
+         GetLocalDerefinementMatrices(elem_geoms[i], localR[elem_geoms[i]]);
+      }
   }

   SparseMatrix *R = new SparseMatrix(ndofs*vdim, old_ndofs*vdim);
@@ -2125,14 +2231,34 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,

   bool is_dg = FEColl()->GetContType() == FiniteElementCollection::DISCONTINUOUS;
   int num_marked = 0;
+   const FiniteElement *fe = nullptr;
+   DenseMatrix localRVO; //for variable order only
   for (int k = 0; k < dtrans.embeddings.Size(); k++)
   {
      const Embedding &emb = dtrans.embeddings[k];
      Geometry::Type geom = mesh->GetElementBaseGeometry(emb.parent);
-      DenseMatrix &lR = localR[geom](emb.matrix);
+
+      if (IsVariableOrder())
+      {
+         fe = GetFE(emb.parent);
+         const DenseTensor &pmats = dtrans.point_matrices[geom];
+         const int ldof = fe->GetDof();
+
+         IsoparametricTransformation isotr;
+         isotr.SetIdentityTransformation(geom);
+
+         localRVO.SetSize(ldof, ldof);
+         isotr.SetPointMat(pmats(emb.matrix));
+         // Local restriction is size ldofxldof assuming that the parent and
+         // child are of same polynomial order.
+         fe->GetLocalRestriction(isotr, localRVO);
+      }
+      DenseMatrix &lR = IsVariableOrder() ? localRVO : localR[geom](emb.matrix);

      elem_dof->GetRow(emb.parent, dofs);
      old_elem_dof->GetRow(k, old_dofs);
+      MFEM_VERIFY(old_dofs.Size() == dofs.Size(),
+                  "Parent and child must have same #dofs.");

      for (int vd = 0; vd < vdim; vd++)
      {
@@ -2158,7 +2284,7 @@ SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
      }
   }

-   if (!is_dg)
+   if (!is_dg && !IsVariableOrder())
   {
      MFEM_VERIFY(num_marked == R->Height(),
                  "internal error: not all rows of R were set.");
@@ -2216,6 +2342,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh_, NURBSExtension *NURBSext_,

   const NURBSFECollection *nurbs_fec =
      dynamic_cast<const NURBSFECollection *>(fec_);
+
   if (nurbs_fec)
   {
      MFEM_VERIFY(mesh_->NURBSext, "NURBS FE space requires a NURBS mesh.");
@@ -2312,12 +2439,63 @@ void FiniteElementSpace::UpdateNURBS()
   face_dof = NULL;
   face_to_be.DeleteAll();

+   // Depending on the element type create the appropriate extensions
+   // for the individual components.
   dynamic_cast<const NURBSFECollection *>(fec)->Reset();

-   ndofs = NURBSext->GetNDof();
-   elem_dof = NURBSext->GetElementDofTable();
-   bdr_elem_dof = NURBSext->GetBdrElementDofTable();
+   if (dynamic_cast<const NURBS_HDivFECollection *>(fec))
+   {
+      VNURBSext.SetSize(mesh->Dimension());
+      for (int d = 0; d < mesh->Dimension(); d++)
+      {
+         VNURBSext[d] = NURBSext->GetDivExtension(d);
+      }
+   }

+   if (dynamic_cast<const NURBS_HCurlFECollection *>(fec))
+   {
+      VNURBSext.SetSize(mesh->Dimension());
+      for (int d = 0; d < mesh->Dimension(); d++)
+      {
+         VNURBSext[d] = NURBSext->GetCurlExtension(d);
+      }
+   }
+
+   // If required: concatenate the dof tables of the individual components into
+   // one dof table for the vector fespace.
+   if (VNURBSext.Size() == 2)
+   {
+      int offset1 = VNURBSext[0]->GetNDof();
+      ndofs = VNURBSext[0]->GetNDof() + VNURBSext[1]->GetNDof();
+
+      // Merge Tables
+      elem_dof = new Table(*VNURBSext[0]->GetElementDofTable(),
+                           *VNURBSext[1]->GetElementDofTable(),offset1 );
+
+      bdr_elem_dof = new Table(*VNURBSext[0]->GetBdrElementDofTable(),
+                               *VNURBSext[1]->GetBdrElementDofTable(),offset1);
+   }
+   else if (VNURBSext.Size() == 3)
+   {
+      int offset1 = VNURBSext[0]->GetNDof();
+      int offset2 = offset1 + VNURBSext[1]->GetNDof();
+      ndofs = offset2 + VNURBSext[2]->GetNDof();
+
+      // Merge Tables
+      elem_dof = new Table(*VNURBSext[0]->GetElementDofTable(),
+                           *VNURBSext[1]->GetElementDofTable(),offset1,
+                           *VNURBSext[2]->GetElementDofTable(),offset2);
+
+      bdr_elem_dof = new Table(*VNURBSext[0]->GetBdrElementDofTable(),
+                               *VNURBSext[1]->GetBdrElementDofTable(),offset1,
+                               *VNURBSext[2]->GetBdrElementDofTable(),offset2);
+   }
+   else
+   {
+      ndofs = NURBSext->GetNDof();
+      elem_dof = NURBSext->GetElementDofTable();
+      bdr_elem_dof = NURBSext->GetBdrElementDofTable();
+   }
   mesh_sequence = mesh->GetSequence();
   sequence++;
 }
@@ -3319,11 +3497,21 @@ void FiniteElementSpace::Destroy()
   dof_elem_array.DeleteAll();
   dof_ldof_array.DeleteAll();

+   for (int i = 0; i < VNURBSext.Size(); i++)
+   {
+      delete VNURBSext[i];
+   }
+
   if (NURBSext)
   {
      if (own_ext) { delete NURBSext; }
      delete face_dof;
      face_to_be.DeleteAll();
+      if (VNURBSext.Size() > 0 )
+      {
+         delete elem_dof;
+         delete bdr_elem_dof;
+      }
   }
   else
   {
@@ -3335,6 +3523,8 @@ void FiniteElementSpace::Destroy()
      delete [] bdofs;
   }
   ceed::RemoveBasisAndRestriction(this);
+
+
 }

 void FiniteElementSpace::DestroyDoFTransArray()
@@ -3353,19 +3543,27 @@ void FiniteElementSpace::GetTransferOperator(

   if (T.Type() == Operator::MFEM_SPARSEMAT)
   {
-      Mesh::GeometryList elem_geoms(*mesh);
-
-      DenseTensor localP[Geometry::NumGeom];
-      for (int i = 0; i < elem_geoms.Size(); i++)
+      if (!IsVariableOrder())
      {
-         GetLocalRefinementMatrices(coarse_fes, elem_geoms[i],
-                                    localP[elem_geoms[i]]);
+         Mesh::GeometryList elem_geoms(*mesh);
+
+         DenseTensor localP[Geometry::NumGeom];
+         for (int i = 0; i < elem_geoms.Size(); i++)
+         {
+            GetLocalRefinementMatrices(coarse_fes, elem_geoms[i],
+                                       localP[elem_geoms[i]]);
+         }
+         T.Reset(RefinementMatrix_main(coarse_fes.GetNDofs(),
+                                       coarse_fes.GetElementToDofTable(),
+                                       coarse_fes.
+                                       GetElementToFaceOrientationTable(),
+                                       localP));
+      }
+      else
+      {
+         T.Reset(VariableOrderRefinementMatrix(coarse_fes.GetNDofs(),
+                                               coarse_fes.GetElementToDofTable()));
      }
-      T.Reset(RefinementMatrix_main(coarse_fes.GetNDofs(),
-                                    coarse_fes.GetElementToDofTable(),
-                                    coarse_fes.
-                                    GetElementToFaceOrientationTable(),
-                                    localP));
   }
   else
   {
@@ -3416,19 +3614,33 @@ void FiniteElementSpace::GetTrueTransferOperator(

 void FiniteElementSpace::UpdateElementOrders()
 {
-   const CoarseFineTransformations &cf_tr = mesh->GetRefinementTransforms();
-
   Array<char> new_order(mesh->GetNE());
   switch (mesh->GetLastOperation())
   {
      case Mesh::REFINE:
      {
+         const CoarseFineTransformations &cf_tr = mesh->GetRefinementTransforms();
         for (int i = 0; i < mesh->GetNE(); i++)
         {
            new_order[i] = elem_order[cf_tr.embeddings[i].parent];
         }
         break;
      }
+      case Mesh::DEREFINE:
+      {
+         const CoarseFineTransformations &cf_tr =
+            mesh->ncmesh->GetDerefinementTransforms();
+         Table coarse_to_fine;
+         cf_tr.MakeCoarseToFineTable(coarse_to_fine);
+         Array<int> tabrow;
+         for (int i = 0; i < coarse_to_fine.Size(); i++)
+         {
+            coarse_to_fine.GetRow(i, tabrow);
+            //For now we require that all children are of same polynomial order.
+            new_order[i] = elem_order[tabrow[0]];
+         }
+         break;
+      }
      default:
         MFEM_ABORT("not implemented yet");
   }
@@ -3523,11 +3735,23 @@ void FiniteElementSpace::Update(bool want_transform)
         {
            BuildConformingInterpolation();
            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
-            if (cP && cR)
+            if (IsVariableOrder())
            {
-               Th.SetOperatorOwner(false);
-               Th.Reset(new TripleProductOperator(cP.get(), cR.get(), Th.Ptr(),
-                                                  false, false, true));
+               if (cP && cR_hp)
+               {
+                  Th.SetOperatorOwner(false);
+                  Th.Reset(new TripleProductOperator(cP.get(), cR_hp.get(), Th.Ptr(),
+                                                     false, false, true));
+               }
+            }
+            else
+            {
+               if (cP && cR)
+               {
+                  Th.SetOperatorOwner(false);
+                  Th.Reset(new TripleProductOperator(cP.get(), cR.get(), Th.Ptr(),
+                                                     false, false, true));
+               }
            }
            break;
         }
@@ -3640,6 +3864,7 @@ FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)
   input >> ord;

   NURBSFECollection *nurbs_fec = dynamic_cast<NURBSFECollection*>(r_fec);
+   if (nurbs_fec) { nurbs_fec->SetDim(m->Dimension()); }
   NURBSExtension *nurbs_ext = NULL;
   if (fes_format == 90) // original format, v0.9
   {
@@ -268,6 +268,10 @@ protected:
   Array<int> dof_elem_array, dof_ldof_array;

   NURBSExtension *NURBSext;
+   /** array of NURBS extension for H(div) and H(curl) vector elements.
+       For each direction an extension is created from the base NURBSext,
+       with an increase in order in the appropriate direction. */
+   Array<NURBSExtension*> VNURBSext;
   int own_ext;
   mutable Array<int> face_to_be; // NURBS FE space only

@@ -469,6 +473,11 @@ protected:
                                       const Table *coarse_elem_fos,
                                       const DenseTensor localP[]) const;

+   /* This method returns the Refinement matrix (i.e., the embedding)
+      from a coarse variable-order fes to a fine fes (after a geometric refinement) */
+   SparseMatrix *VariableOrderRefinementMatrix(const int coarse_ndofs,
+                                               const Table &coarse_elem_dof) const;
+
   void GetLocalRefinementMatrices(Geometry::Type geom,
                                   DenseTensor &localP) const;
   void GetLocalDerefinementMatrices(Geometry::Type geom,
@@ -517,6 +526,8 @@ protected:
                                               const Array<int> *perm);

 public:
+
+
   /** @brief Default constructor: the object is invalid until initialized using
       the method Load(). */
   FiniteElementSpace();
@@ -12,6 +12,8 @@
 // Implementation of GridFunction

 #include "gridfunc.hpp"
+#include "linearform.hpp"
+#include "bilinearform.hpp"
 #include "quadinterpolator.hpp"
 #include "../mesh/nurbs.hpp"
 #include "../general/text.hpp"
@@ -39,7 +41,7 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
   UseDevice(true);

   fes = new FiniteElementSpace;
-   fec = fes->Load(m, input);
+   fec_owned = fes->Load(m, input);

   skip_comment_lines(input, '#');
   istream::int_type next_char = input.peek();
@@ -81,10 +83,10 @@ GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)
   int vdim, ordering;

   fes = gf_array[0]->FESpace();
-   fec = FiniteElementCollection::New(fes->FEColl()->Name());
+   fec_owned = FiniteElementCollection::New(fes->FEColl()->Name());
   vdim = fes->GetVDim();
   ordering = fes->GetOrdering();
-   fes = new FiniteElementSpace(m, fec, vdim, ordering);
+   fes = new FiniteElementSpace(m, fec_owned, vdim, ordering);
   SetSize(fes->GetVSize());

   if (m->NURBSext)
@@ -153,11 +155,11 @@ GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)

 void GridFunction::Destroy()
 {
-   if (fec)
+   if (fec_owned)
   {
      delete fes;
-      delete fec;
-      fec = NULL;
+      delete fec_owned;
+      fec_owned = NULL;
   }
 }

@@ -325,10 +327,9 @@ int GridFunction::VectorDim() const
   const FiniteElement *fe;
   if (!fes->GetNE())
   {
-      const FiniteElementCollection *fe_coll = fes->FEColl();
      static const Geometry::Type geoms[3] =
      { Geometry::SEGMENT, Geometry::TRIANGLE, Geometry::TETRAHEDRON };
-      fe = fe_coll->
+      fe = fes->FEColl()->
           FiniteElementForGeometry(geoms[fes->GetMesh()->Dimension()-1]);
   }
   else
@@ -350,7 +351,8 @@ int GridFunction::CurlDim() const
   {
      static const Geometry::Type geoms[3] =
      { Geometry::SEGMENT, Geometry::TRIANGLE, Geometry::TETRAHEDRON };
-      fe = fec->FiniteElementForGeometry(geoms[fes->GetMesh()->Dimension()-1]);
+      fe = fes->FEColl()->
+           FiniteElementForGeometry(geoms[fes->GetMesh()->Dimension()-1]);
   }
   else
   {
@@ -1321,9 +1323,9 @@ void GridFunction::ProjectVectorFieldOn(GridFunction &vec_field, int comp)
   }
 }

-void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
-                                                      GridFunction &der,
-                                                      Array<int> &zones_per_dof)
+void GridFunction::AccumulateAndCountDerivativeValues(
+   int comp, int der_comp, GridFunction &der,
+   Array<int> &zones_per_dof) const
 {
   FiniteElementSpace * der_fes = der.FESpace();
   ElementTransformation * transf;
@@ -1374,7 +1376,8 @@ void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
   }
 }

-void GridFunction::GetDerivative(int comp, int der_comp, GridFunction &der)
+void GridFunction::GetDerivative(int comp, int der_comp,
+                                 GridFunction &der) const
 {
   Array<int> overlap;
   AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
@@ -2061,41 +2064,37 @@ void GridFunction::AccumulateAndCountBdrValues(
   Coefficient *coeff[], VectorCoefficient *vcoeff, const Array<int> &attr,
   Array<int> &values_counter)
 {
-   int i, j, fdof, d, ind, vdim;
-   real_t val;
-   const FiniteElement *fe;
-   ElementTransformation *transf;
   Array<int> vdofs;
   Vector vc;

   values_counter.SetSize(Size());
   values_counter = 0;

-   vdim = fes->GetVDim();
-
+   const int vdim = fes->GetVDim();
   HostReadWrite();

-   for (i = 0; i < fes->GetNBE(); i++)
+   for (int i = 0; i < fes->GetNBE(); i++)
   {
      if (attr[fes->GetBdrAttribute(i) - 1] == 0) { continue; }

-      fe = fes->GetBE(i);
-      fdof = fe->GetDof();
-      transf = fes->GetBdrElementTransformation(i);
+      const FiniteElement *fe = fes->GetBE(i);
+      const int fdof = fe->GetDof();
+      ElementTransformation *transf = fes->GetBdrElementTransformation(i);
      const IntegrationRule &ir = fe->GetNodes();
      fes->GetBdrElementVDofs(i, vdofs);

-      for (j = 0; j < fdof; j++)
+      for (int j = 0; j < fdof; j++)
      {
         const IntegrationPoint &ip = ir.IntPoint(j);
         transf->SetIntPoint(&ip);
         if (vcoeff) { vcoeff->Eval(vc, *transf, ip); }
-         for (d = 0; d < vdim; d++)
+         for (int d = 0; d < vdim; d++)
         {
            if (!vcoeff && !coeff[d]) { continue; }

-            val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
-            if ( (ind = vdofs[fdof*d+j]) < 0 )
+            real_t val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
+            int ind = vdofs[fdof*d+j];
+            if ( ind < 0 )
            {
               val = -val, ind = -1-ind;
            }
@@ -2117,10 +2116,11 @@ void GridFunction::AccumulateAndCountBdrValues(
   // iff A_ij != 0. It is sufficient to resolve just the first level of
   // dependency, since A is a projection matrix: A^n = A due to cR.cP = I.
   // Cases like these arise in 3D when boundary edges are constrained by
-   // (depend on) internal faces/elements. We use the virtual method
-   // GetBoundaryClosure from NCMesh to resolve the dependencies.
-
-   if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3)
+   // (depend on) internal faces/elements, or for internal boundaries in 2 or
+   // 3D. We use the virtual method GetBoundaryClosure from NCMesh to resolve
+   // the dependencies.
+   if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 ||
+                                fes->GetMesh()->Dimension() == 3))
   {
      Vector vals;
      Mesh *mesh = fes->GetMesh();
@@ -2128,26 +2128,19 @@ void GridFunction::AccumulateAndCountBdrValues(
      Array<int> bdr_edges, bdr_vertices, bdr_faces;
      ncmesh->GetBoundaryClosure(attr, bdr_vertices, bdr_edges, bdr_faces);

-      for (i = 0; i < bdr_edges.Size(); i++)
+      auto mark_dofs = [&](ElementTransformation &transf, const FiniteElement &fe)
      {
-         int edge = bdr_edges[i];
-         fes->GetEdgeVDofs(edge, vdofs);
-         if (vdofs.Size() == 0) { continue; }
-
-         transf = mesh->GetEdgeTransformation(edge);
-         transf->Attribute = -1; // TODO: set the boundary attribute
-         fe = fes->GetEdgeElement(edge);
         if (!vcoeff)
         {
-            vals.SetSize(fe->GetDof());
-            for (d = 0; d < vdim; d++)
+            vals.SetSize(fe.GetDof());
+            for (int d = 0; d < vdim; d++)
            {
               if (!coeff[d]) { continue; }

-               fe->Project(*coeff[d], *transf, vals);
+               fe.Project(*coeff[d], transf, vals);
               for (int k = 0; k < vals.Size(); k++)
               {
-                  ind = vdofs[d*vals.Size()+k];
+                  const int ind = vdofs[d*vals.Size()+k];
                  if (++values_counter[ind] == 1)
                  {
                     (*this)(ind) = vals(k);
@@ -2161,11 +2154,11 @@ void GridFunction::AccumulateAndCountBdrValues(
         }
         else // vcoeff != NULL
         {
-            vals.SetSize(vdim*fe->GetDof());
-            fe->Project(*vcoeff, *transf, vals);
+            vals.SetSize(vdim*fe.GetDof());
+            fe.Project(*vcoeff, transf, vals);
            for (int k = 0; k < vals.Size(); k++)
            {
-               ind = vdofs[k];
+               const int ind = vdofs[k];
               if (++values_counter[ind] == 1)
               {
                  (*this)(ind) = vals(k);
@@ -2176,6 +2169,26 @@ void GridFunction::AccumulateAndCountBdrValues(
               }
            }
         }
+      };
+
+      for (auto edge : bdr_edges)
+      {
+         fes->GetEdgeVDofs(edge, vdofs);
+         if (vdofs.Size() == 0) { continue; }
+
+         ElementTransformation *transf = mesh->GetEdgeTransformation(edge);
+         const FiniteElement *fe = fes->GetEdgeElement(edge);
+         mark_dofs(*transf, *fe);
+      }
+
+      for (auto face : bdr_faces)
+      {
+         fes->GetFaceVDofs(face, vdofs);
+         if (vdofs.Size() == 0) { continue; }
+
+         ElementTransformation *transf = mesh->GetFaceTransformation(face);
+         const FiniteElement *fe = fes->GetFaceElement(face);
+         mark_dofs(*transf, *fe);
      }
   }
 }
@@ -2228,26 +2241,37 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
      accumulate_dofs(dofs, lvec, *this, values_counter);
   }

-   if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3)
+   if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 ||
+                                fes->GetMesh()->Dimension() == 3))
   {
      Mesh *mesh = fes->GetMesh();
      NCMesh *ncmesh = mesh->ncmesh;
      Array<int> bdr_edges, bdr_vertices, bdr_faces;
      ncmesh->GetBoundaryClosure(bdr_attr, bdr_vertices, bdr_edges, bdr_faces);

-      for (int i = 0; i < bdr_edges.Size(); i++)
+      for (auto edge : bdr_edges)
      {
-         int edge = bdr_edges[i];
         fes->GetEdgeDofs(edge, dofs);
         if (dofs.Size() == 0) { continue; }

         T = mesh->GetEdgeTransformation(edge);
-         T->Attribute = -1; // TODO: set the boundary attribute
         fe = fes->GetEdgeElement(edge);
         lvec.SetSize(fe->GetDof());
         fe->Project(vcoeff, *T, lvec);
         accumulate_dofs(dofs, lvec, *this, values_counter);
      }
+
+      for (auto face : bdr_faces)
+      {
+         fes->GetFaceDofs(face, dofs);
+         if (dofs.Size() == 0) { continue; }
+
+         T = mesh->GetFaceTransformation(face);
+         fe = fes->GetFaceElement(face);
+         lvec.SetSize(fe->GetDof());
+         fe->Project(vcoeff, *T, lvec);
+         accumulate_dofs(dofs, lvec, *this, values_counter);
+      }
   }
 }

@@ -2350,19 +2374,48 @@ void GridFunction::ProjectCoefficient(Coefficient &coeff)

   if (delta_c == NULL)
   {
-      Array<int> vdofs;
-      Vector vals;
-
-      for (int i = 0; i < fes->GetNE(); i++)
+      if (fes->GetNURBSext() == NULL)
      {
-         doftrans = fes->GetElementVDofs(i, vdofs);
-         vals.SetSize(vdofs.Size());
-         fes->GetFE(i)->Project(coeff, *fes->GetElementTransformation(i), vals);
-         if (doftrans)
+         Array<int> vdofs;
+         Vector vals;
+
+         for (int i = 0; i < fes->GetNE(); i++)
         {
-            doftrans->TransformPrimal(vals);
+            doftrans = fes->GetElementVDofs(i, vdofs);
+            vals.SetSize(vdofs.Size());
+            fes->GetFE(i)->Project(coeff, *fes->GetElementTransformation(i), vals);
+            if (doftrans)
+            {
+               doftrans->TransformPrimal(vals);
+            }
+            SetSubVector(vdofs, vals);
         }
-         SetSubVector(vdofs, vals);
+      }
+      else
+      {
+         // Define and assemble linear form
+         LinearForm b(fes);
+         b.AddDomainIntegrator(new DomainLFIntegrator(coeff));
+         b.Assemble();
+
+         // Define and assemble bilinear form
+         BilinearForm a(fes);
+         a.AddDomainIntegrator(new MassIntegrator());
+         a.Assemble();
+
+         // Set solver and preconditioner
+         SparseMatrix A(a.SpMat());
+         GSSmoother  prec(A);
+         CGSolver cg;
+         cg.SetOperator(A);
+         cg.SetPreconditioner(prec);
+         cg.SetRelTol(1e-12);
+         cg.SetMaxIter(1000);
+         cg.SetPrintLevel(0);
+
+         // Solve and get solution
+         *this = 0.0;
+         cg.Mult(b,*this);
      }
   }
   else
@@ -2403,22 +2456,54 @@ void GridFunction::ProjectCoefficient(

 void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff)
 {
-   int i;
-   Array<int> vdofs;
-   Vector vals;
-
-   DofTransformation * doftrans = NULL;
-
-   for (i = 0; i < fes->GetNE(); i++)
+   if (fes->GetNURBSext() == NULL)
   {
-      doftrans = fes->GetElementVDofs(i, vdofs);
-      vals.SetSize(vdofs.Size());
-      fes->GetFE(i)->Project(vcoeff, *fes->GetElementTransformation(i), vals);
-      if (doftrans)
+
+      int i;
+      Array<int> vdofs;
+      Vector vals;
+
+      DofTransformation * doftrans = NULL;
+
+      for (i = 0; i < fes->GetNE(); i++)
      {
-         doftrans->TransformPrimal(vals);
+         doftrans = fes->GetElementVDofs(i, vdofs);
+         vals.SetSize(vdofs.Size());
+         fes->GetFE(i)->Project(vcoeff, *fes->GetElementTransformation(i), vals);
+         if (doftrans)
+         {
+            doftrans->TransformPrimal(vals);
+         }
+         SetSubVector(vdofs, vals);
      }
-      SetSubVector(vdofs, vals);
+
+   }
+
+   else
+   {
+      // Define and assemble linear form
+      LinearForm b(fes);
+      b.AddDomainIntegrator(new VectorFEDomainLFIntegrator(vcoeff));
+      b.Assemble();
+
+      // Define and assemble bilinear form
+      BilinearForm a(fes);
+      a.AddDomainIntegrator(new VectorFEMassIntegrator());
+      a.Assemble();
+
+      // Set solver and preconditioner
+      SparseMatrix A(a.SpMat());
+      GSSmoother  prec(A);
+      CGSolver cg;
+      cg.SetOperator(A);
+      cg.SetPreconditioner(prec);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(1000);
+      cg.SetPrintLevel(0);
+
+      // Solve and get solution
+      *this = 0.0;
+      cg.Mult(b,*this);
   }
 }

@@ -2966,6 +3051,57 @@ real_t GridFunction::ComputeDivError(
   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
 }

+real_t GridFunction::ComputeLaplaceError(
+   Coefficient *exlap, const IntegrationRule *irs[]) const
+{
+   real_t error = 0.0, a;
+   const FiniteElement *fe;
+   ElementTransformation *Tr;
+   Array<int> dofs;
+   int intorder, fdof;
+   Vector laplace;
+
+   for (int i = 0; i < fes->GetNE(); i++)
+   {
+      laplace.SetSize(fdof);
+      fe = fes->GetFE(i);
+      Tr = fes->GetElementTransformation(i);
+      intorder = 2*fe->GetOrder() + 3;
+      const IntegrationRule *ir;
+      if (irs)
+      {
+         ir = irs[fe->GetGeomType()];
+      }
+      else
+      {
+         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
+      }
+      fes->GetElementDofs(i, dofs);
+      fdof = fe->GetDof();
+      laplace.SetSize(fdof);
+      for (int j = 0; j < ir->GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(j);
+         Tr->SetIntPoint(&ip);
+         fe->CalcPhysLaplacian(*Tr, laplace);
+         a = 0;
+         for (int k = 0; k < fdof; k++)
+            if (dofs[k] >= 0)
+            {
+               a += (*this)(dofs[k]) * laplace(k);
+            }
+            else
+            {
+               a -= (*this)(-1-dofs[k]) * laplace(k);
+            }
+         a -= exlap->Eval(*Tr, ip);
+         error += ip.weight * Tr->Weight() * a * a;
+      }
+   }
+
+   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
+}
+
 real_t GridFunction::ComputeDGFaceJumpError(Coefficient *exsol,
                                            Coefficient *ell_coeff,
                                            class JumpScaling jump_scaling,
@@ -3904,7 +4040,7 @@ void GridFunction::LegacyNCReorder()
      mesh->GetEdgeVertices(i, ev);
      if (old_vertex[ev[0]] > old_vertex[ev[1]])
      {
-         const int *ind = fec->DofOrderForOrientation(Geometry::SEGMENT, -1);
+         const int *ind = fes->FEColl()->DofOrderForOrientation(Geometry::SEGMENT, -1);

         fes->GetEdgeInteriorDofs(i, dofs);
         for (int k = 0; k < dofs.Size(); k++)
@@ -30,14 +30,14 @@ namespace mfem
 class GridFunction : public Vector
 {
 protected:
-   /// FE space on which the grid function lives. Owned if #fec is not NULL.
+   /// FE space on which the grid function lives. Owned if #fec_owned is not NULL.
   FiniteElementSpace *fes;

   /** @brief Used when the grid function is read from a file. It can also be
       set explicitly, see MakeOwner().

       If not NULL, this pointer is owned by the GridFunction. */
-   FiniteElementCollection *fec;
+   FiniteElementCollection *fec_owned;

   long fes_sequence; // see FiniteElementSpace::sequence, Mesh::sequence

@@ -72,16 +72,16 @@ protected:

 public:

-   GridFunction() { fes = NULL; fec = NULL; fes_sequence = 0; UseDevice(true); }
+   GridFunction() { fes = NULL; fec_owned = NULL; fes_sequence = 0; UseDevice(true); }

   /// Copy constructor. The internal true-dof vector #t_vec is not copied.
   GridFunction(const GridFunction &orig)
-      : Vector(orig), fes(orig.fes), fec(NULL), fes_sequence(orig.fes_sequence)
+      : Vector(orig), fes(orig.fes), fec_owned(NULL), fes_sequence(orig.fes_sequence)
   { UseDevice(true); }

   /// Construct a GridFunction associated with the FiniteElementSpace @a *f.
   GridFunction(FiniteElementSpace *f) : Vector(f->GetVSize())
-   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+   { fes = f; fec_owned = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }

   /// Construct a GridFunction using previously allocated array @a data.
   /** The GridFunction does not assume ownership of @a data which is assumed to
@@ -91,13 +91,13 @@ public:
    */
   GridFunction(FiniteElementSpace *f, real_t *data)
      : Vector(data, f->GetVSize())
-   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+   { fes = f; fec_owned = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }

   /** @brief Construct a GridFunction using previously allocated Vector @a base
       starting at the given offset, @a base_offset. */
   GridFunction(FiniteElementSpace *f, Vector &base, int base_offset = 0)
      : Vector(base, base_offset, f->GetVSize())
-   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+   { fes = f; fec_owned = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }

   /// Construct a GridFunction on the given Mesh, using the data from @a input.
   /** The content of @a input should be in the format created by the method
@@ -116,12 +116,12 @@ public:
   GridFunction &operator=(const GridFunction &rhs)
   { return operator=((const Vector &)rhs); }

-   /// Make the GridFunction the owner of #fec and #fes.
-   /** If the new FiniteElementCollection, @a fec_, is NULL, ownership of #fec
+   /// Make the GridFunction the owner of #fec_owned and #fes.
+   /** If the new FiniteElementCollection, @a fec_, is NULL, ownership of #fec_owned
       and #fes is taken away. */
-   void MakeOwner(FiniteElementCollection *fec_) { fec = fec_; }
+   void MakeOwner(FiniteElementCollection *fec_) { fec_owned = fec_; }

-   FiniteElementCollection *OwnFEC() { return fec; }
+   FiniteElementCollection *OwnFEC() { return fec_owned; }

   int VectorDim() const;
   int CurlDim() const;
@@ -321,7 +321,7 @@ public:
       @param[out] der       The resulting derivative (scalar function). The
                             FiniteElementSpace of this function must be set
                             before the call. */
-   void GetDerivative(int comp, int der_comp, GridFunction &der);
+   void GetDerivative(int comp, int der_comp, GridFunction &der) const;

   real_t GetDivergence(ElementTransformation &tr) const;

@@ -387,7 +387,8 @@ public:
   /** @brief Project @a coeff Coefficient to @a this GridFunction. The
       projection computation depends on the choice of the FiniteElementSpace
       #fes. Note that this is usually interpolation at the degrees of freedom
-       in each element (not L2 projection). */
+       in each element (not L2 projection). For NURBS spaces these degrees of
+       freedom are not available and L2 projection is resorted to as fallback. */
   virtual void ProjectCoefficient(Coefficient &coeff);

   /** @brief Project @a coeff Coefficient to @a this GridFunction, using one
@@ -398,7 +399,8 @@ public:
   /** @brief Project @a vcoeff VectorCoefficient to @a this GridFunction. The
       projection computation depends on the choice of the FiniteElementSpace
       #fes. Note that this is usually interpolation at the degrees of freedom
-       in each element (not L2 projection).*/
+       in each element (not L2 projection). For NURBS spaces these degrees of
+       freedom are not available and L2 projection is resorted to as fallback. */
   void ProjectCoefficient(VectorCoefficient &vcoeff);

   /** @brief Project @a vcoeff VectorCoefficient to @a this GridFunction, using
@@ -443,7 +445,7 @@ protected:
       GetDerivative() method; see its documentation. */
   void AccumulateAndCountDerivativeValues(int comp, int der_comp,
                                           GridFunction &der,
-                                           Array<int> &zones_per_dof);
+                                           Array<int> &zones_per_dof) const;

   void AccumulateAndCountBdrValues(Coefficient *coeff[],
                                    VectorCoefficient *vcoeff,
@@ -531,6 +533,10 @@ public:
   virtual real_t ComputeDivError(Coefficient *exdiv,
                                  const IntegrationRule *irs[] = NULL) const;

+   /// Returns ||lap u_ex - lap u_h||_L2 for H1 elements
+   virtual real_t ComputeLaplaceError(Coefficient *exlap,
+                                      const IntegrationRule *irs[] = NULL) const;
+
   /// Returns the Face Jumps error for L2 elements. The error can be weighted
   /// by a constant nu, by nu/h, or nu*p^2/h, depending on the value of
   /// @a jump_scaling.
@@ -1352,6 +1352,85 @@ void OversetFindPointsGSLIB::Interpolate(const Vector &point_pos,
   Interpolate(field_in, field_out);
 }

+GSOPGSLIB::GSOPGSLIB(Array<long long> &ids)
+{
+   gsl_comm = new gslib::comm;
+   cr       = new gslib::crystal;
+#ifdef MFEM_USE_MPI
+   int initialized;
+   MPI_Initialized(&initialized);
+   if (!initialized) { MPI_Init(NULL, NULL); }
+   MPI_Comm comm = MPI_COMM_WORLD;
+   comm_init(gsl_comm, comm);
+#else
+   comm_init(gsl_comm, 0);
+#endif
+   crystal_init(cr, gsl_comm);
+   UpdateIdentifiers(ids);
+}
+
+#ifdef MFEM_USE_MPI
+GSOPGSLIB::GSOPGSLIB(MPI_Comm comm_, Array<long long> &ids)
+   : cr(NULL), gsl_comm(NULL)
+{
+   gsl_comm = new gslib::comm;
+   cr      = new gslib::crystal;
+   comm_init(gsl_comm, comm_);
+   crystal_init(cr, gsl_comm);
+   UpdateIdentifiers(ids);
+}
+#endif
+
+GSOPGSLIB::~GSOPGSLIB()
+{
+   crystal_free(cr);
+   gslib_gs_free(gsl_data);
+   comm_free(gsl_comm);
+   delete gsl_comm;
+   delete cr;
+}
+
+void GSOPGSLIB::UpdateIdentifiers(const Array<long long> &ids)
+{
+   long long minval = ids.Min();
+#ifdef MFEM_USE_MPI
+   MPI_Allreduce(MPI_IN_PLACE, &minval, 1, MPI_LONG_LONG_INT,
+                 MPI_MIN, gsl_comm->c);
+#endif
+   MFEM_VERIFY(minval >= 0, "Unique identifier cannot be negative.");
+   if (gsl_data != NULL) { gslib_gs_free(gsl_data); }
+   num_ids = ids.Size();
+   gsl_data = gslib_gs_setup(ids.GetData(),
+                             ids.Size(),
+                             gsl_comm, 0,
+                             gslib::gs_crystal_router, 0);
+}
+
+void GSOPGSLIB::GS(Vector &senddata, GSOp op)
+{
+   MFEM_VERIFY(senddata.Size() == num_ids,
+               "Incompatible setup and GOP operation.");
+   if (op == GSOp::ADD)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_add,0,gsl_data,0);
+   }
+   else if (op == GSOp::MUL)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_mul,0,gsl_data,0);
+   }
+   else if (op == GSOp::MAX)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_max,0,gsl_data,0);
+   }
+   else if (op == GSOp::MIN)
+   {
+      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_min,0,gsl_data,0);
+   }
+   else
+   {
+      MFEM_ABORT("Invalid GSOp operation.");
+   }
+}

 } // namespace mfem

@@ -23,13 +23,16 @@ struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
 struct crystal;
+struct gs_data;
 }

 namespace mfem
 {

 /** \brief FindPointsGSLIB can robustly evaluate a GridFunction on an arbitrary
- *  collection of points. There are three key functions in FindPointsGSLIB:
+ *  collection of points.
+ *
+ *  There are three key functions in FindPointsGSLIB:
 *
 *  1. Setup - constructs the internal data structures of gslib.
 *
@@ -226,6 +229,7 @@ public:

 /** \brief OversetFindPointsGSLIB enables use of findpts for arbitrary number of
    overlapping grids.
+
    The parameters in this class are the same as FindPointsGSLIB with the
    difference of additional inputs required to account for more than 1 mesh. */
 class OversetFindPointsGSLIB : public FindPointsGSLIB
@@ -290,6 +294,63 @@ public:
   using FindPointsGSLIB::Interpolate;
 };

+/** \brief  Class for gather-scatter (gs) operations on Vectors based on
+    corresponding global identifiers.
+
+    This functionality is useful for gs-ops on DOF values across processor
+    boundary, where the global identifier would be the corresponding true DOF
+    index. Operations currently supported are min, max, sum, and multiplication.
+    Note: identifier 0 does not participate in the gather-scatter operation and
+    a given identifier can be included multiple times on a given rank.
+    For example, consider a vector, v:
+    - v = [0.3, 0.4, 0.25, 0.7] on rank1,
+    - v = [0.6, 0.1] on rank 2,
+    - v = [-0.2, 0.3, 0.7, 0.] on rank 3.
+
+    Consider a corresponding Array<int>, a:
+    - a = [1, 2, 3, 1] on rank 1,
+    - a = [3, 2] on rank 2,
+    - a = [1, 2, 0, 3] on rank 3.
+
+    A gather-scatter "minimum" operation, done as follows:
+    GSOPGSLIB gs = GSOPGSLIB(MPI_COMM_WORLD, a);
+    gs.GS(v, GSOp::MIN);
+    would return into v:
+    - v = [-0.2, 0.1, 0., -0.2] on rank 1,
+    - v = [0., 0.1] on rank 2,
+    - v = [-0.2, 0.1, 0.7, 0.] on rank 3,
+    where the values have been compared across all processors based on the
+    integer identifier. */
+class GSOPGSLIB
+{
+protected:
+   struct gslib::crystal *cr;               // gslib's internal data
+   struct gslib::comm *gsl_comm;            // gslib's internal data
+   struct gslib::gs_data *gsl_data = NULL;
+   int num_ids;
+
+public:
+   GSOPGSLIB(Array<long long> &ids);
+
+#ifdef MFEM_USE_MPI
+   GSOPGSLIB(MPI_Comm comm_, Array<long long> &ids);
+#endif
+
+   virtual ~GSOPGSLIB();
+
+   /// Supported operation types. See class description.
+   enum GSOp {ADD, MUL, MIN, MAX};
+
+   /// Update the identifiers used for the gather-scatter operator.
+   /// Same @a ids get grouped together and id == 0 does not participate.
+   /// See class description.
+   void UpdateIdentifiers(const Array<long long> &ids);
+
+   /// Gather-Scatter operation on senddata. Must match length of unique
+   /// identifiers used in the constructor. See class description.
+   void GS(Vector &senddata, GSOp op);
+};
+
 } // namespace mfem

 #endif // MFEM_USE_GSLIB
@@ -18,7 +18,6 @@
 namespace mfem
 {

-
 void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
                                                     ElementTransformation &Tr,
                                                     const Vector &elfun,
@@ -29,7 +28,7 @@ void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
   const int dof = el.GetDof();

 #ifdef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration

   // shape function value at an integration point
   Vector shape(dof);
@@ -62,7 +61,7 @@ void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
      ir = &IntRules.Get(Tr.GetGeometryType(), order);
   }

-   // loop over interation points
+   // loop over integration points
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
@@ -92,7 +91,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
   const int dof2 = el2.GetDof();

 #ifdef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration

   // shape function value at an integration point - first elem
   Vector shape1(dof1);
@@ -122,7 +121,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
   DenseMatrix elvect2_mat(elvect.GetData() + dof1 * num_equations, dof2,
                           num_equations);

-   // obtain integration rule. If integration is rule is given, then use it.
+   // Obtain integration rule. If integration is rule is given, then use it.
   // Otherwise, get (2*p + IntOrderOffset) order integration rule
   const IntegrationRule *ir = IntRule;
   if (!ir)
@@ -149,7 +148,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
      if (nor.Size() == 1)  // if 1D, use 1 or -1.
      {
         // This assume the 1D integration point is in (0,1). This may not work
-         // if this chages.
+         // if this changes.
         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
      }
      else
@@ -18,43 +18,36 @@
 namespace mfem
 {

-//                     MFEM Hyperbolic Conservation Laws
+// This file contains general hyperbolic conservation element/face form
+// integrators. HyperbolicFormIntegrator and RiemannSolver are defined.
 //
-// Description:
+// HyperbolicFormIntegrator is a NonlinearFormIntegrator that implements
+// element weak divergence and interface flux
 //
-//    This file contains general hyperbolic conservation element/face form
-//    integrators.
+//     ∫_T F(u):∇v,   -∫_e F̂(u)⋅[[v]]
 //
-//    HyperbolicFormIntegrator and RiemannSolver are defined.
-//    HyperbolicFormIntegrator is a NonlinearFormIntegrator that implements
-//    element weak divergence and interface flux
+// Here, T is an element, e is an edge, and [[⋅]] is jump. This form integrator
+// is coupled with RiemannSolver that implements the numerical flux F̂. For
+// RiemannSolver, the Rusanov flux, also known as local Lax-Friedrichs flux, is
+// provided.
 //
-//       ∫_T F(u):∇v,   -∫_e F̂(u)⋅[[v]]
+// To implement a specific hyperbolic conservation laws, users can create
+// derived classes from FluxFunction with overloaded ComputeFlux. One can
+// optionally overload ComputeFluxDotN to avoid creating dense matrix when
+// computing normal flux. Several example equations are also defined including:
+// advection, Burgers', shallow water, and Euler equations. Users can control
+// the quadrature rule by either providing the integration rule, or integration
+// order offset. Integration will use 2*p + IntOrderOffset order quadrature
+// rule.
 //
-//    Here, T is an element, e is an edge, and [[⋅]] is jump. This form
-//    integrator is coupled with RiemannSolver that implements the numerical
-//    flux F̂. For RiemannSolver, the Rusanov flux, also known as local
-//    Lax-Friedrichs flux, is provided.
-//
-//    To implement a specific hyperbolic conservation laws, users can create
-//    derived classes from FluxFunction with overloaded ComputeFlux. One can
-//    optionally overload ComputeFluxDotN to avoid creating dense matrix when
-//    computing normal flux. Several example equations are also defined
-//    including: advection, Burgers', shallow water, and Euler equations. Users
-//    can control the quadrature rule by either providing the integration rule,
-//    or integration order offset. Integration will use 2*p + IntOrderOffset
-//    order quadrature rule.
-//
-//    At each call of HyperbolicFormIntegrator::AssembleElementVector
-//    HyperbolicFormIntegrator::AssembleFaceVector, the maximum characteristic
-//    speed will be updated. This will not be reinitialized automatically.
-//    To reinitialize, use HyperbolicFormIntegrator::ResetMaxCharSpeed. See,
-//    ex18.hpp.
-//
-//    Note: To avoid communication overhead, we update the maximum
-//    characteristic speed within each process. Use a proper MPI routine to
-//    gather the information.
+// At each call of HyperbolicFormIntegrator::AssembleElementVector
+// HyperbolicFormIntegrator::AssembleFaceVector, the maximum characteristic
+// speed will be updated. This will not be reinitialized automatically. To
+// reinitialize, use HyperbolicFormIntegrator::ResetMaxCharSpeed. See, ex18.hpp.
 //
+// Note: To avoid communication overhead, we update the maximum characteristic
+// speed within each MPI process only. Use the appropriate MPI routine to gather
+// the information.

 /**
 * @brief Abstract class for hyperbolic flux for a system of hyperbolic
@@ -88,7 +81,7 @@ public:
   virtual real_t ComputeFlux(const Vector &state, ElementTransformation &Tr,
                              DenseMatrix &flux) const = 0;
   /**
-    * @brief Compute normal flux. Optionally overloadded in the
+    * @brief Compute normal flux. Optionally overloaded in the
    * derived class to avoid creating full dense matrix for flux.
    *
    * @param[in] state state at the current integration point
@@ -168,13 +161,13 @@ protected:
 class HyperbolicFormIntegrator : public NonlinearFormIntegrator
 {
 private:
-   // The maximum characterstic speed, updated during element/face vector assembly
+   // The maximum characteristic speed, updated during element/face vector assembly
   real_t max_char_speed;
   const RiemannSolver &rsolver;   // Numerical flux that maps F(u±,x) to hat(F)
   const FluxFunction &fluxFunction;
   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
 #ifndef MFEM_THREAD_SAFE
-   // Local storages for element integration
+   // Local storage for element integration
   Vector shape;              // shape function value at an integration point
   Vector state;              // state value at an integration point
   DenseMatrix flux;          // flux value at an integration point
@@ -563,7 +563,7 @@ void DiffusionIntegrator::AssemblePatchMatrix_fullQuadrature(
   cdofs.SetSize(maxw[0], maxw[1], maxw[2]);

   // Compute sparsity of the sparse matrix
-   smati = new int[ndof+1];
+   smati = Memory<int>(ndof+1);
   smati[0] = 0;

   for (int dof_j=0; dof_j<ndof; ++dof_j)
@@ -586,8 +586,8 @@ void DiffusionIntegrator::AssemblePatchMatrix_fullQuadrature(
      nnz += ndd;
   }

-   smatj = new int[nnz];
-   smata = new real_t[nnz];
+   smatj = Memory<int>(nnz);
+   smata = Memory<real_t>(nnz);

   for (int i=0; i<nnz; ++i)
   {
@@ -973,7 +973,7 @@ void DiffusionIntegrator::AssemblePatchMatrix_reducedQuadrature(
   cdofs.SetSize(maxw[0], maxw[1], maxw[2]);

   // Compute sparsity of the sparse matrix
-   smati = new int[ndof+1];
+   smati = Memory<int>(ndof+1);
   smati[0] = 0;

   for (int dof_j=0; dof_j<ndof; ++dof_j)
@@ -996,8 +996,8 @@ void DiffusionIntegrator::AssemblePatchMatrix_reducedQuadrature(
      nnz += ndd;
   }

-   smatj = new int[nnz];
-   smata = new real_t[nnz];
+   smatj = Memory<int>(nnz);
+   smata = Memory<real_t>(nnz);

   for (int i=0; i<nnz; ++i)
   {
@@ -157,7 +157,7 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
   static constexpr int aSize = aUpper-aLower;
   static constexpr bool isComponent = (i_block >= 0);

-   //Assuming all elements are the same
+   // Assuming all elements are the same
   const auto &ir = QVec.GetIntRule(0);
   const QuadratureInterpolator *E_To_Q_Map = fespace.GetQuadratureInterpolator(
                                                 ir);
@@ -180,7 +180,7 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
         auto invJ = inv(make_tensor<d, d>(
         [&](int i, int j) { return J(p, i, j, e); }));
         tensor<real_t, aSize, d> gradx;
-         //load grad(x) into gradx
+         // load grad(x) into gradx
         if (isComponent)
         {
            for (int i = 0; i < d; i++)
@@ -198,11 +198,11 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
               }
            }
         }
-         //compute divergence
+         // compute divergence
         real_t div = 0.;
         for (int i = aLower; i < aUpper; i++)
         {
-            //take size of gradx into account
+            // take size of gradx into account
            const int iIndex = isComponent ? 0 : i;
            div += gradx(iIndex,i);
         }
@@ -211,11 +211,11 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
         {
            for (int q = qLower; q < qUpper; q++)
            {
-               //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-               //this contraction could be made slightly cheaper using Voigt
-               //notation, but repeated entries are summed for simplicity.
+               // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+               // this contraction could be made slightly cheaper using Voigt
+               // notation, but repeated entries are summed for simplicity.
               real_t contraction = 0.;
-               //not sure how to combine cases
+               // not sure how to combine cases
               if (isComponent)
               {
                  for (int a = 0; a < d; a++)
@@ -276,7 +276,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
                                   const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
-   //Assuming all elements are the same
+   // Assuming all elements are the same
   const auto &ir = QVec.GetIntRule(0);
   static constexpr int d = dim;
   const int numPoints = ir.GetNPoints();
@@ -299,9 +299,9 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
            {
               for (int q = 0; q < d; q++)
               {
-                  //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-                  //this contraction could be made slightly cheaper using Voigt
-                  //notation, but repeated entries are summed for simplicity.
+                  // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                  // this contraction could be made slightly cheaper using Voigt
+                  // notation, but repeated entries are summed for simplicity.
                  real_t contraction = 0.;
                  for (int a = 0; a < d; a++)
                  {
@@ -321,7 +321,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
      }
   });

-   //Reduce quadrature function to an E-Vector
+   // Reduce quadrature function to an E-Vector
   const auto QRead = Reshape(QVec.Read(), numPoints, d, d, d, numEls);
   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
@@ -348,7 +348,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
   });
 }

-//Templated implementation of ElasticityAssembleEA.
+// Templated implementation of ElasticityAssembleEA.
 template<int dim>
 void ElasticityAssembleEA_(const int i_block,
                           const int j_block,
@@ -360,7 +360,7 @@ void ElasticityAssembleEA_(const int i_block,
                           const DofToQuad &maps,
                           Vector &emat)
 {
-   //Assuming all elements are the same
+   // Assuming all elements are the same
   static constexpr int d = dim;
   const int numPoints = ir.GetNPoints();
   const int numEls = lambda.Size()/numPoints;
@@ -386,7 +386,7 @@ void ElasticityAssembleEA_(const int i_block,
               {
                  for (int m = 0; m < d; m++)
                  {
-                     //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                     // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
                     real_t contraction = 0.;
                     for (int a = 0; a < d; a++)
                     {
@@ -101,7 +101,7 @@ void MomentFittingIntRules::InitVolume(int order, Coefficient& levelset,
         }
      }

-      // assamble the matrix
+      // assemble the matrix
      DenseMatrix Mat(nBasisVolume, ir.GetNPoints());
      for (int ip = 0; ip < ir.GetNPoints(); ip++)
      {
@@ -118,7 +118,7 @@ void MomentFittingIntRules::InitVolume(int order, Coefficient& levelset,
         Mat.SetCol(ip, shape);
      }

-      // compute the svd for the matrix
+      // compute the SVD for the matrix
      VolumeSVD = new DenseMatrixSVD(Mat, 'A', 'A');
      VolumeSVD->Eval(Mat);
   }
@@ -220,8 +220,8 @@ void MomentFittingIntRules::ComputeSurfaceWeights1D(ElementTransformation& Tr)
   {
      IntegrationPoint ip2;
      ip2.x = .5;
-      while (LvlSet->Eval(Tr, ip2) > 1e-12
-             || LvlSet->Eval(Tr, ip2) < -1e-12)
+      while (LvlSet->Eval(Tr, ip2) > tol_1
+             || LvlSet->Eval(Tr, ip2) < -tol_1)
      {
         if (LvlSet->Eval(Tr, ip0) * LvlSet->Eval(Tr, ip2) < 0.)
         {
@@ -237,12 +237,12 @@ void MomentFittingIntRules::ComputeSurfaceWeights1D(ElementTransformation& Tr)
      intp.x = ip2.x;
      intp.weight = 1. / Tr.Weight();
   }
-   else if (LvlSet->Eval(Tr, ip0) > 0. && LvlSet->Eval(Tr, ip1) <= 1e-12)
+   else if (LvlSet->Eval(Tr, ip0) > 0. && LvlSet->Eval(Tr, ip1) <= tol_1)
   {
      intp.x = 1.;
      intp.weight = 1. / Tr.Weight();
   }
-   else if (LvlSet->Eval(Tr, ip1) > 0. && LvlSet->Eval(Tr, ip0) <= 1e-12)
+   else if (LvlSet->Eval(Tr, ip1) > 0. && LvlSet->Eval(Tr, ip0) <= tol_1)
   {
      intp.x = 0.;
      intp.weight = 1. / Tr.Weight();
@@ -290,8 +290,8 @@ void MomentFittingIntRules::ComputeVolumeWeights1D(ElementTransformation& Tr,
         }
      }
   }
-   else if (LvlSet->Eval(Tr, ip0) <= -1e-12
-            || LvlSet->Eval(Tr, ip1) <= -1e-12)
+   else if (LvlSet->Eval(Tr, ip0) <= -tol_1
+            || LvlSet->Eval(Tr, ip1) <= -tol_1)
   {
      for (int ip = 0; ip < ir.GetNPoints(); ip++)
      {
@@ -356,24 +356,24 @@ void MomentFittingIntRules::ComputeSurfaceWeights2D(ElementTransformation& Tr)
      IntegrationPoint ipB;
      Trafo.TransformBack(pointB, ipB);

-      if (LvlSet->Eval(Trafo, ipA) < -1e-12
-          || LvlSet->Eval(Trafo, ipB) < -1e-12)
+      if (LvlSet->Eval(Trafo, ipA) < -tol_1
+          || LvlSet->Eval(Trafo, ipB) < -tol_1)
      {
         interior = false;
      }

-      if (LvlSet->Eval(Trafo, ipA) > -1e-12
-          && LvlSet->Eval(Trafo, ipB) > -1e-12)
+      if (LvlSet->Eval(Trafo, ipA) > -tol_1
+          && LvlSet->Eval(Trafo, ipB) > -tol_1)
      {
         layout = Layout::inside;
      }
-      else if (LvlSet->Eval(Trafo, ipA) > 1e-15
+      else if (LvlSet->Eval(Trafo, ipA) > tol_2
               && LvlSet->Eval(Trafo, ipB) <= 0.)
      {
         layout = Layout::intersected;
      }
      else if (LvlSet->Eval(Trafo, ipA) <= 0.
-               && LvlSet->Eval(Trafo, ipB) > 1e-15)
+               && LvlSet->Eval(Trafo, ipB) > tol_2)
      {
         layout = Layout::intersected;
         Vector temp(pointA.Size());
@@ -399,10 +399,10 @@ void MomentFittingIntRules::ComputeSurfaceWeights2D(ElementTransformation& Tr)
         IntegrationPoint ip;
         Trafo.TransformBack(mid, ip);

-         while (LvlSet->Eval(Trafo, ip) > 1e-12
-                || LvlSet->Eval(Trafo, ip) < -1e-12)
+         while (LvlSet->Eval(Trafo, ip) > tol_1
+                || LvlSet->Eval(Trafo, ip) < -tol_1)
         {
-            if (LvlSet->Eval(Trafo, ip) > 1e-12)
+            if (LvlSet->Eval(Trafo, ip) > tol_1)
            {
               pointC = mid;
            }
@@ -539,7 +539,7 @@ void MomentFittingIntRules::ComputeSurfaceWeights2D(ElementTransformation& Tr)
      temp2 = 0.;
      for (int i = 0; i < nBasis; i++)
      {
-         if (SVD.Singularvalue(i) > 1e-12)
+         if (SVD.Singularvalue(i) > tol_1)
         {
            temp2(i) = temp(i) / SVD.Singularvalue(i);
         }
@@ -606,24 +606,24 @@ void MomentFittingIntRules::ComputeVolumeWeights2D(ElementTransformation& Tr,
      IntegrationPoint ipB;
      Trafo.TransformBack(pointB, ipB);

-      if (LvlSet->Eval(Trafo, ipA) < -1e-12
-          || LvlSet->Eval(Trafo, ipB) < -1e-12)
+      if (LvlSet->Eval(Trafo, ipA) < -tol_1
+          || LvlSet->Eval(Trafo, ipB) < -tol_1)
      {
         interior = false;
      }

-      if (LvlSet->Eval(Trafo, ipA) > -1e-12
-          && LvlSet->Eval(Trafo, ipB) > -1e-12)
+      if (LvlSet->Eval(Trafo, ipA) > -tol_1
+          && LvlSet->Eval(Trafo, ipB) > -tol_1)
      {
         layout = Layout::inside;
      }
-      else if (LvlSet->Eval(Trafo, ipA) > 1e-15
+      else if (LvlSet->Eval(Trafo, ipA) > tol_2
               && LvlSet->Eval(Trafo, ipB) <= 0.)
      {
         layout = Layout::intersected;
      }
      else if (LvlSet->Eval(Trafo, ipA) <= 0.
-               && LvlSet->Eval(Trafo, ipB) > 1e-15)
+               && LvlSet->Eval(Trafo, ipB) > tol_2)
      {
         layout = Layout::intersected;
         Vector temp(pointA.Size());
@@ -648,10 +648,10 @@ void MomentFittingIntRules::ComputeVolumeWeights2D(ElementTransformation& Tr,
         IntegrationPoint ip;
         Trafo.TransformBack(mid, ip);

-         while (LvlSet->Eval(Trafo, ip) > 1e-12
-                || LvlSet->Eval(Trafo, ip) < -1e-12)
+         while (LvlSet->Eval(Trafo, ip) > tol_1
+                || LvlSet->Eval(Trafo, ip) < -tol_1)
         {
-            if (LvlSet->Eval(Trafo, ip) > 1e-12)
+            if (LvlSet->Eval(Trafo, ip) > tol_1)
            {
               pointC = mid;
            }
@@ -786,7 +786,7 @@ void MomentFittingIntRules::ComputeVolumeWeights2D(ElementTransformation& Tr,
      VolumeSVD->LeftSingularvectors().MultTranspose(RHS, temp);
      for (int i = 0; i < nBasisVolume; i++)
      {
-         if (VolumeSVD->Singularvalue(i) > 1e-12)
+         if (VolumeSVD->Singularvalue(i) > tol_1)
         {
            temp2(i) = temp(i) / VolumeSVD->Singularvalue(i);
         }
@@ -865,18 +865,18 @@ void MomentFittingIntRules::ComputeSurfaceWeights3D(ElementTransformation& Tr)
      IntegrationPoint ipD;
      Trafo.TransformBack(pointD, ipD);

-      if (LvlSet->Eval(Trafo, ipA) < -1e-12
-          || LvlSet->Eval(Trafo, ipB) < -1e-12
-          || LvlSet->Eval(Trafo, ipC) < -1e-12
-          || LvlSet->Eval(Trafo, ipD) < -1e-12)
+      if (LvlSet->Eval(Trafo, ipA) < -tol_1
+          || LvlSet->Eval(Trafo, ipB) < -tol_1
+          || LvlSet->Eval(Trafo, ipC) < -tol_1
+          || LvlSet->Eval(Trafo, ipD) < -tol_1)
      {
         interior = false;
      }

-      if (LvlSet->Eval(Trafo, ipA) > -1e-12
-          || LvlSet->Eval(Trafo, ipB) > -1e-12
-          || LvlSet->Eval(Trafo, ipC) > -1e-12
-          || LvlSet->Eval(Trafo, ipD) > -1e-12)
+      if (LvlSet->Eval(Trafo, ipA) > -tol_1
+          || LvlSet->Eval(Trafo, ipB) > -tol_1
+          || LvlSet->Eval(Trafo, ipC) > -tol_1
+          || LvlSet->Eval(Trafo, ipD) > -tol_1)
      {
         element_int = true;
      }
@@ -978,7 +978,7 @@ void MomentFittingIntRules::ComputeSurfaceWeights3D(ElementTransformation& Tr)
      temp2 = 0.;
      for (int i = 0; i < nBasis; i++)
      {
-         if (SVD.Singularvalue(i) > 1e-12)
+         if (SVD.Singularvalue(i) > tol_1)
         {
            temp2(i) = temp(i) / SVD.Singularvalue(i);
         }
@@ -1047,18 +1047,18 @@ void MomentFittingIntRules::ComputeVolumeWeights3D(ElementTransformation& Tr,
      IntegrationPoint ipD;
      Trafo.TransformBack(pointD, ipD);

-      if (LvlSet->Eval(Trafo, ipA) < -1e-12
-          || LvlSet->Eval(Trafo, ipB) < -1e-12
-          || LvlSet->Eval(Trafo, ipC) < -1e-12
-          || LvlSet->Eval(Trafo, ipD) < -1e-12)
+      if (LvlSet->Eval(Trafo, ipA) < -tol_1
+          || LvlSet->Eval(Trafo, ipB) < -tol_1
+          || LvlSet->Eval(Trafo, ipC) < -tol_1
+          || LvlSet->Eval(Trafo, ipD) < -tol_1)
      {
         interior = false;
      }

-      if (LvlSet->Eval(Trafo, ipA) > -1e-12
-          || LvlSet->Eval(Trafo, ipB) > -1e-12
-          || LvlSet->Eval(Trafo, ipC) > -1e-12
-          || LvlSet->Eval(Trafo, ipD) > -1e-12)
+      if (LvlSet->Eval(Trafo, ipA) > -tol_1
+          || LvlSet->Eval(Trafo, ipB) > -tol_1
+          || LvlSet->Eval(Trafo, ipC) > -tol_1
+          || LvlSet->Eval(Trafo, ipD) > -tol_1)
      {
         element_int = true;
      }
@@ -1159,7 +1159,7 @@ void MomentFittingIntRules::ComputeVolumeWeights3D(ElementTransformation& Tr,
      VolumeSVD->LeftSingularvectors().MultTranspose(RHS, temp);
      temp2 = 0.;
      for (int i = 0; i < nBasisVolume; i++)
-         if (VolumeSVD->Singularvalue(i) > 1e-12)
+         if (VolumeSVD->Singularvalue(i) > tol_1)
         {
            temp2(i) = temp(i) / VolumeSVD->Singularvalue(i);
         }
@@ -1239,7 +1239,7 @@ void MomentFittingIntRules::OrthoBasis2D(const IntegrationPoint& ip,

   shape.SetSize(nBasis, 2);

-   // evaluate basis inthe point
+   // evaluate basis in the point
   DenseMatrix preshape(nBasis, 2);
   DivFreeBasis2D(ip, shape);

@@ -1597,6 +1597,6 @@ void MomentFittingIntRules::GetSurfaceWeights(ElementTransformation& Tr,
   }
 }

-#endif //MFEM_USE_LAPACK
+#endif // MFEM_USE_LAPACK

 }
@@ -36,6 +36,17 @@ protected:
   /// Space order for the LS projection.
   int lsOrder;

+   /// @name Tolerances used for point comparisons
+   ///@{
+#ifdef MFEM_USE_DOUBLE
+   static constexpr real_t tol_1 = 1e-12;
+   static constexpr real_t tol_2 = 1e-15;
+#elif defined(MFEM_USE_SINGLE)
+   static constexpr real_t tol_1 = 1e-5;
+   static constexpr real_t tol_2 = 1e-7;
+#endif
+   ///@}
+
   /** @brief Constructor to set up the generated cut IntegrationRules.

       @param [in] order  Order of the constructed IntegrationRule.
@@ -123,6 +123,35 @@ void DomainLFGradIntegrator::AssembleDeltaElementVect(
   dshape.Mult(Qvec, elvect);
 }

+void DomainLFLaplaceIntegrator::AssembleRHSElementVect(const FiniteElement &el,
+                                                       ElementTransformation &Tr,
+                                                       Vector &elvect)
+{
+   int dof = el.GetDof();
+
+   laplace.SetSize(dof);       // vector of size dof
+   elvect.SetSize(dof);
+   elvect = 0.0;
+
+   const IntegrationRule *ir = NULL;//IntRule;
+   if (ir == NULL)
+   {
+      ir = &IntRules.Get(el.GetGeomType(), oa * el.GetOrder() + ob + 4);
+   }
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      Tr.SetIntPoint (&ip);
+      real_t val = Tr.Weight() * Q.Eval(Tr, ip) * alpha;
+
+      el.CalcPhysLaplacian(Tr, laplace);
+
+      add(elvect, ip.weight * val, laplace, elvect);
+   }
+}
+
 void BoundaryLFIntegrator::AssembleRHSElementVect(
   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -174,6 +174,29 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

+/// Class for domain integrator $ L(v) := (f, \Delta v) $
+class DomainLFLaplaceIntegrator : public LinearFormIntegrator
+{
+private:
+   Vector laplace;
+   Coefficient &Q;
+   real_t alpha;
+   int oa, ob;
+public:
+   /// Constructs the domain integrator $ (Q, \nabla v) $
+   DomainLFLaplaceIntegrator(Coefficient &QF, real_t alp = 1.0, int a = 2,
+                             int b = 0)
+      : Q(QF), oa(a), ob(b)   { alpha = alp; }
+
+   /** Given a particular Finite Element and a transformation (Tr)
+       computes the element right hand side element vector, elvect. */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect);
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
+};
+

 /// Class for boundary integration $ L(v) := (g, v) $
 class BoundaryLFIntegrator : public LinearFormIntegrator
@@ -291,7 +291,7 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   const auto ltdof_ldof = HypreRead(R->GetMemoryJ());

   // Go from E-vector format directly to T-vector format
-   MFEM_HYPRE_FORALL(i, ntdofs,
+   mfem::hypre_forall(ntdofs, [=] MFEM_HOST_DEVICE (int i)
   {
      const int j = d_offsets[ltdof_ldof[i]];
      for (int c = 0; c < sdim; ++c)
@@ -269,13 +269,13 @@ void BatchedLOR_H1::Assemble3D()
               real_t vx[8], vy[8], vz[8];
               LORVertexCoordinates3D<ORDER>(X, iel_ho, kx, ky, kz, vx, vy, vz);

-               //MFEM_UNROLL(2)
+               // MFEM_UNROLL(2)
               for (int iqz=0; iqz<2; ++iqz)
               {
-                  //MFEM_UNROLL(2)
+                  // MFEM_UNROLL(2)
                  for (int iqy=0; iqy<2; ++iqy)
                  {
-                     //MFEM_UNROLL(2)
+                     // MFEM_UNROLL(2)
                     for (int iqx=0; iqx<2; ++iqx)
                     {
                        const real_t x = iqx;
@@ -307,21 +307,21 @@ void BatchedLOR_H1::Assemble3D()
                  }
               }

-               //MFEM_UNROLL(2)
+               // MFEM_UNROLL(2)
               for (int iqx=0; iqx<2; ++iqx)
               {
-                  //MFEM_UNROLL(2)
+                  // MFEM_UNROLL(2)
                  for (int jz=0; jz<2; ++jz)
                  {
                     // Note loop starts at iz=jz here, taking advantage of
                     // symmetries.
-                     //MFEM_UNROLL(2)
+                     // MFEM_UNROLL(2)
                     for (int iz=jz; iz<2; ++iz)
                     {
-                        //MFEM_UNROLL(2)
+                        // MFEM_UNROLL(2)
                        for (int iqy=0; iqy<2; ++iqy)
                        {
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int iqz=0; iqz<2; ++iqz)
                           {
                              const real_t mq = const_mq ? MQ(0,0,0,0) : MQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
@@ -356,10 +356,10 @@ void BatchedLOR_H1::Assemble3D()
                              real_t wdetJ = Q(6,iqz,iqy,iqx);
                              mass_A(iqy,iz,jz,iqx) += mq*wdetJ*biz*bjz;
                           }
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int jy=0; jy<2; ++jy)
                           {
-                              //MFEM_UNROLL(2)
+                              // MFEM_UNROLL(2)
                              for (int iy=0; iy<2; ++iy)
                              {
                                 const real_t biy = (iy == iqy) ? 1.0 : 0.0;
@@ -382,16 +382,16 @@ void BatchedLOR_H1::Assemble3D()
                              }
                           }
                        }
-                        //MFEM_UNROLL(2)
+                        // MFEM_UNROLL(2)
                        for (int jy=0; jy<2; ++jy)
                        {
-                           //MFEM_UNROLL(2)
+                           // MFEM_UNROLL(2)
                           for (int jx=0; jx<2; ++jx)
                           {
-                              //MFEM_UNROLL(2)
+                              // MFEM_UNROLL(2)
                              for (int iy=0; iy<2; ++iy)
                              {
-                                 //MFEM_UNROLL(2)
+                                 // MFEM_UNROLL(2)
                                 for (int ix=0; ix<2; ++ix)
                                 {
                                    const real_t bix = (ix == iqx) ? 1.0 : 0.0;
@@ -431,7 +431,7 @@ void BatchedLOR_H1::Assemble3D()
               // Assemble the local matrix into the macro-element sparse matrix
               // in a format similar to coordinate format. The (I,J) arrays
               // are implicit (not stored explicitly).
-               //MFEM_UNROLL(8)
+               // MFEM_UNROLL(8)
               for (int ii_loc=0; ii_loc<nv; ++ii_loc)
               {
                  const int ix = ii_loc%2;
@@ -0,0 +1,828 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "normal_deriv_restriction.hpp"
+#include "fespace.hpp"
+#include "pgridfunc.hpp"
+#include "fe/face_map_utils.hpp"
+#include "../general/forall.hpp"
+
+namespace mfem
+{
+
+/// Compute the face index to volume index map "face_to_vol" in 2D
+static void NormalDerivativeSetupFaceIndexMap2D(
+   int nf, int d, const Array<int>& face_to_elem, Array<int>& face_to_vol)
+{
+   const auto f2e = Reshape(face_to_elem.HostRead(), 2, 2, nf);
+   auto f2v = Reshape(face_to_vol.HostWrite(), d, 2, nf);
+
+   for (int f = 0; f < nf; ++f)
+   {
+      const int fid0 = f2e(0, 1, f);
+      const int fid1 = f2e(1, 1, f);
+      for (int side = 0; side < 2; ++side)
+      {
+         const int el = f2e(side, 0, f);
+
+         if (el < 0)
+         {
+            for (int p = 0; p < d; ++p)
+            {
+               f2v(p, side, f) = -1;
+            }
+         }
+         else
+         {
+            for (int p = 0; p < d; ++p)
+            {
+               int i, j;
+               internal::FaceIdxToVolIdx2D(p, d, fid0, fid1, side, i, j);
+
+               f2v(p, side, f) = i + d * j;
+            }
+         }
+      }
+   }
+}
+
+/// Compute the face index to volume index map "face_to_vol" in 3D
+static void NormalDerivativeSetupFaceIndexMap3D(
+   int nf, int d, const Array<int>& face_to_elem, Array<int>& face_to_vol)
+{
+   const auto f2e = Reshape(face_to_elem.HostRead(), 2, 3, nf);
+   auto f2v = Reshape(face_to_vol.HostWrite(), d*d, 2, nf);
+
+   for (int f = 0; f < nf; ++f)
+   {
+      const int fid0 = f2e(0, 1, f);
+      const int fid1 = f2e(1, 1, f);
+      for (int side = 0; side < 2; ++side)
+      {
+         const int el = f2e(side, 0, f);
+         const int orientation = f2e(side, 2, f);
+
+         if (el < 0)
+         {
+            for (int p = 0; p < d*d; ++p)
+            {
+               f2v(p, side, f) = -1;
+            }
+         }
+         else
+         {
+            for (int p = 0; p < d*d; ++p)
+            {
+               int i, j, k; // 3D lexicographic index of quad point p
+               internal::FaceIdxToVolIdx3D(p, d, fid0, fid1, side, orientation, i, j, k);
+
+               f2v(p, side, f) = i + d * (j + d * k);
+            }
+         }
+      }
+   }
+}
+
+L2NormalDerivativeFaceRestriction::L2NormalDerivativeFaceRestriction(
+   const FiniteElementSpace &fes_,
+   const ElementDofOrdering f_ordering,
+   const FaceType face_type_)
+   : fes(fes_),
+     face_type(face_type_),
+     dim(fes.GetMesh()->Dimension()),
+     nf(fes.GetNFbyType(face_type)),
+     ne(fes.GetNE())
+{
+   MFEM_VERIFY(f_ordering == ElementDofOrdering::LEXICOGRAPHIC,
+               "Non-lexicographic ordering not currently supported in "
+               "L2NormalDerivativeFaceRestriction.");
+
+   Mesh &mesh = *fes.GetMesh();
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const int d = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR).ndof;
+
+   if (dim == 2)
+   {
+      // (el0, el1, fid0, fid1)
+      face_to_elem.SetSize(nf * 4);
+      face_to_vol.SetSize(2 * nf * d);
+   }
+   else if (dim == 3)
+   {
+      // (el0, el1, fid0, fid1, or0, or1)
+      face_to_elem.SetSize(nf * 6);
+      face_to_vol.SetSize(2 * nf * d * d);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension.");
+   }
+   auto f2e = Reshape(face_to_elem.HostWrite(), 2, (dim == 2) ? 2 : 3, nf);
+
+   // Populate the face_to_elem array. The elem_indicator will be used to count
+   // the number of elements that are adjacent to faces of the given type.
+   Array<int> elem_indicator(ne);
+   elem_indicator = 0;
+
+   int f_ind = 0;
+   for (int f = 0; f < fes.GetNF(); ++f)
+   {
+      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
+
+      if (face.IsOfFaceType(face_type))
+      {
+         f2e(0, 0, f_ind) = face.element[0].index;
+         f2e(0, 1, f_ind) = face.element[0].local_face_id;
+         if (dim == 3)
+         {
+            f2e(0, 2, f_ind) = face.element[0].orientation;
+         }
+
+         elem_indicator[face.element[0].index] = 1;
+
+         if (face_type == FaceType::Interior)
+         {
+            const int el_idx_1 = face.element[1].index;
+            if (face.IsShared())
+            {
+               // Indicate shared face by index >= ne
+               f2e(1, 0, f_ind) = ne + el_idx_1;
+            }
+            else
+            {
+               // Face is not shared
+               f2e(1, 0, f_ind) = el_idx_1;
+               elem_indicator[el_idx_1] = 1;
+            }
+            f2e(1, 1, f_ind) = face.element[1].local_face_id;
+
+            if (dim == 3)
+            {
+               f2e(1, 2, f_ind) = face.element[1].orientation;
+            }
+         }
+         else
+         {
+            f2e(1, 0, f_ind) = -1;
+            f2e(1, 1, f_ind) = -1;
+
+            if (dim == 3)
+            {
+               f2e(1, 2, f_ind) = -1;
+            }
+         }
+
+         f_ind++;
+      }
+   }
+
+   // evaluate face to vol map
+   if (dim == 2)
+   {
+      NormalDerivativeSetupFaceIndexMap2D(nf, d, face_to_elem, face_to_vol);
+   }
+   else if (dim == 3)
+   {
+      NormalDerivativeSetupFaceIndexMap3D(nf, d, face_to_elem, face_to_vol);
+   }
+
+   // Number of elements adjacent to faces of face_type
+   ne_type = elem_indicator.Sum();
+
+   // In 2D: (el, f0,f1,f2,f3, s0,s1,s2,s3)
+   // In 3D: (el, f0,f1,f2,f3,f4,f5, s0,s1,s2,s3,s4,s5)
+   const int elem_data_sz = (dim == 2) ? 9 : 13;
+
+   elem_to_face.SetSize(elem_data_sz * ne_type);
+   elem_to_face = -1;
+
+   auto e2f = Reshape(elem_to_face.HostWrite(), elem_data_sz, ne_type);
+   elem_indicator.PartialSum();
+
+   const int nsides = (face_type == FaceType::Interior) ? 2 : 1;
+   const int side_begin = (dim == 2) ? 5 : 7;
+   for (int f = 0; f < nf; ++f)
+   {
+      for (int side = 0; side < nsides; ++side)
+      {
+         const int el = f2e(side, 0, f);
+         // Skip shared faces
+         if (el < ne)
+         {
+            const int face_id = f2e(side, 1, f);
+
+            const int e = elem_indicator[el] - 1;
+            e2f(0, e) = el;
+            e2f(1 + face_id, e) = f;
+            e2f(side_begin + face_id, e) = side;
+         }
+      }
+   }
+}
+
+void L2NormalDerivativeFaceRestriction::Mult(const Vector &x, Vector &y) const
+{
+   if (nf == 0) { return; }
+   switch (dim)
+   {
+      case 2:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: Mult2D<1>(x, y); break;
+            case 2: Mult2D<2>(x, y); break;
+            case 3: Mult2D<3>(x, y); break;
+            case 4: Mult2D<4>(x, y); break;
+            case 5: Mult2D<5>(x, y); break;
+            case 6: Mult2D<6>(x, y); break;
+            case 7: Mult2D<7>(x, y); break;
+            case 8: Mult2D<8>(x, y); break;
+            default: Mult2D(x, y); break;
+         }
+      }
+      break;
+      case 3:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: Mult3D<1>(x, y); break;
+            case 2: Mult3D<2>(x, y); break;
+            case 3: Mult3D<3>(x, y); break;
+            case 4: Mult3D<4>(x, y); break;
+            case 5: Mult3D<5>(x, y); break;
+            case 6: Mult3D<6>(x, y); break;
+            case 7: Mult3D<7>(x, y); break;
+            case 8: Mult3D<8>(x, y); break;
+            default: Mult3D(x, y); break; // fallback
+         }
+         break;
+      }
+      default: MFEM_ABORT("Dimension not supported."); break;
+   }
+}
+
+void L2NormalDerivativeFaceRestriction::AddMultTranspose(
+   const Vector &x, Vector &y, const real_t a) const
+{
+   if (nf == 0) { return; }
+   switch (dim)
+   {
+      case 2:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: AddMultTranspose2D<1>(x, y, a); break;
+            case 2: AddMultTranspose2D<2>(x, y, a); break;
+            case 3: AddMultTranspose2D<3>(x, y, a); break;
+            case 4: AddMultTranspose2D<4>(x, y, a); break;
+            case 5: AddMultTranspose2D<5>(x, y, a); break;
+            case 6: AddMultTranspose2D<6>(x, y, a); break;
+            case 7: AddMultTranspose2D<7>(x, y, a); break;
+            case 8: AddMultTranspose2D<8>(x, y, a); break;
+            default: AddMultTranspose2D(x, y, a); break;
+         }
+      }
+      break;
+      case 3:
+      {
+         const int d1d = fes.GetElementOrder(0) + 1;
+         switch (d1d)
+         {
+            case 1: AddMultTranspose3D<1>(x, y, a); break;
+            case 2: AddMultTranspose3D<2>(x, y, a); break;
+            case 3: AddMultTranspose3D<3>(x, y, a); break;
+            case 4: AddMultTranspose3D<4>(x, y, a); break;
+            case 5: AddMultTranspose3D<5>(x, y, a); break;
+            case 6: AddMultTranspose3D<6>(x, y, a); break;
+            case 7: AddMultTranspose3D<7>(x, y, a); break;
+            case 8: AddMultTranspose3D<8>(x, y, a); break;
+            default: AddMultTranspose3D(x, y, a); break; // fallback
+         }
+         break;
+      }
+      default: MFEM_ABORT("Not yet implemented"); break;
+   }
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::Mult2D(const Vector &x, Vector &y) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+   const int num_elem = ne;
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+
+   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, face_type);
+   const int ne_shared = face_nbr_data.Size() / d / d / vd;
+
+   MFEM_VERIFY(q == d, "");
+   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
+
+   // derivative of 1D basis function
+   const auto G_ = Reshape(maps.G.Read(), q, d);
+   // (el0, el1, fid0, fid1)
+   const auto f2e = Reshape(face_to_elem.Read(), 2, 2, nf);
+
+   const auto f2v = Reshape(face_to_vol.Read(), q, 2, nf);
+
+   // if byvdim, d_x has shape (vdim, nddof, nddof, ne)
+   // otherwise, d_x has shape (nddof, nddof, ne, vdim)
+   const auto d_x = Reshape(x.Read(), t?vd:d, d, t?d:ne, t?ne:vd);
+   const auto d_x_shared = Reshape(face_nbr_data.Read(),
+                                   t?vd:d, d, t?d:ne_shared, t?ne_shared:vd);
+   auto d_y = Reshape(y.Write(), q, vd, 2, nf);
+
+   mfem::forall_2D(nf, 2, q, [=] MFEM_HOST_DEVICE (int f) -> void
+   {
+      constexpr int MD = (T_D1D) ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED real_t G_s[MD*MD];
+      DeviceMatrix G(G_s, q, d);
+
+      MFEM_SHARED int E[2];
+      MFEM_SHARED int FID[2];
+      MFEM_SHARED int F2V[2][MD];
+
+      if (MFEM_THREAD_ID(x) == 0)
+      {
+         MFEM_FOREACH_THREAD(j, y, d)
+         {
+            for (int i = 0; i < q; ++i)
+            {
+               G(i, j) = G_(i, j);
+            }
+         }
+      }
+
+      MFEM_FOREACH_THREAD(side, x, 2)
+      {
+         if (MFEM_THREAD_ID(y) == 0)
+         {
+            E[side] = f2e(side, 0, f);
+            FID[side] = f2e(side, 1, f);
+         }
+
+         MFEM_FOREACH_THREAD(j, y, d)
+         {
+            F2V[side][j] = f2v(j, side, f);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, x, 2)
+      {
+         const int el = E[side];
+         const bool shared = (el >= num_elem);
+         const auto &d_x_e = shared ? d_x_shared : d_x;
+         const int el_idx = shared ? el - num_elem : el;
+
+         const int face_id = FID[side];
+
+         MFEM_FOREACH_THREAD(p, y, q)
+         {
+            if (el < 0)
+            {
+               for (int c = 0; c < vd; ++c)
+               {
+                  d_y(p, c, side, f) = 0.0;
+               }
+            }
+            else
+            {
+               const int ij = F2V[side][p];
+               const int i = ij % q;
+               const int j = ij / q;
+
+               for (int c=0; c < vd; ++c)
+               {
+                  real_t grad_n = 0;
+                  for (int kk=0; kk < d; ++kk)
+                  {
+                     const int k = (face_id == 0 || face_id == 2) ? i : kk;
+                     const int l = (face_id == 0 || face_id == 2) ? kk : j;
+                     const real_t g = (face_id == 0 || face_id == 2) ? G(j,l) : G(i,k);
+                     grad_n += g * d_x_e(t?c:k, t?k:l, t?l:el_idx, t?el_idx:c);
+                  }
+                  d_y(p, c, side, f) = grad_n;
+               }
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::Mult3D(const Vector &x, Vector &y) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+   const int num_elem = ne;
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+   const int q2d = q * q;
+
+   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, face_type);
+   const int ne_shared = face_nbr_data.Size() / d / d / d / vd;
+
+   MFEM_VERIFY(q == d, "");
+   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
+
+   const auto G_ = Reshape(maps.G.Read(), q, d);
+   // (el0, el1, fid0, fid1, or0, or1)
+   const auto f2e = Reshape(face_to_elem.Read(), 2, 3, nf);
+   const auto f2v = Reshape(face_to_vol.Read(), q2d, 2, nf);
+
+   // t ? (vdim, d, d, d, ne) : (d, d, d, ne, vdim)
+   const auto d_x = Reshape(x.Read(), t?vd:d, d, d, t?d:ne, t?ne:vd);
+   const auto d_x_shared = Reshape(face_nbr_data.Read(),
+                                   t?vd:d, d, d, t?d:ne_shared, t?ne_shared:vd);
+   auto d_y = Reshape(y.Write(), q2d, vd, 2, nf);
+
+   mfem::forall_2D(nf, q2d, 2, [=] MFEM_HOST_DEVICE (int f) -> void
+   {
+      static constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED real_t G_s[MD*MD];
+      DeviceMatrix G(G_s, d, q);
+
+      MFEM_SHARED int E[2];
+      MFEM_SHARED int FID[2];
+      MFEM_SHARED int F2V[2][MD*MD];
+
+      // Load G matrix into shared memory
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         MFEM_FOREACH_THREAD(j, x, d*q)
+         {
+            const int p = j % q;
+            const int k = j / q;
+            G(k, p) = G_(p, k);
+         }
+      }
+
+      MFEM_FOREACH_THREAD(side, y, 2)
+      {
+         if (MFEM_THREAD_ID(x) == 0)
+         {
+            E[side] = f2e(side, 0, f);
+            FID[side] = f2e(side, 1, f);
+         }
+         MFEM_FOREACH_THREAD(j, x, q2d)
+         {
+            F2V[side][j] = f2v(j, side, f);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, y, 2)
+      {
+         const int el = E[side];
+         const bool shared = (el >= num_elem);
+         const auto &d_x_e = shared ? d_x_shared : d_x;
+         const int el_idx = shared ? el - num_elem : el;
+
+         const int face_id = FID[side];
+
+         // Is this face parallel to the x-y plane in reference coordinates?
+         const bool xy_plane = (face_id == 0 || face_id == 5);
+         const bool xz_plane = (face_id == 1 || face_id == 3);
+         const bool yz_plane = (face_id == 2 || face_id == 4);
+
+         MFEM_FOREACH_THREAD(p, x, q2d)
+         {
+            if (el_idx < 0)
+            {
+               for (int c = 0; c < vd; ++c)
+               {
+                  d_y(p, c, side, f) = 0.0;
+               }
+            }
+            else
+            {
+               const int ijk = F2V[side][p];
+               const int k = ijk / q2d;
+               const int i = ijk % q;
+               const int j = (ijk - q2d*k) / q;
+
+               // the fixed 1D index of the normal component of the face
+               // quadrature point
+               const int g_row = yz_plane ? i : xz_plane ? j : k;
+
+               for (int c = 0; c < vd; ++c)
+               {
+                  real_t grad_n = 0.0;
+
+                  for (int kk = 0; kk < d; ++kk)
+                  {
+                     // (l, m, n) 3D lexicographic index of interior points used
+                     // in evaluating normal derivatives
+                     const int l = yz_plane ? kk : i;
+                     const int m = xz_plane ? kk : j;
+                     const int n = xy_plane ? kk : k;
+
+                     const real_t g = G(kk, g_row);
+
+                     grad_n += g * d_x_e(t?c:l, t?l:m, t?m:n, t?n:el_idx, t?el_idx:c);
+                  }
+                  d_y(p, c, side, f) = grad_n;
+               }
+            }
+         }
+      }
+   });
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::AddMultTranspose2D(
+   const Vector &y, Vector &x, const real_t a) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+
+   // derivative of 1D basis function
+   auto G_ = Reshape(maps.G.Read(), q, d);
+
+   // entries of e2f: (el,f0,f1,f2,f3,s0,s1,s2,s3)
+   auto e2f = Reshape(elem_to_face.Read(), 9, ne_type);
+
+   auto f2v = Reshape(face_to_vol.Read(), d, 2, nf);
+
+   // if byvdim, d_x has shape (vdim, nddof, nddof, ne)
+   // otherwise, d_x has shape (nddof, nddof, ne, vdim)
+   auto d_x = Reshape(x.ReadWrite(), t?vd:d, d, t?d:ne, t?ne:vd);
+   auto d_y = Reshape(y.Read(), q, vd, 2, nf);
+
+   mfem::forall_2D(ne_type, d, d, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED real_t y_s[MD];
+      MFEM_SHARED int pp[MD];
+      MFEM_SHARED int jj;
+      if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0) { jj = 0; }
+
+      MFEM_SHARED real_t BG[MD*MD];
+      DeviceMatrix G(BG, q, d);
+
+      MFEM_SHARED real_t x_s[MD*MD];
+      DeviceMatrix xx(x_s, d, d);
+
+      MFEM_SHARED int el; // global element index
+      MFEM_SHARED int faces[4];
+      MFEM_SHARED int sides[4];
+
+      MFEM_FOREACH_THREAD(i,x,d)
+      {
+         MFEM_FOREACH_THREAD(p,y,q)
+         {
+            G(p,i) = a * G_(p,i);
+            xx(p,i) = 0.0;
+         }
+      }
+
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         if (MFEM_THREAD_ID(x) == 0)
+         {
+            el = e2f(0, e);
+         }
+
+         MFEM_FOREACH_THREAD(i, x, 4)
+         {
+            faces[i] = e2f(1 + i, e);
+            sides[i] = e2f(5 + i, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int face_id=0; face_id < 4; ++face_id)
+      {
+         const int f = faces[face_id];
+
+         if (f < 0) { continue; }
+
+         const int side = sides[face_id];
+
+         if (MFEM_THREAD_ID(y) == 0)
+         {
+            MFEM_FOREACH_THREAD(p,x,d)
+            {
+               y_s[p] = d_y(p, 0, side, f);
+
+               const int ij = f2v(p, side, f);
+               const int i = ij % q;
+               const int j = ij / q;
+
+               pp[(face_id == 0 || face_id == 2) ? i : j] = p;
+               if (MFEM_THREAD_ID(x) == 0)
+               {
+                  jj = (face_id == 0 || face_id == 2) ? j : i;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(k,x,d)
+         {
+            MFEM_FOREACH_THREAD(l,y,d)
+            {
+               const int p = (face_id == 0 || face_id == 2) ? pp[k] : pp[l];
+               const int kk = (face_id == 0 || face_id == 2) ? l : k;
+               const real_t g = G(jj, kk);
+               xx(k,l) += g * y_s[p];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(k,x,d)
+      {
+         MFEM_FOREACH_THREAD(l,y,d)
+         {
+            const int c = 0;
+            d_x(t?c:k, t?k:l, t?l:el, t?el:c) += xx(k,l);
+         }
+      }
+   });
+}
+
+template <int T_D1D>
+void L2NormalDerivativeFaceRestriction::AddMultTranspose3D(
+   const Vector &y, Vector &x, const real_t a) const
+{
+   const int vd = fes.GetVDim();
+   const bool t = fes.GetOrdering() == Ordering::byVDIM;
+
+   MFEM_VERIFY(vd == 1, "vdim > 1 not supported.");
+
+   const FiniteElement &fe = *fes.GetFE(0);
+   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
+
+   const int q = maps.nqpt;
+   const int d = maps.ndof;
+   const int q2d = q * q;
+
+   MFEM_VERIFY(q == d, "");
+   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
+
+   auto G_ = Reshape(maps.G.Read(), q, d);
+
+   // (el, f0,f1,f2,f3,f4,f5, s0,s1,s2,s3,s4,s5)
+   auto e2f = Reshape(elem_to_face.Read(), 13, ne_type);
+
+   auto f2v = Reshape(face_to_vol.Read(), q2d, 2, nf);
+
+   auto d_x = Reshape(x.ReadWrite(), t?vd:d, d, d, t?d:ne, t?ne:vd);
+   const auto d_y = Reshape(y.Read(), q2d, vd, 2, nf);
+
+   mfem::forall_2D(ne_type, q, q, [=] MFEM_HOST_DEVICE (int e) -> void
+   {
+      static constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED int pp[MD][MD];
+      MFEM_SHARED real_t y_s[MD*MD];
+      MFEM_SHARED int jj;
+      if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0) { jj = 0; }
+
+      MFEM_SHARED real_t xx_s[MD*MD*MD];
+      auto xx = Reshape(xx_s, d, d, d);
+
+      MFEM_SHARED real_t G_s[MD*MD];
+      DeviceMatrix G(G_s, q, d);
+
+      MFEM_SHARED int el;
+      MFEM_SHARED int faces[6];
+      MFEM_SHARED int sides[6];
+
+      // Load G into shared memory
+      MFEM_FOREACH_THREAD(j, x, d)
+      {
+         MFEM_FOREACH_THREAD(i, y, q)
+         {
+            G(i, j) = a * G_(i, j);
+            G(i, j) = a * G_(i, j);
+            G(i, j) = a * G_(i, j);
+         }
+      }
+
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         if (MFEM_THREAD_ID(x) == 0)
+         {
+            el = e2f(0, e); // global element index
+         }
+
+         MFEM_FOREACH_THREAD(i, x, 6)
+         {
+            faces[i] = e2f(1 + i, e);
+            sides[i] = e2f(7 + i, e);
+         }
+      }
+
+      MFEM_FOREACH_THREAD(k, x, d)
+      {
+         MFEM_FOREACH_THREAD(j, y, d)
+         {
+            for (int i = 0; i < d; ++i)
+            {
+               xx(i, j, k) = 0.0;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      for (int face_id = 0; face_id < 6; ++face_id)
+      {
+         const int f = faces[face_id];
+
+         if (f < 0)
+         {
+            continue;
+         }
+
+         const int side = sides[face_id];
+
+         // is this face parallel to the x-y plane in reference coordinates?
+         const bool xy_plane = (face_id == 0 || face_id == 5);
+         const bool xz_plane = (face_id == 1 || face_id == 3);
+
+         MFEM_FOREACH_THREAD(p1, x, q)
+         {
+            MFEM_FOREACH_THREAD(p2, y, q)
+            {
+               const int p = p1 + q * p2;
+               y_s[p] = d_y(p, 0, side, f);
+
+               const int ijk = f2v(p, side, f);
+               const int k = ijk / q2d;
+               const int i = ijk % q;
+               const int j = (ijk - q2d*k) / q;
+
+               pp[(xy_plane || xz_plane) ? i : j][(xy_plane) ? j : k] = p;
+               if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0)
+               {
+                  jj = (xy_plane) ? k : (xz_plane) ? j : i;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(n, x, d)
+         {
+            MFEM_FOREACH_THREAD(m, y, d)
+            {
+               for (int l = 0; l < d; ++l)
+               {
+                  const int p = (xy_plane) ? pp[l][m] : (xz_plane) ? pp[l][n] : pp[m][n];
+                  const int kk = (xy_plane) ? n : (xz_plane) ? m : l;
+                  const real_t g = G(jj, kk);
+                  xx(l, m, n) += g * y_s[p];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // map back to global array
+      MFEM_FOREACH_THREAD(n, x, d)
+      {
+         MFEM_FOREACH_THREAD(m, y, d)
+         {
+            for (int l = 0; l < d; ++l)
+            {
+               const int c = 0;
+               d_x(t?c:l, t?l:m, t?m:n, t?n:el, t?el:c) += xx(l, m, n);
+            }
+         }
+      }
+   });
+}
+
+} // namespace mfem
@@ -0,0 +1,85 @@
+// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_NORMAL_DERIV_RESTRICTION
+#define MFEM_NORMAL_DERIV_RESTRICTION
+
+#include "../mesh/mesh.hpp"
+
+namespace mfem
+{
+
+class FiniteElementSpace;
+enum class ElementDofOrdering;
+
+/// @brief Class to compute face normal derivatives (in reference coordinate) of
+/// an L2 grid function (used internally by L2FaceRestriction).
+class L2NormalDerivativeFaceRestriction
+{
+protected:
+   const FiniteElementSpace &fes; ///< The L2 finite element space.
+   const FaceType face_type; ///< Face type: either boundary or interior.
+   const int dim; ///< Dimension of the mesh.
+   const int nf; ///< Number of faces of the given @a face_type.
+   const int ne; ///< Number of elements.
+   int ne_type; ///< Number of elements with faces of type face type
+
+   Array<int> face_to_elem; ///< Face-wise information array.
+   Array<int> elem_to_face; ///< Element-wise information array.
+   Array<int> face_to_vol; ///< maps face index to volume index
+
+public:
+   /// @brief Constructor.
+   /// @param[in] fes_ The associated FiniteElementSpace (should be L2/DG).
+   /// @param[in] f_ordering Request a specific face dof ordering. Currently
+   /// only ElementDofOrdering::LEXICOGRAPHIC is supported.
+   /// @param[in] face_type_ Type of faces to compute restriction (interior or boundary).
+   L2NormalDerivativeFaceRestriction(const FiniteElementSpace &fes_,
+                                     const ElementDofOrdering f_ordering,
+                                     const FaceType face_type_);
+
+   /// @brief Computes the normal derivatives on the @a face_type faces of the mesh.
+   /// @param[in] x The L-vector degrees of freedom.
+   /// @param[out] y The face E(like)-vector degrees of freedom of the format
+   /// (face_dofs x vdim x 2 x nf) where nf is the number of faces of type @a
+   /// face_type. The face_dofs are ordered according to @a f_ordering specified
+   /// in the constructor.
+   void Mult(const Vector &x, Vector &y) const;
+
+   /// @brief Computes the transpose of the action of Mult(), accumulating into
+   ///        @a y with coefficient @a a.
+   /// @param x Face E-vector layout (face_dofs x vdim x 2 x nf).
+   /// @param y L-vector layout.
+   /// @param a Optional coefficient (y = y + a*R^t*x)
+   void AddMultTranspose(const Vector &x, Vector &y,
+                         const real_t a = 1.0) const;
+
+   /// @name Internal compute kernels. Public because of nvcc restriction.
+   ///@{
+
+   template <int T_D1D = 0>
+   void Mult2D(const Vector &x, Vector &y) const;
+
+   template <int T_D1D = 0>
+   void AddMultTranspose2D(const Vector &x, Vector &y, const real_t a) const;
+
+   template <int T_D1D = 0>
+   void Mult3D(const Vector &x, Vector &y) const;
+
+   template <int T_D1D = 0>
+   void AddMultTranspose3D(const Vector &x, Vector &y, const real_t a) const;
+
+   /// @}
+};
+
+}
+
+#endif // MFEM_RESTRICTION
@@ -368,6 +368,72 @@ const
   y.Add(a, Ytmp);
 }

+real_t ParBilinearForm::ParInnerProduct(const ParGridFunction &x,
+                                        const ParGridFunction &y) const
+{
+   MFEM_ASSERT(mat != NULL, "local matrix must be assembled");
+
+   real_t loc = InnerProduct(x, y);
+   real_t glob = 0.;
+
+   MPI_Allreduce(&loc, &glob, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                 pfes->GetComm());
+
+   return glob;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(const ParGridFunction &x,
+                                         const ParGridFunction &y) const
+{
+   MFEM_ASSERT(x.ParFESpace() == pfes, "the parallel spaces must match");
+   MFEM_ASSERT(y.ParFESpace() == pfes, "the parallel spaces must match");
+
+   HypreParVector *x_p = x.ParallelProject();
+   HypreParVector *y_p = y.ParallelProject();
+
+   real_t res = TrueInnerProduct(*x_p, *y_p);
+
+   delete x_p;
+   delete y_p;
+
+   return res;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(HypreParVector &x,
+                                         HypreParVector &y) const
+{
+   MFEM_VERIFY(p_mat.Ptr() != NULL, "parallel matrix must be assembled");
+
+   if (p_mat->GetType() != Operator::Hypre_ParCSR)
+   {
+      return TrueInnerProduct((const Vector&)x, (const Vector&)y);
+   }
+
+   HypreParVector *Ax = new HypreParVector(pfes);
+   HypreParMatrix *A = p_mat.As<HypreParMatrix>();
+
+   A->Mult(x, *Ax);
+
+   real_t res = mfem::InnerProduct(y, *Ax);
+
+   delete Ax;
+
+   return res;
+}
+
+real_t ParBilinearForm::TrueInnerProduct(const Vector &x,
+                                         const Vector &y) const
+{
+   MFEM_VERIFY(p_mat.Ptr() != NULL, "parallel matrix must be assembled");
+
+   Vector Ax(pfes->GetTrueVSize());
+   p_mat->Mult(x, Ax);
+
+   real_t res = mfem::InnerProduct(pfes->GetComm(), y, Ax);
+
+   return res;
+}
+
 void ParBilinearForm::FormLinearSystem(
   const Array<int> &ess_tdof_list, Vector &x, Vector &b,
   OperatorHandle &A, Vector &X, Vector &B, int copy_interior)
@@ -173,6 +173,37 @@ public:
       vectors on the true dofs. */
   void TrueAddMult(const Vector &x, Vector &y, const real_t a = 1.0) const;

+   /// Compute $ y^T M x $
+   /** @warning The calculation is performed on local dofs, assuming that
+       the local vectors are consistent with the prolongations of the true
+       vectors (see ParGridFunction::Distribute()). If this is not the case,
+       use TrueInnerProduct(const ParGridFunction &, const ParGridFunction &)
+       instead.
+       @note It is assumed that the local matrix is assembled and it has
+       not been replaced by the parallel matrix through FormSystemMatrix().
+       @see TrueInnerProduct(const ParGridFunction&, const ParGridFunction&) */
+   real_t ParInnerProduct(const ParGridFunction &x,
+                          const ParGridFunction &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (grid function version)
+   /** @note The ParGridFunction%s are restricted to the true-vectors for
+       for calculation.
+       @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix().
+       @see ParInnerProduct(const ParGridFunction&, const ParGridFunction&) */
+   real_t TrueInnerProduct(const ParGridFunction &x,
+                           const ParGridFunction &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (Hypre vector version)
+   /** @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix(). */
+   real_t TrueInnerProduct(HypreParVector &x, HypreParVector &y) const;
+
+   /// Compute $ y^T M x $ on true dofs (true-vector version)
+   /** @note It is assumed that the parallel system matrix is assembled,
+       see FormSystemMatrix(). */
+   real_t TrueInnerProduct(const Vector &x, const Vector &y) const;
+
   /// Return the parallel FE space associated with the ParBilinearForm.
   ParFiniteElementSpace *ParFESpace() const { return pfes; }

@@ -861,17 +861,17 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
         }
      }

-      HYPRE_Int *i_diag = new HYPRE_Int[ldof+1];
-      HYPRE_Int *j_diag = new HYPRE_Int[ltdof];
-      real_t    *d_diag = new real_t[ltdof];
+      HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
+      real_t    *d_diag = Memory<real_t>(ltdof);
      int diag_counter;

-      HYPRE_Int *i_offd = new HYPRE_Int[ldof+1];
-      HYPRE_Int *j_offd = new HYPRE_Int[nnz_offd];
-      real_t    *d_offd = new real_t[nnz_offd];
+      HYPRE_Int *i_offd = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_offd = Memory<HYPRE_Int>(nnz_offd);
+      real_t    *d_offd = Memory<real_t>(nnz_offd);
      int offd_counter;

-      HYPRE_BigInt *cmap   = new HYPRE_BigInt[ldof-ltdof];
+      HYPRE_BigInt *cmap   = Memory<HYPRE_BigInt>(ldof-ltdof);

      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
      HYPRE_BigInt *row_starts = GetDofOffsets();
@@ -39,9 +39,10 @@ ParGridFunction::ParGridFunction(ParMesh *pmesh, const GridFunction *gf,
 {
   const FiniteElementSpace *glob_fes = gf->FESpace();
   // duplicate the FiniteElementCollection from 'gf'
-   fec = FiniteElementCollection::New(glob_fes->FEColl()->Name());
+   fec_owned = FiniteElementCollection::New(glob_fes->FEColl()->Name());
   // create a local ParFiniteElementSpace from the global one:
-   fes = pfes = new ParFiniteElementSpace(pmesh, glob_fes, partitioning, fec);
+   fes = pfes = new ParFiniteElementSpace(pmesh, glob_fes, partitioning,
+                                          fec_owned);
   SetSize(pfes->GetVSize());

   if (partitioning)
@@ -81,7 +82,7 @@ ParGridFunction::ParGridFunction(ParMesh *pmesh, std::istream &input)
   : GridFunction(pmesh, input)
 {
   // Convert the FiniteElementSpace, fes, to a ParFiniteElementSpace:
-   pfes = new ParFiniteElementSpace(pmesh, fec, fes->GetVDim(),
+   pfes = new ParFiniteElementSpace(pmesh, fec_owned, fes->GetVDim(),
                                    fes->GetOrdering());
   delete fes;
   fes = pfes;
@@ -249,6 +250,8 @@ void ParGridFunction::ExchangeFaceNbrData()
   auto send_data_ptr = mpi_gpu_aware ? send_data.Read() : send_data.HostRead();
   auto face_nbr_data_ptr = mpi_gpu_aware ? face_nbr_data.Write() :
                            face_nbr_data.HostWrite();
+   // Wait for the kernel to be done since it updates what's sent and it may be async
+   if (mpi_gpu_aware) { MFEM_STREAM_SYNC; }
   for (int fn = 0; fn < num_face_nbrs; fn++)
   {
      int nbr_rank = pmesh->GetFaceNbrRank(fn);
@@ -518,7 +521,7 @@ void ParGridFunction::CountElementsPerVDof(Array<int> &elem_per_vdof) const
 }

 void ParGridFunction::GetDerivative(int comp, int der_comp,
-                                    ParGridFunction &der)
+                                    ParGridFunction &der) const
 {
   Array<int> overlap;
   AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
@@ -713,10 +716,10 @@ void ParGridFunction::ProjectBdrCoefficient(
         }
      }
   }
+   gcomm.Bcast<int>(values_counter.HostReadWrite());
   for (int i = 0; i < values_counter.Size(); i++)
   {
-      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
-                  bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
                  "internal error");
   }
 #endif
@@ -753,10 +756,10 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
 #ifdef MFEM_DEBUG
   Array<int> ess_vdofs_marker;
   pfes->GetEssentialVDofs(bdr_attr, ess_vdofs_marker);
+   gcomm.Bcast<int>(values_counter.HostReadWrite());
   for (int i = 0; i < values_counter.Size(); i++)
   {
-      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
-                  bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
                  "internal error: " << pfes->GetLocalTDofNumber(i) << ' ' << bool(
                     values_counter[i]));
   }
@@ -231,7 +231,7 @@ public:
   void CountElementsPerVDof(Array<int> &elem_per_vdof) const override;

   /// Parallel version of GridFunction::GetDerivative(); see its documentation.
-   void GetDerivative(int comp, int der_comp, ParGridFunction &der);
+   void GetDerivative(int comp, int der_comp, ParGridFunction &der) const;

   /** Sets the output vector @a dof_vals to the values of the degrees of
       freedom of element @a el. If @a el is greater than or equal to the number
@@ -262,7 +262,6 @@ public:
                              const Array<int> &attr) override
   { ProjectBdrCoefficient(coeff, NULL, attr); }

-   // Only the values in the master are guaranteed to be correct!
   void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                     const Array<int> &bdr_attr) override;

@@ -18,6 +18,7 @@
 #include "pgridfunc.hpp"
 #include "pfespace.hpp"
 #include "fespace.hpp"
+#include "fe/face_map_utils.hpp"
 #include "../general/forall.hpp"

 namespace mfem
@@ -277,21 +278,22 @@ void ParNCH1FaceRestriction::ComputeGatherIndices(
   gather_offsets[0] = 0;
 }

-ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
+ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &pfes_,
                                           ElementDofOrdering f_ordering,
                                           FaceType type,
                                           L2FaceValues m,
                                           bool build)
-   : L2FaceRestriction(fes, f_ordering, type, m, false)
+   : L2FaceRestriction(pfes_, f_ordering, type, m, false),
+     pfes(pfes_)
 {
   if (!build) { return; }
   if (nf==0) { return; }

-   CheckFESpace(f_ordering);
+   CheckFESpace();

-   ComputeScatterIndicesAndOffsets(f_ordering, type);
+   ComputeScatterIndicesAndOffsets();

-   ComputeGatherIndices(f_ordering, type);
+   ComputeGatherIndices();
 }

 ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
@@ -307,14 +309,8 @@ void ParL2FaceRestriction::DoubleValuedConformingMult(
   MFEM_ASSERT(
      m == L2FaceValues::DoubleValued,
      "This method should be called when m == L2FaceValues::DoubleValued.");
-   const ParFiniteElementSpace &pfes =
-      static_cast<const ParFiniteElementSpace&>(this->fes);
-   ParGridFunction x_gf;
-   x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(&pfes),
-                const_cast<Vector&>(x), 0);
-   // Face-neighbor information is only needed for interior faces. For boundary
-   // faces, no communication is required.
-   if (type == FaceType::Interior) { x_gf.ExchangeFaceNbrData(); }
+
+   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, type);

   // Early return only after calling ParGridFunction::ExchangeFaceNbrData,
   // otherwise MPI communication can hang.
@@ -329,7 +325,7 @@ void ParL2FaceRestriction::DoubleValuedConformingMult(
   auto d_indices1 = scatter_indices1.Read();
   auto d_indices2 = scatter_indices2.Read();
   auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
-   auto d_x_shared = Reshape(x_gf.FaceNbrData().Read(),
+   auto d_x_shared = Reshape(face_nbr_data.Read(),
                             t?vd:nsdofs, t?nsdofs:vd);
   auto d_y = Reshape(y.Write(), nface_dofs, vd, 2, nf);
   mfem::forall(nfdofs, [=] MFEM_HOST_DEVICE (int i)
@@ -567,13 +563,9 @@ void ParL2FaceRestriction::FillJAndData(const Vector &ea_data,
   });
 }

-void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();
-   const ParFiniteElementSpace &pfes =
-      static_cast<const ParFiniteElementSpace&>(this->fes);

   // Initialization of the offsets
   for (int i = 0; i <= ndofs; ++i)
@@ -622,9 +614,7 @@ void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets(
 }


-void ParL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParL2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -666,11 +656,11 @@ ParNCL2FaceRestriction::ParNCL2FaceRestriction(const ParFiniteElementSpace &fes,
   if (nf==0) { return; }
   x_interp.UseDevice(true);

-   CheckFESpace(f_ordering);
+   CheckFESpace();

-   ComputeScatterIndicesAndOffsets(f_ordering, type);
+   ComputeScatterIndicesAndOffsets();

-   ComputeGatherIndices(f_ordering, type);
+   ComputeGatherIndices();
 }

 void ParNCL2FaceRestriction::SingleValuedNonconformingMult(
@@ -979,9 +969,7 @@ void ParNCL2FaceRestriction::FillJAndData(const Vector &ea_data,
   MFEM_ABORT("Not yet implemented.");
 }

-void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -1064,9 +1052,7 @@ void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
   interpolations.InitializeNCInterpConfig();
 }

-void ParNCL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void ParNCL2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -139,9 +139,11 @@ public: // For nvcc
 class ParL2FaceRestriction : virtual public L2FaceRestriction
 {
 protected:
+   const ParFiniteElementSpace &pfes;
+
   /** @brief Constructs an ParL2FaceRestriction.

-       @param[in] fes        The ParFiniteElementSpace on which this operates
+       @param[in] pfes_      The ParFiniteElementSpace on which this operates
       @param[in] f_ordering Request a specific face dof ordering
       @param[in] type       Request internal or boundary faces dofs
       @param[in] m          Request the face dofs for elem1, or both elem1 and
@@ -149,7 +151,7 @@ protected:
       @param[in] build      Request the ParL2FaceRestriction to compute the
                             scatter/gather indices. False should only be used
                             when inheriting from ParL2FaceRestriction. */
-   ParL2FaceRestriction(const ParFiniteElementSpace& fes,
+   ParL2FaceRestriction(const ParFiniteElementSpace &pfes_,
                        ElementDofOrdering f_ordering,
                        FaceType type,
                        L2FaceValues m,
@@ -229,22 +231,14 @@ public:
 private:
   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
       for the gathering: E-vector to L-vector.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();

 public:
   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -381,22 +375,14 @@ private:
   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
       for the gathering: E-vector to L-vector, and the interpolators from
       coarse to fine face for master non-comforming faces.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();

 public:
   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -215,6 +215,10 @@ public:
   /// quadrature point, oriented relative to "element 1".
   int GetPermutedIndex(int idx, int iq) const override;

+   /// @brief Get the face index (in the standard Mesh numbering) associated
+   /// with face @a idx in the FaceQuadratureSpace.
+   int GetMeshFaceIndex(int idx) const { return face_indices[idx]; }
+
   /// @brief Returns the index associated with the face described by @a T.
   ///
   /// The index may differ from the mesh face or boundary element index
@@ -10,17 +10,16 @@
 // CONTRIBUTING.md for details.

 #include "restriction.hpp"
+#include "normal_deriv_restriction.hpp"
 #include "gridfunc.hpp"
 #include "fespace.hpp"
+#include "pgridfunc.hpp"
+#include "qspace.hpp"
+#include "fe/face_map_utils.hpp"
 #include "../general/forall.hpp"
+
 #include <climits>

-#ifdef MFEM_USE_MPI
-
-#include "pfespace.hpp"
-
-#endif
-
 namespace mfem
 {

@@ -907,112 +906,6 @@ void ConformingFaceRestriction::SetFaceDofsGatherIndices(
   }
 }

-static int ToLexOrdering2D(const int face_id, const int size1d, const int i)
-{
-   if (face_id==2 || face_id==3)
-   {
-      return size1d-1-i;
-   }
-   else
-   {
-      return i;
-   }
-}
-
-static int PermuteFace2D(const int face_id1, const int face_id2,
-                         const int orientation,
-                         const int size1d, const int index)
-{
-   int new_index;
-   // Convert from lex ordering
-   if (face_id1==2 || face_id1==3)
-   {
-      new_index = size1d-1-index;
-   }
-   else
-   {
-      new_index = index;
-   }
-   // Permute based on face orientations
-   if (orientation==1)
-   {
-      new_index = size1d-1-new_index;
-   }
-   return ToLexOrdering2D(face_id2, size1d, new_index);
-}
-
-static int ToLexOrdering3D(const int face_id, const int size1d, const int i,
-                           const int j)
-{
-   if (face_id==2 || face_id==1 || face_id==5)
-   {
-      return i + j*size1d;
-   }
-   else if (face_id==3 || face_id==4)
-   {
-      return (size1d-1-i) + j*size1d;
-   }
-   else // face_id==0
-   {
-      return i + (size1d-1-j)*size1d;
-   }
-}
-
-static int PermuteFace3D(const int face_id1, const int face_id2,
-                         const int orientation,
-                         const int size1d, const int index)
-{
-   int i=0, j=0, new_i=0, new_j=0;
-   i = index%size1d;
-   j = index/size1d;
-   // Convert from lex ordering
-   if (face_id1==3 || face_id1==4)
-   {
-      i = size1d-1-i;
-   }
-   else if (face_id1==0)
-   {
-      j = size1d-1-j;
-   }
-   // Permute based on face orientations
-   switch (orientation)
-   {
-      case 0:
-         new_i = i;
-         new_j = j;
-         break;
-      case 1:
-         new_i = j;
-         new_j = i;
-         break;
-      case 2:
-         new_i = j;
-         new_j = (size1d-1-i);
-         break;
-      case 3:
-         new_i = (size1d-1-i);
-         new_j = j;
-         break;
-      case 4:
-         new_i = (size1d-1-i);
-         new_j = (size1d-1-j);
-         break;
-      case 5:
-         new_i = (size1d-1-j);
-         new_j = (size1d-1-i);
-         break;
-      case 6:
-         new_i = (size1d-1-j);
-         new_j = i;
-         break;
-      case 7:
-         new_i = i;
-         new_j = (size1d-1-j);
-         break;
-   }
-   return ToLexOrdering3D(face_id2, size1d, new_i, new_j);
-}
-
 // Permute dofs or quads on a face for e2 to match with the ordering of e1
 int PermuteFaceL2(const int dim, const int face_id1,
                  const int face_id2, const int orientation,
@@ -1023,9 +916,9 @@ int PermuteFaceL2(const int dim, const int face_id1,
      case 1:
         return 0;
      case 2:
-         return PermuteFace2D(face_id1, face_id2, orientation, size1d, index);
+         return internal::PermuteFace2D(face_id1, face_id2, orientation, size1d, index);
      case 3:
-         return PermuteFace3D(face_id1, face_id2, orientation, size1d, index);
+         return internal::PermuteFace3D(face_id1, face_id2, orientation, size1d, index);
      default:
         MFEM_ABORT("Unsupported dimension.");
         return 0;
@@ -1038,6 +931,7 @@ L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
                                     const L2FaceValues m,
                                     bool build)
   : fes(fes),
+     ordering(f_ordering),
     nf(fes.GetNFbyType(type)),
     ne(fes.GetNE()),
     vdim(fes.GetVDim()),
@@ -1060,11 +954,9 @@ L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
   width = fes.GetVSize();
   if (!build) { return; }

-   CheckFESpace(f_ordering);
-
-   ComputeScatterIndicesAndOffsets(f_ordering,type);
-
-   ComputeGatherIndices(f_ordering, type);
+   CheckFESpace();
+   ComputeScatterIndicesAndOffsets();
+   ComputeGatherIndices();
 }

 L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
@@ -1327,7 +1219,7 @@ void L2FaceRestriction::AddFaceMatricesToElementMatrices(const Vector &fea_data,
   }
 }

-void L2FaceRestriction::CheckFESpace(const ElementDofOrdering f_ordering)
+void L2FaceRestriction::CheckFESpace()
 {
 #ifdef MFEM_USE_MPI

@@ -1351,7 +1243,7 @@ void L2FaceRestriction::CheckFESpace(const ElementDofOrdering f_ordering)
               "Only Gauss-Lobatto and Bernstein basis are supported in "
               "L2FaceRestriction.");
   if (nf==0) { return; }
-   const bool dof_reorder = (f_ordering == ElementDofOrdering::LEXICOGRAPHIC);
+   const bool dof_reorder = (ordering == ElementDofOrdering::LEXICOGRAPHIC);
   if (!dof_reorder)
   {
      MFEM_ABORT("Non-Tensor L2FaceRestriction not yet implemented.");
@@ -1371,9 +1263,7 @@ void L2FaceRestriction::CheckFESpace(const ElementDofOrdering f_ordering)
 #endif
 }

-void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType face_type)
+void L2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();
   // Initialization of the offsets
@@ -1389,16 +1279,16 @@ void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
      MFEM_ASSERT(!face.IsShared(),
                  "Unexpected shared face in L2FaceRestriction.");
-      if ( face.IsOfFaceType(face_type) )
+      if ( face.IsOfFaceType(type) )
      {
         SetFaceDofsScatterIndices1(face,f_ind);
         if ( m==L2FaceValues::DoubleValued )
         {
-            if ( face_type==FaceType::Interior && face.IsInterior() )
+            if ( type==FaceType::Interior && face.IsInterior() )
            {
               PermuteAndSetFaceDofsScatterIndices2(face,f_ind);
            }
-            else if ( face_type==FaceType::Boundary && face.IsBoundary() )
+            else if ( type==FaceType::Boundary && face.IsBoundary() )
            {
               SetBoundaryDofsScatterIndices2(face,f_ind);
            }
@@ -1415,9 +1305,7 @@ void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
   }
 }

-void L2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType face_type)
+void L2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();
   // Computation of gather_indices
@@ -1427,11 +1315,11 @@ void L2FaceRestriction::ComputeGatherIndices(
      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
      MFEM_ASSERT(!face.IsShared(),
                  "Unexpected shared face in L2FaceRestriction.");
-      if ( face.IsOfFaceType(face_type) )
+      if ( face.IsOfFaceType(type) )
      {
         SetFaceDofsGatherIndices1(face,f_ind);
         if ( m==L2FaceValues::DoubleValued &&
-              face_type==FaceType::Interior &&
+              type==FaceType::Interior &&
              face.IsLocal())
         {
            PermuteAndSetFaceDofsGatherIndices2(face,f_ind);
@@ -1598,6 +1486,28 @@ void L2FaceRestriction::PermuteAndSetFaceDofsGatherIndices2(
   }
 }

+void L2FaceRestriction::NormalDerivativeMult(const Vector &x, Vector &y) const
+{
+   EnsureNormalDerivativeRestriction();
+   normal_deriv_restr->Mult(x, y);
+}
+
+void L2FaceRestriction::NormalDerivativeAddMultTranspose(const Vector &x,
+                                                         Vector &y) const
+{
+   EnsureNormalDerivativeRestriction();
+   normal_deriv_restr->AddMultTranspose(x, y);
+}
+
+void L2FaceRestriction::EnsureNormalDerivativeRestriction() const
+{
+   if (!normal_deriv_restr)
+   {
+      normal_deriv_restr.reset(
+         new L2NormalDerivativeFaceRestriction(fes, ordering, type));
+   }
+}
+
 InterpolationManager::InterpolationManager(const FiniteElementSpace &fes,
                                           ElementDofOrdering ordering,
                                           FaceType type)
@@ -1775,11 +1685,11 @@ NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
   if (!build) { return; }
   x_interp.UseDevice(true);

-   CheckFESpace(f_ordering);
+   CheckFESpace();

-   ComputeScatterIndicesAndOffsets(f_ordering, type);
+   ComputeScatterIndicesAndOffsets();

-   ComputeGatherIndices(f_ordering, type);
+   ComputeGatherIndices();
 }

 NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
@@ -2259,18 +2169,16 @@ int ToLexOrdering(const int dim, const int face_id, const int size1d,
      case 1:
         return 0;
      case 2:
-         return ToLexOrdering2D(face_id, size1d, index);
+         return internal::ToLexOrdering2D(face_id, size1d, index);
      case 3:
-         return ToLexOrdering3D(face_id, size1d, index%size1d, index/size1d);
+         return internal::ToLexOrdering3D(face_id, size1d, index%size1d, index/size1d);
      default:
         MFEM_ABORT("Unsupported dimension.");
         return 0;
   }
 }

-void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
 {
   Mesh &mesh = *fes.GetMesh();

@@ -2334,9 +2242,7 @@ void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
   interpolations.InitializeNCInterpConfig();
 }

-void NCL2FaceRestriction::ComputeGatherIndices(
-   const ElementDofOrdering f_ordering,
-   const FaceType type)
+void NCL2FaceRestriction::ComputeGatherIndices()
 {
   Mesh &mesh = *fes.GetMesh();
   // Computation of gather_indices
@@ -2376,4 +2282,33 @@ void NCL2FaceRestriction::ComputeGatherIndices(
   gather_offsets[0] = 0;
 }

+Vector GetLVectorFaceNbrData(
+   const FiniteElementSpace &fes, const Vector &x, FaceType ftype)
+{
+#ifdef MFEM_USE_MPI
+   if (ftype == FaceType::Interior)
+   {
+      if (auto *pfes = const_cast<ParFiniteElementSpace*>
+                       (dynamic_cast<const ParFiniteElementSpace*>(&fes)))
+      {
+         if (auto *x_gf = const_cast<ParGridFunction*>
+                          (dynamic_cast<const ParGridFunction*>(&x)))
+         {
+            Vector &gf_face_nbr = x_gf->FaceNbrData();
+            if (gf_face_nbr.Size() == 0) { x_gf->ExchangeFaceNbrData(); }
+            gf_face_nbr.Read();
+            return Vector(gf_face_nbr, 0, gf_face_nbr.Size());
+         }
+         else
+         {
+            ParGridFunction gf(pfes, const_cast<Vector&>(x));
+            gf.ExchangeFaceNbrData();
+            return std::move(gf.FaceNbrData());
+         }
+      }
+   }
+#endif
+   return Vector();
+}
+
 } // namespace mfem
@@ -14,6 +14,7 @@

 #include "../linalg/operator.hpp"
 #include "../mesh/mesh.hpp"
+#include "normal_deriv_restriction.hpp"

 namespace mfem
 {
@@ -21,6 +22,8 @@ namespace mfem
 class FiniteElementSpace;
 enum class ElementDofOrdering;

+class FaceQuadratureSpace;
+
 /// Abstract base class that defines an interface for element restrictions.
 class ElementRestrictionOperator : public Operator
 {
@@ -220,6 +223,45 @@ public:
      y = 0.0;
      AddMultTranspose(x, y);
   }
+
+   /** @brief For each face, sets @a y to the partial derivative of @a x with
+              respect to the reference coordinate whose direction is
+              perpendicular to the face on the reference element.
+
+    @details This is not the normal derivative in physical coordinates, but can
+             be mapped to the physical normal derivative using the element
+             Jacobian and the tangential derivatives (in reference coordinates)
+             which can be computed from the face values (provided by Mult).
+
+             Note that due to the polynomial degree of the element mapping, the
+             physical normal derivative may be a higher degree polynomial than
+             the restriction of the values to the face. However, the normal
+             derivative in reference coordinates has degree-1, and therefore can
+             be exactly represented with the degrees of freedom of a face
+             E-vector.
+
+    @param[in]     x The L-vector degrees of freedom.
+    @param[in,out] y The reference normal derivative degrees of freedom. Is
+                     E-vector like.
+    */
+   virtual void NormalDerivativeMult(const Vector &x, Vector &y) const
+   {
+      MFEM_ABORT("Not implemented for this restriction operator.");
+   }
+
+   /** @brief Add the face reference-normal derivative degrees of freedom in @a
+              x to the element degrees of freedom in @a y.
+
+       @details see NormalDerivativeMult.
+
+       @param[in]     x The degrees of freedom of the face reference-normal
+                        derivative. Is E-vector like.
+       @param[in,out] y The L-vector degrees of freedom.
+   */
+   virtual void NormalDerivativeAddMultTranspose(const Vector &x, Vector &y) const
+   {
+      MFEM_ABORT("Not implemented for this restriction operator.");
+   }
 };

 /// @brief Operator that extracts face degrees of freedom for H1, ND, or RT
@@ -365,6 +407,7 @@ class L2FaceRestriction : public FaceRestriction
 {
 protected:
   const FiniteElementSpace &fes;
+   const ElementDofOrdering ordering;
   const int nf; // Number of faces of the requested type
   const int ne; // Number of elements
   const int vdim; // vdim
@@ -379,6 +422,7 @@ protected:
   Array<int> scatter_indices2; // Scattering indices for element 2 on each face
   Array<int> gather_offsets; // offsets for the gathering indices of each dof
   Array<int> gather_indices; // gathering indices for each dof
+   mutable std::unique_ptr<L2NormalDerivativeFaceRestriction> normal_deriv_restr;

   /** @brief Constructs an L2FaceRestriction.

@@ -487,34 +531,49 @@ public:
   virtual void AddFaceMatricesToElementMatrices(const Vector &fea_data,
                                                 Vector &ea_data) const;

+   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
+      face E-Vector.
+
+      @param[in]  x The L-vector degrees of freedom.
+      @param[out] y The face E-Vector degrees of freedom with the given format:
+                    (face_dofs x vdim x 2 x nf) where nf is the number of
+                    interior or boundary faces requested by @a type in the
+                    constructor. The face_dofs are ordered according to the
+                    given ElementDofOrdering. */
+   void NormalDerivativeMult(const Vector &x, Vector &y) const override;
+
+   /** @brief Add the face reference-normal derivative degrees of freedom in @a
+              x to the element degrees of freedom in @a y.
+
+       @details see NormalDerivativeMult.
+
+       @param[in]     x The degrees of freedom of the face reference-normal
+                        derivative. Is E-vector like.
+       @param[in,out] y The L-vector degrees of freedom.
+   */
+   void NormalDerivativeAddMultTranspose(const Vector &x,
+                                         Vector &y) const override;
 private:
   /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
       for the gathering: E-vector to L-vector.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();
+
+   /// Create the internal normal derivative restriction operator if needed.
+   void EnsureNormalDerivativeRestriction() const;

 protected:
   mutable Array<int> face_map; // Used in the computation of GetFaceDofs

   /** @brief Verify that L2FaceRestriction is built from an L2 FESpace.
-
-       @param[in] f_ordering The requested face dof ordering.
   */
-   void CheckFESpace(const ElementDofOrdering f_ordering);
+   void CheckFESpace();

   /** @brief Set the scattering indices of elem1, and increment the offsets for
       the face described by the @a face. The ordering of the face dofs of elem1
@@ -938,22 +997,14 @@ private:
   /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
       for the gathering: E-vector to L-vector, and the interpolators from
       coarse to fine face for master non-comforming faces.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
-                                        const FaceType type);
+   void ComputeScatterIndicesAndOffsets();

   /** @brief Compute the gather indices: E-vector to L-vector.

       Note: Requires the gather offsets to be computed.
-
-       @param[in] f_ordering Request a specific face dof ordering.
-       @param[in] type       Request internal or boundary faces dofs.
   */
-   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
-                             const FaceType type);
+   void ComputeGatherIndices();

 public:
   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -1015,7 +1066,6 @@ public:
   void DoubleValuedNonconformingTransposeInterpolationInPlace(Vector& x) const;
 };

-
 /** @brief Convert a dof face index from Native ordering to lexicographic
    ordering for quads and hexes.

@@ -1044,6 +1094,20 @@ int PermuteFaceL2(const int dim, const int face_id1,
                  const int face_id2, const int orientation,
                  const int size1d, const int index);

+/// @brief Return the face-neighbor data given the L-vector @a x.
+///
+/// If the input vector @a x is a ParGridFunction with non-empty face-neighbor
+/// data, return an alias to ParGridFunction::FaceNbrData() (avoiding an
+/// unneeded call to ParGridFunction::ExchangeFaceNbrData).
+///
+/// Otherwise, create a temporary ParGridFunction, exchange the face-neighbor
+/// data, and return the resulting vector.
+///
+/// If @a fes is not a parallel space, or if @a ftype is not FaceType::Interior,
+/// return an empty vector.
+Vector GetLVectorFaceNbrData(
+   const FiniteElementSpace &fes, const Vector &x, FaceType ftype);
+
 }

 #endif // MFEM_RESTRICTION
@@ -503,7 +503,7 @@ public:

      Array<int> vdofs;
      const Array<int> *dof_map = sol_fe.GetDofMap();
-      const int *dof_map_ = dof_map->GetData();
+      const int *dof_map_ = (dof_map) ? dof_map->GetData() : NULL;
      DenseMatrix M_loc_perm(dofs*vdim,dofs*vdim); // initialized with zeros

      const int NE = mesh.GetNE();
--- a/Show More
+++ b/Show More