alternative preconditioner

add mesh file
minor
2024-12-04 10:03:43 -05:00 · 2024-12-04 09:40:55 -05:00 · 2024-01-22 10:52:39 -08:00 · 2024-01-12 13:07:25 -08:00 · 2024-01-12 13:06:59 -08:00 · 2024-01-04 16:09:52 -08:00
194 changed files with 136188 additions and 5111 deletions
@@ -128,6 +128,13 @@ examples/amgx/sol.gf
 examples/amgx/mesh.*
 examples/amgx/sol.*

+examples/caliper/ex1
+examples/caliper/ex1p
+examples/caliper/refined.mesh
+examples/caliper/sol.gf
+examples/caliper/mesh.*
+examples/caliper/sol.*
+
 examples/ginkgo/ex1
 examples/ginkgo/refined.mesh
 examples/ginkgo/sol.gf
@@ -276,6 +283,10 @@ miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
 miniapps/nurbs/sin-fit.mesh
 miniapps/nurbs/CurveInt
+miniapps/nurbs/nurbs_naca_cmesh
+miniapps/nurbs/naca-cmesh.mesh
+miniapps/nurbs/glvis_naca-cmesh.mesh
+miniapps/nurbs/Naca_cmesh

 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -300,7 +311,7 @@ miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/plor-transfer
 miniapps/tools/get-values
-miniapps/tools/check-tmop-metric
+miniapps/tools/tmop-check-metric
 miniapps/tools/tmop-metric-magnitude
 miniapps/tools/nodal-transfer
 miniapps/tools/ParaView
@@ -15,11 +15,24 @@ Discretization improvements
 ---------------------------
 - Introduced support for higher order non conformal Nedelec elements on
  simplices in ParMesh.
+- Introduced support for internal boundary elements in nonconformal adapted
+  meshes.
+
+- Added functionality for construction of cut-surface and cut-volume
+  IntegrationRules through a moment-fitting approach. The cut is specified by
+  the zero level set of a Coefficient. See fem/intrules_cut.hpp and Example 38.

 Miscellaneous
 -------------
 - The ReadCubit Genesis mesh importer has been rewritten to improve readability.

+- Updated the Doxygen documentation style, which now requires Doxygen version
+  1.9.8 or later. See the doc/ directory.
+
+- Improved thread safety for global variables in the library, for example
+  IntegrationRules IntRules, RefinedIntRules, GeometryRefiner
+  GlobGeometryRefiner, and FiniteElement::dof2quad_array.
+

 Version 4.6, released on September 27, 2023
 ===========================================
@@ -87,6 +100,8 @@ Linear and nonlinear solvers

 - Added HIP support to the PETSc and SUNDIALS interfaces.

+- Efficient GPU-accelerated LOR assembly now supports surface meshes.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new H(div) solver miniapp demonstrating the use of a matrix-free
@@ -139,10 +139,9 @@ if (MFEM_USE_CUDA)
      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
   endif()
   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
+   find_package(CUDAToolkit REQUIRED)
   set(CUSPARSE_FOUND TRUE)
-   set(CUSPARSE_LIBRARIES "cusparse")
-   set(CUBLAS_FOUND TRUE)
-   set(CUBLAS_LIBRARIES "cublas")
+   get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
 endif()

 if (XSDK_ENABLE_C)
@@ -531,7 +530,7 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
+    ADIOS2 CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
@@ -641,16 +640,34 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
  foreach(Header mfem.hpp mfem-performance.hpp)
    message(STATUS
      "Writing substitute header --> \"${Header}\"")
-    file(WRITE "${PROJECT_BINARY_DIR}/${Header}"
+    file(WRITE "${PROJECT_BINARY_DIR}/${Header}.tmp"
 "// Auto-generated file.
 #define MFEM_CONFIG_FILE \"${PROJECT_BINARY_DIR}/config/_config.hpp\"
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")
+
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
+      "${PROJECT_BINARY_DIR}/${Header}.tmp"
+      "${PROJECT_BINARY_DIR}/${Header}"
+    )
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
+      "${PROJECT_BINARY_DIR}/${Header}.tmp"
+    )
+
    # This version will be installed in the top include directory:
-    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
+    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
 "// Auto-generated file.
 #include \"mfem/${Header}\"
 ")
+
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
+      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
+      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
+    )
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
+      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
+    )
+
  endforeach()
 endif()

@@ -659,8 +659,7 @@ The specific libraries and their options are:
  requires the PT-Scotch and Scalapack libraries as well as ParMETIS, which
  includes METIS 5 in its distribution. Starting with STRUMPACK v2.2.0, ParMETIS
  and PT-Scotch are optional dependencies.
-  The support for STRUMPACK was added in MFEM v3.3.2 and it requires STRUMPACK
-  2.0.0 or later.
+  The support for STRUMPACK was added in MFEM v3.3.2.
  URL: http://portal.nersc.gov/project/sparse/strumpack
  Options: STRUMPACK_OPT, STRUMPACK_LIB.
  Versions: STRUMPACK >= 3.0.0.
@@ -797,7 +796,7 @@ The specific libraries and their options are:
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.10.
+  Versions: libCEED >= 0.12.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported.
@@ -14,9 +14,13 @@
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
+#   - HYPRE_USING_CUDA (internal)
 #   - HYPRE_USING_HIP (internal)

 if (HYPRE_FOUND)
+  if (HYPRE_USING_CUDA)
+    find_package(CUDAToolkit REQUIRED)
+  endif()
  if (HYPRE_USING_HIP)
    find_package(rocsparse REQUIRED)
    find_package(rocrand REQUIRED)
@@ -27,6 +31,20 @@ endif()
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
  "Paths to headers required by HYPRE." "Libraries required by HYPRE."
+  CHECK_BUILD HYPRE_USING_CUDA FALSE
+  "
+#undef HYPRE_USING_CUDA
+#include <HYPRE_config.h>
+
+#ifndef HYPRE_USING_CUDA
+#error HYPRE is built without CUDA.
+#endif
+
+int main()
+{
+   return 0;
+}
+"
  CHECK_BUILD HYPRE_USING_HIP FALSE
  "
 #undef HYPRE_USING_HIP
@@ -57,6 +75,16 @@ if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
  endif()
 endif()

+if (HYPRE_FOUND AND HYPRE_USING_CUDA)
+  find_package(CUDAToolkit REQUIRED)
+  get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
+  get_target_property(CURAND_LIBRARIES CUDA::curand LOCATION)
+  list(APPEND HYPRE_LIBRARIES ${CUSPARSE_LIBRARIES} ${CURAND_LIBRARIES})
+  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
+      "HYPRE libraries + dependencies." FORCE)
+  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
+endif()
+
 if (HYPRE_FOUND AND HYPRE_USING_HIP)
  find_package(rocsparse REQUIRED)
  find_package(rocrand REQUIRED)
@@ -106,12 +106,7 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-if (MFEM_USE_CUDA)
-   # This is only necessary when hypre is built with cuda:
-   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
-       "Libraries that HYPRE depends on.")
-endif()
-# HIP dependency for HYPRE is handled in FindHYPRE.cmake.
+# CUDA and HIP dependencies for HYPRE are handled in FindHYPRE.cmake.

 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")

@@ -157,7 +152,8 @@ set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
 # STRUMPACK may also depend on "OpenMP", depending on how it was compiled.
 # Starting with v2.2.0 of STRUMPACK, ParMETIS and Scotch are optional.
 set(STRUMPACK_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
-    "ScaLAPACK" "Scotch/ptscotch/ptscotcherr/scotch/scotcherr" CACHE STRING
+    "Scotch/ptscotch/ptscotcherr/scotch/scotcherr"
+    "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
    "Additional packages required by STRUMPACK.")
 # If the MPI package does not find all required Fortran libraries:
 # set(STRUMPACK_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
@@ -38,14 +38,14 @@ all: header config-mk
 MPI = $(MFEM_USE_MPI:NO=)
 GHV_CXX ?= $(MFEM_CXX)
 GHV = get_hypre_version
-GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
+GHV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
 SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
 SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
 MUMPS = $(MFEM_USE_MUMPS:NO=)
 GMV_CXX ?= $(MFEM_CXX)
 GMV = get_mumps_version
-GMV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))
+GMV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))

 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
@@ -110,3 +110,4 @@ config-mk:

 clean:
 	rm -f $(CONFIG_HPP) $(CONFIG_MK) sample-runs-build.log
+	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out
@@ -315,7 +315,7 @@ function extract_sample_runs()
   sruns=`grep -v "^//.*  mpirun .* ${app}" "${src}" |
          grep "^//.*  ${app}" |
          sed -e "s/.*  ${app}/${vg_app}/g"`
-   runs="${sruns}${pruns}"
+   runs="${sruns}"$'\n'"${pruns}"
   if [ "$skip_gen_meshes" == "yes" ]; then
      runs=`printf "%s" "$runs" | grep -v ".* -m .*\.gen"`
   fi
@@ -112,6 +112,7 @@ namespace mfem {
 * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
 * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: Topology optimization
 * - <a class="el" href="ex37p_8cpp_source.html">Example 37p</a>: parallel topology optimization
+ * - <a class="el" href="ex38_8cpp_source.html">Example 38</a>: cut-surface and cut-volume integration
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -14,7 +14,7 @@ If not already available, Doxygen can be downloaded from

   http://www.doxygen.org

-We recommend using version 1.8 or later.
+We recommend using version 1.9.8 or later.

 To build the documentation, simply type "make" in the doc/ directory. This will
 create the file CodeDocumentation.html, which can be viewed in any web browser.
@@ -0,0 +1,3 @@
+html {
+    --content-maxwidth: auto;
+}
@@ -0,0 +1,78 @@
+<!-- HTML header for doxygen 1.9.6-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="$langISO">
+<head>
+<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
+<meta http-equiv="X-UA-Compatible" content="IE=11"/>
+<meta name="generator" content="Doxygen $doxygenversion"/>
+<meta name="viewport" content="width=device-width, initial-scale=1"/>
+<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME-->
+<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME-->
+<link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/>
+<!--BEGIN DISABLE_INDEX-->
+  <!--BEGIN FULL_SIDEBAR-->
+<script type="text/javascript">var page_layout=1;</script>
+  <!--END FULL_SIDEBAR-->
+<!--END DISABLE_INDEX-->
+<script type="text/javascript" src="$relpath^jquery.js"></script>
+<script type="text/javascript" src="$relpath^dynsections.js"></script>
+$treeview
+$search
+$mathjax
+$darkmode
+<link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
+$extrastylesheet
+<script type="text/javascript" src="$relpath^doxygen-awesome-darkmode-toggle.js"></script>
+<script type="text/javascript">
+    DoxygenAwesomeDarkModeToggle.init()
+</script>
+</head>
+<body>
+<!--BEGIN DISABLE_INDEX-->
+  <!--BEGIN FULL_SIDEBAR-->
+<div id="side-nav" class="ui-resizable side-nav-resizable"><!-- do not remove this div, it is closed by doxygen! -->
+  <!--END FULL_SIDEBAR-->
+<!--END DISABLE_INDEX-->
+
+<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
+
+<!--BEGIN TITLEAREA-->
+<div id="titlearea">
+<table cellspacing="0" cellpadding="0">
+ <tbody>
+ <tr id="projectrow">
+  <!--BEGIN PROJECT_LOGO-->
+  <td id="projectlogo"><img alt="Logo" src="$relpath^$projectlogo"/></td>
+  <!--END PROJECT_LOGO-->
+  <!--BEGIN PROJECT_NAME-->
+  <td id="projectalign">
+   <div id="projectname">$projectname<!--BEGIN PROJECT_NUMBER--><span id="projectnumber">&#160;$projectnumber</span><!--END PROJECT_NUMBER-->
+   </div>
+   <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">$projectbrief</div><!--END PROJECT_BRIEF-->
+  </td>
+  <!--END PROJECT_NAME-->
+  <!--BEGIN !PROJECT_NAME-->
+   <!--BEGIN PROJECT_BRIEF-->
+    <td>
+    <div id="projectbrief">$projectbrief</div>
+    </td>
+   <!--END PROJECT_BRIEF-->
+  <!--END !PROJECT_NAME-->
+  <!--BEGIN DISABLE_INDEX-->
+   <!--BEGIN SEARCHENGINE-->
+     <!--BEGIN !FULL_SIDEBAR-->
+    <td>$searchbox</td>
+     <!--END !FULL_SIDEBAR-->
+   <!--END SEARCHENGINE-->
+  <!--END DISABLE_INDEX-->
+ </tr>
+  <!--BEGIN SEARCHENGINE-->
+   <!--BEGIN FULL_SIDEBAR-->
+   <tr><td colspan="2">$searchbox</td></tr>
+   <!--END FULL_SIDEBAR-->
+  <!--END SEARCHENGINE-->
+ </tbody>
+</table>
+</div>
+<!--END TITLEAREA-->
+<!-- end header part -->
@@ -0,0 +1,157 @@
+/**
+
+Doxygen Awesome
+https://github.com/jothepro/doxygen-awesome-css
+
+MIT License
+
+Copyright (c) 2021 - 2023 jothepro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+class DoxygenAwesomeDarkModeToggle extends HTMLElement {
+    // SVG icons from https://fonts.google.com/icons
+    // Licensed under the Apache 2.0 license:
+    // https://www.apache.org/licenses/LICENSE-2.0.html
+    static lightModeIcon = `<svg xmlns="http://www.w3.org/2000/svg" enable-background="new 0 0 24 24" height="24px" viewBox="0 0 24 24" width="24px" fill="#FCBF00"><rect fill="none" height="24" width="24"/><circle cx="12" cy="12" opacity=".3" r="3"/><path d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"/></svg>`
+    static darkModeIcon = `<svg xmlns="http://www.w3.org/2000/svg" enable-background="new 0 0 24 24" height="24px" viewBox="0 0 24 24" width="24px" fill="#FE9700"><rect fill="none" height="24" width="24"/><path d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27 C17.45,17.19,14.93,19,12,19c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z" opacity=".3"/><path d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"/></svg>`
+    static title = "Toggle Light/Dark Mode"
+
+    static prefersLightModeInDarkModeKey = "prefers-light-mode-in-dark-mode"
+    static prefersDarkModeInLightModeKey = "prefers-dark-mode-in-light-mode"
+
+    static _staticConstructor = function() {
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.userPreference)
+        // Update the color scheme when the browsers preference changes
+        // without user interaction on the website.
+        window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', event => {
+            DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
+        })
+        // Update the color scheme when the tab is made visible again.
+        // It is possible that the appearance was changed in another tab 
+        // while this tab was in the background.
+        document.addEventListener("visibilitychange", visibilityState => {
+            if (document.visibilityState === 'visible') {
+                DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
+            }
+        });
+    }()
+
+    static init() {
+        $(function() {
+            $(document).ready(function() {
+                const toggleButton = document.createElement('doxygen-awesome-dark-mode-toggle')
+                toggleButton.title = DoxygenAwesomeDarkModeToggle.title
+                toggleButton.updateIcon()
+
+                window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', event => {
+                    toggleButton.updateIcon()
+                })
+                document.addEventListener("visibilitychange", visibilityState => {
+                    if (document.visibilityState === 'visible') {
+                        toggleButton.updateIcon()
+                    }
+                });
+
+                $(document).ready(function(){
+                    document.getElementById("MSearchBox").parentNode.appendChild(toggleButton)
+                })
+                $(window).resize(function(){
+                    document.getElementById("MSearchBox").parentNode.appendChild(toggleButton)
+                })
+            })
+        })
+    }
+
+    constructor() {
+        super();
+        this.onclick=this.toggleDarkMode
+    }
+
+    /**
+     * @returns `true` for dark-mode, `false` for light-mode system preference
+     */
+    static get systemPreference() {
+        return window.matchMedia('(prefers-color-scheme: dark)').matches
+    }
+
+    /**
+     * @returns `true` for dark-mode, `false` for light-mode user preference
+     */
+    static get userPreference() {
+        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) || 
+        (DoxygenAwesomeDarkModeToggle.systemPreference && !localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey))
+    }
+
+    static set userPreference(userPreference) {
+        DoxygenAwesomeDarkModeToggle.darkModeEnabled = userPreference
+        if(!userPreference) {
+            if(DoxygenAwesomeDarkModeToggle.systemPreference) {
+                localStorage.setItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey, true)
+            } else {
+                localStorage.removeItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)
+            }
+        } else {
+            if(!DoxygenAwesomeDarkModeToggle.systemPreference) {
+                localStorage.setItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey, true)
+            } else {
+                localStorage.removeItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey)
+            }
+        }
+        DoxygenAwesomeDarkModeToggle.onUserPreferenceChanged()
+    }
+
+    static enableDarkMode(enable) {
+        if(enable) {
+            DoxygenAwesomeDarkModeToggle.darkModeEnabled = true
+            document.documentElement.classList.add("dark-mode")
+            document.documentElement.classList.remove("light-mode")
+        } else {
+            DoxygenAwesomeDarkModeToggle.darkModeEnabled = false
+            document.documentElement.classList.remove("dark-mode")
+            document.documentElement.classList.add("light-mode")
+        }
+    }
+
+    static onSystemPreferenceChanged() {
+        DoxygenAwesomeDarkModeToggle.darkModeEnabled = DoxygenAwesomeDarkModeToggle.userPreference
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.darkModeEnabled)
+    }
+
+    static onUserPreferenceChanged() {
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.darkModeEnabled)
+    }
+
+    toggleDarkMode() {
+        DoxygenAwesomeDarkModeToggle.userPreference = !DoxygenAwesomeDarkModeToggle.userPreference
+        this.updateIcon()
+    }
+
+    updateIcon() {
+        if(DoxygenAwesomeDarkModeToggle.darkModeEnabled) {
+            this.innerHTML = DoxygenAwesomeDarkModeToggle.darkModeIcon
+        } else {
+            this.innerHTML = DoxygenAwesomeDarkModeToggle.lightModeIcon
+        }
+    }
+}
+
+customElements.define("doxygen-awesome-dark-mode-toggle", DoxygenAwesomeDarkModeToggle);
@@ -16,7 +16,7 @@ DOXYGEN_CONF = CodeDocumentation.conf
 # doxygen uses: graphviz, latex
 html: $(DOXYGEN_CONF)
 	@# Generate the html documentation
-	@( cat $(DOXYGEN_CONF) ; echo "$(MFEM_DOXYGEN_FLAGS)" ) | doxygen -
+	@( cat $(DOXYGEN_CONF) ; printf "$(MFEM_DOXYGEN_FLAGS)\n" ) | doxygen -
 	@echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > CodeDocumentation.html
 	@cat warnings.log 1>&2
 	@# Generate the log of undocumented methods
@@ -45,6 +45,12 @@ list(APPEND ALL_EXE_SRCS
  ex37.cpp
  )

+if(MFEM_USE_LAPACK)
+list(APPEND ALL_EXE_SRCS
+  ex38.cpp
+  )
+endif()
+
 if (MFEM_USE_MPI)
  list(APPEND ALL_EXE_SRCS
    ex0p.cpp
@@ -262,12 +262,13 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
      if (sp_solver)
      {
-         STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+         STRUMPACKSolver * strumpack = new STRUMPACKSolver(MPI_COMM_WORLD, argc, argv);
         strumpack->SetPrintFactorStatistics(true);
         strumpack->SetPrintSolveStatistics(false);
         strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
         strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-         strumpack->DisableMatching();
+         strumpack->SetMatching(strumpack::MatchingJob::NONE);
+         strumpack->SetCompression(strumpack::CompressionType::NONE);
         strumpack->SetOperator(*Arow);
         strumpack->SetFromCommandLine();
         precond = strumpack;
@@ -26,13 +26,12 @@
 using namespace std;
 using namespace mfem;

-/** After spatial discretization, the conduction model can be written as:
+/** After spatial discretization, the wave model can be written as:
 *
 *     d^2u/dt^2 = M^{-1}(-Ku)
 *
- *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K is the diffusion operator with diffusivity depending on u:
- *  (\kappa + \alpha u).
+ *  where u is the vector representing the temperature, M is the mass,
+ *  and K is the stiffness matrix.
 *
 *  Class WaveOperator represents the right-hand side of the above ODE.
 */
@@ -301,7 +300,7 @@ int main(int argc, char *argv[])
   Vector dudt;
   dudt_gf.GetTrueDofs(dudt);

-   // 7. Initialize the conduction operator and the visualization.
+   // 7. Initialize the wave operator and the visualization.
   Array<int> ess_bdr;
   if (mesh->bdr_attributes.Size())
   {
@@ -356,7 +355,7 @@ int main(int argc, char *argv[])
      else
      {
         sout.precision(precision);
-         sout << "solution\n" << *mesh << dudt_gf;
+         sout << "solution\n" << *mesh << u_gf;
         sout << "pause\n";
         sout << flush;
         cout << "GLVis visualization paused."
@@ -170,6 +170,7 @@ int main(int argc, char *argv[])
   bool herm_conv = true;
   bool slu_solver  = false;
   bool mumps_solver = false;
+   bool strumpack_solver = false;
   bool visualization = 1;
   bool pa = false;
   const char *device_config = "cpu";
@@ -200,6 +201,11 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_MUMPS
   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
                  "--no-mumps-solver", "Use the MUMPS Solver.");
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&strumpack_solver, "-strumpack", "--strumpack-solver",
+                  "-no-strumpack", "--no-strumpack-solver",
+                  "Use the STRUMPACK Solver.");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -209,13 +215,14 @@ int main(int argc, char *argv[])
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.Parse();
-   if (slu_solver && mumps_solver)
+   if (slu_solver + mumps_solver + strumpack_solver > 1)
   {
      if (myid == 0)
-         cout << "WARNING: Both SuperLU and MUMPS have been selected,"
-              << " please choose either one." << endl
+         cout << "WARNING: More than one of SuperLU, MUMPS, and STRUMPACK have"
+              << " been selected, please choose only one." << endl
              << "         Defaulting to SuperLU." << endl;
      mumps_solver = false;
+      strumpack_solver = false;
   }

   if (iprob > 4) { iprob = 4; }
@@ -474,6 +481,24 @@ int main(int argc, char *argv[])
      delete A;
   }
 #endif
+#ifdef MFEM_USE_STRUMPACK
+   if (!pa && strumpack_solver)
+   {
+      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      STRUMPACKRowLocMatrix SA(*A);
+      STRUMPACKSolver strumpack(MPI_COMM_WORLD, argc, argv);
+      strumpack.SetPrintFactorStatistics(false);
+      strumpack.SetPrintSolveStatistics(false);
+      strumpack.SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack.SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      strumpack.SetMatching(strumpack::MatchingJob::NONE);
+      strumpack.SetCompression(strumpack::CompressionType::NONE);
+      strumpack.SetFromCommandLine();
+      strumpack.SetOperator(SA);
+      strumpack.Mult(B, X);
+      delete A;
+   }
+#endif
 #ifdef MFEM_USE_MUMPS
   if (!pa && mumps_solver)
   {
@@ -493,7 +518,7 @@ int main(int argc, char *argv[])
   //
   //    In PML:   1/mu (abs(1/det(J) J^T J) Curl E, Curl F)
   //              + omega^2 * epsilon (abs(det(J) * (J^T J)^-1) * E, F)
-   if (pa || (!slu_solver && !mumps_solver))
+   if (pa || (!slu_solver && !mumps_solver && !strumpack_solver))
   {
      ConstantCoefficient absomeg(pow(omega, 2) * epsilon);
      RestrictedCoefficient restr_absomeg(absomeg,attr);
@@ -0,0 +1,696 @@
+//                                MFEM Example 38
+//
+// Compile with: make ex38
+//
+// Sample runs:
+// (since all sample runs require LAPACK, the * symbol is used to exclude them
+// from the automatically generated internal MFEM tests).
+//              * ex38
+//              * ex38 -i volumetric1d
+//              * ex38 -i surface2d
+//              * ex38 -i surface2d -o 4 -r 5
+//              * ex38 -i volumetric2d
+//              * ex38 -i volumetric2d -o 4 -r 5
+//              * ex38 -i surface3d
+//              * ex38 -i surface3d -o 4 -r 5
+//              * ex38 -i volumetric3d
+//              * ex38 -i volumetric3d -o 4 -r 5
+//
+// Description: This example code demonstrates the use of MFEM to integrate
+//              functions over implicit interfaces and subdomains bounded by
+//              implicit interfaces.
+//
+//              The quadrature rules are constructed by means of moment-fitting.
+//              The interface is given by the zero isoline of a level-set
+//              function ϕ and the subdomain is given as the domain where ϕ>0
+//              holds. The algorithm for construction of the quadrature rules
+//              was introduced by Mueller, Kummer and Oberlack [1].
+//
+//              This example also showcases how to set up integrators using the
+//              integration rules on implicit surfaces and subdomains.
+//
+// [1] Mueller, B., Kummer, F. and Oberlack, M. (2013) Highly accurate surface
+//     and volume integration on implicit domains by means of moment-fitting.
+//     Int. J. Numer. Meth. Engr. (96) 512-528. DOI:10.1002/nme.4569
+
+#include "mfem.hpp"
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+/// @brief Integration rule the example should demonstrate
+enum class IntegrationType { Volumetric1D, Surface2D, Volumetric2D,
+                             Surface3D, Volumetric3D
+                           };
+IntegrationType itype;
+
+/// @brief Level-set function defining the implicit interface
+double lvlset(const Vector& X)
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return .55 - X(0);
+      case IntegrationType::Surface2D:
+         return 1. - (pow(X(0), 2.) + pow(X(1), 2.));
+      case IntegrationType::Volumetric2D:
+         return 1. - (pow(X(0) / 1.5, 2.) + pow(X(1) / .75, 2.));
+      case IntegrationType::Surface3D:
+         return 1. - (pow(X(0), 2.) + pow(X(1), 2.) + pow(X(2), 2.));
+      case IntegrationType::Volumetric3D:
+         return 1. - (pow(X(0) / 1.5, 2.) + pow(X(1) / .75, 2.) + pow(X(2) / .5, 2.));
+      default:
+         return 1.;
+   }
+}
+
+/// @brief Function that should be integrated
+double integrand(const Vector& X)
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return 1.;
+      case IntegrationType::Surface2D:
+         return 3. * pow(X(0), 2.) - pow(X(1), 2.);
+      case IntegrationType::Volumetric2D:
+         return 1.;
+      case IntegrationType::Surface3D:
+         return 4. - 3. * pow(X(0), 2.) + 2. * pow(X(1), 2.) - pow(X(2), 2.);
+      case IntegrationType::Volumetric3D:
+         return 1.;
+      default:
+         return 0.;
+   }
+}
+
+/// @brief Analytic surface integral
+double Surface()
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return 1.;
+      case IntegrationType::Surface2D:
+         return 2. * M_PI;
+      case IntegrationType::Volumetric2D:
+         return 7.26633616541076;
+      case IntegrationType::Surface3D:
+         return 40. / 3. * M_PI;
+      case IntegrationType::Volumetric3D:
+         return 9.90182151329315;
+      default:
+         return 0.;
+   }
+}
+
+/// @brief Analytic volume integral over subdomain with positive level-set
+double Volume()
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return .55;
+      case IntegrationType::Surface2D:
+         return NAN;
+      case IntegrationType::Volumetric2D:
+         return 9. / 8. * M_PI;
+      case IntegrationType::Surface3D:
+         return NAN;
+      case IntegrationType::Volumetric3D:
+         return 3. / 4. * M_PI;
+      default:
+         return 0.;
+   }
+}
+
+#ifdef MFEM_USE_LAPACK
+/**
+ @brief Class for surface IntegrationRule
+
+ This class demonstrates how IntegrationRules computed as CutIntegrationRules
+ can be saved to reduce the impact by computing them from scratch each time.
+ */
+class SIntegrationRule : public IntegrationRule
+{
+protected:
+   /// @brief Space Dimension of the IntegrationRule
+   int dim;
+   /// @brief Column-wise matrix of the quadtrature weights
+   DenseMatrix Weights;
+   /// @brief Column-wise matrix of the transformation weights of the normal
+   DenseMatrix SurfaceWeights;
+
+public:
+   /**
+    @brief Constructor of SIntegrationRule
+
+    The surface integrationRules are computed and saved in the constructor.
+
+    @param [in] Order Order of the IntegrationRule
+    @param [in] LvlSet Level-set defining the implicit interface
+    @param [in] lsOrder Polynomial degree for approx of level-set function
+    @param [in] mesh Pointer to the mesh that is used
+   */
+   SIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
+   {
+      dim = mesh->Dimension();
+
+      IsoparametricTransformation Tr;
+      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
+      mesh->GetElementTransformation(0, &Tr);
+      IntegrationRule ir;
+      MFIRs.GetSurfaceIntegrationRule(Tr, ir);
+      if (dim >1)
+      {
+         Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
+      }
+      else
+      {
+         Weights.SetSize(2, mesh->GetNE());
+      }
+      SurfaceWeights.SetSize(ir.GetNPoints(), mesh->GetNE());
+      Vector w;
+      MFIRs.GetSurfaceWeights(Tr, ir, w);
+      SurfaceWeights.SetCol(0, w);
+      SetSize(ir.GetNPoints());
+
+      for (int ip = 0; ip < GetNPoints(); ip++)
+      {
+         IntPoint(ip).index = ip;
+         IntegrationPoint &intp = IntPoint(ip);
+         intp.x = ir.IntPoint(ip).x;
+         intp.y = ir.IntPoint(ip).y;
+         intp.z = ir.IntPoint(ip).z;
+         if (dim > 1)
+         {
+            Weights(ip, 0) = ir.IntPoint(ip).weight;
+         }
+         else
+         {
+            Weights(0, 0) = ir.IntPoint(ip).x;
+            Weights(1, 0) = ir.IntPoint(ip).weight;
+         }
+      }
+
+
+      for (int elem = 1; elem < mesh->GetNE(); elem++)
+      {
+         mesh->GetElementTransformation(elem, &Tr);
+         MFIRs.GetSurfaceIntegrationRule(Tr, ir);
+         Vector w;
+         MFIRs.GetSurfaceWeights(Tr, ir, w);
+         SurfaceWeights.SetCol(elem, w);
+
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            if (dim > 1)
+            {
+               Weights(ip, elem) = ir.IntPoint(ip).weight;
+            }
+            else
+            {
+               Weights(0, elem) = ir.IntPoint(ip).x;
+               Weights(1, elem) = ir.IntPoint(ip).weight;
+            }
+         }
+      }
+   }
+
+   /**
+    @brief Set the weights for the given element and multiply them with the
+    transformation of the interface
+    */
+   void SetElementinclSurfaceWeight(int Element)
+   {
+      if (dim == 1)
+      {
+         IntegrationPoint &intp = IntPoint(0);
+         intp.x = Weights(0, Element);
+         intp.weight = Weights(1, Element);
+         cout << intp.x << " " << Element << endl;
+      }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element) * SurfaceWeights(ip, Element);
+         }
+   }
+
+   /// @brief Set the weights for the given element
+   void SetElement(int Element)
+   {
+      if (dim == 1)
+      {
+         IntegrationPoint &intp = IntPoint(0);
+         intp.x = Weights(0, Element);
+         intp.weight = Weights(1, Element);
+      }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element);
+         }
+   }
+
+   /// @brief Destructor of SIntegrationRule
+   ~SIntegrationRule() {}
+};
+
+/**
+ @brief Class for volume IntegrationRule
+
+ This class demonstrates how IntegrationRules computed as CutIntegrationRules
+ can be saved to reduce the impact by computing them from scratch each time.
+ */
+class CIntegrationRule : public IntegrationRule
+{
+protected:
+   /// @brief Space Dimension of the IntegrationRule
+   int dim;
+   /// @brief Column-wise matrix of the quadtrature weights
+   DenseMatrix Weights;
+
+public:
+   /**
+    @brief Constructor of CIntegrationRule
+
+    The volume integrationRules are computed and saved in the constructor.
+
+    @param [in] Order Order of the IntegrationRule
+    @param [in] LvlSet Level-set defining the implicit interface
+    @param [in] lsOrder Polynomial degree for approx of level-set function
+    @param [in] mesh Pointer to the mesh that is used
+   */
+   CIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
+   {
+      dim = mesh->Dimension();
+
+      IsoparametricTransformation Tr;
+      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
+      mesh->GetElementTransformation(0, &Tr);
+      IntegrationRule ir;
+      MFIRs.GetVolumeIntegrationRule(Tr, ir);
+      if (dim > 1)
+      {
+         Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
+      }
+      else
+      {
+         Weights.SetSize(2 * ir.GetNPoints(), mesh->GetNE());
+      }
+
+      SetSize(ir.GetNPoints());
+      for (int ip = 0; ip < GetNPoints(); ip++)
+      {
+         IntPoint(ip).index = ip;
+         IntegrationPoint &intp = IntPoint(ip);
+         intp.x = ir.IntPoint(ip).x;
+         intp.y = ir.IntPoint(ip).y;
+         intp.z = ir.IntPoint(ip).z;
+         if (dim > 1)
+         {
+            Weights(ip, 0) = ir.IntPoint(ip).weight;
+         }
+         else
+         {
+            Weights(2 * ip, 0) = ir.IntPoint(ip).x;
+            Weights(2 * ip + 1, 0) = ir.IntPoint(ip).weight;
+         }
+      }
+
+      for (int elem = 1; elem < mesh->GetNE(); elem++)
+      {
+         mesh->GetElementTransformation(elem, &Tr);
+         MFIRs.GetVolumeIntegrationRule(Tr, ir);
+
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            if (dim > 1)
+            {
+               Weights(ip, elem) = ir.IntPoint(ip).weight;
+            }
+            else
+            {
+               Weights(2 * ip, elem) = ir.IntPoint(ip).x;
+               Weights(2 * ip + 1, elem) = ir.IntPoint(ip).weight;
+            }
+         }
+      }
+   }
+
+   /// @brief Set the weights for the given element
+   void SetElement(int Element)
+   {
+      if (dim == 1)
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.x = Weights(2 * ip, Element);
+            intp.weight = Weights(2 * ip + 1, Element);
+         }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element);
+         }
+   }
+
+   /// @brief Destructor of CIntegrationRule
+   ~CIntegrationRule() {}
+};
+/**
+ @brief Class for surface linearform integrator
+
+ Integrator to demonstrate the use of the surface integration rule on an
+ implicit surface defined by a level-set.
+ */
+class SurfaceLFIntegrator : public LinearFormIntegrator
+{
+protected:
+   /// @brief vector to evaluate the basis functions
+   Vector shape;
+
+   /// @brief surface integration rule
+   SIntegrationRule* SIntRule;
+
+   /// @brief coefficient representing the level-set defining the interface
+   Coefficient &LevelSet;
+
+   /// @brief coefficient representing the integrand
+   Coefficient &Q;
+
+public:
+   /**
+    @brief Constructor for the surface linear form integrator
+
+    Constructor for the surface linear form integrator to demonstrate the use
+    of the surface integration rule by means of moment-fitting.
+
+    @param [in] q coefficient representing the inegrand
+    @param [in] levelset level-set defining the implicit interfac
+    @param [in] ir surface integrtion rule to be used
+    */
+   SurfaceLFIntegrator(Coefficient &q, Coefficient &levelset,
+                       SIntegrationRule* ir)
+      : LinearFormIntegrator(), SIntRule(ir), LevelSet(levelset), Q(q) { }
+
+   /**
+    @brief Assembly of the element vector
+
+    Assemble the element vector of for the right hand side on the element given
+    by the FiniteElement and ElementTransformation.
+
+    @param [in] el finite Element the vector belongs to
+    @param [in] Tr transformation of finite element
+    @param [out] elvect vector containing the
+   */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
+   {
+      int dof = el.GetDof();
+      shape.SetSize(dof);
+      elvect.SetSize(dof);
+      elvect = 0.;
+
+      // Update the surface integration rule for the current element
+      SIntRule->SetElementinclSurfaceWeight(Tr.ElementNo);
+
+      for (int ip = 0; ip < SIntRule->GetNPoints(); ip++)
+      {
+         Tr.SetIntPoint((&(SIntRule->IntPoint(ip))));
+         double val = Tr.Weight() * Q.Eval(Tr, SIntRule->IntPoint(ip));
+         el.CalcShape(SIntRule->IntPoint(ip), shape);
+         add(elvect, SIntRule->IntPoint(ip).weight * val, shape, elvect);
+      }
+   }
+};
+
+/**
+ @brief Class for subdomain linearform integrator
+
+ Integrator to demonstrate the use of the subdomain integration rule within
+ an area defined by an implicit surface defined by a level-set.
+ */
+class SubdomainLFIntegrator : public LinearFormIntegrator
+{
+protected:
+   /// @brief vector to evaluate the basis functions
+   Vector shape;
+
+   /// @brief surface integration rule
+   CIntegrationRule* CIntRule;
+
+   /// @brief coefficient representing the level-set defining the interface
+   Coefficient &LevelSet;
+
+   /// @brief coefficient representing the integrand
+   Coefficient &Q;
+
+public:
+   /**
+    @brief Constructor for the volumetric subdomain linear form integrator
+
+    Constructor for the subdomain linear form integrator to demonstrate the use
+    of the volumetric subdomain integration rule by means of moment-fitting.
+
+    @param [in] q coefficient representing the inegrand
+    @param [in] levelset level-set defining the implicit interfac
+    @param [in] ir subdomain integrtion rule to be used
+    */
+   SubdomainLFIntegrator(Coefficient &q, Coefficient &levelset,
+                         CIntegrationRule* ir)
+      : LinearFormIntegrator(), CIntRule(ir), LevelSet(levelset), Q(q) { }
+
+   /**
+    @brief Assembly of the element vector
+
+    Assemble the element vector of for the right hand side on the element given
+    by the FiniteElement and ElementTransformation.
+
+    @param [in] el finite Element the vector belongs to
+    @param [in] Tr transformation of finite element
+    @param [out] elvect vector containing the
+   */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
+   {
+      int dof = el.GetDof();
+      shape.SetSize(dof);
+      elvect.SetSize(dof);
+      elvect = 0.;
+
+      // Update the subdomain integration rule
+      CIntRule->SetElement(Tr.ElementNo);
+
+      for (int ip = 0; ip < CIntRule->GetNPoints(); ip++)
+      {
+         Tr.SetIntPoint((&(CIntRule->IntPoint(ip))));
+         double val = Tr.Weight()
+                      * Q.Eval(Tr, CIntRule->IntPoint(ip));
+         el.CalcPhysShape(Tr, shape);
+         add(elvect, CIntRule->IntPoint(ip).weight * val, shape, elvect);
+      }
+   }
+};
+#endif // MFEM_USE_LAPACK
+
+int main(int argc, char *argv[])
+{
+#ifndef MFEM_USE_LAPACK
+   cout << "MFEM must be build with LAPACK for this example." << endl;
+   return EXIT_FAILURE;
+#else
+   // 1. Parse he command-line options.
+   int ref_levels = 3;
+   int order = 2;
+   const char *inttype = "surface2d";
+   bool visualization = true;
+   itype = IntegrationType::Surface2D;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order", "Order of quadrature rule");
+   args.AddOption(&ref_levels, "-r", "--refine", "Number of meh refinements");
+   args.AddOption(&inttype, "-i", "--integrationtype",
+                  "IntegrationType to demonstrate");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.ParseCheck();
+
+   if (strcmp(inttype, "volumetric1d") == 0
+       || strcmp(inttype, "Volumetric1D") == 0)
+   {
+      itype = IntegrationType::Volumetric1D;
+   }
+   else if (strcmp(inttype, "surface2d") == 0
+            || strcmp(inttype, "Surface2D") == 0)
+   {
+      itype = IntegrationType::Surface2D;
+   }
+   else if (strcmp(inttype, "volumetric2d") == 0
+            || strcmp(inttype, "Volumetric2D") == 0)
+   {
+      itype = IntegrationType::Volumetric2D;
+   }
+   else if (strcmp(inttype, "surface3d") == 0
+            || strcmp(inttype, "Surface3d") == 0)
+   {
+      itype = IntegrationType::Surface3D;
+   }
+   else if (strcmp(inttype, "volumetric3d") == 0
+            || strcmp(inttype, "Volumetric3d") == 0)
+   {
+      itype = IntegrationType::Volumetric3D;
+   }
+
+   // 2. Construct and refine the mesh.
+   Mesh *mesh;
+   if (itype == IntegrationType::Volumetric1D)
+   {
+      mesh = new Mesh("../data/inline-segment.mesh");
+   }
+   if (itype == IntegrationType::Surface2D
+       || itype == IntegrationType::Volumetric2D)
+   {
+      mesh = new Mesh(2, 4, 1, 0, 2);
+      mesh->AddVertex(-1.6,-1.6);
+      mesh->AddVertex(1.6,-1.6);
+      mesh->AddVertex(1.6,1.6);
+      mesh->AddVertex(-1.6,1.6);
+      mesh->AddQuad(0,1,2,3);
+      mesh->FinalizeQuadMesh(1, 0, 1);
+   }
+   else if (itype == IntegrationType::Surface3D
+            || itype == IntegrationType::Volumetric3D)
+   {
+      mesh = new Mesh(3, 8, 1, 0, 3);
+      mesh->AddVertex(-1.6,-1.6,-1.6);
+      mesh->AddVertex(1.6,-1.6,-1.6);
+      mesh->AddVertex(1.6,1.6,-1.6);
+      mesh->AddVertex(-1.6,1.6,-1.6);
+      mesh->AddVertex(-1.6,-1.6,1.6);
+      mesh->AddVertex(1.6,-1.6,1.6);
+      mesh->AddVertex(1.6,1.6,1.6);
+      mesh->AddVertex(-1.6,1.6,1.6);
+      mesh->AddHex(0,1,2,3,4,5,6,7);
+      mesh->FinalizeHexMesh(1, 0, 1);
+   }
+
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 3. Define the necessary finite element space on the mesh.
+   H1_FECollection fe_coll(1, mesh->Dimension());
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, &fe_coll);
+
+   // 4. Construction Coefficients for the level set and the integrand.
+   FunctionCoefficient levelset(lvlset);
+   FunctionCoefficient u(integrand);
+
+   // 5. Define the necessary Integration rules on element 0.
+   IsoparametricTransformation Tr;
+   mesh->GetElementTransformation(0, &Tr);
+   SIntegrationRule* sir = new SIntegrationRule(order, levelset, 2, mesh);
+   CIntegrationRule* cir = NULL;
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      cir = new CIntegrationRule(order, levelset, 2, mesh);
+   }
+
+   // 6. Define and assemble the linear forms on the finite element space.
+   LinearForm surface(fespace);
+   LinearForm volume(fespace);
+
+   surface.AddDomainIntegrator(new SurfaceLFIntegrator(u, levelset, sir));
+   surface.Assemble();
+
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      volume.AddDomainIntegrator(new SubdomainLFIntegrator(u, levelset, cir));
+      volume.Assemble();
+   }
+
+   // 7. Print information, computed values and errors to the console.
+   int qorder = 0;
+   int nbasis = 2 * (order + 1) + (int)(order * (order + 1) / 2);
+   IntegrationRules irs(0, Quadrature1D::GaussLegendre);
+   IntegrationRule ir = irs.Get(Geometry::SQUARE, qorder);
+   for (; ir.GetNPoints() <= nbasis; qorder++)
+   {
+      ir = irs.Get(Geometry::SQUARE, qorder);
+   }
+   cout << "============================================" << endl;
+   cout << "Mesh size dx:                       ";
+   if (itype != IntegrationType::Volumetric1D)
+   {
+      cout << 3.2 / pow(2., (double)ref_levels) << endl;
+   }
+   else
+   {
+      cout << .25 / pow(2., (double)ref_levels) << endl;
+   }
+   if (itype == IntegrationType::Surface2D
+       || itype == IntegrationType::Volumetric2D)
+   {
+      cout << "Number of div free basis functions: " << nbasis << endl;
+      cout << "Number of quadrature points:        " << ir.GetNPoints() << endl;
+   }
+   cout << scientific << setprecision(2);
+   cout << "============================================" << endl;
+   cout << "Computed value of surface integral: " << surface.Sum() << endl;
+   cout << "True value of surface integral:     " << Surface() << endl;
+   cout << "Absolut Error (Surface):            ";
+   cout << abs(surface.Sum() - Surface()) << endl;
+   cout << "Relative Error (Surface):           ";
+   cout << abs(surface.Sum() - Surface()) / Surface() << endl;
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      cout << "--------------------------------------------" << endl;
+      cout << "Computed value of volume integral:  " << volume.Sum() << endl;
+      cout << "True value of volume integral:      " << Volume() << endl;
+      cout << "Absolut Error (Volume):             ";
+      cout << abs(volume.Sum() - Volume()) << endl;
+      cout << "Relative Error (Volume):            ";
+      cout << abs(volume.Sum() - Volume()) / Volume() << endl;
+   }
+   cout << "============================================" << endl;
+
+   // 8. Plot the level-set function on a high order finite element space.
+   if (visualization)
+   {
+      H1_FECollection fe_coll2(5, mesh->Dimension());
+      FiniteElementSpace fespace2(mesh, &fe_coll2);
+      FunctionCoefficient levelset_coeff(levelset);
+      GridFunction lgf(&fespace2);
+      lgf.ProjectCoefficient(levelset_coeff);
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << lgf << flush;
+      sol_sock << "keys pppppppppppppppppppppppppppcmmlRj\n";
+      sol_sock << "levellines " << 0. << " " << 0. << " " << 1 << "\n" << flush;
+   }
+
+   delete sir;
+   delete cir;
+   delete fespace;
+   delete mesh;
+   return EXIT_SUCCESS;
+#endif //MFEM_USE_LAPACK
+}
@@ -3,28 +3,28 @@
 //
 // Compile with: make ex1
 //
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/star-mixed.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/fichera-mixed.mesh
-//               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
+// Sample runs:  ex1 -m ../../data/square-disc.mesh
+//               ex1 -m ../../data/star.mesh
+//               ex1 -m ../../data/star-mixed.mesh
+//               ex1 -m ../../data/escher.mesh
+//               ex1 -m ../../data/fichera.mesh
+//               ex1 -m ../../data/fichera-mixed.mesh
+//               ex1 -m ../../data/toroid-wedge.mesh
+//               ex1 -m ../../data/square-disc-p2.vtk -o 2
+//               ex1 -m ../../data/square-disc-p3.mesh -o 3
+//               ex1 -m ../../data/square-disc-nurbs.mesh -o -1
+//               ex1 -m ../../data/star-mixed-p2.mesh -o 2
+//               ex1 -m ../../data/disc-nurbs.mesh -o -1
+//               ex1 -m ../../data/pipe-nurbs.mesh -o -1
+//               ex1 -m ../../data/fichera-mixed-p2.mesh -o 2
+//               ex1 -m ../../data/star-surf.mesh
+//               ex1 -m ../../data/square-disc-surf.mesh
+//               ex1 -m ../../data/inline-segment.mesh
+//               ex1 -m ../../data/amr-quad.mesh
+//               ex1 -m ../../data/amr-hex.mesh
+//               ex1 -m ../../data/fichera-amr.mesh
+//               ex1 -m ../../data/mobius-strip.mesh
+//               ex1 -m ../../data/mobius-strip.mesh -o -1 -sc
 //
 // Device sample runs:
 //               ex1 -pa -d cuda
@@ -32,7 +32,7 @@
 //               ex1 -pa -d occa-cuda
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
-//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
+//               ex1 -m ../../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -22,15 +22,19 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
-  ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-  ex31 ex33 ex34 ex36 ex37
+   ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
+   ex31 ex33 ex34 ex36 ex37
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
-  ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
-  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-	ex37p
+   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
+   ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
+   ex37p
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-  ex24p ex25p ex26p ex34p ex35p
+   ex24p ex25p ex26p ex34p ex35p
+
+ifeq ($(MFEM_USE_LAPACK),YES)
+   SEQ_EXAMPLES += ex38
+endif

 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
@@ -273,12 +273,13 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
         if (sp_solver)
         {
-            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            STRUMPACKSolver * strumpack = new STRUMPACKSolver(MPI_COMM_WORLD, argc, argv);
            strumpack->SetPrintFactorStatistics(true);
            strumpack->SetPrintSolveStatistics(false);
            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            strumpack->DisableMatching();
+            strumpack->SetMatching(strumpack::MatchingJob::NONE);
+            strumpack->SetCompression(strumpack::CompressionType::NONE);
            strumpack->SetOperator(*Arow);
            strumpack->SetFromCommandLine();
            precond = strumpack;
@@ -0,0 +1,246 @@
+//                       MFEM Example 1 - Parallel Version
+//
+// Compile with: make stokes
+//
+// Sample runs:
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void rhs_func(const Vector & x, Vector &f)
+{
+   f.SetSize(x.Size());
+   f = 0.0;
+}
+
+void u0_func(const Vector & x, Vector &u0)
+{
+   u0.SetSize(x.Size());
+   u0 = 0.0;
+   // u0[0] = -sin(M_PI * x[1]);
+   u0[0] = -x[1]*(1-x[1])*x[2]*(1-x[2]);
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/inline-hex.mesh";
+   int sref = 0;
+   int pref = 0;
+   bool visualization = true;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sref, "-sr", "--refinements-serial",
+                  "Number of serial refinements");
+   args.AddOption(&pref, "-pr", "--refinements-parallel",
+                  "Number of parallel refinements");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // Mesh mesh(mesh_file, 1, 1);
+
+   Mesh mesh = Mesh::MakeCartesian3D(8,1,1,mfem::Element::HEXAHEDRON,8.0,1.0,1.0);
+   int dim = mesh.Dimension();
+
+   for (int l = 0; l < sref; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int l = 0; l < pref; l++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+   FiniteElementCollection *ufec = new H1_FECollection(2,dim);
+   FiniteElementCollection *pfec = new H1_FECollection(1,dim);
+
+   ParFiniteElementSpace ufes(&pmesh, ufec,dim,mfem::Ordering::byVDIM);
+   ParFiniteElementSpace pfes(&pmesh, pfec);
+   Array<ParFiniteElementSpace *> pfespaces;
+   pfespaces.Append(&ufes);
+   pfespaces.Append(&pfes);
+   HYPRE_BigInt usize = ufes.GlobalTrueVSize();
+   HYPRE_BigInt psize = pfes.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of velocity dofs: " << usize << endl;
+      cout << "Number of pressure dofs: " << psize << endl;
+   }
+
+   ParGridFunction u_gf(&ufes); u_gf = 0.0;
+   ParGridFunction p_gf(&pfes); p_gf = 0.0;
+   VectorFunctionCoefficient u0_cf(dim, u0_func);
+   Array<int> ess_bdr_in(pmesh.bdr_attributes.Max()); ess_bdr_in = 0;
+   Array<int> ess_bdr(pmesh.bdr_attributes.Max()); ess_bdr = 1;
+   ess_bdr_in[2] = 1; // x = 1 plane
+   ess_bdr[2] = 0; // x = 1 plane
+   ess_bdr[4] = 0; // x = 0 plane
+   Vector vzero(dim); vzero = 0.0;
+   VectorConstantCoefficient zero_cf(vzero);
+   u_gf.ProjectBdrCoefficient(zero_cf,ess_bdr);
+   u_gf.ProjectBdrCoefficient(u0_cf,ess_bdr_in);
+
+   // essential dofs;
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listp;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> essbdr(pmesh.bdr_attributes.Max());
+      essbdr = 1; essbdr[4] = 0;
+      ufes.GetEssentialTrueDofs(essbdr, ess_tdof_list);
+      essbdr = 0; essbdr[4] = 1;
+      pfes.GetEssentialTrueDofs(essbdr, ess_tdof_listp);
+   }
+
+   ParBilinearForm prec_pq(&pfes);
+   ConstantCoefficient one(1.0);
+   prec_pq.AddDomainIntegrator(new MassIntegrator(one));
+   prec_pq.Assemble();
+   HypreParMatrix Mpq;
+   prec_pq.FormSystemMatrix(ess_tdof_listp,Mpq);
+
+   for (int i = 0; i < ess_tdof_listp.Size(); i++)
+   {
+      ess_tdof_listp[i] += ufes.GetTrueVSize();
+   }
+   ess_tdof_list.Append(ess_tdof_listp);
+
+   // linear and bilinear forms
+   ParLinearForm b_v(&ufes);
+   VectorFunctionCoefficient rhs_cf(dim, rhs_func);
+   b_v.AddDomainIntegrator(new VectorDomainLFIntegrator(rhs_cf));
+   b_v.Assemble();
+
+   ParLinearForm b_q(&pfes);
+   b_q.Assemble();
+
+
+   ConstantCoefficient negone(-1.0);
+   // (∇ u, ∇ v)
+   ParBilinearForm a_uv(&ufes);
+   a_uv.AddDomainIntegrator(new VectorDiffusionIntegrator(one));
+
+   ParMixedBilinearForm a_pv(&pfes,&ufes);
+   a_pv.AddDomainIntegrator(new TransposeIntegrator(new VectorDivergenceIntegrator(
+                                                       negone)));
+
+   ParMixedBilinearForm a_uq(&ufes,&pfes);
+   a_uq.AddDomainIntegrator(new VectorDivergenceIntegrator(negone));
+
+   ParBilinearForm a_pq(&pfes);
+   ConstantCoefficient zero(0.0);
+   a_pq.AddDomainIntegrator(new MassIntegrator(zero));
+
+   ParBlockForm bform(pfespaces);
+   bform.SetBlock(&a_uv,0,0);
+   bform.SetBlock(&a_pv,0,1);
+   bform.SetBlock(&a_uq,1,0);
+   bform.SetBlock(&a_pq,1,1);
+
+   Array<int> tdof_offsets(3);
+   tdof_offsets[0] = 0;
+   tdof_offsets[1] = ufes.TrueVSize();
+   tdof_offsets[2] = pfes.TrueVSize();
+   tdof_offsets.PartialSum();
+   Array<int> dof_offsets(3);
+   dof_offsets[0] = 0;
+   dof_offsets[1] = ufes.GetVSize();
+   dof_offsets[2] = pfes.GetVSize();
+   dof_offsets.PartialSum();
+
+   Vector b(dof_offsets.Last());
+   b.SetVector(b_v,0);
+   b.SetVector(b_q,dof_offsets[1]);
+   Vector x(dof_offsets.Last());
+   x.SetVector(u_gf,0);
+   x.SetVector(p_gf,dof_offsets[1]);
+
+   OperatorPtr Ah;
+   Vector B,X;
+   bform.Assemble();
+   bform.FormLinearSystem(ess_tdof_list,x, b, Ah, X, B);
+
+   BlockOperator * A = Ah.As<BlockOperator>();
+
+   BlockDiagonalPreconditioner prec(tdof_offsets);
+   HypreBoomerAMG amg_v((HypreParMatrix&)A->GetBlock(0,0));
+   amg_v.SetSystemsOptions(dim);
+
+   HypreBoomerAMG amg_p(Mpq);
+
+   prec.SetDiagonalBlock(0,&amg_v);
+   prec.SetDiagonalBlock(1,&amg_p);
+
+
+   // ((HypreParMatrix*)&A->GetBlock(0,0))->Print("A00.mat");
+   // // ((HypreParMatrix*)&A->GetBlock(0,1))->Print("A01.mat");
+   // ((HypreParMatrix*)&A->GetBlock(1,0))->Print("A10.mat");
+   // // ((HypreParMatrix*)&A->GetBlock(1,1))->Print("A11.mat");
+   // return 0;
+
+
+   MINRESSolver solver(MPI_COMM_WORLD);
+   solver.SetRelTol(1e-12);
+   solver.SetMaxIter(60);
+   solver.SetPrintLevel(1);
+   solver.SetPreconditioner(prec);
+   solver.SetOperator(*A);
+   solver.Mult(B, X);
+
+
+   BlockVector Xb(X.GetData(),tdof_offsets);
+   u_gf.SetFromTrueDofs(Xb.GetBlock(0));
+   p_gf.SetFromTrueDofs(Xb.GetBlock(1));
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << u_gf << flush;
+
+      socketstream solp_sock(vishost, visport);
+      solp_sock << "parallel " << num_procs << " " << myid << "\n";
+      solp_sock.precision(8);
+      solp_sock << "solution\n" << pmesh << p_gf << flush;
+
+   }
+
+   delete ufec;
+   delete pfec;
+
+   return 0;
+}
@@ -0,0 +1,260 @@
+//                       MFEM Example 1 - Parallel Version
+//
+// Compile with: make stokes
+//
+// Sample runs:
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void rhs_func(const Vector & x, Vector &f)
+{
+   f.SetSize(x.Size());
+   f = 0.0;
+}
+
+void u0_func(const Vector & x, Vector &u0)
+{
+   u0.SetSize(x.Size());
+   u0 = 0.0;
+   double r0 = 2.393;
+   u0[0] = -(r0*r0 - x[1]*x[1] - x[2]*x[2]);
+   // u0[0] = +1;
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "anu.msh";
+   int sref = 0;
+   int pref = 0;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sref, "-sr", "--refinements-serial",
+                  "Number of serial refinements");
+   args.AddOption(&pref, "-pr", "--refinements-parallel",
+                  "Number of parallel refinements");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+
+   int dim = mesh.Dimension();
+
+   for (int l = 0; l < sref; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int l = 0; l < pref; l++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+
+   FiniteElementCollection *ufec = new H1_FECollection(2,dim);
+   FiniteElementCollection *pfec = new H1_FECollection(1,dim);
+
+   ParFiniteElementSpace ufes(&pmesh, ufec,dim,mfem::Ordering::byVDIM);
+   ParFiniteElementSpace pfes(&pmesh, pfec);
+   Array<ParFiniteElementSpace *> pfespaces;
+   pfespaces.Append(&ufes);
+   pfespaces.Append(&pfes);
+
+   HYPRE_BigInt usize = ufes.GlobalTrueVSize();
+   HYPRE_BigInt psize = pfes.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of velocity dofs: " << usize << endl;
+      cout << "Number of pressure dofs: " << psize << endl;
+   }
+
+   ParGridFunction u_gf(&ufes); u_gf = 0.0;
+   ParGridFunction p_gf(&pfes); p_gf = 0.0;
+   VectorFunctionCoefficient u0_cf(dim, u0_func);
+   Array<int> ess_bdr_in(pmesh.bdr_attributes.Max()); ess_bdr_in = 0;
+   Array<int> ess_bdr(pmesh.bdr_attributes.Max()); ess_bdr = 1;
+   ess_bdr_in[1] = 1; // inflow
+   ess_bdr[0] = 0; // rest of bdr (except the outflow)
+   Vector vzero(dim); vzero = 0.0;
+   VectorConstantCoefficient zero_cf(vzero);
+   u_gf.ProjectBdrCoefficient(zero_cf,ess_bdr);
+   u_gf.ProjectBdrCoefficient(u0_cf,ess_bdr_in);
+
+   // essential dofs;
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listp;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> essbdr(pmesh.bdr_attributes.Max());
+      essbdr = 1; essbdr[2] = 0;
+      ufes.GetEssentialTrueDofs(essbdr, ess_tdof_list);
+      essbdr = 0; essbdr[2] = 1;
+      // pfes.GetEssentialTrueDofs(essbdr, ess_tdof_listp);
+   }
+   ConstantCoefficient one(1.0);
+
+   ParBilinearForm prec_pq(&pfes);
+   prec_pq.AddDomainIntegrator(new MassIntegrator(one));
+   prec_pq.Assemble();
+   HypreParMatrix Mpq;
+   prec_pq.FormSystemMatrix(ess_tdof_listp,Mpq);
+
+   for (int i = 0; i < ess_tdof_listp.Size(); i++)
+   {
+      ess_tdof_listp[i] += ufes.GetTrueVSize();
+   }
+   ess_tdof_list.Append(ess_tdof_listp);
+
+   // linear and bilinear forms
+   ParLinearForm b_v(&ufes);
+   VectorFunctionCoefficient rhs_cf(dim, rhs_func);
+   b_v.AddDomainIntegrator(new VectorDomainLFIntegrator(rhs_cf));
+   b_v.Assemble();
+
+   ParLinearForm b_q(&pfes);
+   b_q.Assemble();
+
+
+   ConstantCoefficient negone(-1.0);
+
+   // (∇ u, ∇ v)
+   ParBilinearForm a_uv(&ufes);
+   a_uv.AddDomainIntegrator(new VectorDiffusionIntegrator(one));
+   ParMixedBilinearForm a_pv(&pfes,&ufes);
+   a_pv.AddDomainIntegrator(new TransposeIntegrator(new VectorDivergenceIntegrator(
+                                                       negone)));
+
+   ParMixedBilinearForm a_uq(&ufes,&pfes);
+   a_uq.AddDomainIntegrator(new VectorDivergenceIntegrator(negone));
+
+   ParBilinearForm a_pq(&pfes);
+
+   ParBlockForm bform(pfespaces);
+   bform.SetBlock(&a_uv,0,0);
+   bform.SetBlock(&a_pv,0,1);
+   bform.SetBlock(&a_uq,1,0);
+   bform.SetBlock(&a_pq,1,1);
+
+
+   Array<int> tdof_offsets(3);
+   tdof_offsets[0] = 0;
+   tdof_offsets[1] = ufes.TrueVSize();
+   tdof_offsets[2] = pfes.TrueVSize();
+   tdof_offsets.PartialSum();
+   Array<int> dof_offsets(3);
+   dof_offsets[0] = 0;
+   dof_offsets[1] = ufes.GetVSize();
+   dof_offsets[2] = pfes.GetVSize();
+   dof_offsets.PartialSum();
+
+   Vector b(dof_offsets.Last());
+   b.SetVector(b_v,0);
+   b.SetVector(b_q,dof_offsets[1]);
+   Vector x(dof_offsets.Last());
+   x.SetVector(u_gf,0);
+   x.SetVector(p_gf,dof_offsets[1]);
+
+   OperatorPtr Ah;
+   Vector B,X;
+   bform.Assemble();
+   bform.FormLinearSystem(ess_tdof_list,x, b, Ah, X, B);
+
+   BlockOperator * A = Ah.As<BlockOperator>();
+
+   BlockDiagonalPreconditioner prec(tdof_offsets);
+   HypreParMatrix A00 = (HypreParMatrix&)A->GetBlock(0,0);
+   HypreParMatrix A01 = (HypreParMatrix&)A->GetBlock(0,1);
+   HypreBoomerAMG amg_v(A00);
+   amg_v.SetSystemsOptions(dim);
+   prec.SetDiagonalBlock(0,&amg_v);
+
+   HypreParVector A00_diag(MPI_COMM_WORLD, A00.GetGlobalNumRows(),
+                                 A00.GetRowStarts());
+   A00.GetDiag(A00_diag);
+   HypreParMatrix S_tmp(A01);
+   S_tmp.InvScaleRows(A00_diag);
+   HypreParMatrix *S = ParMult(A01.Transpose(), &S_tmp, true);
+   HypreBoomerAMG amg_p(*S);
+   // HypreBoomerAMG amg_p(Mpq);
+   prec.SetDiagonalBlock(1,&amg_p);
+
+   MINRESSolver solver(MPI_COMM_WORLD);
+   solver.SetRelTol(1e-12);
+   solver.SetMaxIter(20000);
+   solver.SetPrintLevel(1);
+   solver.SetPreconditioner(prec);
+   solver.SetOperator(*A);
+   solver.Mult(B, X);
+
+   BlockVector Xb(X.GetData(),tdof_offsets);
+   u_gf.SetFromTrueDofs(Xb.GetBlock(0));
+   p_gf.SetFromTrueDofs(Xb.GetBlock(1));
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   bool paraview = true;
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(mesh_file, &pmesh);
+      paraview_dc->SetPrefixPath("ParaView");
+      paraview_dc->SetLevelsOfDetail(2);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("velocity",&u_gf);
+      paraview_dc->RegisterField("pressure",&p_gf);
+      paraview_dc->Save();
+   }
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << u_gf << flush;
+
+      socketstream solp_sock(vishost, visport);
+      solp_sock << "parallel " << num_procs << " " << myid << "\n";
+      solp_sock.precision(8);
+      solp_sock << "solution\n" << pmesh << p_gf << flush;
+
+   }
+
+   delete ufec;
+   delete pfec;
+
+   return 0;
+}
@@ -77,6 +77,7 @@ set(SRCS
  gridfunc.cpp
  hybridization.cpp
  intrules.cpp
+  intrules_cut.cpp
  ceed/interface/basis.cpp
  ceed/interface/restriction.cpp
  ceed/interface/operator.cpp
@@ -96,9 +97,6 @@ set(SRCS
  lor/lor_ads.cpp
  lor/lor_ams.cpp
  lor/lor_batched.cpp
-  lor/lor_h1.cpp
-  lor/lor_nd.cpp
-  lor/lor_rt.cpp
  multigrid.cpp
  nonlinearform.cpp
  nonlinearform_ext.cpp
@@ -186,6 +184,7 @@ set(HDRS
  gridfunc.hpp
  hybridization.hpp
  intrules.hpp
+  intrules_cut.hpp
  ceed/interface/basis.hpp
  ceed/interface/integrator.hpp
  ceed/interface/interface.hpp
@@ -269,7 +268,8 @@ if (MFEM_USE_MPI)
    pgridfunc.cpp
    plinearform.cpp
    pnonlinearform.cpp
-    prestriction.cpp)
+    prestriction.cpp
+    pblockform.cpp)
  # If this list (HDRS -> HEADERS) is used for install, we probably want the
  # headers added all the time.
  list(APPEND HDRS
@@ -278,7 +278,8 @@ if (MFEM_USE_MPI)
    pgridfunc.hpp
    plinearform.hpp
    pnonlinearform.hpp
-    prestriction.hpp)
+    prestriction.hpp
+    pblockform.hpp)
 endif()

 convert_filenames_to_full_paths(SRCS)
@@ -101,6 +101,7 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)

   // Copy the pointers to the integrators
   domain_integs = bf->domain_integs;
+   domain_integs_marker = bf->domain_integs_marker;

   boundary_integs = bf->boundary_integs;
   boundary_integs_marker = bf->boundary_integs_marker;
@@ -433,6 +434,9 @@ void BilinearForm::Assemble(int skip_zeros)
      // Element-wise integration
      for (int i = 0; i < fes -> GetNE(); i++)
      {
+         // Set both doftrans (potentially needed to assemble the element
+         // matrix) and vdofs, which is also needed when the element matrices
+         // are pre-assembled.
         doftrans = fes->GetElementVDofs(i, vdofs);
         if (element_matrices)
         {
@@ -441,6 +445,8 @@ void BilinearForm::Assemble(int skip_zeros)
         else
         {
            const int elem_attr = fes->GetMesh()->GetAttribute(i);
+            eltrans = fes->GetElementTransformation(i);
+
            elmat.SetSize(0);
            for (int k = 0; k < domain_integs.Size(); k++)
            {
@@ -448,9 +454,8 @@ void BilinearForm::Assemble(int skip_zeros)
                    (*(domain_integs_marker[k]))[elem_attr-1] == 1)
                   && !domain_integs[k]->Patchwise())
               {
-                  const FiniteElement &fe = *fes->GetFE(i);
-                  eltrans = fes->GetElementTransformation(i);
-                  domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+                  domain_integs[k]->AssembleElementMatrix(*fes->GetFE(i),
+                                                          *eltrans, elemmat);
                  if (elmat.Size() == 0)
                  {
                     elmat = elemmat;
@@ -1072,7 +1077,8 @@ void BilinearForm::EliminateEssentialBCFromDofs(
 void BilinearForm::EliminateEssentialBCFromDofs (const Array<int> &ess_dofs,
                                                 DiagonalPolicy dpolicy)
 {
-   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+   MFEM_ASSERT(ess_dofs.Size() == height,
+               "incorrect dof Array size: " << ess_dofs.Size() << ' ' << height);

   for (int i = 0; i < ess_dofs.Size(); i++)
      if (ess_dofs[i] < 0)
@@ -1084,7 +1090,8 @@ void BilinearForm::EliminateEssentialBCFromDofs (const Array<int> &ess_dofs,
 void BilinearForm::EliminateEssentialBCFromDofsDiag (const Array<int> &ess_dofs,
                                                     double value)
 {
-   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+   MFEM_ASSERT(ess_dofs.Size() == height,
+               "incorrect dof Array size: " << ess_dofs.Size() << ' ' << height);

   for (int i = 0; i < ess_dofs.Size(); i++)
      if (ess_dofs[i] < 0)
@@ -1222,11 +1229,14 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,

   // Copy the pointers to the integrators
   domain_integs = mbf->domain_integs;
-   boundary_integs = mbf->boundary_integs;
-   trace_face_integs = mbf->trace_face_integs;
-   boundary_trace_face_integs = mbf->boundary_trace_face_integs;
+   domain_integs_marker = mbf->domain_integs_marker;

+   boundary_integs = mbf->boundary_integs;
   boundary_integs_marker = mbf->boundary_integs_marker;
+
+   trace_face_integs = mbf->trace_face_integs;
+
+   boundary_trace_face_integs = mbf->boundary_trace_face_integs;
   boundary_trace_face_integs_marker = mbf->boundary_trace_face_integs_marker;

   assembly = AssemblyLevel::LEGACY;
@@ -1349,6 +1359,14 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const
 void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
   domain_integs.Append (bfi);
+   domain_integs_marker.Append(NULL); // NULL marker means apply everywhere
+}
+
+void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi,
+                                             Array<int> &elem_marker)
+{
+   domain_integs.Append (bfi);
+   domain_integs_marker.Append(&elem_marker);
 }

 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
@@ -1383,7 +1401,7 @@ void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
   boundary_trace_face_integs_marker.Append(&bdr_marker);
 }

-void MixedBilinearForm::Assemble (int skip_zeros)
+void MixedBilinearForm::Assemble(int skip_zeros)
 {
   if (ext)
   {
@@ -1405,8 +1423,20 @@ void MixedBilinearForm::Assemble (int skip_zeros)

   if (domain_integs.Size())
   {
+      for (int k = 0; k < domain_integs.Size(); k++)
+      {
+         if (domain_integs_marker[k] != NULL)
+         {
+            MFEM_VERIFY(domain_integs_marker[k]->Size() ==
+                        (mesh->attributes.Size() ? mesh->attributes.Max() : 0),
+                        "invalid element marker for domain integrator #"
+                        << k << ", counting from zero");
+         }
+      }
+
      for (int i = 0; i < test_fes -> GetNE(); i++)
      {
+         const int elem_attr = mesh->GetAttribute(i);
         dom_dof_trans = trial_fes -> GetElementVDofs (i, trial_vdofs);
         ran_dof_trans = test_fes  -> GetElementVDofs (i, test_vdofs);
         eltrans = test_fes -> GetElementTransformation (i);
@@ -1415,10 +1445,14 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         elmat = 0.0;
         for (int k = 0; k < domain_integs.Size(); k++)
         {
-            domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
-                                                        *test_fes  -> GetFE(i),
-                                                        *eltrans, elemmat);
-            elmat += elemmat;
+            if (domain_integs_marker[k] == NULL ||
+                (*(domain_integs_marker[k]))[elem_attr-1] == 1)
+            {
+               domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
+                                                           *test_fes  -> GetFE(i),
+                                                           *eltrans, elemmat);
+               elmat += elemmat;
+            }
         }
         if (ran_dof_trans || dom_dof_trans)
         {
@@ -1941,41 +1975,56 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      return;
   }

-   Array<int> dom_vdofs, ran_vdofs;
-   ElementTransformation *T;
+   ElementTransformation *eltrans;
   DofTransformation * dom_dof_trans;
   DofTransformation * ran_dof_trans;
-   const FiniteElement *dom_fe, *ran_fe;
-   DenseMatrix totelmat, elmat;
+   DenseMatrix elmat;
+
+   Mesh *mesh = test_fes->GetMesh();

   if (mat == NULL)
   {
      mat = new SparseMatrix(height, width);
   }

-   if (domain_integs.Size() > 0)
+   if (domain_integs.Size())
   {
+      for (int k = 0; k < domain_integs.Size(); k++)
+      {
+         if (domain_integs_marker[k] != NULL)
+         {
+            MFEM_VERIFY(domain_integs_marker[k]->Size() ==
+                        (mesh->attributes.Size() ? mesh->attributes.Max() : 0),
+                        "invalid element marker for domain integrator #"
+                        << k << ", counting from zero");
+         }
+      }
+
      for (int i = 0; i < test_fes->GetNE(); i++)
      {
-         dom_dof_trans = trial_fes->GetElementVDofs(i, dom_vdofs);
-         ran_dof_trans = test_fes->GetElementVDofs(i, ran_vdofs);
-         T = test_fes->GetElementTransformation(i);
-         dom_fe = trial_fes->GetFE(i);
-         ran_fe = test_fes->GetFE(i);
+         const int elem_attr = mesh->GetAttribute(i);
+         dom_dof_trans = trial_fes->GetElementVDofs(i, trial_vdofs);
+         ran_dof_trans = test_fes->GetElementVDofs(i, test_vdofs);
+         eltrans = test_fes->GetElementTransformation(i);

-         domain_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                  totelmat);
-         for (int j = 1; j < domain_integs.Size(); j++)
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
+         for (int k = 0; k < domain_integs.Size(); k++)
         {
-            domain_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                     elmat);
-            totelmat += elmat;
+            if (domain_integs_marker[k] == NULL ||
+                (*(domain_integs_marker[k]))[elem_attr-1] == 1)
+            {
+               domain_integs[k]->AssembleElementMatrix2(*trial_fes->GetFE(i),
+                                                        *test_fes->GetFE(i),
+                                                        *eltrans, elemmat);
+               elmat += elemmat;
+            }
         }
         if (ran_dof_trans || dom_dof_trans)
         {
-            TransformPrimal(ran_dof_trans, dom_dof_trans, totelmat);
+            TransformPrimal(ran_dof_trans, dom_dof_trans, elemmat);
         }
-         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
+         mat->SetSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
      }
   }

@@ -1984,21 +2033,20 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      const int nfaces = test_fes->GetMesh()->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
      {
-         trial_fes->GetFaceVDofs(i, dom_vdofs);
-         test_fes->GetFaceVDofs(i, ran_vdofs);
-         T = test_fes->GetMesh()->GetFaceTransformation(i);
-         dom_fe = trial_fes->GetFaceElement(i);
-         ran_fe = test_fes->GetFaceElement(i);
+         trial_fes->GetFaceVDofs(i, trial_vdofs);
+         test_fes->GetFaceVDofs(i, test_vdofs);
+         eltrans = test_fes->GetMesh()->GetFaceTransformation(i);

-         trace_face_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                      totelmat);
-         for (int j = 1; j < trace_face_integs.Size(); j++)
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
+         for (int k = 0; k < trace_face_integs.Size(); k++)
         {
-            trace_face_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                         elmat);
-            totelmat += elmat;
+            trace_face_integs[k]->AssembleElementMatrix2(*trial_fes->GetFaceElement(i),
+                                                         *test_fes->GetFaceElement(i),
+                                                         *eltrans, elemmat);
+            elmat += elemmat;
         }
-         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
+         mat->SetSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }
 }
@@ -100,7 +100,7 @@ protected:
   /// Includes all by default.
   /// 0 - ignore attribute
   /// 1 - include attribute
-   Array<Array<int>*>             domain_integs_marker;
+   Array<Array<int>*> domain_integs_marker; ///< Entries are not owned.

   /// Set of Boundary Integrators to be applied.
   Array<BilinearFormIntegrator*> boundary_integs;
@@ -722,10 +722,13 @@ protected:

   /// Domain integrators.
   Array<BilinearFormIntegrator*> domain_integs;
+   /// Entries are not owned.
+   Array<Array<int>*> domain_integs_marker;

   /// Boundary integrators.
   Array<BilinearFormIntegrator*> boundary_integs;
-   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.
+   /// Entries are not owned.
+   Array<Array<int>*> boundary_integs_marker;

   /// Trace face (skeleton) integrators.
   Array<BilinearFormIntegrator*> trace_face_integs;
@@ -805,12 +808,16 @@ public:
   /// Adds a domain integrator. Assumes ownership of @a bfi.
   void AddDomainIntegrator(BilinearFormIntegrator *bfi);

+   /// Adds a domain integrator. Assumes ownership of @a bfi.
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi,
+                            Array<int> &elem_marker);
+
   /// Adds a boundary integrator. Assumes ownership of @a bfi.
   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi);

   /// Adds a boundary integrator. Assumes ownership of @a bfi.
-   void AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
-                               Array<int> &bdr_marker);
+   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi,
+                              Array<int> &bdr_marker);

   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.

@@ -820,14 +827,18 @@ public:
   void AddTraceFaceIntegrator(BilinearFormIntegrator *bfi);

   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi);
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi);

   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi,
-                                   Array<int> &bdr_marker);
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi,
+                                  Array<int> &bdr_marker);

   /// Access all integrators added with AddDomainIntegrator().
   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }
+   /** @brief Access all domain markers added with AddDomainIntegrator().
+       If no marker was specified when the integrator was added, the
+       corresponding pointer (to Array<int>) will be NULL. */
+   Array<Array<int>*> *GetDBFI_Marker() { return &domain_integs_marker; }

   /// Access all integrators added with AddBoundaryIntegrator().
   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
@@ -1065,6 +1076,9 @@ public:
   /// Adds a domain interpolator. Assumes ownership of @a di.
   void AddDomainInterpolator(DiscreteInterpolator *di)
   { AddDomainIntegrator(di); }
+   void AddDomainInterpolator(DiscreteInterpolator *di,
+                              Array<int> &elem_marker)
+   { AddDomainIntegrator(di, elem_marker); }

   /// Adds a trace face interpolator. Assumes ownership of @a di.
   void AddTraceFaceInterpolator(DiscreteInterpolator *di)
@@ -1072,6 +1086,7 @@ public:

   /// Access all interpolators added with AddDomainInterpolator().
   Array<BilinearFormIntegrator*> *GetDI() { return &domain_integs; }
+   Array<Array<int>*> *GetDI_Marker() { return &domain_integs_marker; }

   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
   /** This method must be called before assembly. */
@@ -303,7 +303,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
      std::unordered_map<int,int> f_to_be;
      for (int i = 0; i < mesh.GetNBE(); ++i)
      {
-         const int f = mesh.GetBdrElementEdgeIndex(i);
+         const int f = mesh.GetBdrElementFaceIndex(i);
         f_to_be[f] = i;
      }
      const int nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary);
@@ -1340,10 +1340,11 @@ void MassIntegrator::AssembleElementMatrix2(
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
-      trial_fe.CalcShape(ip, shape);
-      test_fe.CalcShape(ip, te_shape);
-
      Trans.SetIntPoint (&ip);
+
+      trial_fe.CalcPhysShape(Trans, shape);
+      test_fe.CalcPhysShape(Trans, te_shape);
+
      w = Trans.Weight() * ip.weight;
      if (Q)
      {
@@ -64,7 +64,7 @@ public:
   /// Assemble diagonal and add it to Vector @a diag.
   virtual void AssembleDiagonalPA(Vector &diag);

-   /// Assemble diagonal of ADA^T (A is this integrator) and add it to @a diag.
+   /// Assemble diagonal of \f$ADA^{\mathrm{T}}\f$ (\f$A\f$ is this integrator) and add it to @a diag.
   virtual void AssembleDiagonalPA_ADAt(const Vector &D, Vector &diag);

   /// Method for partially assembled action.
@@ -137,8 +137,8 @@ public:
                                      DenseMatrix &elmat);

   /** Compute the local matrix representation of a bilinear form
-       a(u,v) defined on different trial (given by u) and test
-       (given by v) spaces. The rows in the local matrix correspond
+       \f$a(u,v)\f$ defined on different trial (given by \f$u\f$) and test
+       (given by \f$v\f$) spaces. The rows in the local matrix correspond
       to the test dofs and the columns -- to the trial dofs. */
   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
                                       const FiniteElement &test_fe,
@@ -706,9 +706,9 @@ private:

 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, v) in either 1D, 2D,
-    or 3D and where Q is an optional scalar coefficient, u and v are each in H1
-    or L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q u, v)\f$ in either 1D, 2D,
+    or 3D and where \f$Q\f$ is an optional scalar coefficient, \f$u\f$ and \f$v\f$ are each in \f$H^1\f$
+    or \f$L_2\f$. */
 class MixedScalarMassIntegrator : public MixedScalarIntegrator
 {
 public:
@@ -717,9 +717,9 @@ public:
      : MixedScalarIntegrator(q) { same_calc_shape = true; }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, v) in either 2D, or
-    3D and where Q is a vector coefficient, u is in H1 or L2 and v is in H(Curl)
-    or H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} u, v)\f$ in either 2D, or
+    3D and where \f$\vec{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H^1\f$ or \f$L_2\f$ and \f$v\f$ is in \f$H\f$(curl)
+    or \f$H\f$(div). */
 class MixedVectorProductIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -727,8 +727,8 @@ public:
      : MixedScalarVectorIntegrator(vq) {}
 };

-/** Class for integrating the bilinear form a(u,v) := (Q D u, v) in 1D where Q
-    is an optional scalar coefficient, u is in H1, and v is in L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla u, v)\f$ in 1D where Q
+    is an optional scalar coefficient, \f$u\f$ is in \f$H^1\f$, and \f$v\f$ is in \f$L_2\f$. */
 class MixedScalarDerivativeIntegrator : public MixedScalarIntegrator
 {
 public:
@@ -762,8 +762,8 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := -(Q u, D v) in 1D where Q
-    is an optional scalar coefficient, u is in L2, and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := -(Q u, \nabla v)\f$ in 1D where \f$Q\f$
+    is an optional scalar coefficient, \f$u\f$ is in \f$L_2\f$, and \f$v\f$ is in \f$H^1\f$. */
 class MixedScalarWeakDerivativeIntegrator : public MixedScalarIntegrator
 {
 public:
@@ -799,8 +799,8 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q div u, v) in either 2D
-    or 3D where Q is an optional scalar coefficient, u is in H(Div), and v is a
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla \cdot u, v)\f$ in either 2D
+    or 3D where \f$Q\f$ is an optional scalar coefficient, \f$u\f$ is in \f$H\f$(div), and \f$v\f$ is a
    scalar field. */
 class MixedScalarDivergenceIntegrator : public MixedScalarIntegrator
 {
@@ -821,7 +821,7 @@ protected:
   inline virtual const char * FiniteElementTypeFailureMessage() const
   {
      return "MixedScalarDivergenceIntegrator:  "
-             "Trial must be H(Div) and the test space must be a "
+             "Trial must be \f$H\f$(div) and the test space must be a "
             "scalar field";
   }

@@ -836,9 +836,8 @@ protected:
   { trial_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V div u, v) in either 2D
-    or 3D where V is a vector coefficient, u is in H(Div), and v is a vector
-    field. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \nabla \cdot u, v)\f$ in either 2D
+    or 3D where \f$\vec{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H\f$(div), and \f$v\f$ is in \f$H\f$(div). */
 class MixedVectorDivergenceIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -874,9 +873,9 @@ protected:
   { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := -(Q u, div v) in either 2D
-    or 3D where Q is an optional scalar coefficient, u is in L2 or H1, and v is
-    in H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := -(Q u, \nabla \cdot v)\f$ in either 2D
+    or 3D where \f$Q\f$ is an optional scalar coefficient, \f$u\f$ is in \f$L_2\f$ or \f$H^1\f$, and \f$v\f$ is
+    in \f$H\f$(div). */
 class MixedScalarWeakGradientIntegrator : public MixedScalarIntegrator
 {
 public:
@@ -914,9 +913,9 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) in 2D where
-    Q is an optional scalar coefficient, u is in H(Curl), and v is in L2 or
-    H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \mathrm{curl}(u), v)\f$ in 2D where
+    \f$Q\f$ is an optional scalar coefficient, \f$u\f$ is in \f$H\f$(curl), and \f$v\f$ is in \f$L_2\f$ or
+    \f$H^1\f$. */
 class MixedScalarCurlIntegrator : public MixedScalarIntegrator
 {
 public:
@@ -968,9 +967,9 @@ protected:
   int dim, ne, dofs1D, quad1D, dofs1Dtest;
 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, curl v) in 2D where
-    Q is an optional scalar coefficient, u is in L2 or H1, and v is in
-    H(Curl). Partial assembly (PA) is supported but could be further optimized
+/** Class for integrating the bilinear form \f$a(u,v) := (Q u, \mathrm{curl}(v))\f$ in 2D where
+    \f$Q\f$ is an optional scalar coefficient, \f$u\f$ is in \f$L_2\f$ or \f$H^1\f$, and \f$v\f$ is in
+    \f$H\f$(curl). Partial assembly (PA) is supported but could be further optimized
    by using more efficient threading and shared memory.
 */
 class MixedScalarWeakCurlIntegrator : public MixedScalarIntegrator
@@ -1006,9 +1005,9 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, v) in either 2D or
-    3D and where Q is an optional coefficient (of type scalar, matrix, or
-    diagonal matrix) u and v are each in H(Curl) or H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q u, v)\f$ in either 2D or
+    3D and where \f$Q\f$ is an optional coefficient (of type scalar, matrix, or
+    diagonal matrix) \f$u\f$ and \f$v\f$ are each in \f$H\f$(curl) or \f$H\f$(div). */
 class MixedVectorMassIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1021,8 +1020,8 @@ public:
      : MixedVectorIntegrator(mq) { same_calc_shape = true; }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x u, v) in 3D and where
-    V is a vector coefficient u and v are each in H(Curl) or H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times u, v)\f$ in 3D and where
+    \f$\vec{V}\f$ is a vector coefficient \f$u\f$ and \f$v\f$ are each in \f$H\f$(curl) or \f$H\f$(div). */
 class MixedCrossProductIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1030,9 +1029,9 @@ public:
      : MixedVectorIntegrator(vq, false) { same_calc_shape = true; }
 };

-/** Class for integrating the bilinear form a(u,v) := (V . u, v) in 2D or 3D and
-    where V is a vector coefficient u is in H(Curl) or H(Div) and v is in H1 or
-    L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \cdot u, v)\f$ in 2D or 3D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in \f$H^1\f$ or
+    \f$L_2\f$. */
 class MixedDotProductIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1055,9 +1054,9 @@ public:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (-V . u, Div v) in 2D or
-    3D and where V is a vector coefficient u is in H(Curl) or H(Div) and v is in
-    RT. */
+/** Class for integrating the bilinear form \f$a(u,v) := (-\vec{V} \cdot u, \nabla \cdot v)\f$ in 2D or
+    3D and where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in
+    \f$H\f$(div). */
 class MixedWeakGradDotIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1093,8 +1092,8 @@ public:
   { scalar_fe.CalcPhysDivShape(Trans, shape); shape *= -1.0; }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x u, Grad v) in 3D and
-    where V is a vector coefficient u is in H(Curl) or H(Div) and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (v \vec{V} \times u, \nabla v)\f$ in 3D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in \f$H^1\f$. */
 class MixedWeakDivCrossIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1127,9 +1126,9 @@ public:
   { test_fe.CalcPhysDShape(Trans, shape); shape *= -1.0; }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q Grad u, Grad v) in 3D
-    or in 2D and where Q is a scalar or matrix coefficient u and v are both in
-    H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla u, \nabla v)\f$ in 3D
+    or in 2D and where \f$Q\f$ is a scalar or matrix coefficient \f$u\f$ and \f$v\f$ are both in
+    \f$H^1\f$. */
 class MixedGradGradIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1185,8 +1184,8 @@ public:
   { test_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Grad u, Grad v) in 3D
-    or in 2D and where V is a vector coefficient u and v are both in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \nabla u, \nabla v)\f$ in 3D
+    or in 2D and where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ and \f$v\f$ are both in \f$H^1\f$. */
 class MixedCrossGradGradIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1227,9 +1226,9 @@ public:
   { test_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q Curl u, Curl v) in 3D
-    and where Q is a scalar or matrix coefficient u and v are both in
-    H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \mathrm{curl}(u), \mathrm{curl}(v))\f$ in 3D
+    and where \f$Q\f$ is a scalar or matrix coefficient \f$u\f$ and \f$v\f$ are both in
+    \f$H\f$(curl). */
 class MixedCurlCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1276,8 +1275,8 @@ public:
   { test_fe.CalcPhysCurlShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Curl u, Curl v) in 3D
-    and where V is a vector coefficient u and v are both in H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \mathrm{curl}(u), \mathrm{curl}(v))\f$ in 3D
+    and where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ and \f$v\f$ are both in \f$H\f$(curl). */
 class MixedCrossCurlCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1320,8 +1319,8 @@ public:
   { test_fe.CalcPhysCurlShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Curl u, Grad v) in 3D
-    and where V is a vector coefficient u is in H(Curl) and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \mathrm{curl}(u), \nabla \cdot v)\f$ in 3D
+    and where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) and \f$v\f$ is in \f$H^1\f$. */
 class MixedCrossCurlGradIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1363,8 +1362,8 @@ public:
   { test_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Grad u, Curl v) in 3D
-    and where V is a scalar coefficient u is in H1 and v is in H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (v \times \nabla \cdot u, \mathrm{curl}(v))\f$ in 3D
+    and where \f$v\f$ is a scalar coefficient \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H\f$(curl). */
 class MixedCrossGradCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1406,9 +1405,9 @@ public:
   { test_fe.CalcPhysCurlShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x u, Curl v) in 3D and
-    where V is a vector coefficient u is in H(Curl) or H(Div) and v is in
-    H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times u, \mathrm{curl}(v))\f$ in 3D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in
+    \f$H\f$(curl). */
 class MixedWeakCurlCrossIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1441,9 +1440,9 @@ public:
   { test_fe.CalcPhysCurlShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x u, Curl v) in 2D and
-    where V is a vector coefficient u is in H(Curl) or H(Div) and v is in
-    H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times u, \mathrm{curl}(v))\f$ in 2D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in
+    \f$H\f$(curl). */
 class MixedScalarWeakCurlCrossIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1476,9 +1475,9 @@ public:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Grad u, v) in 3D or
-    in 2D and where V is a vector coefficient u is in H1 and v is in H(Curl) or
-    H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \nabla \cdot u, v)\f$ in 3D or
+    in 2D and where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H\f$(curl) or
+    \f$H\f$(div). */
 class MixedCrossGradIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1516,9 +1515,9 @@ public:
   { test_fe.CalcVShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Curl u, v) in 3D and
-    where V is a vector coefficient u is in H(Curl) and v is in H(Curl) or
-    H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \mathrm{curl}(u), v)\f$ in 3D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) and \f$v\f$ is in \f$H\f$(curl) or
+    \f$H\f$(div). */
 class MixedCrossCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1551,9 +1550,9 @@ public:
   { trial_fe.CalcPhysCurlShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Curl u, v) in 2D and
-    where V is a vector coefficient u is in H(Curl) and v is in H(Curl) or
-    H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \mathrm{curl}(u), v)\f$ in 2D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) and \f$v\f$ is in \f$H\f$(curl) or
+    \f$H\f$(div). */
 class MixedScalarCrossCurlIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1586,8 +1585,8 @@ public:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Grad u, v) in 2D and
-    where V is a vector coefficient u is in H1 and v is in H1 or L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \nabla \cdot u, v)\f$ in 2D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H^1\f$ or \f$L_2\f$. */
 class MixedScalarCrossGradIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1620,8 +1619,8 @@ public:
   { vector_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x u, v) in 2D and where
-    V is a vector coefficient u is in ND or RT and v is in H1 or L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times u, v)\f$ in 2D and where
+    \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in \f$H^1\f$ or \f$L_2\f$. */
 class MixedScalarCrossProductIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1645,8 +1644,11 @@ public:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x z u, v) in 2D and
-    where V is a vector coefficient u is in H1 or L2 and v is in ND or RT. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times u \hat{z}, v)\f$ in 2D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H^1\f$ or \f$L_2\f$ and \f$v\f$ is in \f$H\f$(curl) or \f$H\f$(div).
+
+    \todo Documentation what \f$\hat{z}\f$ is (also missing in https://mfem.org/bilininteg/).
+   */
 class MixedScalarWeakCrossProductIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1675,8 +1677,8 @@ public:
   { scalar_fe.CalcPhysShape(Trans, shape); shape *= -1.0; }
 };

-/** Class for integrating the bilinear form a(u,v) := (V . Grad u, v) in 2D or
-    3D and where V is a vector coefficient, u is in H1 and v is in H1 or L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \cdot \nabla u, v)\f$ in 2D or
+    3D and where \f$\vec{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H^1\f$ or \f$L_2\f$. */
 class MixedDirectionalDerivativeIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1708,8 +1710,8 @@ public:
   { vector_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (-V . Grad u, Div v) in 2D
-    or 3D and where V is a vector coefficient, u is in H1 and v is in RT. */
+/** Class for integrating the bilinear form \f$a(u,v) := (-\hat{V} \cdot  \nabla \cdot u, \nabla \cdot v)\f$ in 2D
+    or 3D and where \f$\hat{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H\f$(div). */
 class MixedGradDivIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1747,8 +1749,8 @@ public:
   { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (-V Div u, Grad v) in 2D
-    or 3D and where V is a vector coefficient, u is in RT and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (-\hat{V} \nabla \cdot u, \nabla \cdot v)\f$ in 2D
+    or 3D and where \f$\hat{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H\f$(div) and \f$v\f$ is in \f$H^1\f$. */
 class MixedDivGradIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1787,8 +1789,8 @@ public:
   { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (-V u, Grad v) in 2D or 3D
-    and where V is a vector coefficient, u is in H1 or L2 and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (-\hat{V} u, \nabla \cdot v)\f$ in 2D or 3D
+    and where \f$\hat{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H^1\f$ or \f$L_2\f$ and \f$v\f$ is in \f$H^1\f$. */
 class MixedScalarWeakDivergenceIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1820,9 +1822,9 @@ public:
   { vector_fe.CalcPhysDShape(Trans, shape); shape *= -1.0; }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q grad u, v) in either 2D
-    or 3D and where Q is an optional coefficient (of type scalar, matrix, or
-    diagonal matrix) u is in H1 and v is in H(Curl) or H(Div). Partial assembly
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla u, v)\f$ in either 2D
+    or 3D and where \f$Q\f$ is an optional coefficient (of type scalar, matrix, or
+    diagonal matrix) \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H\f$(curl) or \f$H\f$(div). Partial assembly
    (PA) is supported but could be further optimized by using more efficient
    threading and shared memory.
 */
@@ -1849,7 +1851,7 @@ protected:
   inline virtual const char * FiniteElementTypeFailureMessage() const
   {
      return "MixedVectorGradientIntegrator:  "
-             "Trial spaces must be H1 and the test space must be a "
+             "Trial spaces must be \f$H^1\f$ and the test space must be a "
             "vector field in 2D or 3D";
   }

@@ -1881,9 +1883,9 @@ private:
   int dim, ne, dofs1D, quad1D;
 };

-/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) in 3D and
-    where Q is an optional coefficient (of type scalar, matrix, or diagonal
-    matrix) u is in H(Curl) and v is in H(Div) or H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \mathrm{curl}(u), v)\f$ in 3D and
+    where \f$Q\f$ is an optional coefficient (of type scalar, matrix, or diagonal
+    matrix) \f$u\f$ is in \f$H\f$(curl) and \f$v\f$ is in \f$H\f$(div) or \f$H\f$(curl). */
 class MixedVectorCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1940,9 +1942,9 @@ private:
   int dim, ne, dofs1D, dofs1Dtest,quad1D, testType, trialType, coeffDim;
 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, curl v) in 3D and
-    where Q is an optional coefficient (of type scalar, matrix, or diagonal
-    matrix) u is in H(Div) or H(Curl) and v is in H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q u, \mathrm{curl}(v))\f$ in 3D and
+    where \f$Q\f$ is an optional coefficient (of type scalar, matrix, or diagonal
+    matrix) \f$u\f$ is in \f$H\f$(div) or \f$H\f$(curl) and \f$v\f$ is in \f$H\f$(curl). */
 class MixedVectorWeakCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1997,9 +1999,9 @@ private:
   int dim, ne, dofs1D, quad1D, testType, trialType, coeffDim;
 };

-/** Class for integrating the bilinear form a(u,v) := - (Q u, grad v) in either
-    2D or 3D and where Q is an optional coefficient (of type scalar, matrix, or
-    diagonal matrix) u is in H(Div) or H(Curl) and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := - (Q u, \nabla v)\f$ in either
+    2D or 3D and where \f$Q\f$ is an optional coefficient (of type scalar, matrix, or
+    diagonal matrix) \f$u\f$ is in \f$H\f$(div) or \f$H\f$(curl) and \f$v\f$ is in \f$H^1\f$. */
 class MixedVectorWeakDivergenceIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -2039,11 +2041,11 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q grad u, v) where Q is a
-    scalar coefficient, and v is a vector with components v_i in the same (H1) space
-    as u.
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla u, v)\f$ where \f$Q\f$ is a
+    scalar coefficient, and \f$v\f$ is a vector with components \f$v_i\f$ in the same (\f$H^1\f$) space
+    as \f$u\f$.

-    See also MixedVectorGradientIntegrator when v is in H(curl). */
+    See also MixedVectorGradientIntegrator when \f$v\f$ is in \f$H\f$(curl). */
 class GradientIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2090,7 +2092,7 @@ public:
                                         ElementTransformation &Trans);
 };

-/** Class for integrating the bilinear form a(u,v) := (Q grad u, grad v) where Q
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla u, \nabla v)\f$ where \f$Q\f$
    can be a scalar or a matrix coefficient. */
 class DiffusionIntegrator: public BilinearFormIntegrator
 {
@@ -2257,7 +2259,7 @@ public:
   Coefficient *GetCoefficient() const { return Q; }
 };

-/** Class for local mass matrix assembling a(u,v) := (Q u, v) */
+/** Class for local mass matrix assembling \f$a(u,v) := (Q u, v)\f$ */
 class MassIntegrator: public BilinearFormIntegrator
 {
   friend class DGMassInverse;
@@ -2322,7 +2324,7 @@ public:
   const Coefficient *GetCoefficient() const { return Q; }
 };

-/** Mass integrator (u, v) restricted to the boundary of a domain */
+/** Mass integrator \f$(u, v)\f$ restricted to the boundary of a domain */
 class BoundaryMassIntegrator : public MassIntegrator
 {
 public:
@@ -2336,7 +2338,7 @@ public:
                                   DenseMatrix &elmat);
 };

-/// alpha (q . grad u, v)
+/// \f$ \alpha (Q \cdot \nabla u, v)\f$
 class ConvectionIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2394,7 +2396,7 @@ public:
 // Alias for @ConvectionIntegrator.
 using NonconservativeConvectionIntegrator = ConvectionIntegrator;

-/// -alpha (u, q . grad v), negative transpose of ConvectionIntegrator
+/// \f$-\alpha (u, q \cdot \nabla v)\f$, negative transpose of ConvectionIntegrator
 class ConservativeConvectionIntegrator : public TransposeIntegrator
 {
 public:
@@ -2402,7 +2404,7 @@ public:
      : TransposeIntegrator(new ConvectionIntegrator(q, -a)) { }
 };

-/// alpha (q . grad u, v) using the "group" FE discretization
+/// \f$ \alpha (Q \cdot \nabla u, v)\f$ using the "group" FE discretization
 class GroupConvectionIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2421,8 +2423,8 @@ public:
                                      DenseMatrix &);
 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, v),
-    where u=(u1,...,un) and v=(v1,...,vn); ui and vi are defined
+/** Class for integrating the bilinear form \f$a(u,v) := (Q u, v)\f$,
+    where \f$ u=(u_1,\dots,u_n) \f$ and \f$ v=(v_1,\dots,v_n)\f$, \f$u_i\f$ and \f$v_i\f$ are defined
    by scalar FE through standard transformation. */
 class VectorMassIntegrator: public BilinearFormIntegrator
 {
@@ -2483,10 +2485,10 @@ public:
 };


-/** Class for integrating (div u, p) where u is a vector field given by
-    VectorFiniteElement through Piola transformation (for RT elements); p is
+/** Class for integrating \f$(\nabla \cdot u, p)\f$ where \f$u\f$ is a vector field given by
+    VectorFiniteElement through Piola transformation (for Raviart-Thomas elements); \f$p\f$ is
    scalar function given by FiniteElement through standard transformation.
-    Here, u is the trial function and p is the test function.
+    Here, \f$u\f$ is the trial function and \f$p\f$ is the test function.

    Note: if the test space does not have map type INTEGRAL, then the element
    matrix returned by AssembleElementMatrix2 will not depend on the
@@ -2530,8 +2532,8 @@ public:
 };


-/** Integrator for `(-Q u, grad v)` for Nedelec (`u`) and H1 (`v`) elements.
-    This is equivalent to a weak divergence of the Nedelec basis functions. */
+/** Integrator for \f$(-Q u, \nabla v)\f$ for Nedelec (\f$u\f$) and \f$H^1\f$ (\f$v\f$) elements.
+    This is equivalent to a weak divergence of the \f$H\f$(curl) basis functions. */
 class VectorFEWeakDivergenceIntegrator: public BilinearFormIntegrator
 {
 protected:
@@ -2557,8 +2559,8 @@ public:
                                       DenseMatrix &elmat);
 };

-/** Integrator for (curl u, v) for Nedelec and RT elements. If the trial and
-    test spaces are switched, assembles the form (u, curl v). */
+/** Integrator for \f$(\mathrm{curl}(u), v)\f$ for Nedelec and Raviart-Thomas elements. If the trial and
+    test spaces are switched, assembles the form \f$(u, \mathrm{curl}(v))\f$. */
 class VectorFECurlIntegrator: public BilinearFormIntegrator
 {
 protected:
@@ -2583,7 +2585,7 @@ public:
                                       DenseMatrix &elmat);
 };

-/// Class for integrating (Q D_i(u), v); u and v are scalars
+/// Class for integrating \f$ (Q \partial_i(u), v) \f$ where \f$u\f$ and \f$v\f$ are scalars
 class DerivativeIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2606,7 +2608,7 @@ public:
                                       DenseMatrix &elmat);
 };

-/// Integrator for (curl u, curl v) for Nedelec elements
+/// Integrator for \f$(\mathrm{curl}(u), \mathrm{curl}(v))\f$ for Nedelec elements
 class CurlCurlIntegrator: public BilinearFormIntegrator
 {
 private:
@@ -2671,7 +2673,7 @@ public:
   const Coefficient *GetCoefficient() const { return Q; }
 };

-/** Integrator for (curl u, curl v) for FE spaces defined by 'dim' copies of a
+/** Integrator for \f$(\mathrm{curl}(u), \mathrm{curl}(v))\f$ for FE spaces defined by 'dim' copies of a
    scalar FE space. */
 class VectorCurlCurlIntegrator: public BilinearFormIntegrator
 {
@@ -2692,20 +2694,21 @@ public:
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
-   /// Compute element energy: (1/2) (curl u, curl u)_E
+   /// Compute element energy: \f$ \frac{1}{2} (\mathrm{curl}(u), \mathrm{curl}(u))_E\f$
   virtual double GetElementEnergy(const FiniteElement &el,
                                   ElementTransformation &Tr,
                                   const Vector &elfun);
 };

-/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) where Q is
-    an optional scalar coefficient, and v is a vector with components v_i in
-    the L2 or H1 space. This integrator handles 3 cases:
-    (a) u ∈ H(curl) in 3D, v is a 3D vector with components v_i in L^2 or H^1
-    (b) u ∈ H(curl) in 2D, v is a scalar field in L^2 or H^1
-    (c) u is a scalar field in H^1, i.e, curl u := [0 1;-1 0]grad u and v is a
-        2D vector field with components v_i in L^2 or H^1 space.
-    Note: Case (b) can also be handled by MixedScalarCurlIntegrator  */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \mathrm{curl}(u), v)\f$ where \f$Q\f$ is
+    an optional scalar coefficient, and \f$v\f$ is a vector with components \f$v_i\f$ in
+    the \f$L_2\f$ or \f$H^1\f$ space. This integrator handles 3 cases:
+    1. u ∈ \f$H\f$(curl) in 3D, \f$v\f$ is a 3D vector with components \f$v_i\f$ in \f$L^2\f$ or \f$H^1\f$
+    2. u ∈ \f$H\f$(curl) in 2D, \f$v\f$ is a scalar field in \f$L^2\f$ or \f$H^1\f$
+    3. u is a scalar field in \f$H^1\f$, i.e, \f$\mathrm{curl}(u) := \begin{pmatrix} 0 & 1 \\ -1 & 0 \end{pmatrix}\f$, \f$\nabla u\f$ and \f$v\f$ is a
+        2D vector field with components \f$v_i\f$ in \f$L^2\f$ or \f$H^1\f$ space.
+
+    Note: Case 2 can also be handled by MixedScalarCurlIntegrator  */
 class MixedCurlIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2727,10 +2730,10 @@ public:
                                       DenseMatrix &elmat);
 };

-/** Integrator for (Q u, v), where Q is an optional coefficient (of type scalar,
-    vector (diagonal matrix), or matrix), trial function u is in H(Curl) or
-    H(Div), and test function v is in H(Curl), H(Div), or v=(v1,...,vn), where
-    vi are in H1. */
+/** Integrator for \f$(Q u, v)\f$, where \f$Q\f$ is an optional coefficient (of type scalar,
+    vector (diagonal matrix), or matrix), trial function \f$u\f$ is in \f$H\f$(curl) or
+    \f$H\f$(div), and test function \f$v\f$ is in \f$H\f$(curl), \f$H\f$(div), or \f$v=(v_1,\dots,v_n)\f$, where
+    \f$v_i\f$ are in \f$H^1\f$. */
 class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
@@ -2789,8 +2792,8 @@ public:
   const Coefficient *GetCoefficient() const { return Q; }
 };

-/** Integrator for (Q div u, p) where u=(v1,...,vn) and all vi are in the same
-    scalar FE space; p is also in a (different) scalar FE space.  */
+/** Integrator for \f$(Q \nabla \cdot u, v)\f$ where \f$u=(u_1,\cdots,u_n)\f$ and all \f$u_i\f$ are in the same
+    scalar FE space; \f$v\f$ is also in a (different) scalar FE space.  */
 class VectorDivergenceIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2837,7 +2840,7 @@ public:
                                         ElementTransformation &Trans);
 };

-/// (Q div u, div v) for RT elements
+/// \f$(Q \nabla \cdot u, \nabla \cdot v)\f$ for Raviart-Thomas elements
 class DivDivIntegrator: public BilinearFormIntegrator
 {
 protected:
@@ -2878,10 +2881,10 @@ public:
 };

 /** Integrator for
-
-      (Q grad u, grad v) = sum_i (Q grad u_i, grad v_i) e_i e_i^T
-
-    for vector FE spaces, where e_i is the unit vector in the i-th direction.
+    \f[
+      (Q \nabla u, \nabla v) = \sum_i (Q \nabla u_i, \nabla v_i) e_i e_i^{\mathrm{T}}
+    \f]
+    for vector FE spaces, where \f$e_i\f$ is the unit vector in the \f$i\f$-th direction.
    The resulting local element matrix is square, of size <tt> vdim*dof </tt>,
    where \c vdim is the vector dimension space and \c dof is the local degrees
    of freedom. The integrator is not aware of the true vector dimension and
@@ -2942,7 +2945,7 @@ public:
       \c Vector.

       The element matrix is block-diagonal and each block is integrated with
-       coefficient q_i.
+       coefficient \f$q_{i}\f$.

       If the vector dimension does not match the true dimension of the space,
       the resulting element matrix will be mathematically invalid. */
@@ -2954,7 +2957,7 @@ public:
       \c Matrix.

       The element matrix is populated in each block. Each block is integrated
-       with coefficient q_ij.
+       with coefficient \f$q_{ij}\f$.

       If the vector dimension does not match the true dimension of the space,
       the resulting element matrix will be mathematically invalid. */
@@ -2978,8 +2981,10 @@ public:
 };

 /** Integrator for the linear elasticity form:
-    a(u,v) = (lambda div(u), div(v)) + (2 mu e(u), e(v)),
-    where e(v) = (1/2) (grad(v) + grad(v)^T).
+    \f[
+      a(u,v) = (\lambda \mathrm{div}(u), \mathrm{div}(v)) + (2 \mu \varepsilon(u), \varepsilon(v)),
+    \f]
+    where \f$\varepsilon(v) = \frac{1}{2} (\mathrm{grad}(v) + \mathrm{grad}(v)^{\mathrm{T}})\f$.
    This is a 'Vector' integrator, i.e. defined for FE spaces
    using multiple copies of a scalar FE space. */
 class ElasticityIntegrator : public BilinearFormIntegrator
@@ -2998,8 +3003,8 @@ private:
 public:
   ElasticityIntegrator(Coefficient &l, Coefficient &m)
   { lambda = &l; mu = &m; }
-   /** With this constructor lambda = q_l * m and mu = q_m * m;
-       if dim * q_l + 2 * q_m = 0 then trace(sigma) = 0. */
+   /** With this constructor \f$\lambda = q_l * m\f$ and \f$\mu = q_m * m\f$
+       if \f$dim * q_l + 2 * q_m = 0\f$ then \f$\tr(\sigma) = 0\f$. */
   ElasticityIntegrator(Coefficient &m, double q_l, double q_m)
   { lambda = NULL; mu = &m; q_lambda = q_l; q_mu = q_m; }

@@ -3007,12 +3012,12 @@ public:
                                      ElementTransformation &,
                                      DenseMatrix &);

-   /** Compute the stress corresponding to the local displacement @a u and
+   /** Compute the stress corresponding to the local displacement @a \f$u\f$ and
       interpolate it at the nodes of the given @a fluxelem. Only the symmetric
       part of the stress is stored, so that the size of @a flux is equal to
       the number of DOFs in @a fluxelem times dim*(dim+1)/2. In 2D, the order
-       of the stress components is: s_xx, s_yy, s_xy. In 3D, it is: s_xx, s_yy,
-       s_zz, s_xy, s_xz, s_yz. In other words, @a flux is the local vector for
+       of the stress components is: \f$s_xx, s_yy, s_xy\f$. In 3D, it is: \f$s_xx, s_yy,
+       s_zz, s_xy, s_xz, s_yz\f$. In other words, @a flux is the local vector for
       a FE space with dim*(dim+1)/2 vector components, based on the finite
       element @a fluxelem. The integration rule is taken from @a fluxelem.
       @a ir exists to specific an alternative integration rule. */
@@ -3030,37 +3035,39 @@ public:
       dim*(dim+1)/2 vector components, based on the finite element @a fluxelem.
       The number of components, dim*(dim+1)/2 is such that it represents the
       symmetric part of the (symmetric) stress tensor. The order of the
-       components is: s_xx, s_yy, s_xy in 2D, and s_xx, s_yy, s_zz, s_xy, s_xz,
-       s_yz in 3D. */
+       components is: \f$s_xx, s_yy, s_xy\f$ in 2D, and \f$s_xx, s_yy, s_zz, s_xy, s_xz,
+       s_yz\f$ in 3D. */
   virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                    ElementTransformation &Trans,
                                    Vector &flux, Vector *d_energy = NULL);
 };

 /** Integrator for the DG form:
-    alpha < rho_u (u.n) {v},[w] > + beta < rho_u |u.n| [v],[w] >,
-    where v and w are the trial and test variables, respectively, and rho/u are
-    given scalar/vector coefficients. {v} represents the average value of v on
-    the face and [v] is the jump such that {v}=(v1+v2)/2 and [v]=(v1-v2) for the
-    face between elements 1 and 2. For boundary elements, v2=0. The vector
-    coefficient, u, is assumed to be continuous across the faces and when given
-    the scalar coefficient, rho, is assumed to be discontinuous. The integrator
-    uses the upwind value of rho, rho_u, which is value from the side into which
-    the vector coefficient, u, points.
+    \f[
+      \alpha \langle \rho_u (u \cdot n) \{v\},[w] \rangle + \beta \langle \rho_u |u \cdot n| [v],[w] \rangle,
+    \f]
+    where \f$v\f$ and \f$w\f$ are the trial and test variables, respectively, and \f$\rho\f$/\f$u\f$ are
+    given scalar/vector coefficients. \f$\{v\}\f$ represents the average value of \f$v\f$ on
+    the face and \f$[v]\f$ is the jump such that \f$\{v\}=(v_1+v_2)/2\f$ and \f$[v]=(v_1-v_2)\f$ for the
+    face between elements \f$1\f$ and \f$2\f$. For boundary elements, \f$v2=0\f$. The vector
+    coefficient, \f$u\f$, is assumed to be continuous across the faces and when given
+    the scalar coefficient, \f$\rho\f$, is assumed to be discontinuous. The integrator
+    uses the upwind value of \f$\rho\f$, denoted by \f$\rho_u\f$, which is value from the side into which
+    the vector coefficient, \f$u\f$, points.

-    One use case for this integrator is to discretize the operator -u.grad(v)
+    One use case for this integrator is to discretize the operator \f$-u \cdot \nabla v\f$
    with a DG formulation. The resulting formulation uses the
-    ConvectionIntegrator (with coefficient u, and parameter alpha = -1) and the
-    transpose of the DGTraceIntegrator (with coefficient u, and parameters alpha
-    = 1, beta = -1/2 to use the upwind face flux, see also
+    ConvectionIntegrator (with coefficient \f$u\f$, and parameter \f$\alpha = -1\f$) and the
+    transpose of the DGTraceIntegrator (with coefficient \f$u\f$, and parameters \f$\alpha
+    = 1\f$, \f$\beta = -1/2\f$ to use the upwind face flux, see also
    NonconservativeDGTraceIntegrator). This discretization and the handling of
    the inflow and outflow boundaries is illustrated in Example 9/9p.

-    Another use case for this integrator is to discretize the operator -div(u v)
+    Another use case for this integrator is to discretize the operator \f$-\mathrm{div}(u v)\f$
    with a DG formulation. The resulting formulation is conservative and
-    consists of the ConservativeConvectionIntegrator (with coefficient u, and
-    parameter alpha = -1) plus the DGTraceIntegrator (with coefficient u, and
-    parameters alpha = -1, beta = -1/2 to use the upwind face flux).
+    consists of the ConservativeConvectionIntegrator (with coefficient \f$u\f$, and
+    parameter \f$\alpha = -1\f$) plus the DGTraceIntegrator (with coefficient \f$u\f$, and
+    parameters \f$\alpha = -1\f$, \f$\beta = -1/2\f$ to use the upwind face flux).
    */
 class DGTraceIntegrator : public BilinearFormIntegrator
 {
@@ -3078,11 +3085,11 @@ private:
   Vector shape1, shape2;

 public:
-   /// Construct integrator with rho = 1, b = 0.5*a.
+   /// Construct integrator with \f$\rho = 1\f$, \f$\beta = \alpha/2\f$.
   DGTraceIntegrator(VectorCoefficient &u_, double a)
   { rho = NULL; u = &u_; alpha = a; beta = 0.5*a; }

-   /// Construct integrator with rho = 1.
+   /// Construct integrator with \f$\rho = 1\f$.
   DGTraceIntegrator(VectorCoefficient &u_, double a, double b)
   { rho = NULL; u = &u_; alpha = a; beta = b; }

@@ -3127,8 +3134,9 @@ using ConservativeDGTraceIntegrator = DGTraceIntegrator;

 /** Integrator that represents the face terms used for the non-conservative
    DG discretization of the convection equation:
-    -alpha < rho_u (u.n) {v},[w] > + beta < rho_u |u.n| [v],[w] >.
-
+    \f[
+      -\alpha \langle \rho_u (u \cdot n) \{v\},[w] \rangle + \beta \langle \rho_u |u \cdot n| [v],[w] \rangle.
+    \f]
    This integrator can be used with together with ConvectionIntegrator to
    implement an upwind DG discretization in non-conservative form, see ex9 and
    ex9p. */
@@ -3147,17 +3155,19 @@ public:
 };

 /** Integrator for the DG form:
-
-        - < {(Q grad(u)).n}, [v] > + sigma < [u], {(Q grad(v)).n} >
-        + kappa < {h^{-1} Q} [u], [v] >
-
-    where Q is a scalar or matrix diffusion coefficient and u, v are the trial
-    and test spaces, respectively. The parameters sigma and kappa determine the
+    \f[
+        - \langle \{(Q \nabla u) \cdot n\}, [v] \rangle + \sigma \langle [u], \{(Q \nabla v) \cdot n \} \rangle
+        + \kappa \langle \{h^{-1} Q\} [u], [v] \rangle
+    \f]
+    where \f$Q\f$ is a scalar or matrix diffusion coefficient and \f$u\f$, \f$v\f$ are the trial
+    and test spaces, respectively. The parameters \f$\sigma\f$ and \f$\kappa\f$ determine the
    DG method to be used (when this integrator is added to the "broken"
    DiffusionIntegrator):
-    * sigma = -1, kappa >= kappa0: symm. interior penalty (IP or SIPG) method,
-    * sigma = +1, kappa > 0: non-symmetric interior penalty (NIPG) method,
-    * sigma = +1, kappa = 0: the method of Baumann and Oden. */
+    - \f$\sigma = -1\f$, \f$\kappa \geq \kappa_0\f$: symm. interior penalty (IP or SIPG) method,
+    - \f$\sigma = +1\f$, \f$\kappa > 0\f$: non-symmetric interior penalty (NIPG) method,
+    - \f$\sigma = +1\f$, \f$\kappa = 0\f$: the method of Baumann and Oden.
+
+    \todo Clarify used notation. */
 class DGDiffusionIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -3184,11 +3194,11 @@ public:
 };

 /** Integrator for the "BR2" diffusion stabilization term
-
-    sum_e eta (r_e([u]), r_e([v]))
-
-    where r_e is the lifting operator defined on each edge e (potentially
-    weighted by a coefficient Q). The parameter eta can be chosen to be one to
+    \f[
+      \sum_e \eta (r_e([u]), r_e([v]))
+    \f]
+    where \f$r_e\f$ is the lifting operator defined on each edge \f$e\f$ (potentially
+    weighted by a coefficient \f$Q\f$). The parameter eta can be chosen to be one to
    obtain a stable discretization. The constructor for this integrator requires
    the finite element space because the lifting operator depends on the
    element-wise inverse mass matrix.
@@ -3275,8 +3285,8 @@ public:
    \begin{split}
    \sigma(u) &= \lambda \nabla \cdot u I + 2 \mu \varepsilon(u) \\
              &= \lambda \nabla \cdot u I + 2 \mu \frac{1}{2} (\nabla u + \nabla
-                 u^T) \\
-              &= \lambda \nabla \cdot u I + \mu (\nabla u + \nabla u^T)
+                 u^{\mathrm{T}}) \\
+              &= \lambda \nabla \cdot u I + \mu (\nabla u + \nabla u^{\mathrm{T}})
    \end{split}
    \f]

@@ -3353,8 +3363,8 @@ protected:
      DenseMatrix &elmat, DenseMatrix &jmat);
 };

-/** Integrator for the DPG form: < v, [w] > over all faces (the interface) where
-    the trial variable v is defined on the interface and the test variable w is
+/** Integrator for the DPG form:\f$ \langle v, [w] \rangle \f$ over all faces (the interface) where
+    the trial variable \f$v\f$ is defined on the interface and the test variable \f$w\f$ is
    defined inside the elements, generally in a DG space. */
 class TraceJumpIntegrator : public BilinearFormIntegrator
 {
@@ -3371,9 +3381,9 @@ public:
                                   DenseMatrix &elmat);
 };

-/** Integrator for the form: < v, [w.n] > over all faces (the interface) where
-    the trial variable v is defined on the interface and the test variable w is
-    in an H(div)-conforming space. */
+/** Integrator for the form:\f$ \langle v, [w \cdot n] \rangle \f$ over all faces (the interface) where
+    the trial variable \f$v\f$ is defined on the interface and the test variable \f$w\f$ is
+    in an \f$H\f$(div)-conforming space. */
 class NormalTraceJumpIntegrator : public BilinearFormIntegrator
 {
 private:
@@ -3390,10 +3400,10 @@ public:
                                   DenseMatrix &elmat);
 };

-/** Integrator for the DPG form: < v, w > over a face (the interface) where
-    the trial variable v is defined on the interface
-    (H^-1/2 i.e., v:=u⋅n normal trace of H(div))
-    and the test variable w is in an H1-conforming space. */
+/** Integrator for the DPG form:\f$ \langle v, w \rangle \f$ over a face (the interface) where
+    the trial variable \f$v\f$ is defined on the interface
+    (\f$H^{-1/2}\f$ i.e., \f$v := u \cdot n\f$ normal trace of \f$H\f$(div))
+    and the test variable \f$w\f$ is in an \f$H^1\f$-conforming space. */
 class TraceIntegrator : public BilinearFormIntegrator
 {
 private:
@@ -3407,9 +3417,9 @@ public:
                                DenseMatrix &elmat);
 };

-/** Integrator for the form: < v, w.n > over a face (the interface) where
-    the trial variable v is defined on the interface (H^1/2, i.e., trace of H1)
-    and the test variable w is in an H(div)-conforming space. */
+/** Integrator for the form: \f$ \langle v, w \cdot n \rangle \f$ over a face (the interface) where
+    the trial variable \f$v\f$ is defined on the interface (\f$H^{1/2}\f$, i.e., trace of \f$H^1\f$)
+    and the test variable \f$w\f$ is in an \f$H\f$(div)-conforming space. */
 class NormalTraceIntegrator : public BilinearFormIntegrator
 {
 private:
@@ -3426,10 +3436,10 @@ public:
 };


-/** Integrator for the form: < v, w × n > over a face (the interface)
- *  In 3D the trial variable v is defined on the interface (H^-1/2(curl), trace of H(curl))
- *  In 2D it's defined on the interface (H^1/2, trace of H1)
- *  The test variable w is in an H(curl)-conforming space. */
+/** Integrator for the form: \f$\langle v, w \times n \rangle\f$ over a face (the interface)
+ *  In 3D the trial variable \f$v\f$ is defined on the interface (\f$H^{-1/2}\f$(curl), trace of \f$H\f$(curl))
+ *  In 2D it's defined on the interface (\f$H^{1/2}\f$, trace of \f$H^1\f$)
+ *  The test variable \f$w\f$ is in an \f$H\f$(curl)-conforming space. */
 class TangentTraceIntegrator : public BilinearFormIntegrator
 {
 private:
@@ -3477,8 +3487,8 @@ class DiscreteInterpolator : public BilinearFormIntegrator { };


 /** Class for constructing the gradient as a DiscreteLinearOperator from an
-    H1-conforming space to an H(curl)-conforming space. The range space can be
-    vector L2 space as well. */
+    \f$H^1\f$-conforming space to an \f$H\f$(curl)-conforming space. The range space can be
+    vector \f$L_2\f$ space as well. */
 class GradientInterpolator : public DiscreteInterpolator
 {
 public:
@@ -3495,8 +3505,8 @@ public:

   /** @brief Setup method for PA data.

-       @param[in] trial_fes   H1 Lagrange space
-       @param[in] test_fes    H(curl) Nedelec space
+       @param[in] trial_fes   \f$H^1\f$ Lagrange space
+       @param[in] test_fes    \f$H\f$(curl) Nedelec space
    */
   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
                           const FiniteElementSpace &test_fes);
@@ -3570,7 +3580,7 @@ public:
    the global discrete divergence matrix.

    Note: Since the dofs in the L2_FECollection are nodal values, the local
-    discrete divergence matrix (with an RT-type domain space) will depend on
+    discrete divergence matrix (with an \f$H\f$(div)-type domain space) will depend on
    the transformation. On the other hand, the local matrix returned by
    VectorFEDivergenceIntegrator is independent of the transformation. */
 class DivergenceInterpolator : public DiscreteInterpolator
@@ -3585,8 +3595,8 @@ public:


 /** A trace face interpolator class for interpolating the normal component of
-    the domain space, e.g. vector H1, into the range space, e.g. the trace of
-    RT which uses FiniteElement::INTEGRAL map type. */
+    the domain space, e.g. vector \f$H^1\f$, into the range space, e.g. the trace of
+    \f$H\f$(div) which uses FiniteElement::INTEGRAL map type. */
 class NormalInterpolator : public DiscreteInterpolator
 {
 public:
@@ -3648,7 +3658,7 @@ protected:
 };

 /** Interpolator of the 2D cross product between a vector coefficient and an
-    H(curl)-conforming field onto an L2-conforming field. */
+    \f$H\f$(curl)-conforming field onto an \f$L_2\f$-conforming field. */
 class ScalarCrossProductInterpolator : public DiscreteInterpolator
 {
 public:
@@ -3664,8 +3674,8 @@ protected:
 };

 /** Interpolator of the cross product between a vector coefficient and an
-    H(curl)-conforming field onto an H(div)-conforming field. The range space
-    can also be vector L2. */
+    \f$H\f$(curl)-conforming field onto an \f$H\f$(div)-conforming field. The range space
+    can also be vector \f$L_2\f$. */
 class VectorCrossProductInterpolator : public DiscreteInterpolator
 {
 public:
@@ -3681,8 +3691,8 @@ protected:
 };

 /** Interpolator of the inner product between a vector coefficient and an
-    H(div)-conforming field onto an L2-conforming field. The range space can
-    also be H1. */
+    \f$H\f$(div)-conforming field onto an \f$L_2\f$-conforming field. The range space can
+    also be \f$H^1\f$. */
 class VectorInnerProductInterpolator : public DiscreteInterpolator
 {
 public:
@@ -12,10 +12,14 @@
 #ifndef MFEM_LIBCEED_CEED
 #define MFEM_LIBCEED_CEED

+#include "../../../config/config.hpp"
+
 #ifdef MFEM_USE_CEED
+
 #include <ceed.h>
-#if !CEED_VERSION_GE(0,10,0)
-#error MFEM requires a libCEED version >= 0.10.0
+
+#if !CEED_VERSION_GE(0,12,0)
+#error MFEM requires a libCEED version >= 0.12.0
 #endif

 namespace mfem
@@ -294,14 +294,14 @@ public:
                                nelem, nqpts, ncomp, strides,
                                &quadCoeff->restr);
         CeedOperatorSetField(build_oper, "coeff", quadCoeff->restr,
-                              CEED_BASIS_COLLOCATED, quadCoeff->coeffVector);
+                              CEED_BASIS_NONE, quadCoeff->coeffVector);
      }
      CeedOperatorSetField(build_oper, "dx", mesh_restr,
                           mesh_basis, CEED_VECTOR_ACTIVE);
      CeedOperatorSetField(build_oper, "weights", CEED_ELEMRESTRICTION_NONE,
                           mesh_basis, CEED_VECTOR_NONE);
      CeedOperatorSetField(build_oper, "qdata", restr_i,
-                           CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                           CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);

      // Compute the quadrature data for the operator.
      CeedOperatorApply(build_oper, node_coords, qdata, CEED_REQUEST_IMMEDIATE);
@@ -355,7 +355,7 @@ public:
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "u", trial_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "u", trial_restr, trial_basis, CEED_VECTOR_ACTIVE);
@@ -369,14 +369,14 @@ public:
            break;
      }
      // qdata
-      CeedOperatorSetField(oper, "qdata", restr_i, CEED_BASIS_COLLOCATED,
+      CeedOperatorSetField(oper, "qdata", restr_i, CEED_BASIS_NONE,
                           qdata);
      // output
      switch (op.test_op)
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "v", test_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "v", test_restr, test_basis, CEED_VECTOR_ACTIVE);
@@ -685,14 +685,14 @@ public:
                                nelem, nqpts, ncomp, strides,
                                &quadCoeff->restr);
         CeedOperatorSetField(oper, "coeff", quadCoeff->restr,
-                              CEED_BASIS_COLLOCATED, quadCoeff->coeffVector);
+                              CEED_BASIS_NONE, quadCoeff->coeffVector);
      }
      // input
      switch (op.trial_op)
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "u", trial_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "u", trial_restr, trial_basis,
@@ -718,7 +718,7 @@ public:
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "v", test_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "v", test_restr, test_basis,
@@ -132,24 +132,24 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedOperatorGetCeed(oper, &ceed); CeedChk(ierr);
+   ierr = CeedOperatorGetCeed(oper, &ceed); PCeedChk(ierr);

   CeedQFunction qf;
   bool isComposite;
-   ierr = CeedOperatorIsComposite(oper, &isComposite); CeedChk(ierr);
+   ierr = CeedOperatorIsComposite(oper, &isComposite); PCeedChk(ierr);
   CeedOperator *subops;
   if (isComposite)
   {
 #if CEED_VERSION_GE(0, 10, 2)
-      ierr = CeedCompositeOperatorGetSubList(oper, &subops); CeedChk(ierr);
+      ierr = CeedCompositeOperatorGetSubList(oper, &subops); PCeedChk(ierr);
 #else
-      ierr = CeedOperatorGetSubList(oper, &subops); CeedChk(ierr);
+      ierr = CeedOperatorGetSubList(oper, &subops); PCeedChk(ierr);
 #endif
-      ierr = CeedOperatorGetQFunction(subops[0], &qf); CeedChk(ierr);
+      ierr = CeedOperatorGetQFunction(subops[0], &qf); PCeedChk(ierr);
   }
   else
   {
-      ierr = CeedOperatorGetQFunction(oper, &qf); CeedChk(ierr);
+      ierr = CeedOperatorGetQFunction(oper, &qf); PCeedChk(ierr);
   }
   CeedInt numinputfields, numoutputfields;
   ierr = CeedQFunctionGetNumArgs(qf, &numinputfields, &numoutputfields);
@@ -157,12 +157,12 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
   if (isComposite)
   {
      ierr = CeedOperatorGetFields(subops[0], &numinputfields, &inputfields,
-                                   &numoutputfields, NULL); CeedChk(ierr);
+                                   &numoutputfields, NULL); PCeedChk(ierr);
   }
   else
   {
      ierr = CeedOperatorGetFields(oper, &numinputfields, &inputfields,
-                                   &numoutputfields, NULL); CeedChk(ierr);
+                                   &numoutputfields, NULL); PCeedChk(ierr);
   }

   CeedVector if_vector;
@@ -170,7 +170,7 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
   int found_index = -1;
   for (int i = 0; i < numinputfields; ++i)
   {
-      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector); CeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector); PCeedChk(ierr);
      if (if_vector == CEED_VECTOR_ACTIVE)
      {
         if (found)
@@ -13,12 +13,15 @@
 #define MFEM_LIBCEED_UTIL

 #include "../../../config/config.hpp"
+#include "../../../general/error.hpp"
+
 #include <functional>
 #include <string>
 #include <tuple>
 #include <unordered_map>

 #include "ceed.hpp"
+
 #ifdef MFEM_USE_CEED
 #include <ceed/backend.h>  // for CeedOperatorField
 #endif
@@ -170,10 +173,12 @@ namespace internal
 {

 #ifdef MFEM_USE_CEED
+
 /** @warning These maps have a tendency to create bugs when adding new "types"
    of CeedBasis and CeedElemRestriction. */
 extern ceed::BasisMap ceed_basis_map;
 extern ceed::RestrMap ceed_restr_map;
+
 #endif

 } // namespace internal
@@ -99,7 +99,7 @@ int CeedOperatorGetSize(CeedOperator oper, CeedInt * size)
 {
   CeedSize in_len, out_len;
   int ierr = CeedOperatorGetActiveVectorLengths(oper, &in_len, &out_len);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   *size = (CeedInt)in_len;
   MFEM_VERIFY(in_len == out_len, "not a square CeedOperator");
   MFEM_VERIFY(in_len == *size, "size overflow");
@@ -378,67 +378,68 @@ int AlgebraicInterpolation::Initialize(

   CeedSize height, width;
   ierr = CeedElemRestrictionGetLVectorSize(erestrictu_coarse, &width);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   ierr = CeedElemRestrictionGetLVectorSize(erestrictu_fine, &height);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // interpolation qfunction
   const int bp3_ncompu = 1;
   CeedQFunction l_qf_restrict, l_qf_prolong;
   ierr = CeedQFunctionCreateIdentity(ceed, bp3_ncompu, CEED_EVAL_NONE,
-                                      CEED_EVAL_INTERP, &l_qf_restrict); CeedChk(ierr);
+                                      CEED_EVAL_INTERP, &l_qf_restrict); PCeedChk(ierr);
   ierr = CeedQFunctionCreateIdentity(ceed, bp3_ncompu, CEED_EVAL_INTERP,
-                                      CEED_EVAL_NONE, &l_qf_prolong); CeedChk(ierr);
+                                      CEED_EVAL_NONE, &l_qf_prolong); PCeedChk(ierr);

   qf_restrict = l_qf_restrict;
   qf_prolong = l_qf_prolong;

   CeedVector c_fine_multiplicity;
-   ierr = CeedVectorCreate(ceed, height, &c_fine_multiplicity); CeedChk(ierr);
-   ierr = CeedVectorSetValue(c_fine_multiplicity, 0.0); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &c_fine_multiplicity); PCeedChk(ierr);
+   ierr = CeedVectorSetValue(c_fine_multiplicity, 0.0); PCeedChk(ierr);

   // Create the restriction operator
   // Restriction - Fine to coarse
   ierr = CeedOperatorCreate(ceed, qf_restrict, CEED_QFUNCTION_NONE,
-                             CEED_QFUNCTION_NONE, &op_restrict); CeedChk(ierr);
+                             CEED_QFUNCTION_NONE, &op_restrict); PCeedChk(ierr);
   ierr = CeedOperatorSetField(op_restrict, "input", erestrictu_fine,
-                               CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                               CEED_BASIS_NONE, CEED_VECTOR_ACTIVE); PCeedChk(ierr);
   ierr = CeedOperatorSetField(op_restrict, "output", erestrictu_coarse,
-                               basisctof, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                               basisctof, CEED_VECTOR_ACTIVE); PCeedChk(ierr);

   // Interpolation - Coarse to fine
   // Create the prolongation operator
   ierr =  CeedOperatorCreate(ceed, qf_prolong, CEED_QFUNCTION_NONE,
-                              CEED_QFUNCTION_NONE, &op_interp); CeedChk(ierr);
+                              CEED_QFUNCTION_NONE, &op_interp); PCeedChk(ierr);
   ierr =  CeedOperatorSetField(op_interp, "input", erestrictu_coarse,
-                                basisctof, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                                basisctof, CEED_VECTOR_ACTIVE); PCeedChk(ierr);
   ierr = CeedOperatorSetField(op_interp, "output", erestrictu_fine,
-                               CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                               CEED_BASIS_NONE, CEED_VECTOR_ACTIVE); PCeedChk(ierr);

   ierr = CeedElemRestrictionGetMultiplicity(erestrictu_fine,
-                                             c_fine_multiplicity); CeedChk(ierr);
-   ierr = CeedVectorCreate(ceed, height, &fine_multiplicity_r); CeedChk(ierr);
+                                             c_fine_multiplicity); PCeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &fine_multiplicity_r); PCeedChk(ierr);

   CeedScalar* fine_r_data;
   const CeedScalar* fine_data;
   ierr = CeedVectorGetArrayWrite(fine_multiplicity_r, CEED_MEM_HOST,
-                                  &fine_r_data); CeedChk(ierr);
+                                  &fine_r_data); PCeedChk(ierr);
   ierr = CeedVectorGetArrayRead(c_fine_multiplicity, CEED_MEM_HOST,
-                                 &fine_data); CeedChk(ierr);
+                                 &fine_data); PCeedChk(ierr);
   for (CeedSize i = 0; i < height; ++i)
   {
      fine_r_data[i] = 1.0 / fine_data[i];
   }

-   ierr = CeedVectorRestoreArray(fine_multiplicity_r, &fine_r_data); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(fine_multiplicity_r, &fine_r_data);
+   PCeedChk(ierr);
   ierr = CeedVectorRestoreArrayRead(c_fine_multiplicity, &fine_data);
-   CeedChk(ierr);
-   ierr = CeedVectorDestroy(&c_fine_multiplicity); CeedChk(ierr);
+   PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&c_fine_multiplicity); PCeedChk(ierr);

-   ierr = CeedVectorCreate(ceed, height, &fine_work); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &fine_work); PCeedChk(ierr);

-   ierr = CeedVectorCreate(ceed, height, &v_); CeedChk(ierr);
-   ierr = CeedVectorCreate(ceed, width, &u_); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &v_); PCeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, width, &u_); PCeedChk(ierr);

   return 0;
 }
@@ -447,12 +448,12 @@ int AlgebraicInterpolation::Finalize()
 {
   int ierr;

-   ierr = CeedQFunctionDestroy(&qf_restrict); CeedChk(ierr);
-   ierr = CeedQFunctionDestroy(&qf_prolong); CeedChk(ierr);
-   ierr = CeedOperatorDestroy(&op_interp); CeedChk(ierr);
-   ierr = CeedOperatorDestroy(&op_restrict); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&fine_multiplicity_r); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&fine_work); CeedChk(ierr);
+   ierr = CeedQFunctionDestroy(&qf_restrict); PCeedChk(ierr);
+   ierr = CeedQFunctionDestroy(&qf_prolong); PCeedChk(ierr);
+   ierr = CeedOperatorDestroy(&op_interp); PCeedChk(ierr);
+   ierr = CeedOperatorDestroy(&op_restrict); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&fine_multiplicity_r); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&fine_work); PCeedChk(ierr);

   return 0;
 }
@@ -498,8 +499,8 @@ int CeedVectorPointwiseMult(CeedVector a, const CeedVector b)
   CeedVectorGetCeed(a, &ceed);

   CeedSize length, length2;
-   ierr = CeedVectorGetLength(a, &length); CeedChk(ierr);
-   ierr = CeedVectorGetLength(b, &length2); CeedChk(ierr);
+   ierr = CeedVectorGetLength(a, &length); PCeedChk(ierr);
+   ierr = CeedVectorGetLength(b, &length2); PCeedChk(ierr);
   if (length != length2)
   {
      return CeedError(ceed, 1, "Vector sizes don't match");
@@ -516,14 +517,14 @@ int CeedVectorPointwiseMult(CeedVector a, const CeedVector b)
   }
   CeedScalar *a_data;
   const CeedScalar *b_data;
-   ierr = CeedVectorGetArray(a, mem, &a_data); CeedChk(ierr);
-   ierr = CeedVectorGetArrayRead(b, mem, &b_data); CeedChk(ierr);
+   ierr = CeedVectorGetArray(a, mem, &a_data); PCeedChk(ierr);
+   ierr = CeedVectorGetArrayRead(b, mem, &b_data); PCeedChk(ierr);
   MFEM_VERIFY(int(length) == length, "length overflow");
   mfem::forall(length, [=] MFEM_HOST_DEVICE (int i)
   {a_data[i] *= b_data[i];});

-   ierr = CeedVectorRestoreArray(a, &a_data); CeedChk(ierr);
-   ierr = CeedVectorRestoreArrayRead(b, &b_data); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(a, &a_data); PCeedChk(ierr);
+   ierr = CeedVectorRestoreArrayRead(b, &b_data); PCeedChk(ierr);

   return 0;
 }
@@ -11,12 +11,12 @@

 #include "full-assembly.hpp"

+#ifdef MFEM_USE_CEED
+
 #include "../../../linalg/sparsemat.hpp"
 #include "../interface/util.hpp"
 #include "../interface/ceed.hpp"

-#ifdef MFEM_USE_CEED
-
 namespace mfem
 {

@@ -45,32 +45,32 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
+   ierr = CeedOperatorGetCeed(op, &ceed); PCeedChk(ierr);

   // Assemble QFunction
   CeedQFunction qf;
-   ierr = CeedOperatorGetQFunction(op, &qf); CeedChk(ierr);
+   ierr = CeedOperatorGetQFunction(op, &qf); PCeedChk(ierr);
   CeedInt numinputfields, numoutputfields;
-   CeedChk(ierr);
+   PCeedChk(ierr);
   CeedVector assembledqf;
   CeedElemRestriction rstr_q;
   ierr = CeedOperatorLinearAssembleQFunction(
-             op, &assembledqf, &rstr_q, CEED_REQUEST_IMMEDIATE); CeedChk(ierr);
+             op, &assembledqf, &rstr_q, CEED_REQUEST_IMMEDIATE); PCeedChk(ierr);

   CeedSize qflength;
-   ierr = CeedVectorGetLength(assembledqf, &qflength); CeedChk(ierr);
+   ierr = CeedVectorGetLength(assembledqf, &qflength); PCeedChk(ierr);

   CeedOperatorField *input_fields;
   CeedOperatorField *output_fields;
   ierr = CeedOperatorGetFields(op, &numinputfields, &input_fields,
                                &numoutputfields, &output_fields);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // Determine active input basis
   CeedQFunctionField *qffields;
   ierr = CeedQFunctionGetFields(qf, &numinputfields, &qffields,
                                 &numoutputfields, NULL);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   CeedInt numemodein = 0, ncomp, dim = 1;
   CeedEvalMode *emodein = NULL;
   CeedBasis basisin = NULL;
@@ -78,28 +78,28 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
   for (CeedInt i=0; i<numinputfields; i++)
   {
      CeedVector vec;
-      ierr = CeedOperatorFieldGetVector(input_fields[i], &vec); CeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(input_fields[i], &vec); PCeedChk(ierr);
      if (vec == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorFieldGetBasis(input_fields[i], &basisin);
-         CeedChk(ierr);
-         ierr = CeedBasisGetNumComponents(basisin, &ncomp); CeedChk(ierr);
-         ierr = CeedBasisGetDimension(basisin, &dim); CeedChk(ierr);
+         PCeedChk(ierr);
+         ierr = CeedBasisGetNumComponents(basisin, &ncomp); PCeedChk(ierr);
+         ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
         ierr = CeedOperatorFieldGetElemRestriction(input_fields[i], &rstrin);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         CeedEvalMode emode;
         ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         switch (emode)
         {
            case CEED_EVAL_NONE:
            case CEED_EVAL_INTERP:
-               ierr = CeedHackRealloc(numemodein + 1, &emodein); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodein + 1, &emodein); PCeedChk(ierr);
               emodein[numemodein] = emode;
               numemodein += 1;
               break;
            case CEED_EVAL_GRAD:
-               ierr = CeedHackRealloc(numemodein + dim, &emodein); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodein + dim, &emodein); PCeedChk(ierr);
               for (CeedInt d=0; d<dim; d++)
               {
                  emodein[numemodein+d] = emode;
@@ -116,7 +116,7 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)

   // Determine active output basis
   ierr = CeedQFunctionGetFields(qf, &numinputfields, NULL, &numoutputfields,
-                                 &qffields); CeedChk(ierr);
+                                 &qffields); PCeedChk(ierr);
   CeedInt numemodeout = 0;
   CeedEvalMode *emodeout = NULL;
   CeedBasis basisout = NULL;
@@ -124,27 +124,27 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
   for (CeedInt i=0; i<numoutputfields; i++)
   {
      CeedVector vec;
-      ierr = CeedOperatorFieldGetVector(output_fields[i], &vec); CeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(output_fields[i], &vec); PCeedChk(ierr);
      if (vec == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorFieldGetBasis(output_fields[i], &basisout);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         ierr = CeedOperatorFieldGetElemRestriction(output_fields[i], &rstrout);
-         CeedChk(ierr);
-         CeedChk(ierr);
+         PCeedChk(ierr);
+         PCeedChk(ierr);
         CeedEvalMode emode;
         ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         switch (emode)
         {
            case CEED_EVAL_NONE:
            case CEED_EVAL_INTERP:
-               ierr = CeedHackRealloc(numemodeout + 1, &emodeout); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodeout + 1, &emodeout); PCeedChk(ierr);
               emodeout[numemodeout] = emode;
               numemodeout += 1;
               break;
            case CEED_EVAL_GRAD:
-               ierr = CeedHackRealloc(numemodeout + dim, &emodeout); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodeout + dim, &emodeout); PCeedChk(ierr);
               for (CeedInt d=0; d<dim; d++)
               {
                  emodeout[numemodeout+d] = emode;
@@ -161,47 +161,47 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)

   CeedInt nelem, elemsize, nqpts;
   CeedSize nnodes;
-   ierr = CeedElemRestrictionGetNumElements(rstrin, &nelem); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetElementSize(rstrin, &elemsize); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetLVectorSize(rstrin, &nnodes); CeedChk(ierr);
-   ierr = CeedBasisGetNumQuadraturePoints(basisin, &nqpts); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetNumElements(rstrin, &nelem); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetElementSize(rstrin, &elemsize); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetLVectorSize(rstrin, &nnodes); PCeedChk(ierr);
+   ierr = CeedBasisGetNumQuadraturePoints(basisin, &nqpts); PCeedChk(ierr);

   // Determine elem_dof relation
   CeedVector index_vec;
-   ierr = CeedVectorCreate(ceed, nnodes, &index_vec); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, nnodes, &index_vec); PCeedChk(ierr);
   CeedScalar *array;
   ierr = CeedVectorGetArrayWrite(index_vec, CEED_MEM_HOST, &array);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   for (CeedSize i = 0; i < nnodes; ++i)
   {
      array[i] = i;
   }
-   ierr = CeedVectorRestoreArray(index_vec, &array); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(index_vec, &array); PCeedChk(ierr);
   CeedVector elem_dof;
-   ierr = CeedVectorCreate(ceed, nelem * elemsize, &elem_dof); CeedChk(ierr);
-   ierr = CeedVectorSetValue(elem_dof, 0.0); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, nelem * elemsize, &elem_dof); PCeedChk(ierr);
+   ierr = CeedVectorSetValue(elem_dof, 0.0); PCeedChk(ierr);
   CeedElemRestrictionApply(rstrin, CEED_NOTRANSPOSE, index_vec,
-                            elem_dof, CEED_REQUEST_IMMEDIATE); CeedChk(ierr);
+                            elem_dof, CEED_REQUEST_IMMEDIATE); PCeedChk(ierr);
   const CeedScalar * elem_dof_a;
   ierr = CeedVectorGetArrayRead(elem_dof, CEED_MEM_HOST, &elem_dof_a);
-   CeedChk(ierr);
-   ierr = CeedVectorDestroy(&index_vec); CeedChk(ierr);
+   PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&index_vec); PCeedChk(ierr);

   // loop over elements and put in SparseMatrix
   // SparseMatrix * out = new SparseMatrix(nnodes, nnodes);
   MFEM_ASSERT(out->Height() == nnodes, "Sizes don't match!");
   MFEM_ASSERT(out->Width() == nnodes, "Sizes don't match!");
   const CeedScalar *interpin, *gradin;
-   ierr = CeedBasisGetInterp(basisin, &interpin); CeedChk(ierr);
-   ierr = CeedBasisGetGrad(basisin, &gradin); CeedChk(ierr);
+   ierr = CeedBasisGetInterp(basisin, &interpin); PCeedChk(ierr);
+   ierr = CeedBasisGetGrad(basisin, &gradin); PCeedChk(ierr);

   const CeedScalar * assembledqfarray;
   ierr = CeedVectorGetArrayRead(assembledqf, CEED_MEM_HOST, &assembledqfarray);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   CeedInt layout[3];
-   ierr = CeedElemRestrictionGetELayout(rstr_q, &layout); CeedChk(ierr);
-   ierr = CeedElemRestrictionDestroy(&rstr_q); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetELayout(rstr_q, &layout); PCeedChk(ierr);
+   ierr = CeedElemRestrictionDestroy(&rstr_q); PCeedChk(ierr);

   // enforce structurally symmetric for later elimination
   const int skip_zeros = 0;
@@ -280,13 +280,13 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
      out->AddSubMatrix(rows, rows, elem_mat, skip_zeros);
   }

-   ierr = CeedVectorRestoreArrayRead(elem_dof, &elem_dof_a); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&elem_dof); CeedChk(ierr);
+   ierr = CeedVectorRestoreArrayRead(elem_dof, &elem_dof_a); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&elem_dof); PCeedChk(ierr);
   ierr = CeedVectorRestoreArrayRead(assembledqf, &assembledqfarray);
-   CeedChk(ierr);
-   ierr = CeedVectorDestroy(&assembledqf); CeedChk(ierr);
-   ierr = CeedHackFree(&emodein); CeedChk(ierr);
-   ierr = CeedHackFree(&emodeout); CeedChk(ierr);
+   PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&assembledqf); PCeedChk(ierr);
+   ierr = CeedHackFree(&emodein); PCeedChk(ierr);
+   ierr = CeedHackFree(&emodeout); PCeedChk(ierr);

   return 0;
 }
@@ -297,7 +297,7 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)

   CeedSize in_len, out_len;
   ierr = CeedOperatorGetActiveVectorLengths(op, &in_len, &out_len);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   const int nnodes = in_len;
   MFEM_VERIFY(in_len == out_len, "not a square CeedOperator");
   MFEM_VERIFY(in_len == nnodes, "size overflow");
@@ -305,26 +305,26 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)
   SparseMatrix *out = new SparseMatrix(nnodes, nnodes);

   bool isComposite;
-   ierr = CeedOperatorIsComposite(op, &isComposite); CeedChk(ierr);
+   ierr = CeedOperatorIsComposite(op, &isComposite); PCeedChk(ierr);
   if (isComposite)
   {
      CeedInt numsub;
      CeedOperator *subops;
 #if CEED_VERSION_GE(0, 10, 2)
      CeedCompositeOperatorGetNumSub(op, &numsub);
-      ierr = CeedCompositeOperatorGetSubList(op, &subops); CeedChk(ierr);
+      ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr);
 #else
      CeedOperatorGetNumSub(op, &numsub);
-      ierr = CeedOperatorGetSubList(op, &subops); CeedChk(ierr);
+      ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr);
 #endif
      for (int i = 0; i < numsub; ++i)
      {
-         ierr = CeedSingleOperatorFullAssemble(subops[i], out); CeedChk(ierr);
+         ierr = CeedSingleOperatorFullAssemble(subops[i], out); PCeedChk(ierr);
      }
   }
   else
   {
-      ierr = CeedSingleOperatorFullAssemble(op, out); CeedChk(ierr);
+      ierr = CeedSingleOperatorFullAssemble(op, out); PCeedChk(ierr);
   }
   // enforce structurally symmetric for later elimination
   const int skip_zeros = 0;
@@ -338,4 +338,4 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)

 } // namespace mfem

-#endif
+#endif // MFEM_USE_CEED
@@ -19,6 +19,8 @@
 namespace mfem
 {

+class SparseMatrix;
+
 namespace ceed
 {

@@ -34,6 +36,6 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat);

 } // namespace mfem

-#endif
+#endif // MFEM_USE_CEED

-#endif
+#endif // MFEM_CEED_ASSEMBLE_HPP
@@ -15,8 +15,8 @@
 #include "../interface/util.hpp"

 #ifdef MFEM_USE_CEED
-#include <ceed/backend.h>

+#include <ceed/backend.h>
 #include <math.h>
 // todo: should probably use Ceed memory wrappers instead of calloc/free?
 #include <stdlib.h>
@@ -86,14 +86,14 @@ int CeedATPMGElemRestriction(int order,
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedElemRestrictionGetCeed(er_in, &ceed); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetCeed(er_in, &ceed); PCeedChk(ierr);

   CeedInt numelem, numcomp, elemsize;
   CeedSize numnodes;
-   ierr = CeedElemRestrictionGetNumElements(er_in, &numelem); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetLVectorSize(er_in, &numnodes); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetElementSize(er_in, &elemsize); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetNumComponents(er_in, &numcomp); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetNumElements(er_in, &numelem); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetLVectorSize(er_in, &numnodes); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetElementSize(er_in, &elemsize); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetNumComponents(er_in, &numcomp); PCeedChk(ierr);
   if (numcomp != 1)
   {
      // todo: multi-component will require more thought
@@ -107,31 +107,31 @@ int CeedATPMGElemRestriction(int order,

   CeedVector in_lvec, in_evec;
   ierr = CeedElemRestrictionCreateVector(er_in, &in_lvec, &in_evec);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // Create the elem_dof array from the given high-order ElemRestriction
   // by using it to map the L-vector indices to an E-vector
   CeedScalar * lvec_data;
   ierr = CeedVectorGetArrayWrite(in_lvec, CEED_MEM_HOST, &lvec_data);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   for (CeedSize i = 0; i < numnodes; ++i)
   {
      lvec_data[i] = (CeedScalar) i;
   }
-   ierr = CeedVectorRestoreArray(in_lvec, &lvec_data); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(in_lvec, &lvec_data); PCeedChk(ierr);
   CeedInt in_layout[3];
-   ierr = CeedElemRestrictionGetELayout(er_in, &in_layout); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetELayout(er_in, &in_layout); PCeedChk(ierr);
   if (in_layout[0] == 0 && in_layout[1] == 0 && in_layout[2] == 0)
   {
      return CeedError(ceed, 1, "Cannot interpret e-vector ordering of given"
                       "CeedElemRestriction!");
   }
   ierr = CeedElemRestrictionApply(er_in, CEED_NOTRANSPOSE, in_lvec, in_evec,
-                                   CEED_REQUEST_IMMEDIATE); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&in_lvec); CeedChk(ierr);
+                                   CEED_REQUEST_IMMEDIATE); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&in_lvec); PCeedChk(ierr);
   const CeedScalar * in_elem_dof;
   ierr = CeedVectorGetArrayRead(in_evec, CEED_MEM_HOST, &in_elem_dof);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // Create a map (dof_map) that maps high-order ldof indices to
   // low-order ldof indices, with -1 indicating no correspondence
@@ -469,13 +469,13 @@ int CeedATPMGElemRestriction(int order,
                       "CeedATPMGElemRestriction does not yet support this dimension.");
   }

-   ierr = CeedVectorRestoreArrayRead(in_evec, &in_elem_dof); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&in_evec); CeedChk(ierr);
+   ierr = CeedVectorRestoreArrayRead(in_evec, &in_elem_dof); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&in_evec); PCeedChk(ierr);

   ierr = CeedElemRestrictionCreate(ceed, numelem, coarse_elemsize, numcomp,
                                    0, running_out_ldof_count,
                                    CEED_MEM_HOST, CEED_COPY_VALUES, out_elem_dof,
-                                    er_out); CeedChk(ierr);
+                                    er_out); PCeedChk(ierr);

   delete [] out_elem_dof;

@@ -491,7 +491,7 @@ int CeedBasisATPMGCoarseToFine(Ceed ceed, int P1d, int dim, int order_reduction,
   // calling the following Ceed function)
   int ierr;
   ierr = CeedBasisCreateTensorH1Lagrange(ceed, dim, 1, P1d - order_reduction, P1d,
-                                          CEED_GAUSS_LOBATTO, basisc2f); CeedChk(ierr);
+                                          CEED_GAUSS_LOBATTO, basisc2f); PCeedChk(ierr);
   return 0;
 }

@@ -501,13 +501,13 @@ int CeedBasisATPMGCoarseToFine(CeedBasis basisin,
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedBasisGetCeed(basisin, &ceed); CeedChk(ierr);
+   ierr = CeedBasisGetCeed(basisin, &ceed); PCeedChk(ierr);

   CeedInt dim, P1d;
-   ierr = CeedBasisGetDimension(basisin, &dim); CeedChk(ierr);
-   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); CeedChk(ierr);
+   ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
+   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); PCeedChk(ierr);
   ierr = CeedBasisATPMGCoarseToFine(ceed, P1d, dim, order_reduction,
-                                     basisc2f); CeedChk(ierr);
+                                     basisc2f); PCeedChk(ierr);
   return 0;
 }

@@ -518,20 +518,20 @@ int CeedBasisATPMGCoarsen(CeedBasis basisin,
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedBasisGetCeed(basisin, &ceed); CeedChk(ierr);
+   ierr = CeedBasisGetCeed(basisin, &ceed); PCeedChk(ierr);

   CeedInt dim, ncomp, P1d, Q1d;
-   ierr = CeedBasisGetDimension(basisin, &dim); CeedChk(ierr);
-   ierr = CeedBasisGetNumComponents(basisin, &ncomp); CeedChk(ierr);
-   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); CeedChk(ierr);
-   ierr = CeedBasisGetNumQuadraturePoints1D(basisin, &Q1d); CeedChk(ierr);
+   ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
+   ierr = CeedBasisGetNumComponents(basisin, &ncomp); PCeedChk(ierr);
+   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); PCeedChk(ierr);
+   ierr = CeedBasisGetNumQuadraturePoints1D(basisin, &Q1d); PCeedChk(ierr);

   CeedInt coarse_P1d = P1d - order_reduction;

   const CeedScalar *interp1d;
-   ierr = CeedBasisGetInterp1D(basisin, &interp1d); CeedChk(ierr);
+   ierr = CeedBasisGetInterp1D(basisin, &interp1d); PCeedChk(ierr);
   const CeedScalar * grad1d;
-   ierr = CeedBasisGetGrad1D(basisin, &grad1d); CeedChk(ierr);
+   ierr = CeedBasisGetGrad1D(basisin, &grad1d); PCeedChk(ierr);

   CeedScalar * coarse_interp1d = new CeedScalar[coarse_P1d * Q1d];
   CeedScalar * coarse_grad1d = new CeedScalar[coarse_P1d * Q1d];
@@ -542,14 +542,14 @@ int CeedBasisATPMGCoarsen(CeedBasis basisin,
   /* one way you might be able to tell is to just run this algorithm
      with coarse_P1d = 2 (i.e., linear) and look for symmetry in the coarse
      basis matrix? */
-   ierr = CeedLobattoQuadrature(P1d, fine_nodal_points, NULL); CeedChk(ierr);
+   ierr = CeedLobattoQuadrature(P1d, fine_nodal_points, NULL); PCeedChk(ierr);
   for (int i = 0; i < P1d; ++i)
   {
      fine_nodal_points[i] = 0.5 * fine_nodal_points[i] + 0.5; // cheating
   }

   const CeedScalar *interp_ctof;
-   ierr = CeedBasisGetInterp1D(basisc2f, &interp_ctof); CeedChk(ierr);
+   ierr = CeedBasisGetInterp1D(basisc2f, &interp_ctof); PCeedChk(ierr);

   for (int i = 0; i < Q1d; ++i)
   {
@@ -568,12 +568,12 @@ int CeedBasisATPMGCoarsen(CeedBasis basisin,
   }

   const CeedScalar * qref1d;
-   ierr = CeedBasisGetQRef(basisin, &qref1d); CeedChk(ierr);
+   ierr = CeedBasisGetQRef(basisin, &qref1d); PCeedChk(ierr);
   const CeedScalar * qweight1d;
-   ierr = CeedBasisGetQWeights(basisin, &qweight1d); CeedChk(ierr);
+   ierr = CeedBasisGetQWeights(basisin, &qweight1d); PCeedChk(ierr);
   ierr = CeedBasisCreateTensorH1(ceed, dim, ncomp,
                                  coarse_P1d, Q1d, coarse_interp1d, coarse_grad1d,
-                                  qref1d, qweight1d, basisout); CeedChk(ierr);
+                                  qref1d, qweight1d, basisout); PCeedChk(ierr);

   delete [] fine_nodal_points;
   delete [] coarse_interp1d;
@@ -593,19 +593,19 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,

   int ierr;
   Ceed ceed;
-   ierr = CeedOperatorGetCeed(oper, &ceed); CeedChk(ierr);
+   ierr = CeedOperatorGetCeed(oper, &ceed); PCeedChk(ierr);

   CeedQFunction qf;
-   ierr = CeedOperatorGetQFunction(oper, &qf); CeedChk(ierr);
+   ierr = CeedOperatorGetQFunction(oper, &qf); PCeedChk(ierr);
   CeedInt numinputfields, numoutputfields;
   CeedQFunctionField *inputqfields, *outputqfields;
   ierr = CeedQFunctionGetFields(qf, &numinputfields, &inputqfields,
                                 &numoutputfields, &outputqfields);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   CeedOperatorField *inputfields, *outputfields;
   ierr = CeedOperatorGetFields(oper, &numinputfields, &inputfields,
                                &numoutputfields, &outputfields);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   CeedElemRestriction * er_input = new CeedElemRestriction[numinputfields];
   CeedElemRestriction * er_output = new CeedElemRestriction[numoutputfields];
@@ -619,10 +619,11 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
   for (int i = 0; i < numinputfields; ++i)
   {
      ierr = CeedOperatorFieldGetElemRestriction(inputfields[i],
-                                                 &er_input[i]); CeedChk(ierr);
-      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector[i]); CeedChk(ierr);
+                                                 &er_input[i]); PCeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector[i]);
+      PCeedChk(ierr);
      ierr = CeedOperatorFieldGetBasis(inputfields[i], &basis_input[i]);
-      CeedChk(ierr);
+      PCeedChk(ierr);
      if (if_vector[i] == CEED_VECTOR_ACTIVE)
      {
         if (active_input_basis < 0)
@@ -638,11 +639,11 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
   for (int i = 0; i < numoutputfields; ++i)
   {
      ierr = CeedOperatorFieldGetElemRestriction(outputfields[i],
-                                                 &er_output[i]); CeedChk(ierr);
+                                                 &er_output[i]); PCeedChk(ierr);
      ierr = CeedOperatorFieldGetVector(outputfields[i], &of_vector[i]);
-      CeedChk(ierr);
+      PCeedChk(ierr);
      ierr = CeedOperatorFieldGetBasis(outputfields[i], &basis_output[i]);
-      CeedChk(ierr);
+      PCeedChk(ierr);
      if (of_vector[i] == CEED_VECTOR_ACTIVE)
      {
         // should already be coarsened
@@ -659,36 +660,36 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,

   CeedOperator coper;
   ierr = CeedOperatorCreate(ceed, qf, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE,
-                             &coper); CeedChk(ierr);
+                             &coper); PCeedChk(ierr);

   for (int i = 0; i < numinputfields; ++i)
   {
      char * fieldname;
-      ierr = CeedQFunctionFieldGetName(inputqfields[i], &fieldname); CeedChk(ierr);
+      ierr = CeedQFunctionFieldGetName(inputqfields[i], &fieldname); PCeedChk(ierr);
      if (if_vector[i] == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorSetField(coper, fieldname, coarse_er, cbasis,
-                                     if_vector[i]); CeedChk(ierr);
+                                     if_vector[i]); PCeedChk(ierr);
      }
      else
      {
         ierr = CeedOperatorSetField(coper, fieldname, er_input[i], basis_input[i],
-                                     if_vector[i]); CeedChk(ierr);
+                                     if_vector[i]); PCeedChk(ierr);
      }
   }
   for (int i = 0; i < numoutputfields; ++i)
   {
      char * fieldname;
-      ierr = CeedQFunctionFieldGetName(outputqfields[i], &fieldname); CeedChk(ierr);
+      ierr = CeedQFunctionFieldGetName(outputqfields[i], &fieldname); PCeedChk(ierr);
      if (of_vector[i] == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorSetField(coper, fieldname, coarse_er, cbasis,
-                                     of_vector[i]); CeedChk(ierr);
+                                     of_vector[i]); PCeedChk(ierr);
      }
      else
      {
         ierr = CeedOperatorSetField(coper, fieldname, er_output[i], basis_output[i],
-                                     of_vector[i]); CeedChk(ierr);
+                                     of_vector[i]); PCeedChk(ierr);
      }
   }
   delete [] er_input;
@@ -711,21 +712,21 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
   int ierr;

   CeedQFunction qf;
-   ierr = CeedOperatorGetQFunction(oper, &qf); CeedChk(ierr);
+   ierr = CeedOperatorGetQFunction(oper, &qf); PCeedChk(ierr);
   CeedInt numinputfields, numoutputfields;
   CeedOperatorField *inputfields;
   ierr = CeedOperatorGetFields(oper, &numinputfields, &inputfields,
                                &numoutputfields, NULL);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   CeedBasis basis;
-   ierr = CeedOperatorGetActiveBasis(oper, &basis); CeedChk(ierr);
+   ierr = CeedOperatorGetActiveBasis(oper, &basis); PCeedChk(ierr);
   ierr = CeedBasisATPMGCoarseToFine(basis, basis_ctof_out, order_reduction);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   ierr = CeedBasisATPMGCoarsen(basis, *basis_ctof_out, coarse_basis_out,
-                                order_reduction); CeedChk(ierr);
+                                order_reduction); PCeedChk(ierr);
   ierr = CeedATPMGOperator(oper, order_reduction, coarse_er, *coarse_basis_out,
-                            *basis_ctof_out, out); CeedChk(ierr);
+                            *basis_ctof_out, out); PCeedChk(ierr);
   return 0;
 }

@@ -734,11 +735,11 @@ int CeedOperatorGetOrder(CeedOperator oper, CeedInt * order)
   int ierr;

   CeedOperatorField active_field;
-   ierr = CeedOperatorGetActiveField(oper, &active_field); CeedChk(ierr);
+   ierr = CeedOperatorGetActiveField(oper, &active_field); PCeedChk(ierr);
   CeedBasis basis;
-   ierr = CeedOperatorFieldGetBasis(active_field, &basis); CeedChk(ierr);
+   ierr = CeedOperatorFieldGetBasis(active_field, &basis); PCeedChk(ierr);
   int P1d;
-   ierr = CeedBasisGetNumNodes1D(basis, &P1d); CeedChk(ierr);
+   ierr = CeedBasisGetNumNodes1D(basis, &P1d); PCeedChk(ierr);
   *order = P1d - 1;

   return 0;
@@ -753,13 +754,13 @@ int CeedATPMGBundle(CeedOperator oper, int order_reduction,
 {
   int ierr;
   CeedInt order;
-   ierr = CeedOperatorGetOrder(oper, &order); CeedChk(ierr);
+   ierr = CeedOperatorGetOrder(oper, &order); PCeedChk(ierr);
   CeedElemRestriction ho_er;
-   ierr = CeedOperatorGetActiveElemRestriction(oper, &ho_er); CeedChk(ierr);
+   ierr = CeedOperatorGetActiveElemRestriction(oper, &ho_er); PCeedChk(ierr);
   ierr = CeedATPMGElemRestriction(order, order_reduction, ho_er, er_out, dof_map);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   ierr = CeedATPMGOperator(oper, order_reduction, *er_out, coarse_basis_out,
-                            basis_ctof_out, coarse_oper); CeedChk(ierr);
+                            basis_ctof_out, coarse_oper); PCeedChk(ierr);
   return 0;
 }

@@ -14,175 +14,166 @@
 namespace mfem
 {

+void DofTransformation::TransformPrimal(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Size();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->TransformPrimal(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->TransformPrimal(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void DofTransformation::InvTransformPrimal(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Height();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->InvTransformPrimal(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->InvTransformPrimal(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void DofTransformation::TransformDual(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Size();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->TransformDual(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->TransformDual(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void DofTransformation::InvTransformDual(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Size();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->InvTransformDual(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->InvTransformDual(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
 void TransformPrimal(const DofTransformation *ran_dof_trans,
                     const DofTransformation *dom_dof_trans,
                     DenseMatrix &elmat)
 {
-   if (ran_dof_trans && dom_dof_trans)
-   {
-      ran_dof_trans->TransformPrimalCols(elmat);
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else if (ran_dof_trans)
+   // No action if both transformations are NULL
+   if (ran_dof_trans)
   {
      ran_dof_trans->TransformPrimalCols(elmat);
   }
-   else if (dom_dof_trans)
+   if (dom_dof_trans)
   {
      dom_dof_trans->TransformDualRows(elmat);
   }
-   else
-   {
-      // If both transformations are NULL this function should not be called
-   }
 }

 void TransformDual(const DofTransformation *ran_dof_trans,
                   const DofTransformation *dom_dof_trans,
                   DenseMatrix &elmat)
 {
-   if (ran_dof_trans && dom_dof_trans)
-   {
-      ran_dof_trans->TransformDualCols(elmat);
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else if (ran_dof_trans)
+   // No action if both transformations are NULL
+   if (ran_dof_trans)
   {
      ran_dof_trans->TransformDualCols(elmat);
   }
-   else if (dom_dof_trans)
+   if (dom_dof_trans)
   {
      dom_dof_trans->TransformDualRows(elmat);
   }
-   else
-   {
-      // If both transformations are NULL this function should not be called
-   }
-}
-
-void StatelessVDofTransformation::TransformPrimal(const Array<int> & face_ori,
-                                                  double *v) const
-{
-   int size = sdoftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES || vdim_ == 1)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->TransformPrimal(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->TransformPrimal(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void StatelessVDofTransformation::InvTransformPrimal(
-   const Array<int> & face_ori,
-   double *v) const
-{
-   int size = sdoftrans_->Height();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->InvTransformPrimal(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->InvTransformPrimal(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void StatelessVDofTransformation::TransformDual(const Array<int> & face_ori,
-                                                double *v) const
-{
-   int size = sdoftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->TransformDual(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->TransformDual(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void StatelessVDofTransformation::InvTransformDual(const Array<int> & face_ori,
-                                                   double *v) const
-{
-   int size = sdoftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->InvTransformDual(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->InvTransformDual(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
 }

 // ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
-const double ND_StatelessDofTransformation::T_data[24] =
+const double ND_DofTransformation::T_data[24] =
 {
   1.0,  0.0,  0.0,  1.0,
   -1.0, -1.0,  0.0,  1.0,
@@ -192,11 +183,11 @@ const double ND_StatelessDofTransformation::T_data[24] =
   0.0,  1.0,  1.0,  0.0
 };

-const DenseTensor ND_StatelessDofTransformation
-::T(const_cast<double*>(ND_StatelessDofTransformation::T_data), 2, 2, 6);
+const DenseTensor ND_DofTransformation
+::T(const_cast<double *>(ND_DofTransformation::T_data), 2, 2, 6);

 // ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
-const double ND_StatelessDofTransformation::TInv_data[24] =
+const double ND_DofTransformation::TInv_data[24] =
 {
   1.0,  0.0,  0.0,  1.0,
   -1.0, -1.0,  0.0,  1.0,
@@ -206,12 +197,11 @@ const double ND_StatelessDofTransformation::TInv_data[24] =
   0.0,  1.0,  1.0,  0.0
 };

-const DenseTensor ND_StatelessDofTransformation
-::TInv(const_cast<double*>(TInv_data), 2, 2, 6);
+const DenseTensor ND_DofTransformation
+::TInv(const_cast<double *>(TInv_data), 2, 2, 6);

-ND_StatelessDofTransformation::ND_StatelessDofTransformation(int size, int p,
-                                                             int num_edges,
-                                                             int num_tri_faces)
+ND_DofTransformation::ND_DofTransformation(int size, int p, int num_edges,
+                                           int num_tri_faces)
   : StatelessDofTransformation(size)
   , order(p)
   , nedofs(p)
@@ -221,18 +211,19 @@ ND_StatelessDofTransformation::ND_StatelessDofTransformation(int size, int p,
 {
 }

-void ND_StatelessDofTransformation::TransformPrimal(const Array<int> & Fo,
-                                                    double *v) const
+void ND_DofTransformation::TransformPrimal(const Array<int> & Fo,
+                                           double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -240,23 +231,25 @@ void ND_StatelessDofTransformation::TransformPrimal(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2.UseExternalData(const_cast<double *>(T.GetData(Fo[f])), 2, 2);
+         T2.Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_StatelessDofTransformation::InvTransformPrimal(const Array<int> & Fo,
-                                                       double *v) const
+void ND_DofTransformation::InvTransformPrimal(const Array<int> & Fo,
+                                              double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2Inv;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -264,23 +257,24 @@ void ND_StatelessDofTransformation::InvTransformPrimal(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2Inv.UseExternalData(const_cast<double *>(TInv.GetData(Fo[f])), 2, 2);
+         T2Inv.Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_StatelessDofTransformation::TransformDual(const Array<int> & Fo,
-                                                  double *v) const
+void ND_DofTransformation::TransformDual(const Array<int> & Fo, double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2Inv;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -288,23 +282,25 @@ void ND_StatelessDofTransformation::TransformDual(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2Inv.UseExternalData(const_cast<double *>(TInv.GetData(Fo[f])), 2, 2);
+         T2Inv.MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_StatelessDofTransformation::InvTransformDual(const Array<int> & Fo,
-                                                     double *v) const
+void ND_DofTransformation::InvTransformDual(const Array<int> & Fo,
+                                            double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -312,7 +308,8 @@ void ND_StatelessDofTransformation::InvTransformDual(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2.UseExternalData(const_cast<double *>(T.GetData(Fo[f])), 2, 2);
+         T2.MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }
@@ -80,6 +80,9 @@ public:
   inline int Width() const { return size_; }
   inline int NumCols() const { return size_; }

+   /// If the DofTransformation performs no transformation
+   virtual bool IsIdentity() const = 0;
+
   /** Transform local DoFs to align with the global DoFs. For example, this
       transformation can be used to map the local vector computed by
       FiniteElement::Project() to the transformed vector stored within a
@@ -115,6 +118,8 @@ public:
   inline void InvTransformDual(const Array<int> & face_orientation,
                                Vector &v) const
   { InvTransformDual(face_orientation, v.GetData()); }
+
+   virtual ~StatelessDofTransformation() = default;
 };

 /** The DofTransformation class is an extension of the
@@ -133,35 +138,76 @@ public:
    transferring finite element degrees of freedom between different meshes.
    For examples of its use see the TransferMap used by the SubMesh class.
   */
-class DofTransformation : virtual public StatelessDofTransformation
+class DofTransformation
 {
 protected:
-   Array<int> Fo;
-
-   DofTransformation(int size)
-      : StatelessDofTransformation(size) {}
+   Array<int> Fo_;
+   const StatelessDofTransformation * dof_trans_;
+   int vdim_;
+   int ordering_;

 public:
+   /** @brief Default constructor which requires that SetDofTransformation be
+       called before use. */
+   DofTransformation(int vdim = 1, int ordering = 0)
+      : dof_trans_(NULL)
+      , vdim_(vdim)
+      , ordering_(ordering)
+   {}
+
+   /// Constructor with a known StatelessDofTransformation
+   DofTransformation(const StatelessDofTransformation & dof_trans,
+                     int vdim = 1, int ordering = 0)
+      : dof_trans_(&dof_trans)
+      , vdim_(vdim)
+      , ordering_(ordering)
+   {}

   /** @brief Configure the transformation using face orientations for the
       current element. */
   /// The face_orientation array can be obtained from Mesh::GetElementFaces.
-   inline void SetFaceOrientations(const Array<int> & face_orientation)
-   { Fo = face_orientation; }
+   inline void SetFaceOrientations(const Array<int> & Fo)
+   { Fo_ = Fo; }

-   inline const Array<int> & GetFaceOrientations() const { return Fo; }
+   /// Return the face orientations for the current element
+   inline const Array<int> & GetFaceOrientations() const { return Fo_; }

-   using StatelessDofTransformation::TransformPrimal;
-   using StatelessDofTransformation::InvTransformPrimal;
-   using StatelessDofTransformation::TransformDual;
-   using StatelessDofTransformation::InvTransformDual;
+   /// Set or change the nested StatelessDofTransformation object
+   inline void SetDofTransformation(const StatelessDofTransformation & dof_trans)
+   {
+      dof_trans_ = &dof_trans;
+   }
+   inline void SetDofTransformation(const StatelessDofTransformation * dof_trans)
+   {
+      dof_trans_ = dof_trans;
+   }
+
+   /// Return the nested StatelessDofTransformation object
+   inline const StatelessDofTransformation * GetDofTransformation() const
+   { return dof_trans_; }
+
+   /// Set or change the vdim and ordering parameter
+   inline void SetVDim(int vdim = 1, int ordering = 0)
+   {
+      vdim_ = vdim;
+      ordering_ = ordering;
+   }
+
+   /// Return the current vdim value
+   inline int GetVDim() const { return vdim_; }
+
+   inline int Size() const { return dof_trans_->Size(); }
+   inline int Height() const { return dof_trans_->Height(); }
+   inline int NumRows() const { return dof_trans_->NumRows(); }
+   inline int Width() const { return dof_trans_->Width(); }
+   inline int NumCols() const { return dof_trans_->NumCols(); }
+   inline bool IsIdentity() const { return dof_trans_->IsIdentity(); }

   /** Transform local DoFs to align with the global DoFs. For example, this
       transformation can be used to map the local vector computed by
       FiniteElement::Project() to the transformed vector stored within a
       GridFunction object. */
-   inline void TransformPrimal(double *v) const
-   { TransformPrimal(Fo, v); }
+   void TransformPrimal(double *v) const;
   inline void TransformPrimal(Vector &v) const
   { TransformPrimal(v.GetData()); }

@@ -179,21 +225,18 @@ public:
       transform the vector obtained using GridFunction::GetSubVector before it
       can be used to compute a local interpolation.
   */
-   inline void InvTransformPrimal(double *v) const
-   { InvTransformPrimal(Fo, v); }
+   void InvTransformPrimal(double *v) const;
   inline void InvTransformPrimal(Vector &v) const
   { InvTransformPrimal(v.GetData()); }

   /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
       into a LinearForm object. */
-   inline void TransformDual(double *v) const
-   { TransformDual(Fo, v); }
+   void TransformDual(double *v) const;
   inline void TransformDual(Vector &v) const
   { TransformDual(v.GetData()); }

   /** Inverse Transform dual DoFs */
-   inline void InvTransformDual(double *v) const
-   { InvTransformDual(Fo, v); }
+   void InvTransformDual(double *v) const;
   inline void InvTransformDual(Vector &v) const
   { InvTransformDual(v.GetData()); }

@@ -225,8 +268,6 @@ public:
         TransformDual(V.GetColumn(c));
      }
   }
-
-   virtual ~DofTransformation() = default;
 };

 /** Transform a matrix of DoFs entries from different finite element spaces as
@@ -245,145 +286,6 @@ void TransformDual(const DofTransformation *ran_dof_trans,
                   const DofTransformation *dom_dof_trans,
                   DenseMatrix &elmat);

-/** The StatelessVDofTransformation class implements a nested transformation
-    where an arbitrary StatelessDofTransformation is replicated with a
-    vdim >= 1.
-*/
-class StatelessVDofTransformation : virtual public StatelessDofTransformation
-{
-protected:
-   int vdim_;
-   int ordering_;
-   StatelessDofTransformation * sdoftrans_;
-
-public:
-   /** @brief Default constructor which requires that SetDofTransformation be
-       called before use. */
-   StatelessVDofTransformation(int vdim = 1, int ordering = 0)
-      : StatelessDofTransformation(0)
-      , vdim_(vdim)
-      , ordering_(ordering)
-      , sdoftrans_(NULL)
-   {}
-
-   /// Constructor with a known StatelessDofTransformation
-   StatelessVDofTransformation(StatelessDofTransformation & doftrans,
-                               int vdim = 1,
-                               int ordering = 0)
-      : StatelessDofTransformation(vdim * doftrans.Size())
-      , vdim_(vdim)
-      , ordering_(ordering)
-      , sdoftrans_(&doftrans)
-   {}
-
-   /// Set or change the vdim parameter
-   inline void SetVDim(int vdim)
-   {
-      vdim_ = vdim;
-      if (sdoftrans_)
-      {
-         size_ = vdim_ * sdoftrans_->Size();
-      }
-   }
-
-   /// Return the current vdim value
-   inline int GetVDim() const { return vdim_; }
-
-   /// Set or change the nested StatelessDofTransformation object
-   inline void SetDofTransformation(StatelessDofTransformation & doftrans)
-   {
-      size_ = vdim_ * doftrans.Size();
-      sdoftrans_ = &doftrans;
-   }
-
-   /// Return the nested StatelessDofTransformation object
-   inline StatelessDofTransformation * GetDofTransformation() const
-   { return sdoftrans_; }
-
-   using StatelessDofTransformation::TransformPrimal;
-   using StatelessDofTransformation::InvTransformPrimal;
-   using StatelessDofTransformation::TransformDual;
-   using StatelessDofTransformation::InvTransformDual;
-
-   /** Specializations of these base class methods which account for the vdim
-       and ordering of the full set of DoFs.
-   */
-   void TransformPrimal(const Array<int> & face_ori, double *v) const;
-   void InvTransformPrimal(const Array<int> & face_ori, double *v) const;
-   void TransformDual(const Array<int> & face_ori, double *v) const;
-   void InvTransformDual(const Array<int> & face_ori, double *v) const;
-};
-
-/** The VDofTransformation class implements a nested transformation where an
-    arbitrary DofTransformation is replicated with a vdim >= 1.
-*/
-class VDofTransformation : public StatelessVDofTransformation,
-   public DofTransformation
-{
-protected:
-   DofTransformation * doftrans_;
-
-public:
-   /** @brief Default constructor which requires that SetDofTransformation be
-       called before use. */
-   VDofTransformation(int vdim = 1, int ordering = 0)
-      : StatelessDofTransformation(0)
-      , StatelessVDofTransformation(vdim, ordering)
-      , DofTransformation(0)
-      , doftrans_(NULL)
-   {}
-
-   /// Constructor with a known DofTransformation
-   /// @note The face orientations in @a doftrans will be copied into the
-   /// new VDofTransformation object.
-   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
-                      int ordering = 0)
-      : StatelessDofTransformation(vdim * doftrans.Size())
-      , StatelessVDofTransformation(doftrans, vdim, ordering)
-      , DofTransformation(vdim * doftrans.Size())
-      , doftrans_(&doftrans)
-   {
-      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
-   }
-
-   using StatelessVDofTransformation::SetDofTransformation;
-
-   /// Set or change the nested DofTransformation object
-   /// @note The face orientations in @a doftrans will be copied into the
-   /// VDofTransformation object.
-   void SetDofTransformation(DofTransformation & doftrans)
-   {
-      doftrans_ = &doftrans;
-      StatelessVDofTransformation::SetDofTransformation(doftrans);
-      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
-   }
-
-   /// Return the nested DofTransformation object
-   inline DofTransformation * GetDofTransformation() const { return doftrans_; }
-
-   /// Set new face orientations in both the VDofTransformation and the
-   /// DofTransformation contained within (if there is one).
-   inline void SetFaceOrientations(const Array<int> & face_orientation)
-   {
-      DofTransformation::SetFaceOrientations(face_orientation);
-      if (doftrans_) { doftrans_->SetFaceOrientations(face_orientation); }
-   }
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   inline void TransformPrimal(double *v) const
-   { TransformPrimal(Fo, v); }
-   inline void InvTransformPrimal(double *v) const
-   { InvTransformPrimal(Fo, v); }
-   inline void TransformDual(double *v) const
-   { TransformDual(Fo, v); }
-   inline void InvTransformDual(double *v) const
-   { InvTransformDual(Fo, v); }
-};
-
 /** Abstract base class for high-order Nedelec spaces on elements with
    triangular faces.

@@ -396,7 +298,7 @@ public:
    be accessed as DenseMatrices using the GetFaceTransform() and
    GetFaceInverseTransform() methods.
 */
-class ND_StatelessDofTransformation : virtual public StatelessDofTransformation
+class ND_DofTransformation : public StatelessDofTransformation
 {
 private:
   static const double T_data[24];
@@ -410,8 +312,7 @@ protected:
   const int nedges; // number of edges per element
   const int nfaces; // number of triangular faces per element

-   ND_StatelessDofTransformation(int size, int order,
-                                 int num_edges, int num_tri_faces);
+   ND_DofTransformation(int size, int order, int num_edges, int num_tri_faces);

 public:
   // Return the 2x2 transformation operator for the given face orientation
@@ -421,116 +322,41 @@ public:
   static const DenseMatrix & GetFaceInverseTransform(int ori)
   { return TInv(ori); }

-   void TransformPrimal(const Array<int> & face_orientation,
-                        double *v) const;
+   bool IsIdentity() const override { return nfdofs < 2; }

-   void InvTransformPrimal(const Array<int> & face_orientation,
-                           double *v) const;
-
-   void TransformDual(const Array<int> & face_orientation,
-                      double *v) const;
-
-   void InvTransformDual(const Array<int> & face_orientation,
-                         double *v) const;
+   void TransformPrimal(const Array<int> & Fo, double *v) const override;
+   void InvTransformPrimal(const Array<int> & Fo, double *v) const override;
+   void TransformDual(const Array<int> & Fo, double *v) const override;
+   void InvTransformDual(const Array<int> & Fo, double *v) const override;
 };

 /// Stateless DoF transformation implementation for the Nedelec basis on
 /// triangles
-class ND_TriStatelessDofTransformation : public ND_StatelessDofTransformation
-{
-public:
-   ND_TriStatelessDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2))
-      , ND_StatelessDofTransformation(order*(order + 2), order, 3, 1)
-   {}
-};
-
-/// DoF transformation implementation for the Nedelec basis on triangles
-class ND_TriDofTransformation : public DofTransformation,
-   public ND_TriStatelessDofTransformation
+class ND_TriDofTransformation : public ND_DofTransformation
 {
 public:
   ND_TriDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2))
-      , DofTransformation(order*(order + 2))
-      , ND_TriStatelessDofTransformation(order)
-   {}
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   using ND_TriStatelessDofTransformation::TransformPrimal;
-   using ND_TriStatelessDofTransformation::InvTransformPrimal;
-   using ND_TriStatelessDofTransformation::TransformDual;
-   using ND_TriStatelessDofTransformation::InvTransformDual;
-};
-
-/// DoF transformation implementation for the Nedelec basis on tetrahedra
-class ND_TetStatelessDofTransformation : public ND_StatelessDofTransformation
-{
-public:
-   ND_TetStatelessDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
-      , ND_StatelessDofTransformation(order*(order + 2)*(order + 3)/2, order,
-                                      6, 4)
+      : ND_DofTransformation(order*(order + 2), order, 3, 1)
   {}
 };

 /// DoF transformation implementation for the Nedelec basis on tetrahedra
-class ND_TetDofTransformation : public DofTransformation,
-   public ND_TetStatelessDofTransformation
+class ND_TetDofTransformation : public ND_DofTransformation
 {
 public:
   ND_TetDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
-      , DofTransformation(order*(order + 2)*(order + 3)/2)
-      , ND_TetStatelessDofTransformation(order)
-   {}
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   using ND_TetStatelessDofTransformation::TransformPrimal;
-   using ND_TetStatelessDofTransformation::InvTransformPrimal;
-   using ND_TetStatelessDofTransformation::TransformDual;
-   using ND_TetStatelessDofTransformation::InvTransformDual;
-};
-
-/// DoF transformation implementation for the Nedelec basis on wedge elements
-class ND_WedgeStatelessDofTransformation : public ND_StatelessDofTransformation
-{
-public:
-   ND_WedgeStatelessDofTransformation(int order)
-      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
-      , ND_StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2,
-                                      order, 9, 2)
+      : ND_DofTransformation(order*(order + 2)*(order + 3)/2, order, 6, 4)
   {}
 };

 /// DoF transformation implementation for the Nedelec basis on wedge elements
-class ND_WedgeDofTransformation : public DofTransformation,
-   public ND_WedgeStatelessDofTransformation
+class ND_WedgeDofTransformation : public ND_DofTransformation
 {
 public:
   ND_WedgeDofTransformation(int order)
-      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
-      , DofTransformation(3 * order * ((order + 1) * (order + 2))/2)
-      , ND_WedgeStatelessDofTransformation(order)
+      : ND_DofTransformation(3 * order * ((order + 1) * (order + 2))/2,
+                             order, 9, 2)
   {}
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   using ND_WedgeStatelessDofTransformation::TransformPrimal;
-   using ND_WedgeStatelessDofTransformation::InvTransformPrimal;
-   using ND_WedgeStatelessDofTransformation::TransformDual;
-   using ND_WedgeStatelessDofTransformation::InvTransformDual;
 };

 } // namespace mfem
@@ -355,15 +355,11 @@ int InverseElementTransformation::Transform(const Vector &pt,
         }
         else
         {
-            const int old_type = GlobGeometryRefiner.GetType();
-            GlobGeometryRefiner.SetType(qpts_type);
-            RefinedGeometry &RefG =
-               *GlobGeometryRefiner.Refine(T->GetGeometryType(), order);
+            RefinedGeometry &RefG = *refiner.Refine(T->GetGeometryType(), order);
            int closest_idx = (init_guess_type == ClosestPhysNode) ?
                              FindClosestPhysPoint(pt, RefG.RefPts) :
                              FindClosestRefPoint(pt, RefG.RefPts);
            ip0 = &RefG.RefPts.IntPoint(closest_idx);
-            GlobGeometryRefiner.SetType(old_type);
         }
         break;
      }
@@ -233,7 +233,7 @@ protected:
   // Parameters of the inversion algorithms:
   const IntegrationPoint *ip0;
   int init_guess_type; // algorithm to use
-   int qpts_type; // Quadrature1D type for the initial guess type
+   GeometryRefiner refiner; // geometry refiner for initial guess
   int rel_qpts_order; // num_1D_qpts = max(trans_order+rel_qpts_order,0)+1
   int solver_type; // solution strategy to use
   int max_iter; // max. number of Newton iterations
@@ -276,7 +276,7 @@ public:
      : T(Trans),
        ip0(NULL),
        init_guess_type(Center),
-        qpts_type(Quadrature1D::OpenHalfUniform),
+        refiner(Quadrature1D::OpenHalfUniform),
        rel_qpts_order(-1),
        solver_type(NewtonElementProject),
        max_iter(16),
@@ -301,7 +301,7 @@ public:
   { ip0 = &init_ip; SetInitialGuessType(GivenPoint); }

   /// Set the Quadrature1D type used for the `Closest*` initial guess types.
-   void SetInitGuessPointsType(int q_type) { qpts_type = q_type; }
+   void SetInitGuessPointsType(int q_type) { refiner.SetType(q_type); }

   /// Set the relative order used for the `Closest*` initial guess types.
   /** The number of points in each spatial direction is given by the formula
@@ -361,7 +361,7 @@ public:
 class IsoparametricTransformation : public ElementTransformation
 {
 private:
-   DenseMatrix dshape,d2shape;
+   DenseMatrix dshape, d2shape;
   Vector shape;

   const FiniteElement *FElem;
@@ -74,11 +74,11 @@ public:
 /** @brief The ZienkiewiczZhuEstimator class implements the Zienkiewicz-Zhu
    error estimation procedure.

-    Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
+    [1] Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
    and a posteriori error estimates. Part 1: The recovery technique.
    Int. J. Num. Meth. Engng. 33, 1331-1364 (1992).

-    Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
+    [2] Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
    and a posteriori error estimates. Part 2: Error estimates and adaptivity.
    Int. J. Num. Meth. Engng. 33, 1365-1382 (1992).

@@ -220,8 +220,8 @@ public:
    The required BilinearFormIntegrator must implement the method
    ComputeElementFlux().

-   COMMENTS:
-   *  The present implementation ignores all single-element patches corresponding
+   @note
+   -  The present implementation ignores all single-element patches corresponding
      to boundary faces. This is appropriate for Dirichlet boundaries, but
      suboptimal for Neumann boundaries. Reference 3 shows that a constrained
      least-squares problem, where the reconstructed flux is constrained by the
@@ -229,13 +229,13 @@ public:
      NOTE THAT THIS CONSTRAINED LS PROBLEM IS NOT YET IMPLEMENTED, so it is
      possible that the local error estimates for elements on a Neumann boundary
      are suboptimal.
-   *  The global polynomial basis used for the flux reconstruction is, by default,
+   -  The global polynomial basis used for the flux reconstruction is, by default,
      aligned with the physical Cartesian axis. For patches with 2D elements, this
      has been improved on so that the basis is aligned with the physical patch
      orientation. Reorientation of the flux reconstruction basis is helpful to
      maintain symmetry in the refinement pattern and could be extended to 3D.
-   *  This estimator is ONLY implemented IN SERIAL.
-   *  Anisotropic refinement is NOT YET SUPPORTED.
+   -  This estimator is ONLY implemented IN SERIAL.
+   -  Anisotropic refinement is NOT YET SUPPORTED.

 */
 class LSZienkiewiczZhuEstimator : public ErrorEstimator
@@ -359,135 +359,148 @@ void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,

   // Hessian in physical coords
   lhm.Invert();
-   Mult( hess, lhm, Hessian);
+   Mult(hess, lhm, Hessian);
 }

 const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
                                             DofToQuad::Mode mode) const
 {
+   DofToQuad *d2q = nullptr;
   MFEM_VERIFY(mode == DofToQuad::FULL, "invalid mode requested");

-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-#ifdef MFEM_THREAD_SAFE
-   DenseMatrix vshape(dof, dim);
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
 #endif
-
-   DofToQuad *d2q = new DofToQuad;
-   const int nqpt = ir.GetNPoints();
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = dof;
-   d2q->nqpt = nqpt;
-   if (range_type == SCALAR)
   {
-      d2q->B.SetSize(nqpt*dof);
-      d2q->Bt.SetSize(dof*nqpt);
-
-      Vector shape;
-      vshape.GetColumnReference(0, shape);
-      for (int i = 0; i < nqpt; i++)
+      for (int i = 0; i < dof2quad_array.Size(); i++)
      {
-         const IntegrationPoint &ip = ir.IntPoint(i);
-         CalcShape(ip, shape);
-         for (int j = 0; j < dof; j++)
-         {
-            d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = shape(j);
-         }
+         d2q = dof2quad_array[i];
+         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
      }
-   }
-   else if (range_type == VECTOR)
-   {
-      d2q->B.SetSize(nqpt*dim*dof);
-      d2q->Bt.SetSize(dof*nqpt*dim);
-
-      for (int i = 0; i < nqpt; i++)
+      if (!d2q)
      {
-         const IntegrationPoint &ip = ir.IntPoint(i);
-         CalcVShape(ip, vshape);
-         for (int d = 0; d < dim; d++)
+#ifdef MFEM_THREAD_SAFE
+         DenseMatrix vshape(dof, dim);
+#endif
+         d2q = new DofToQuad;
+         const int nqpt = ir.GetNPoints();
+         d2q->FE = this;
+         d2q->IntRule = &ir;
+         d2q->mode = mode;
+         d2q->ndof = dof;
+         d2q->nqpt = nqpt;
+         switch (range_type)
         {
-            for (int j = 0; j < dof; j++)
+            case SCALAR:
            {
-               d2q->B[i+nqpt*(d+dim*j)] = d2q->Bt[j+dof*(i+nqpt*d)] = vshape(j, d);
-            }
-         }
-      }
-   }
-   else
-   {
-      // Skip B and Bt for unknown range type
-   }
-   switch (deriv_type)
-   {
-      case GRAD:
-      {
-         d2q->G.SetSize(nqpt*dim*dof);
-         d2q->Gt.SetSize(dof*nqpt*dim);
+               d2q->B.SetSize(nqpt*dof);
+               d2q->Bt.SetSize(dof*nqpt);

-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcDShape(ip, vshape);
-            for (int d = 0; d < dim; d++)
-            {
-               for (int j = 0; j < dof; j++)
+               Vector shape;
+               vshape.GetColumnReference(0, shape);
+               for (int i = 0; i < nqpt; i++)
               {
-                  d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j, d);
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcShape(ip, shape);
+                  for (int j = 0; j < dof; j++)
+                  {
+                     d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = shape(j);
+                  }
               }
+               break;
            }
-         }
-         break;
-      }
-      case DIV:
-      {
-         d2q->G.SetSize(nqpt*dof);
-         d2q->Gt.SetSize(dof*nqpt);
-
-         Vector divshape;
-         vshape.GetColumnReference(0, divshape);
-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcDivShape(ip, divshape);
-            for (int j = 0; j < dof; j++)
+            case VECTOR:
            {
-               d2q->G[i+nqpt*j] = d2q->Gt[j+dof*i] = divshape(j);
-            }
-         }
-         break;
-      }
-      case CURL:
-      {
-         d2q->G.SetSize(nqpt*cdim*dof);
-         d2q->Gt.SetSize(dof*nqpt*cdim);
+               d2q->B.SetSize(nqpt*dim*dof);
+               d2q->Bt.SetSize(dof*nqpt*dim);

-         DenseMatrix curlshape(vshape.GetData(), dof, cdim);  // cdim <= dim
-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcCurlShape(ip, curlshape);
-            for (int d = 0; d < cdim; d++)
-            {
-               for (int j = 0; j < dof; j++)
+               for (int i = 0; i < nqpt; i++)
               {
-                  d2q->G[i+nqpt*(d+cdim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcVShape(ip, vshape);
+                  for (int d = 0; d < dim; d++)
+                  {
+                     for (int j = 0; j < dof; j++)
+                     {
+                        d2q->B[i+nqpt*(d+dim*j)] =
+                           d2q->Bt[j+dof*(i+nqpt*d)] = vshape(j, d);
+                     }
+                  }
               }
+               break;
            }
+            case UNKNOWN_RANGE_TYPE:
+               // Skip B and Bt for unknown range type
+               break;
         }
-         break;
+         switch (deriv_type)
+         {
+            case GRAD:
+            {
+               d2q->G.SetSize(nqpt*dim*dof);
+               d2q->Gt.SetSize(dof*nqpt*dim);
+
+               for (int i = 0; i < nqpt; i++)
+               {
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcDShape(ip, vshape);
+                  for (int d = 0; d < dim; d++)
+                  {
+                     for (int j = 0; j < dof; j++)
+                     {
+                        d2q->G[i+nqpt*(d+dim*j)] =
+                           d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j, d);
+                     }
+                  }
+               }
+               break;
+            }
+            case DIV:
+            {
+               d2q->G.SetSize(nqpt*dof);
+               d2q->Gt.SetSize(dof*nqpt);
+
+               Vector divshape;
+               vshape.GetColumnReference(0, divshape);
+               for (int i = 0; i < nqpt; i++)
+               {
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcDivShape(ip, divshape);
+                  for (int j = 0; j < dof; j++)
+                  {
+                     d2q->G[i+nqpt*j] = d2q->Gt[j+dof*i] = divshape(j);
+                  }
+               }
+               break;
+            }
+            case CURL:
+            {
+               d2q->G.SetSize(nqpt*cdim*dof);
+               d2q->Gt.SetSize(dof*nqpt*cdim);
+
+               DenseMatrix curlshape(vshape.GetData(), dof, cdim);  // cdim <= dim
+               for (int i = 0; i < nqpt; i++)
+               {
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcCurlShape(ip, curlshape);
+                  for (int d = 0; d < cdim; d++)
+                  {
+                     for (int j = 0; j < dof; j++)
+                     {
+                        d2q->G[i+nqpt*(d+cdim*j)] =
+                           d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
+                     }
+                  }
+               }
+               break;
+            }
+            case NONE:
+               // Skip G and Gt for unknown derivative type
+               break;
+         }
+         dof2quad_array.Append(d2q);
      }
-      case NONE:
-      default:
-         // Skip G and Gt for unknown derivative type
-         break;
   }
-   dof2quad_array.Append(d2q);
   return *d2q;
 }

@@ -904,14 +917,14 @@ VectorFiniteElement::VectorFiniteElement(int D, Geometry::Type G,
 }

 void VectorFiniteElement::CalcShape(
-   const IntegrationPoint &ip, Vector &shape ) const
+   const IntegrationPoint &ip, Vector &shape) const
 {
   mfem_error("Error: Cannot use scalar CalcShape(...) function with\n"
              "   VectorFiniteElements!");
 }

 void VectorFiniteElement::CalcDShape(
-   const IntegrationPoint &ip, DenseMatrix &dshape ) const
+   const IntegrationPoint &ip, DenseMatrix &dshape) const
 {
   mfem_error("Error: Cannot use scalar CalcDShape(...) function with\n"
              "   VectorFiniteElements!");
@@ -2183,51 +2196,72 @@ void Poly_1D::CalcChebyshev(const int p, const double x, double *u, double *d,

 const double *Poly_1D::GetPoints(const int p, const int btype)
 {
+   Array<double*> *pts;
   BasisType::Check(btype);
   const int qtype = BasisType::GetQuadrature1D(btype);
-
   if (qtype == Quadrature1D::Invalid) { return NULL; }

-   if (points_container.find(btype) == points_container.end())
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (Poly1DGetPoints)
+#endif
   {
-      points_container[btype] = new Array<double*>(h_mt);
+      auto it = points_container.find(btype);
+      if (it != points_container.end())
+      {
+         pts = it->second;
+      }
+      else
+      {
+         pts = new Array<double*>(h_mt);
+         points_container[btype] = pts;
+      }
+      if (pts->Size() <= p)
+      {
+         pts->SetSize(p + 1, NULL);
+      }
+      if ((*pts)[p] == NULL)
+      {
+         (*pts)[p] = new double[p + 1];
+         quad_func.GivePolyPoints(p + 1, (*pts)[p], qtype);
+      }
   }
-   Array<double*> &pts = *points_container[btype];
-   if (pts.Size() <= p)
-   {
-      pts.SetSize(p + 1, NULL);
-   }
-   if (pts[p] == NULL)
-   {
-      pts[p] = new double[p + 1];
-      quad_func.GivePolyPoints(p+1, pts[p], qtype);
-   }
-   return pts[p];
+   return (*pts)[p];
 }

 Poly_1D::Basis &Poly_1D::GetBasis(const int p, const int btype)
 {
+   Array<Basis*> *bases;
   BasisType::Check(btype);

-   if ( bases_container.find(btype) == bases_container.end() )
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (Poly1DGetBasis)
+#endif
   {
-      // we haven't been asked for basis or points of this type yet
-      bases_container[btype] = new Array<Basis*>(h_mt);
+      auto it = bases_container.find(btype);
+      if (it != bases_container.end())
+      {
+         bases = it->second;
+      }
+      else
+      {
+         // we haven't been asked for basis or points of this type yet
+         bases = new Array<Basis*>(h_mt);
+         bases_container[btype] = bases;
+      }
+      if (bases->Size() <= p)
+      {
+         bases->SetSize(p + 1, NULL);
+      }
+      if ((*bases)[p] == NULL)
+      {
+         EvalType etype;
+         if (btype == BasisType::Positive) { etype = Positive; }
+         else if (btype == BasisType::IntegratedGLL) { etype = Integrated; }
+         else { etype = Barycentric; }
+         (*bases)[p] = new Basis(p, GetPoints(p, btype), etype);
+      }
   }
-   Array<Basis*> &bases = *bases_container[btype];
-   if (bases.Size() <= p)
-   {
-      bases.SetSize(p + 1, NULL);
-   }
-   if (bases[p] == NULL)
-   {
-      EvalType etype;
-      if (btype == BasisType::Positive) { etype = Positive; }
-      else if (btype == BasisType::IntegratedGLL) { etype = Integrated; }
-      else { etype = Barycentric; }
-      bases[p] = new Basis(p, GetPoints(p, btype), etype);
-   }
-   return *bases[p];
+   return *(*bases)[p];
 }

 Poly_1D::~Poly_1D()
@@ -2236,7 +2270,7 @@ Poly_1D::~Poly_1D()
        it != points_container.end() ; ++it)
   {
      Array<double*>& pts = *it->second;
-      for ( int i = 0 ; i < pts.Size() ; ++i )
+      for (int i = 0; i < pts.Size(); ++i)
      {
         delete [] pts[i];
      }
@@ -2247,7 +2281,7 @@ Poly_1D::~Poly_1D()
        it != bases_container.end() ; ++it)
   {
      Array<Basis*>& bases = *it->second;
-      for ( int i = 0 ; i < bases.Size() ; ++i )
+      for (int i = 0; i < bases.Size(); ++i)
      {
         delete bases[i];
      }
@@ -2461,39 +2495,47 @@ const DofToQuad &TensorBasisElement::GetTensorDofToQuad(
   DofToQuad::Mode mode, const Poly_1D::Basis &basis, bool closed,
   Array<DofToQuad*> &dof2quad_array)
 {
+   DofToQuad *d2q = nullptr;
   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");

-   for (int i = 0; i < dof2quad_array.Size(); i++)
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
+#endif
   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const int ndof = closed ? fe.GetOrder() + 1 : fe.GetOrder();
-   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
-   d2q->FE = &fe;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = ndof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*ndof);
-   d2q->Bt.SetSize(ndof*nqpt);
-   d2q->G.SetSize(nqpt*ndof);
-   d2q->Gt.SetSize(ndof*nqpt);
-   Vector val(ndof), grad(ndof);
-   for (int i = 0; i < nqpt; i++)
-   {
-      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
-      // of the 1D rule.
-      basis.Eval(ir.IntPoint(i).x, val, grad);
-      for (int j = 0; j < ndof; j++)
+      for (int i = 0; i < dof2quad_array.Size(); i++)
      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
-         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
+         d2q = dof2quad_array[i];
+         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
+      }
+      if (!d2q)
+      {
+         d2q = new DofToQuad;
+         const int ndof = closed ? fe.GetOrder() + 1 : fe.GetOrder();
+         const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
+         d2q->FE = &fe;
+         d2q->IntRule = &ir;
+         d2q->mode = mode;
+         d2q->ndof = ndof;
+         d2q->nqpt = nqpt;
+         d2q->B.SetSize(nqpt*ndof);
+         d2q->Bt.SetSize(ndof*nqpt);
+         d2q->G.SetSize(nqpt*ndof);
+         d2q->Gt.SetSize(ndof*nqpt);
+         Vector val(ndof), grad(ndof);
+         for (int i = 0; i < nqpt; i++)
+         {
+            // The first 'nqpt' points in 'ir' have the same x-coordinates as those
+            // of the 1D rule.
+            basis.Eval(ir.IntPoint(i).x, val, grad);
+            for (int j = 0; j < ndof; j++)
+            {
+               d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
+               d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
+            }
+         }
+         dof2quad_array.Append(d2q);
      }
   }
-   dof2quad_array.Append(d2q);
   return *d2q;
 }

@@ -250,7 +250,7 @@ protected:
   /// Container for all DofToQuad objects created by the FiniteElement.
   /** Multiple DofToQuad objects may be needed when different quadrature rules
       or different DofToQuad::Mode are used. */
-   mutable Array<DofToQuad*> dof2quad_array;
+   mutable Array<DofToQuad *> dof2quad_array;

 public:
   /// Enumeration for range_type and deriv_range_type
@@ -596,7 +596,7 @@ public:
   /** @brief Return a DoF transformation object for this particular type of
       basis.
   */
-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return NULL; }

   /// Deconstruct the FiniteElement
@@ -1026,8 +1026,8 @@ public:
   };

 private:
-   typedef std::map< int, Array<double*>* > PointsMap;
-   typedef std::map< int, Array<Basis*>* > BasisMap;
+   typedef std::map<int, Array<double*>*> PointsMap;
+   typedef std::map<int, Array<Basis*>*> BasisMap;

   MemoryType h_mt;
   PointsMap points_container;
@@ -6031,7 +6031,7 @@ void RT0PyrFiniteElement::CalcVShape(const IntegrationPoint &ip,
   shape(1,2) = z;

   shape(2,0) = x * (2.0 - z) * ozi;
-   shape(2,1) = - y * z * ozi;;
+   shape(2,1) = - y * z * ozi;
   shape(2,2) = z;

   shape(3,0) = - x * z * ozi;
@@ -179,7 +179,7 @@ class ND_TetrahedronElement : public VectorFiniteElement
   Array<int> dof2tk;
   DenseMatrixInverse Ti;

-   mutable ND_TetStatelessDofTransformation doftrans;
+   ND_TetDofTransformation doftrans;

 public:
   /// Construct the ND_TetrahedronElement of order @a p
@@ -201,7 +201,7 @@ public:
                                  ElementTransformation &Trans,
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return &doftrans; }
   using FiniteElement::Project;
   virtual void Project(VectorCoefficient &vc,
@@ -242,7 +242,7 @@ class ND_TriangleElement : public VectorFiniteElement
   Array<int> dof2tk;
   DenseMatrixInverse Ti;

-   mutable ND_TriStatelessDofTransformation doftrans;
+   ND_TriDofTransformation doftrans;

 public:
   /// Construct the ND_TriangleElement of order @a p
@@ -264,7 +264,7 @@ public:
                                  ElementTransformation &Trans,
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return &doftrans; }
   using FiniteElement::Project;
   virtual void Project(VectorCoefficient &vc,
@@ -346,7 +346,7 @@ private:
 #endif
   Array<int> dof2tk, t_dof, s_dof;

-   mutable ND_WedgeStatelessDofTransformation doftrans;
+   ND_WedgeDofTransformation doftrans;

   H1_TriangleElement H1TriangleFE;
   ND_TriangleElement NDTriangleFE;
@@ -379,7 +379,7 @@ public:
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }

-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return &doftrans; }

   using FiniteElement::Project;
@@ -59,7 +59,7 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
   int p = (this)->GetOrder();
   double x = ip.x, y = ip.y;

-   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
+   Poly_1D::Basis &edgeNodalBasis = poly1d.GetBasis(p, BasisType::GaussLobatto);
   Vector nodalX(p+1);
   Vector nodalY(p+1);

@@ -113,10 +113,9 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
   {
      double *legX = new double[p-1];
      double *legY = new double[p-1];
-      Poly_1D *storeLegendre = new Poly_1D();

-      storeLegendre->CalcLegendre(p-2, x, legX);
-      storeLegendre->CalcLegendre(p-2, y, legY);
+      Poly_1D::CalcLegendre(p-2, x, legX);
+      Poly_1D::CalcLegendre(p-2, y, legY);

      int interior_total = 0;
      for (int j = 4; j < p + 1; j++)
@@ -131,7 +130,6 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,

      delete[] legX;
      delete[] legY;
-      delete storeLegendre;
   }
 }

@@ -141,7 +139,7 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
   int p = (this)->GetOrder();
   double x = ip.x, y = ip.y;

-   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
+   Poly_1D::Basis &edgeNodalBasis = poly1d.GetBasis(p, BasisType::GaussLobatto);
   Vector nodalX(p+1);
   Vector DnodalX(p+1);
   Vector nodalY(p+1);
@@ -203,10 +201,9 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
      double *legY = new double[p-1];
      double *DlegX = new double[p-1];
      double *DlegY = new double[p-1];
-      Poly_1D *storeLegendre = new Poly_1D();

-      storeLegendre->CalcLegendre(p-2, x, legX, DlegX);
-      storeLegendre->CalcLegendre(p-2, y, legY, DlegY);
+      Poly_1D::CalcLegendre(p-2, x, legX, DlegX);
+      Poly_1D::CalcLegendre(p-2, y, legY, DlegY);

      int interior_total = 0;
      for (int j = 4; j < p + 1; j++)
@@ -224,7 +221,6 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
      delete[] legY;
      delete[] DlegX;
      delete[] DlegY;
-      delete storeLegendre;
   }
 }

@@ -2896,7 +2896,7 @@ ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
 }

-StatelessDofTransformation *
+const StatelessDofTransformation *
 ND_FECollection::DofTransformationForGeometry(Geometry::Type GeomType) const
 {
   if (!Geometry::IsTensorProduct(GeomType) && this->GetOrder() > 1)
@@ -63,7 +63,7 @@ public:
   /** @brief Returns a DoF transformation object compatible with this basis
       and geometry type.
   */
-   virtual StatelessDofTransformation *
+   virtual const StatelessDofTransformation *
   DofTransformationForGeometry(Geometry::Type GeomType) const
   { return NULL; }

@@ -483,7 +483,7 @@ public:
   int DofForGeometry(Geometry::Type GeomType) const override
   { return ND_dof[GeomType]; }

-   StatelessDofTransformation *
+   const StatelessDofTransformation *
   DofTransformationForGeometry(Geometry::Type GeomType) const override;

   const int *DofOrderForOrientation(Geometry::Type GeomType,
@@ -13,6 +13,7 @@
 #define MFEM_FEM_HPP

 #include "intrules.hpp"
+#include "intrules_cut.hpp"
 #include "geom.hpp"
 #include "fe.hpp"
 #include "fe_coll.hpp"
@@ -54,6 +55,7 @@
 #include "plinearform.hpp"
 #include "pbilinearform.hpp"
 #include "pnonlinearform.hpp"
+#include "pblockform.hpp"
 #endif

 #ifdef MFEM_USE_SIDRE
@@ -63,7 +63,6 @@ FiniteElementSpace::FiniteElementSpace()
     elem_dof(NULL), elem_fos(NULL), bdr_elem_dof(NULL), bdr_elem_fos(NULL),
     face_dof(NULL),
     NURBSext(NULL), own_ext(false),
-     DoFTrans(0), VDoFTrans(vdim, ordering),
     cP_is_set(false),
     Th(Operator::ANY_TYPE),
     sequence(0), mesh_sequence(0), orders_changed(false), relaxed_hp(false)
@@ -72,7 +71,6 @@ FiniteElementSpace::FiniteElementSpace()
 FiniteElementSpace::FiniteElementSpace(const FiniteElementSpace &orig,
                                       Mesh *mesh_,
                                       const FiniteElementCollection *fec_)
-   : VDoFTrans(orig.vdim, orig.ordering)
 {
   mesh_ = mesh_ ? mesh_ : orig.mesh;
   fec_ = fec_ ? fec_ : orig.fec;
@@ -212,7 +210,7 @@ void FiniteElementSpace::GetVDofs(int vd, Array<int>& dofs, int ndofs_) const
   }
 }

-void FiniteElementSpace::DofsToVDofs (Array<int> &dofs, int ndofs_) const
+void FiniteElementSpace::DofsToVDofs(Array<int> &dofs, int ndofs_) const
 {
   if (vdim == 1) { return; }
   if (ndofs_ < 0) { ndofs_ = this->ndofs; }
@@ -264,7 +262,7 @@ int FiniteElementSpace::DofToVDof(int dof, int vd, int ndofs_) const
 }

 // static function
-void FiniteElementSpace::AdjustVDofs (Array<int> &vdofs)
+void FiniteElementSpace::AdjustVDofs(Array<int> &vdofs)
 {
   int n = vdofs.Size(), *vdof = vdofs;
   for (int i = 0; i < n; i++)
@@ -277,36 +275,36 @@ void FiniteElementSpace::AdjustVDofs (Array<int> &vdofs)
   }
 }

+void FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs,
+                                         DofTransformation &doftrans) const
+{
+   GetElementDofs(i, vdofs, doftrans);
+   DofsToVDofs(vdofs);
+   doftrans.SetVDim(vdim, ordering);
+}
+
 DofTransformation *
 FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
 {
-   DofTransformation * doftrans = GetElementDofs(i, vdofs);
+   DoFTrans.SetDofTransformation(NULL);
+   GetElementVDofs(i, vdofs, DoFTrans);
+   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
+}
+
+void FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs,
+                                            DofTransformation &doftrans) const
+{
+   GetBdrElementDofs(i, vdofs, doftrans);
   DofsToVDofs(vdofs);
-   if (vdim == 1 || doftrans == NULL)
-   {
-      return doftrans;
-   }
-   else
-   {
-      VDoFTrans.SetDofTransformation(*doftrans);
-      return &VDoFTrans;
-   }
+   doftrans.SetVDim(vdim, ordering);
 }

 DofTransformation *
 FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
 {
-   DofTransformation * doftrans = GetBdrElementDofs(i, vdofs);
-   DofsToVDofs(vdofs);
-   if (vdim == 1 || doftrans == NULL)
-   {
-      return doftrans;
-   }
-   else
-   {
-      VDoFTrans.SetDofTransformation(*doftrans);
-      return &VDoFTrans;
-   }
+   DoFTrans.SetDofTransformation(NULL);
+   GetBdrElementVDofs(i, vdofs, DoFTrans);
+   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
 }

 void FiniteElementSpace::GetPatchVDofs(int i, Array<int> &vdofs) const
@@ -505,13 +503,11 @@ void FiniteElementSpace::BuildDofToArrays()
   }
 }

-static void mark_dofs(const Array<int> &dofs, Array<int> &mark_array)
+void MarkDofs(const Array<int> &dofs, Array<int> &mark_array)
 {
-   for (int i = 0; i < dofs.Size(); i++)
+   for (auto d : dofs)
   {
-      int k = dofs[i];
-      if (k < 0) { k = -1 - k; }
-      mark_array[k] = -1;
+      mark_array[d >= 0 ? d : -1 - d] = -1;
   }
 }

@@ -519,11 +515,9 @@ void FiniteElementSpace::GetEssentialVDofs(const Array<int> &bdr_attr_is_ess,
                                           Array<int> &ess_vdofs,
                                           int component) const
 {
-   Array<int> vdofs, dofs;
-
+   Array<int> dofs;
   ess_vdofs.SetSize(GetVSize());
   ess_vdofs = 0;
-
   for (int i = 0; i < GetNBE(); i++)
   {
      if (bdr_attr_is_ess[GetBdrAttribute(i)-1])
@@ -531,16 +525,14 @@ void FiniteElementSpace::GetEssentialVDofs(const Array<int> &bdr_attr_is_ess,
         if (component < 0)
         {
            // Mark all components.
-            GetBdrElementVDofs(i, vdofs);
-            mark_dofs(vdofs, ess_vdofs);
+            GetBdrElementVDofs(i, dofs);
         }
         else
         {
            GetBdrElementDofs(i, dofs);
-            for (int d = 0; d < dofs.Size(); d++)
-            { dofs[d] = DofToVDof(dofs[d], component); }
-            mark_dofs(dofs, ess_vdofs);
+            for (auto &d : dofs) { d = DofToVDof(d, component); }
         }
+         MarkDofs(dofs, ess_vdofs);
      }
   }

@@ -548,38 +540,47 @@ void FiniteElementSpace::GetEssentialVDofs(const Array<int> &bdr_attr_is_ess,
   // local DOFs affected by boundary elements on other processors
   if (Nonconforming())
   {
-      Array<int> bdr_verts, bdr_edges;
-      mesh->ncmesh->GetBoundaryClosure(bdr_attr_is_ess, bdr_verts, bdr_edges);
-
-      for (int i = 0; i < bdr_verts.Size(); i++)
+      Array<int> bdr_verts, bdr_edges, bdr_faces;
+      mesh->ncmesh->GetBoundaryClosure(bdr_attr_is_ess, bdr_verts, bdr_edges,
+                                       bdr_faces);
+      for (auto v : bdr_verts)
      {
         if (component < 0)
         {
-            GetVertexVDofs(bdr_verts[i], vdofs);
-            mark_dofs(vdofs, ess_vdofs);
+            GetVertexVDofs(v, dofs);
         }
         else
         {
-            GetVertexDofs(bdr_verts[i], dofs);
-            for (int d = 0; d < dofs.Size(); d++)
-            { dofs[d] = DofToVDof(dofs[d], component); }
-            mark_dofs(dofs, ess_vdofs);
+            GetVertexDofs(v, dofs);
+            for (auto &d : dofs) { d = DofToVDof(d, component); }
         }
+         MarkDofs(dofs, ess_vdofs);
      }
-      for (int i = 0; i < bdr_edges.Size(); i++)
+      for (auto e : bdr_edges)
      {
         if (component < 0)
         {
-            GetEdgeVDofs(bdr_edges[i], vdofs);
-            mark_dofs(vdofs, ess_vdofs);
+            GetEdgeVDofs(e, dofs);
         }
         else
         {
-            GetEdgeDofs(bdr_edges[i], dofs);
-            for (int d = 0; d < dofs.Size(); d++)
-            { dofs[d] = DofToVDof(dofs[d], component); }
-            mark_dofs(dofs, ess_vdofs);
+            GetEdgeDofs(e, dofs);
+            for (auto &d : dofs) { d = DofToVDof(d, component); }
         }
+         MarkDofs(dofs, ess_vdofs);
+      }
+      for (auto f : bdr_faces)
+      {
+         if (component < 0)
+         {
+            GetEntityVDofs(2, f, dofs);
+         }
+         else
+         {
+            GetEntityDofs(2, f, dofs);
+            for (auto &d : dofs) { d = DofToVDof(d, component); }
+         }
+         MarkDofs(dofs, ess_vdofs);
      }
   }
 }
@@ -598,6 +599,30 @@ void FiniteElementSpace::GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
   else
   {
      R->BooleanMult(ess_vdofs, ess_tdofs);
+#ifdef MFEM_DEBUG
+      // Verify that in boolean arithmetic: P^T ess_dofs = R ess_dofs
+      Array<int> ess_tdofs2(ess_tdofs.Size());
+      GetConformingProlongation()->BooleanMultTranspose(ess_vdofs, ess_tdofs2);
+
+      int counter = 0;
+      std::string error_msg = "failed dof: ";
+      for (int i = 0; i < ess_tdofs2.Size(); ++i)
+      {
+         if (bool(ess_tdofs[i]) != bool(ess_tdofs2[i]))
+         {
+            error_msg += std::to_string(i) += "(R ";
+            error_msg += std::to_string(bool(ess_tdofs[i])) += " P^T ";
+            error_msg += std::to_string(bool(ess_tdofs2[i])) += ") ";
+            counter++;
+         }
+      }
+
+      MFEM_ASSERT(R->Height() == GetConformingProlongation()->Width(), "!");
+      MFEM_ASSERT(R->Width() == GetConformingProlongation()->Height(), "!");
+      MFEM_ASSERT(R->Width() == ess_vdofs.Size(), "!");
+      MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter
+                  << ' ' << error_msg);
+#endif
   }
   MarkerToList(ess_tdofs, ess_tdof_list);
 }
@@ -777,9 +802,9 @@ FiniteElementSpace::H2L_GlobalRestrictionMatrix (FiniteElementSpace *lfes)
   return R;
 }

-void FiniteElementSpace
-::AddDependencies(SparseMatrix& deps, Array<int>& master_dofs,
-                  Array<int>& slave_dofs, DenseMatrix& I, int skipfirst)
+void FiniteElementSpace::AddDependencies(
+   SparseMatrix& deps, Array<int>& master_dofs, Array<int>& slave_dofs,
+   DenseMatrix& I, int skipfirst)
 {
   for (int i = skipfirst; i < slave_dofs.Size(); i++)
   {
@@ -802,11 +827,9 @@ void FiniteElementSpace
   }
 }

-void FiniteElementSpace
-::AddEdgeFaceDependencies(SparseMatrix &deps, Array<int> &master_dofs,
-                          const FiniteElement *master_fe,
-                          Array<int> &slave_dofs, int slave_face,
-                          const DenseMatrix *pm) const
+void FiniteElementSpace::AddEdgeFaceDependencies(
+   SparseMatrix &deps, Array<int> &master_dofs, const FiniteElement *master_fe,
+   Array<int> &slave_dofs, int slave_face, const DenseMatrix *pm) const
 {
   // In variable-order spaces in 3D, we need to only constrain interior face
   // DOFs (this is done one level up), since edge dependencies can be more
@@ -948,6 +971,15 @@ int FiniteElementSpace::GetEntityDofs(int entity, int index, Array<int> &dofs,
   }
 }

+int FiniteElementSpace::GetEntityVDofs(int entity, int index, Array<int> &dofs,
+                                       Geometry::Type master_geom,
+                                       int variant) const
+{
+   const int n = GetEntityDofs(entity, index, dofs, master_geom, variant);
+   DofsToVDofs(dofs);
+   return n;
+}
+
 void FiniteElementSpace::BuildConformingInterpolation() const
 {
 #ifdef MFEM_USE_MPI
@@ -1533,12 +1565,12 @@ SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
                                localP);
 }

-FiniteElementSpace::RefinementOperator::RefinementOperator
-(const FiniteElementSpace* fespace, Table* old_elem_dof, Table* old_elem_fos,
- int old_ndofs)
-   : fespace(fespace)
-   , old_elem_dof(old_elem_dof)
-   , old_elem_fos(old_elem_fos)
+FiniteElementSpace::RefinementOperator::RefinementOperator(
+   const FiniteElementSpace* fespace, Table* old_elem_dof, Table* old_elem_fos,
+   int old_ndofs)
+   : fespace(fespace),
+     old_elem_dof(old_elem_dof),
+     old_elem_fos(old_elem_fos)
 {
   MFEM_VERIFY(fespace->GetNE() >= old_elem_dof->Size(),
               "Previous mesh is not coarser.");
@@ -1553,7 +1585,7 @@ FiniteElementSpace::RefinementOperator::RefinementOperator
      fespace->GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
   }

-   ConstructDoFTrans();
+   ConstructDoFTransArray();
 }

 FiniteElementSpace::RefinementOperator::RefinementOperator(
@@ -1578,59 +1610,58 @@ FiniteElementSpace::RefinementOperator::RefinementOperator(
      old_elem_fos = new Table(*coarse_fes->GetElementToFaceOrientationTable());
   }

-   ConstructDoFTrans();
+   ConstructDoFTransArray();
 }

 FiniteElementSpace::RefinementOperator::~RefinementOperator()
 {
   delete old_elem_dof;
   delete old_elem_fos;
-   for (int i=0; i<old_DoFTrans.Size(); i++)
+   for (int i=0; i<old_DoFTransArray.Size(); i++)
   {
-      delete old_DoFTrans[i];
+      delete old_DoFTransArray[i];
   }
 }

-void FiniteElementSpace::RefinementOperator
-::ConstructDoFTrans()
+void FiniteElementSpace::RefinementOperator::ConstructDoFTransArray()
 {
-   old_DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
-   for (int i=0; i<old_DoFTrans.Size(); i++)
+   old_DoFTransArray.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<old_DoFTransArray.Size(); i++)
   {
-      old_DoFTrans[i] = NULL;
+      old_DoFTransArray[i] = NULL;
   }

   const FiniteElementCollection *fec_ref = fespace->FEColl();
   if (dynamic_cast<const ND_FECollection*>(fec_ref))
   {
-      const FiniteElement * nd_tri =
+      const FiniteElement *nd_tri =
         fec_ref->FiniteElementForGeometry(Geometry::TRIANGLE);
      if (nd_tri)
      {
-         old_DoFTrans[Geometry::TRIANGLE] =
+         old_DoFTransArray[Geometry::TRIANGLE] =
            new ND_TriDofTransformation(nd_tri->GetOrder());
      }

-      const FiniteElement * nd_tet =
+      const FiniteElement *nd_tet =
         fec_ref->FiniteElementForGeometry(Geometry::TETRAHEDRON);
      if (nd_tet)
      {
-         old_DoFTrans[Geometry::TETRAHEDRON] =
+         old_DoFTransArray[Geometry::TETRAHEDRON] =
            new ND_TetDofTransformation(nd_tet->GetOrder());
      }

-      const FiniteElement * nd_pri =
+      const FiniteElement *nd_pri =
         fec_ref->FiniteElementForGeometry(Geometry::PRISM);
      if (nd_pri)
      {
-         old_DoFTrans[Geometry::PRISM] =
+         old_DoFTransArray[Geometry::PRISM] =
            new ND_WedgeDofTransformation(nd_pri->GetOrder());
      }
   }
 }

-void FiniteElementSpace::RefinementOperator
-::Mult(const Vector &x, Vector &y) const
+void FiniteElementSpace::RefinementOperator::Mult(const Vector &x,
+                                                  Vector &y) const
 {
   Mesh* mesh_ref = fespace->GetMesh();
   const CoarseFineTransformations &trans_ref =
@@ -1662,6 +1693,7 @@ void FiniteElementSpace::RefinementOperator
            fespace->DofsToVDofs(vd, vdofs);
            old_dofs.Copy(old_vdofs);
            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+
            x.GetSubVector(old_vdofs, subX);
            lP.Mult(subX, subY);
            y.SetSubVector(vdofs, subY);
@@ -1670,40 +1702,30 @@ void FiniteElementSpace::RefinementOperator
      else
      {
         old_elem_fos->GetRow(emb.parent, old_Fo);
-         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
-
-         DofTransformation *new_doftrans = NULL;
-         VDofTransformation *vdoftrans =
-            dynamic_cast<VDofTransformation*>(doftrans);
-         if (vdoftrans)
-         {
-            new_doftrans = doftrans;
-            doftrans = vdoftrans->GetDofTransformation();
-         }
+         old_DoFTrans.SetDofTransformation(*old_DoFTransArray[geom]);
+         old_DoFTrans.SetFaceOrientations(old_Fo);

+         doftrans->SetVDim();
         for (int vd = 0; vd < rvdim; vd++)
         {
            dofs.Copy(vdofs);
            fespace->DofsToVDofs(vd, vdofs);
            old_dofs.Copy(old_vdofs);
            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+
            x.GetSubVector(old_vdofs, subX);
-            old_DoFTrans[geom]->InvTransformPrimal(subX);
+            old_DoFTrans.InvTransformPrimal(subX);
            lP.Mult(subX, subY);
            doftrans->TransformPrimal(subY);
            y.SetSubVector(vdofs, subY);
         }
-
-         if (vdoftrans)
-         {
-            doftrans = new_doftrans;
-         }
+         doftrans->SetVDim(rvdim, fespace->GetOrdering());
      }
   }
 }

-void FiniteElementSpace::RefinementOperator
-::MultTranspose(const Vector &x, Vector &y) const
+void FiniteElementSpace::RefinementOperator::MultTranspose(const Vector &x,
+                                                           Vector &y) const
 {
   y = 0.0;

@@ -1727,7 +1749,7 @@ void FiniteElementSpace::RefinementOperator
      const Geometry::Type geom = mesh_ref->GetElementBaseGeometry(k);
      const DenseMatrix &lP = localP[geom](emb.matrix);

-      DofTransformation * doftrans = fespace->GetElementDofs(k, f_dofs);
+      DofTransformation *doftrans = fespace->GetElementDofs(k, f_dofs);
      old_elem_dof->GetRow(emb.parent, c_dofs);

      if (!doftrans)
@@ -1742,7 +1764,6 @@ void FiniteElementSpace::RefinementOperator
            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);

            x.GetSubVector(f_vdofs, subX);
-
            for (int p = 0; p < f_dofs.Size(); ++p)
            {
               if (processed[DecodeDof(f_dofs[p])])
@@ -1750,7 +1771,6 @@ void FiniteElementSpace::RefinementOperator
                  subX[p] = 0.0;
               }
            }
-
            lP.MultTranspose(subX, subY);
            y.AddElementVector(c_vdofs, subY);
         }
@@ -1760,17 +1780,10 @@ void FiniteElementSpace::RefinementOperator
         subYt.SetSize(lP.Width());

         old_elem_fos->GetRow(emb.parent, old_Fo);
-         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
-
-         DofTransformation *new_doftrans = NULL;
-         VDofTransformation *vdoftrans =
-            dynamic_cast<VDofTransformation*>(doftrans);
-         if (vdoftrans)
-         {
-            new_doftrans = doftrans;
-            doftrans = vdoftrans->GetDofTransformation();
-         }
+         old_DoFTrans.SetDofTransformation(*old_DoFTransArray[geom]);
+         old_DoFTrans.SetFaceOrientations(old_Fo);

+         doftrans->SetVDim();
         for (int vd = 0; vd < rvdim; vd++)
         {
            f_dofs.Copy(f_vdofs);
@@ -1787,16 +1800,11 @@ void FiniteElementSpace::RefinementOperator
                  subX[p] = 0.0;
               }
            }
-
            lP.MultTranspose(subX, subYt);
-            old_DoFTrans[geom]->TransformDual(subYt);
+            old_DoFTrans.TransformDual(subYt);
            y.AddElementVector(c_vdofs, subYt);
         }
-
-         if (vdoftrans)
-         {
-            doftrans = new_doftrans;
-         }
+         doftrans->SetVDim(rvdim, fespace->GetOrdering());
      }

      for (int p = 0; p < f_dofs.Size(); ++p)
@@ -2024,8 +2032,8 @@ FiniteElementSpace::DerefinementOperator::~DerefinementOperator()
   delete coarse_elem_dof;
 }

-void FiniteElementSpace::DerefinementOperator
-::Mult(const Vector &x, Vector &y) const
+void FiniteElementSpace::DerefinementOperator::Mult(const Vector &x,
+                                                    Vector &y) const
 {
   Array<int> c_vdofs, f_vdofs;
   Vector loc_x, loc_y;
@@ -2227,7 +2235,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh_, NURBSExtension *NURBSext_,
      R_transpose.reset();
      cP_is_set = false;

-      ConstructDoFTrans();
+      ConstructDoFTransArray();
   }
   else
   {
@@ -2239,40 +2247,39 @@ void FiniteElementSpace::Constructor(Mesh *mesh_, NURBSExtension *NURBSext_,
   BuildElementToDofTable();
 }

-void FiniteElementSpace::ConstructDoFTrans()
+void FiniteElementSpace::ConstructDoFTransArray()
 {
-   DestroyDoFTrans();
+   DestroyDoFTransArray();

-   VDoFTrans.SetVDim(vdim);
-   DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
-   for (int i=0; i<DoFTrans.Size(); i++)
+   DoFTransArray.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<DoFTransArray.Size(); i++)
   {
-      DoFTrans[i] = NULL;
+      DoFTransArray[i] = NULL;
   }
   if (mesh->Dimension() < 3) { return; }
   if (dynamic_cast<const ND_FECollection*>(fec))
   {
-      const FiniteElement * nd_tri =
+      const FiniteElement *nd_tri =
         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
      if (nd_tri)
      {
-         DoFTrans[Geometry::TRIANGLE] =
+         DoFTransArray[Geometry::TRIANGLE] =
            new ND_TriDofTransformation(nd_tri->GetOrder());
      }

-      const FiniteElement * nd_tet =
+      const FiniteElement *nd_tet =
         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
      if (nd_tet)
      {
-         DoFTrans[Geometry::TETRAHEDRON] =
+         DoFTransArray[Geometry::TETRAHEDRON] =
            new ND_TetDofTransformation(nd_tet->GetOrder());
      }

-      const FiniteElement * nd_pri =
+      const FiniteElement *nd_pri =
         fec->FiniteElementForGeometry(Geometry::PRISM);
      if (nd_pri)
      {
-         DoFTrans[Geometry::PRISM] =
+         DoFTransArray[Geometry::PRISM] =
            new ND_WedgeDofTransformation(nd_pri->GetOrder());
      }
   }
@@ -2324,7 +2331,7 @@ void FiniteElementSpace::BuildNURBSFaceToDofTable() const
   face_to_be = -1;
   for (int b = 0; b < GetNBE(); b++)
   {
-      int f = mesh->GetBdrElementEdgeIndex(b);
+      int f = mesh->GetBdrElementFaceIndex(b);
      face_to_be[f] = b;
   }

@@ -2476,7 +2483,7 @@ void FiniteElementSpace::Construct()

   ndofs = nvdofs + nedofs + nfdofs + nbdofs;

-   ConstructDoFTrans();
+   ConstructDoFTransArray();

   // record the current mesh sequence number to detect refinement etc.
   mesh_sequence = mesh->GetSequence();
@@ -2501,9 +2508,8 @@ int FiniteElementSpace::MinOrder(VarOrderBits bits)
   return 0;
 }

-void FiniteElementSpace
-::CalcEdgeFaceVarOrders(Array<VarOrderBits> &edge_orders,
-                        Array<VarOrderBits> &face_orders) const
+void FiniteElementSpace::CalcEdgeFaceVarOrders(
+   Array<VarOrderBits> &edge_orders, Array<VarOrderBits> &face_orders) const
 {
   MFEM_ASSERT(IsVariableOrder(), "");
   MFEM_ASSERT(Nonconforming(), "");
@@ -2727,8 +2733,8 @@ int FiniteElementSpace::GetNVariants(int entity, int index) const
 static const char* msg_orders_changed =
   "Element orders changed, you need to Update() the space first.";

-DofTransformation *
-FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
+void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs,
+                                        DofTransformation &doftrans) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

@@ -2736,13 +2742,16 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
   {
      elem_dof->GetRow(elem, dofs);

-      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
+      if (DoFTransArray[mesh->GetElementBaseGeometry(elem)])
      {
         Array<int> Fo;
         elem_fos -> GetRow (elem, Fo);
-         DoFTrans[mesh->GetElementBaseGeometry(elem)]->SetFaceOrientations(Fo);
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetElementBaseGeometry(elem)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
-      return DoFTrans[mesh->GetElementBaseGeometry(elem)];
+      return;
   }

   Array<int> V, E, Eo, F, Fo; // TODO: LocalArray
@@ -2766,10 +2775,12 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
      {
         nfd += fec->GetNumDof(mesh->GetFaceGeometry(F[i]), order);
      }
-      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
+      if (DoFTransArray[mesh->GetElementBaseGeometry(elem)])
      {
-         DoFTrans[mesh->GetElementBaseGeometry(elem)]
-         -> SetFaceOrientations(Fo);
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetElementBaseGeometry(elem)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
   }

@@ -2828,54 +2839,18 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
         dofs.Append(bbase + j);
      }
   }
-   return DoFTrans[mesh->GetElementBaseGeometry(elem)];
 }

-void FiniteElementSpace::GetPatchDofs(int patch, Array<int> &dofs) const
+DofTransformation *FiniteElementSpace::GetElementDofs(int elem,
+                                                      Array<int> &dofs) const
 {
-   MFEM_ASSERT(NURBSext,
-               "FiniteElementSpace::GetPatchDofs needs a NURBSExtension");
-   NURBSext->GetPatchDofs(patch, dofs);
+   DoFTrans.SetDofTransformation(NULL);
+   GetElementDofs(elem, dofs, DoFTrans);
+   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
 }

-const FiniteElement *FiniteElementSpace::GetFE(int i) const
-{
-   if (i < 0 || i >= mesh->GetNE())
-   {
-      if (mesh->GetNE() == 0)
-      {
-         MFEM_ABORT("Empty MPI partitions are not permitted!");
-      }
-      MFEM_ABORT("Invalid element id:" << i << "; minimum allowed:" << 0 <<
-                 ", maximum allowed:" << mesh->GetNE()-1);
-   }
-
-   const FiniteElement *FE =
-      fec->GetFE(mesh->GetElementGeometry(i), GetElementOrderImpl(i));
-
-   if (NURBSext)
-   {
-      NURBSext->LoadFE(i, FE);
-   }
-   else
-   {
-#ifdef MFEM_DEBUG
-      // consistency check: fec->GetOrder() and FE->GetOrder() should return
-      // the same value (for standard, constant-order spaces)
-      if (!IsVariableOrder() && FE->GetDim() > 0)
-      {
-         MFEM_ASSERT(FE->GetOrder() == fec->GetOrder(),
-                     "internal error: " <<
-                     FE->GetOrder() << " != " << fec->GetOrder());
-      }
-#endif
-   }
-
-   return FE;
-}
-
-DofTransformation *
-FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
+void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs,
+                                           DofTransformation &doftrans) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

@@ -2883,17 +2858,19 @@ FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
   {
      bdr_elem_dof->GetRow(bel, dofs);

-      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
+      if (DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)])
      {
         Array<int> Fo;
         bdr_elem_fos -> GetRow (bel, Fo);
-         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
-         SetFaceOrientations(Fo);
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
-      return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
+      return;
   }

-   Array<int> V, E, Eo, Fo; // TODO: LocalArray
+   Array<int> V, E, Eo; // TODO: LocalArray
   int F, oF;

   int dim = mesh->Dimension();
@@ -2917,11 +2894,14 @@ FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
   {
      mesh->GetBdrElementFace(bel, &F, &oF);

-      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
+      if (DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)])
      {
-         Fo.Append(oF);
-         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
-         SetFaceOrientations(Fo);
+         mfem::Array<int> Fo(1);
+         Fo[0] = oF;
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
   }

@@ -2963,8 +2943,14 @@ FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
         dofs.Append(EncodeDof(nvdofs + nedofs + fbase, ind[j]));
      }
   }
+}

-   return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
+DofTransformation *FiniteElementSpace::GetBdrElementDofs(int bel,
+                                                         Array<int> &dofs) const
+{
+   DoFTrans.SetDofTransformation(NULL);
+   GetBdrElementDofs(bel, dofs, DoFTrans);
+   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
 }

 int FiniteElementSpace::GetFaceDofs(int face, Array<int> &dofs,
@@ -3134,18 +3120,6 @@ int FiniteElementSpace::GetNumElementInteriorDofs(int i) const
                         GetElementOrderImpl(i));
 }

-void FiniteElementSpace::GetEdgeInteriorDofs(int i, Array<int> &dofs) const
-{
-   MFEM_VERIFY(!IsVariableOrder(), "not implemented");
-
-   int ne = fec->DofForGeometry(Geometry::SEGMENT);
-   dofs.SetSize (ne);
-   for (int j = 0, k = nvdofs+i*ne; j < ne; j++, k++)
-   {
-      dofs[j] = k;
-   }
-}
-
 void FiniteElementSpace::GetFaceInteriorDofs(int i, Array<int> &dofs) const
 {
   MFEM_VERIFY(!IsVariableOrder(), "not implemented");
@@ -3170,6 +3144,61 @@ void FiniteElementSpace::GetFaceInteriorDofs(int i, Array<int> &dofs) const
   }
 }

+void FiniteElementSpace::GetEdgeInteriorDofs(int i, Array<int> &dofs) const
+{
+   MFEM_VERIFY(!IsVariableOrder(), "not implemented");
+
+   int ne = fec->DofForGeometry(Geometry::SEGMENT);
+   dofs.SetSize (ne);
+   for (int j = 0, k = nvdofs+i*ne; j < ne; j++, k++)
+   {
+      dofs[j] = k;
+   }
+}
+
+void FiniteElementSpace::GetPatchDofs(int patch, Array<int> &dofs) const
+{
+   MFEM_ASSERT(NURBSext,
+               "FiniteElementSpace::GetPatchDofs needs a NURBSExtension");
+   NURBSext->GetPatchDofs(patch, dofs);
+}
+
+const FiniteElement *FiniteElementSpace::GetFE(int i) const
+{
+   if (i < 0 || i >= mesh->GetNE())
+   {
+      if (mesh->GetNE() == 0)
+      {
+         MFEM_ABORT("Empty MPI partitions are not permitted!");
+      }
+      MFEM_ABORT("Invalid element id:" << i << "; minimum allowed:" << 0 <<
+                 ", maximum allowed:" << mesh->GetNE()-1);
+   }
+
+   const FiniteElement *FE =
+      fec->GetFE(mesh->GetElementGeometry(i), GetElementOrderImpl(i));
+
+   if (NURBSext)
+   {
+      NURBSext->LoadFE(i, FE);
+   }
+   else
+   {
+#ifdef MFEM_DEBUG
+      // consistency check: fec->GetOrder() and FE->GetOrder() should return
+      // the same value (for standard, constant-order spaces)
+      if (!IsVariableOrder() && FE->GetDim() > 0)
+      {
+         MFEM_ASSERT(FE->GetOrder() == fec->GetOrder(),
+                     "internal error: " <<
+                     FE->GetOrder() << " != " << fec->GetOrder());
+      }
+#endif
+   }
+
+   return FE;
+}
+
 const FiniteElement *FiniteElementSpace::GetBE(int i) const
 {
   int order = fec->GetOrder();
@@ -3242,8 +3271,8 @@ const FiniteElement *FiniteElementSpace::GetEdgeElement(int i,
   return fec->GetFE(Geometry::SEGMENT, eo);
 }

-const FiniteElement *FiniteElementSpace
-::GetTraceElement(int i, Geometry::Type geom_type) const
+const FiniteElement *FiniteElementSpace::GetTraceElement(
+   int i, Geometry::Type geom_type) const
 {
   return fec->TraceFiniteElementForGeometry(geom_type);
 }
@@ -3283,7 +3312,7 @@ void FiniteElementSpace::Destroy()
   }
   E2BFQ_array.SetSize(0);

-   DestroyDoFTrans();
+   DestroyDoFTransArray();

   dof_elem_array.DeleteAll();
   dof_ldof_array.DeleteAll();
@@ -3301,19 +3330,18 @@ void FiniteElementSpace::Destroy()
      delete bdr_elem_dof;
      delete bdr_elem_fos;
      delete face_dof;
-
      delete [] bdofs;
   }
   ceed::RemoveBasisAndRestriction(this);
 }

-void FiniteElementSpace::DestroyDoFTrans()
+void FiniteElementSpace::DestroyDoFTransArray()
 {
-   for (int i = 0; i < DoFTrans.Size(); i++)
+   for (int i = 0; i < DoFTransArray.Size(); i++)
   {
-      delete DoFTrans[i];
+      delete DoFTransArray[i];
   }
-   DoFTrans.SetSize(0);
+   DoFTransArray.SetSize(0);
 }

 void FiniteElementSpace::GetTransferOperator(
@@ -271,8 +271,8 @@ protected:
   int own_ext;
   mutable Array<int> face_to_be; // NURBS FE space only

-   Array<DofTransformation*> DoFTrans;
-   mutable VDofTransformation VDoFTrans;
+   Array<StatelessDofTransformation *> DoFTransArray;
+   mutable DofTransformation DoFTrans;

   /** Matrix representing the prolongation from the global conforming dofs to
       a set of intermediate partially conforming dofs, e.g. the dofs associated
@@ -328,8 +328,8 @@ protected:
   void Construct();
   void Destroy();

-   void ConstructDoFTrans();
-   void DestroyDoFTrans();
+   void ConstructDoFTransArray();
+   void DestroyDoFTransArray();

   void BuildElementToDofTable() const;
   void BuildBdrElementToDofTable() const;
@@ -383,6 +383,10 @@ protected:
   int GetEntityDofs(int entity, int index, Array<int> &dofs,
                     Geometry::Type master_geom = Geometry::INVALID,
                     int variant = 0) const;
+   /// Helper to get vertex, edge or face VDOFs (entity=0,1,2 resp.).
+   int GetEntityVDofs(int entity, int index, Array<int> &dofs,
+                      Geometry::Type master_geom = Geometry::INVALID,
+                      int variant = 0) const;

   // Get degenerate face DOFs: see explanation in method implementation.
   int GetDegenerateFaceDofs(int index, Array<int> &dofs,
@@ -416,10 +420,10 @@ protected:
      Table* old_elem_dof; // Owned.
      Table* old_elem_fos; // Owned.

-      Array<DofTransformation*> old_DoFTrans;
-      mutable VDofTransformation old_VDoFTrans;
+      Array<StatelessDofTransformation*> old_DoFTransArray;
+      mutable DofTransformation old_DoFTrans;

-      void ConstructDoFTrans();
+      void ConstructDoFTransArray();

   public:
      /** Construct the operator based on the elem_dof table of the original
@@ -803,7 +807,16 @@ public:
   /// with triangular faces.
   ///
   /// @note The returned object should NOT be deleted by the caller.
-   virtual DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;
+   DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;
+
+   /// @brief The same as GetElementDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   virtual void GetElementDofs(int elem, Array<int> &dofs,
+                               DofTransformation &doftrans) const;

   /// @brief Returns indices of degrees of freedom for boundary element 'bel'.
   /// The returned indices are offsets into an @ref ldof vector. See also
@@ -817,13 +830,16 @@ public:
   /// with triangular faces.
   ///
   /// @note The returned object should NOT be deleted by the caller.
-   virtual DofTransformation *GetBdrElementDofs(int bel,
-                                                Array<int> &dofs) const;
+   DofTransformation *GetBdrElementDofs(int bel, Array<int> &dofs) const;

-   /** @brief Returns indices of degrees of freedom for NURBS patch index
-    @a patch. Cartesian ordering is used, for the tensor-product degrees of
-    freedom. */
-   void GetPatchDofs(int patch, Array<int> &dofs) const;
+   /// @brief The same as GetBdrElementDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   virtual void GetBdrElementDofs(int bel, Array<int> &dofs,
+                                  DofTransformation &doftrans) const;

   /// @brief Returns the indices of the degrees of freedom for the specified
   /// face, including the DOFs for the edges and the vertices of the face.
@@ -870,6 +886,13 @@ public:
   /// GetElementInteriorVDofs().
   void GetElementInteriorDofs(int i, Array<int> &dofs) const;

+   /// @brief Returns the number of degrees of freedom associated with the
+   /// interior of the specified element.
+   ///
+   /// See GetElementInteriorDofs() for more information or to obtain the
+   /// relevant indices.
+   int GetNumElementInteriorDofs(int i) const;
+
   /// @brief Returns the indices of the degrees of freedom for the interior
   /// of the specified face.
   ///
@@ -882,13 +905,6 @@ public:
   /// GetFaceInteriorVDofs().
   void GetFaceInteriorDofs(int i, Array<int> &dofs) const;

-   /// @brief Returns the number of degrees of freedom associated with the
-   /// interior of the specified element.
-   ///
-   /// See GetElementInteriorDofs() for more information or to obtain the
-   /// relevant indices.
-   int GetNumElementInteriorDofs(int i) const;
-
   /// @brief Returns the indices of the degrees of freedom for the interior
   /// of the specified edge.
   ///
@@ -897,6 +913,11 @@ public:
   void GetEdgeInteriorDofs(int i, Array<int> &dofs) const;
   ///@}

+   /** @brief Returns indices of degrees of freedom for NURBS patch index
+    @a patch. Cartesian ordering is used, for the tensor-product degrees of
+    freedom. */
+   void GetPatchDofs(int patch, Array<int> &dofs) const;
+
   /// @anchor dof2vdof @name DoF To VDoF Conversion methods
   /// These methods convert between local dof and local vector dof using the
   /// appropriate relationship based on the Ordering::Type defined in this
@@ -1023,6 +1044,15 @@ public:
   /// @note The returned object should NOT be deleted by the caller.
   DofTransformation *GetElementVDofs(int i, Array<int> &vdofs) const;

+   /// @brief The same as GetElementVDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   void GetElementVDofs(int i, Array<int> &vdofs,
+                        DofTransformation &doftrans) const;
+
   /// @brief Returns indices of degrees of freedom for @a i'th boundary
   /// element.
   /// The returned indices are offsets into an @ref ldof vector with @b vdim
@@ -1038,6 +1068,15 @@ public:
   /// @note The returned object should NOT be deleted by the caller.
   DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;

+   /// @brief The same as GetBdrElementVDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   void GetBdrElementVDofs(int i, Array<int> &vdofs,
+                           DofTransformation &doftrans) const;
+
   /// Returns indices of degrees of freedom in @a vdofs for NURBS patch @a i.
   void GetPatchVDofs(int i, Array<int> &vdofs) const;

@@ -31,13 +31,13 @@ FmsBasisTypeToMfemBasis(FmsBasisType b)
   switch (b)
   {
      case FMS_NODAL_GAUSS_OPEN:
-         retval = mfem::BasisType::GaussLegendre;;
+         retval = mfem::BasisType::GaussLegendre;
         break;
      case FMS_NODAL_GAUSS_CLOSED:
-         retval = mfem::BasisType::GaussLobatto;;
+         retval = mfem::BasisType::GaussLobatto;
         break;
      case FMS_POSITIVE:
-         retval = mfem::BasisType::Positive;;
+         retval = mfem::BasisType::Positive;
         break;
      case FMS_NODAL_UNIFORM_OPEN:
         retval = mfem::BasisType::OpenUniform;
@@ -1812,22 +1812,22 @@ MeshToFmsMesh(const Mesh *mmesh, FmsMesh *fmesh, FmsComponent *volume)
      switch (betype)
      {
         case Element::POINT:
-            bdr_eles[FMS_VERTEX].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_VERTEX].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::SEGMENT:
-            bdr_eles[FMS_EDGE].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_EDGE].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::TRIANGLE:
-            bdr_eles[FMS_TRIANGLE].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_TRIANGLE].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::QUADRILATERAL:
-            bdr_eles[FMS_QUADRILATERAL].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_QUADRILATERAL].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::TETRAHEDRON:
-            bdr_eles[FMS_TETRAHEDRON].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_TETRAHEDRON].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::HEXAHEDRON:
-            bdr_eles[FMS_HEXAHEDRON].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_HEXAHEDRON].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         default:
            MFEM_WARNING("Unsupported boundary element " << betype << " at boundary index "
@@ -65,10 +65,10 @@ public:

   /** @brief Return an IntegrationRule consisting of all vertices of the given
       Geometry::Type, @a GeomType. */
-   const IntegrationRule *GetVertices(int GeomType);
+   const IntegrationRule *GetVertices(int GeomType) const;

   /// Return the center of the given Geometry::Type, @a GeomType.
-   const IntegrationPoint &GetCenter(int GeomType)
+   const IntegrationPoint &GetCenter(int GeomType) const
   { return GeomCenter[GeomType]; }

   /// Get a random point in the reference element specified by @a GeomType.
@@ -97,9 +97,9 @@ public:

   const DenseMatrix &GetGeomToPerfGeomJac(int GeomType) const
   { return *GeomToPerfGeomJac[GeomType]; }
-   DenseMatrix *GetPerfGeomToGeomJac(int GeomType)
+   const DenseMatrix *GetPerfGeomToGeomJac(int GeomType) const
   { return PerfGeomToGeomJac[GeomType]; }
-   void GetPerfPointMat(int GeomType, DenseMatrix &pm);
+   void GetPerfPointMat(int GeomType, DenseMatrix &pm) const;
   void JacToPerfJac(int GeomType, const DenseMatrix &J,
                     DenseMatrix &PJ) const;

@@ -123,7 +123,7 @@ public:
   }

   /// Return the number of boundary "faces" of a given Geometry::Type.
-   int NumBdr(int GeomType) { return NumBdrArray[GeomType]; }
+   int NumBdr(int GeomType) const { return NumBdrArray[GeomType]; }
 };

 template <> struct
@@ -317,27 +317,27 @@ public:
   int Type;

   RefinedGeometry(int NPts, int NRefG, int NRefE, int NBdrE = 0) :
-      RefPts(NPts), RefGeoms(NRefG), RefEdges(NRefE), NumBdrEdges(NBdrE) { }
+      RefPts(NPts), RefGeoms(NRefG), RefEdges(NRefE), NumBdrEdges(NBdrE) {}
 };

 class GeometryRefiner
 {
 private:
-   int type; // Quadrature1D type (ClosedUniform is default)
+   int Type; // Quadrature1D type (ClosedUniform is default)
   Array<RefinedGeometry *> RGeom[Geometry::NumGeom];
   Array<IntegrationRule *> IntPts[Geometry::NumGeom];

-   RefinedGeometry *FindInRGeom(Geometry::Type Geom, int Times, int ETimes,
-                                int Type);
-   IntegrationRule *FindInIntPts(Geometry::Type Geom, int NPts);
+   RefinedGeometry *FindInRGeom(Geometry::Type Geom, int Times,
+                                int ETimes) const;
+   IntegrationRule *FindInIntPts(Geometry::Type Geom, int NPts) const;

 public:
-   GeometryRefiner();
+   GeometryRefiner(int t = Quadrature1D::ClosedUniform) : Type(t) {}

   /// Set the Quadrature1D type of points to use for subdivision.
-   void SetType(const int t) { type = t; }
+   void SetType(int t) { Type = t; }
   /// Get the Quadrature1D type of points used for subdivision.
-   int GetType() const { return type; }
+   int GetType() const { return Type; }

   RefinedGeometry *Refine(Geometry::Type Geom, int Times, int ETimes = 1);

@@ -345,10 +345,10 @@ public:
   const IntegrationRule *RefineInterior(Geometry::Type Geom, int Times);

   /// Get the Refinement level based on number of points
-   virtual int GetRefinementLevelFromPoints(Geometry::Type Geom, int Npts);
+   static int GetRefinementLevelFromPoints(Geometry::Type Geom, int Npts);

   /// Get the Refinement level based on number of elements
-   virtual int GetRefinementLevelFromElems(Geometry::Type geom, int Npts);
+   static int GetRefinementLevelFromElems(Geometry::Type geom, int Npts);

   ~GeometryRefiner();
 };
@@ -38,9 +38,8 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
   // Grid functions are stored on the device
   UseDevice(true);

-   owned_fes.reset(new FiniteElementSpace);
-   fes = owned_fes.get();
-   fec.reset(fes->Load(m, input));
+   fes = new FiniteElementSpace;
+   fec = fes->Load(m, input);

   skip_comment_lines(input, '#');
   istream::int_type next_char = input.peek();
@@ -82,11 +81,10 @@ GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)
   int vdim, ordering;

   fes = gf_array[0]->FESpace();
-   fec.reset(FiniteElementCollection::New(fes->FEColl()->Name()));
+   fec = FiniteElementCollection::New(fes->FEColl()->Name());
   vdim = fes->GetVDim();
   ordering = fes->GetOrdering();
-   owned_fes.reset(new FiniteElementSpace(m, fec.get(), vdim, ordering));
-   fes = owned_fes.get();
+   fes = new FiniteElementSpace(m, fec, vdim, ordering);
   SetSize(fes->GetVSize());

   if (m->NURBSext)
@@ -155,9 +153,12 @@ GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)

 void GridFunction::Destroy()
 {
-   owned_fes.reset();
-   fec.reset();
-   fes = nullptr;
+   if (fec)
+   {
+      delete fes;
+      delete fec;
+      fec = NULL;
+   }
 }

 void GridFunction::Update()
@@ -2124,8 +2125,8 @@ void GridFunction::AccumulateAndCountBdrValues(
      Vector vals;
      Mesh *mesh = fes->GetMesh();
      NCMesh *ncmesh = mesh->ncmesh;
-      Array<int> bdr_edges, bdr_vertices;
-      ncmesh->GetBoundaryClosure(attr, bdr_vertices, bdr_edges);
+      Array<int> bdr_edges, bdr_vertices, bdr_faces;
+      ncmesh->GetBoundaryClosure(attr, bdr_vertices, bdr_edges, bdr_faces);

      for (i = 0; i < bdr_edges.Size(); i++)
      {
@@ -2231,8 +2232,8 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
   {
      Mesh *mesh = fes->GetMesh();
      NCMesh *ncmesh = mesh->ncmesh;
-      Array<int> bdr_edges, bdr_vertices;
-      ncmesh->GetBoundaryClosure(bdr_attr, bdr_vertices, bdr_edges);
+      Array<int> bdr_edges, bdr_vertices, bdr_faces;
+      ncmesh->GetBoundaryClosure(bdr_attr, bdr_vertices, bdr_edges, bdr_faces);

      for (int i = 0; i < bdr_edges.Size(); i++)
      {
@@ -20,7 +20,6 @@
 #include "../general/adios2stream.hpp"
 #endif
 #include <limits>
-#include <memory>
 #include <ostream>
 #include <string>

@@ -31,13 +30,14 @@ namespace mfem
 class GridFunction : public Vector
 {
 protected:
-   /// FE space on which the grid function lives.
+   /// FE space on which the grid function lives. Owned if #fec is not NULL.
   FiniteElementSpace *fes;

   /** @brief Used when the grid function is read from a file. It can also be
-       set explicitly, see MakeOwner(). */
-   std::shared_ptr<FiniteElementCollection> fec;
-   std::shared_ptr<FiniteElementSpace> owned_fes;
+       set explicitly, see MakeOwner().
+
+       If not NULL, this pointer is owned by the GridFunction. */
+   FiniteElementCollection *fec;

   long fes_sequence; // see FiniteElementSpace::sequence, Mesh::sequence

@@ -74,6 +74,11 @@ public:

   GridFunction() { fes = NULL; fec = NULL; fes_sequence = 0; UseDevice(true); }

+   /// Copy constructor. The internal true-dof vector #t_vec is not copied.
+   GridFunction(const GridFunction &orig)
+      : Vector(orig), fes(orig.fes), fec(NULL), fes_sequence(orig.fes_sequence)
+   { UseDevice(true); }
+
   /// Construct a GridFunction associated with the FiniteElementSpace @a *f.
   GridFunction(FiniteElementSpace *f) : Vector(f->GetVSize())
   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
@@ -102,12 +107,21 @@ public:

   GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces);

+   /// Copy assignment. Only the data of the base class Vector is copied.
+   /** It is assumed that this object and @a rhs use FiniteElementSpace%s that
+       have the same size.
+
+       @note Defining this method overwrites the implicitly defined copy
+       assignment operator. */
+   GridFunction &operator=(const GridFunction &rhs)
+   { return operator=((const Vector &)rhs); }
+
   /// Make the GridFunction the owner of #fec and #fes.
   /** If the new FiniteElementCollection, @a fec_, is NULL, ownership of #fec
       and #fes is taken away. */
-   void MakeOwner(FiniteElementCollection *fec_) { fec.reset(fec_); }
+   void MakeOwner(FiniteElementCollection *fec_) { fec = fec_; }

-   FiniteElementCollection *OwnFEC() { return fec.get(); }
+   FiniteElementCollection *OwnFEC() { return fec; }

   int VectorDim() const;
   int CurlDim() const;
@@ -482,7 +496,6 @@ public:
   virtual void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                             Array<int> &bdr_attr);

-
   virtual double ComputeL2Error(Coefficient *exsol[],
                                 const IntegrationRule *irs[] = NULL,
                                 const Array<int> *elems = NULL) const;
@@ -572,6 +585,10 @@ public:
      return ComputeLpError(infinity(), exsol, NULL, NULL, irs);
   }

+   virtual double ComputeL1Error(Coefficient *exsol[],
+                                 const IntegrationRule *irs[] = NULL) const
+   { return ComputeW11Error(*exsol, NULL, 1, NULL, irs); }
+
   virtual double ComputeL1Error(Coefficient &exsol,
                                 const IntegrationRule *irs[] = NULL) const
   { return ComputeLpError(1.0, exsol, NULL, irs); }
@@ -740,6 +757,9 @@ public:
   /** @brief Write the GridFunction in STL format. Note that the mesh dimension
       must be 2 and that quad elements will be broken into two triangles.*/
   void SaveSTL(std::ostream &out, int TimesToRefine = 1);
+
+   /// Destroys grid function.
+   virtual ~GridFunction() { Destroy(); }
 };


@@ -67,6 +67,7 @@ FindPointsGSLIB::FindPointsGSLIB()

 FindPointsGSLIB::~FindPointsGSLIB()
 {
+   comm_free(gsl_comm);
   delete gsl_comm;
   delete cr;
   for (int i = 0; i < 4; i++)
@@ -159,6 +159,7 @@ static void BLFEvalAssemble(const FiniteElementSpace &fes,
                            const bool normals,
                            Vector &y)
 {
+   if (fes.GetNBE() == 0) { return; }
   Mesh &mesh = *fes.GetMesh();
   const int dim = mesh.Dimension();
   const FiniteElement &el = *fes.GetBE(0);
@@ -214,6 +215,7 @@ void BoundaryLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
                                          const Array<int> &markers,
                                          Vector &b)
 {
+   if (fes.GetNBE() == 0) { return; }
   const FiniteElement &fe = *fes.GetBE(0);
   const int qorder = oa * fe.GetOrder() + ob;
   const Geometry::Type gtype = fe.GetGeomType();
@@ -229,6 +231,7 @@ void BoundaryNormalLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
                                                const Array<int> &markers,
                                                Vector &b)
 {
+   if (fes.GetNBE() == 0) { return; }
   const FiniteElement &fe = *fes.GetBE(0);
   const int qorder = oa * fe.GetOrder() + ob;
   const Geometry::Type gtype = fe.GetGeomType();
@@ -737,7 +737,7 @@ void QuadratureFunctions1D::GivePolyPoints(const int np, double *pts,
         ClosedGL(np, &ir);
         break;
      }
-      default:
+      case Quadrature1D::Invalid:
      {
         MFEM_ABORT("Asking for an unknown type of 1D Quadrature points, "
                    "type = " << type);
@@ -831,7 +831,10 @@ void QuadratureFunctions1D::CalculateUniformWeights(IntegrationRule *ir,
         hinv = p+1;
         ihoffset = 1;
         break;
-      default:
+      case Quadrature1D::GaussLegendre:
+      case Quadrature1D::GaussLobatto:
+      case Quadrature1D::ClosedGL:
+      case Quadrature1D::Invalid:
         MFEM_ABORT("invalid Quadrature1D type: " << type);
   }
   // set w0 = (-1)^p*(p!)/(hinv^p)
@@ -940,10 +943,10 @@ IntegrationRules IntRules(0, Quadrature1D::GaussLegendre);

 IntegrationRules RefinedIntRules(1, Quadrature1D::GaussLegendre);

-IntegrationRules::IntegrationRules(int Ref, int type_):
-   quad_type(type_)
+IntegrationRules::IntegrationRules(int ref, int type)
+   : quad_type(type)
 {
-   refined = Ref;
+   refined = ref;

   if (refined < 0) { own_rules = 0; return; }

@@ -975,11 +978,19 @@ IntegrationRules::IntegrationRules(int Ref, int type_):

   CubeIntRules.SetSize(32, h_mt);
   CubeIntRules = NULL;
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   IntRuleLocks.SetSize(Geometry::NUM_GEOMETRIES, h_mt);
+   for (int i = 0; i < Geometry::NUM_GEOMETRIES; i++)
+   {
+      omp_init_lock(&IntRuleLocks[i]);
+   }
+#endif
 }

 const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
 {
-   Array<IntegrationRule *> *ir_array;
+   Array<IntegrationRule *> *ir_array = NULL;

   switch (GeomType)
   {
@@ -991,9 +1002,9 @@ const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
      case Geometry::CUBE:        ir_array = &CubeIntRules; break;
      case Geometry::PRISM:       ir_array = &PrismIntRules; break;
      case Geometry::PYRAMID:     ir_array = &PyramidIntRules; break;
-      default:
-         mfem_error("IntegrationRules::Get(...) : Unknown geometry type!");
-         ir_array = NULL;
+      case Geometry::INVALID:
+      case Geometry::NUM_GEOMETRIES:
+         MFEM_ABORT("Unknown type of reference element!");
   }

   if (Order < 0)
@@ -1001,36 +1012,35 @@ const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
      Order = 0;
   }

+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_set_lock(&IntRuleLocks[GeomType]);
+#endif
+
   if (!HaveIntRule(*ir_array, Order))
   {
-#ifdef MFEM_USE_LEGACY_OPENMP
-      #pragma omp critical
-#endif
-      {
-         if (!HaveIntRule(*ir_array, Order))
-         {
-            IntegrationRule *ir = GenerateIntegrationRule(GeomType, Order);
+      IntegrationRule *ir = GenerateIntegrationRule(GeomType, Order);
 #ifdef MFEM_DEBUG
-            int RealOrder = Order;
-            while (RealOrder+1 < ir_array->Size() &&
-                   (*ir_array)[RealOrder+1] == ir)
-            {
-               RealOrder++;
-            }
-            MFEM_VERIFY(RealOrder == ir->GetOrder(), "internal error");
-#else
-            MFEM_CONTRACT_VAR(ir);
-#endif
-         }
+      int RealOrder = Order;
+      while (RealOrder+1 < ir_array->Size() && (*ir_array)[RealOrder+1] == ir)
+      {
+         RealOrder++;
      }
+      MFEM_VERIFY(RealOrder == ir->GetOrder(), "internal error");
+#else
+      MFEM_CONTRACT_VAR(ir);
+#endif
   }

+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_unset_lock(&IntRuleLocks[GeomType]);
+#endif
+
   return *(*ir_array)[Order];
 }

 void IntegrationRules::Set(int GeomType, int Order, IntegrationRule &IntRule)
 {
-   Array<IntegrationRule *> *ir_array;
+   Array<IntegrationRule *> *ir_array = NULL;

   switch (GeomType)
   {
@@ -1042,11 +1052,15 @@ void IntegrationRules::Set(int GeomType, int Order, IntegrationRule &IntRule)
      case Geometry::CUBE:        ir_array = &CubeIntRules; break;
      case Geometry::PRISM:       ir_array = &PrismIntRules; break;
      case Geometry::PYRAMID:     ir_array = &PyramidIntRules; break;
-      default:
-         mfem_error("IntegrationRules::Set(...) : Unknown geometry type!");
-         ir_array = NULL;
+      case Geometry::INVALID:
+      case Geometry::NUM_GEOMETRIES:
+         MFEM_ABORT("Unknown type of reference element!");
   }

+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_set_lock(&IntRuleLocks[GeomType]);
+#endif
+
   if (HaveIntRule(*ir_array, Order))
   {
      MFEM_ABORT("Overwriting set rules is not supported!");
@@ -1055,16 +1069,19 @@ void IntegrationRules::Set(int GeomType, int Order, IntegrationRule &IntRule)
   AllocIntRule(*ir_array, Order);

   (*ir_array)[Order] = &IntRule;
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_unset_lock(&IntRuleLocks[GeomType]);
+#endif
 }

-void IntegrationRules::DeleteIntRuleArray(Array<IntegrationRule *> &ir_array)
+void IntegrationRules::DeleteIntRuleArray(
+   Array<IntegrationRule *> &ir_array) const
 {
-   int i;
-   IntegrationRule *ir = NULL;
-
   // Many of the intrules have multiple contiguous copies in the ir_array
   // so we have to be careful to not delete them twice.
-   for (i = 0; i < ir_array.Size(); i++)
+   IntegrationRule *ir = NULL;
+   for (int i = 0; i < ir_array.Size(); i++)
   {
      if (ir_array[i] != NULL && ir_array[i] != ir)
      {
@@ -1076,6 +1093,13 @@ void IntegrationRules::DeleteIntRuleArray(Array<IntegrationRule *> &ir_array)

 IntegrationRules::~IntegrationRules()
 {
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   for (int i = 0; i < Geometry::NUM_GEOMETRIES; i++)
+   {
+      omp_destroy_lock(&IntRuleLocks[i]);
+   }
+#endif
+
   if (!own_rules) { return; }

   DeleteIntRuleArray(PointIntRules);
@@ -1110,10 +1134,11 @@ IntegrationRule *IntegrationRules::GenerateIntegrationRule(int GeomType,
         return PrismIntegrationRule(Order);
      case Geometry::PYRAMID:
         return PyramidIntegrationRule(Order);
-      default:
-         mfem_error("IntegrationRules::Set(...) : Unknown geometry type!");
-         return NULL;
+      case Geometry::INVALID:
+      case Geometry::NUM_GEOMETRIES:
+         MFEM_ABORT("Unknown type of reference element!");
   }
+   return NULL;
 }


@@ -1122,7 +1147,7 @@ IntegrationRule *IntegrationRules::PointIntegrationRule(int Order)
 {
   if (Order > 1)
   {
-      mfem_error("Point Integration Rule of Order > 1 not defined");
+      MFEM_ABORT("Point Integration Rule of Order > 1 not defined");
      return NULL;
   }

@@ -1185,7 +1210,7 @@ IntegrationRule *IntegrationRules::SegmentIntegrationRule(int Order)
         QuadratureFunctions1D::OpenHalfUniform(n, ir);
         break;
      }
-      default:
+      case Quadrature1D::Invalid:
      {
         MFEM_ABORT("unknown Quadrature1D type: " << quad_type);
      }
@@ -1762,8 +1787,8 @@ IntegrationRule *IntegrationRules::PyramidIntegrationRule(int Order)

   for (int k=0; k<npts; k++)
   {
-      const IntegrationPoint & ipc = irc.IntPoint(k);
-      IntegrationPoint & ipp = PyramidIntRules[Order]->IntPoint(k);
+      const IntegrationPoint &ipc = irc.IntPoint(k);
+      IntegrationPoint &ipp = PyramidIntRules[Order]->IntPoint(k);
      ipp.x = ipc.x * (1.0 - ipc.z);
      ipp.y = ipc.y * (1.0 - ipc.z);
      ipp.z = ipc.z;
@@ -1775,8 +1800,8 @@ IntegrationRule *IntegrationRules::PyramidIntegrationRule(int Order)
 // Integration rules for reference prism
 IntegrationRule *IntegrationRules::PrismIntegrationRule(int Order)
 {
-   const IntegrationRule & irt = Get(Geometry::TRIANGLE, Order);
-   const IntegrationRule & irs = Get(Geometry::SEGMENT, Order);
+   const IntegrationRule &irt = Get(Geometry::TRIANGLE, Order);
+   const IntegrationRule &irs = Get(Geometry::SEGMENT, Order);
   int nt = irt.GetNPoints();
   int ns = irs.GetNPoints();
   AllocIntRule(PrismIntRules, Order);
@@ -1790,12 +1815,12 @@ IntegrationRule *IntegrationRules::PrismIntegrationRule(int Order)

   for (int ks=0; ks<ns; ks++)
   {
-      const IntegrationPoint & ips = irs.IntPoint(ks);
+      const IntegrationPoint &ips = irs.IntPoint(ks);
      for (int kt=0; kt<nt; kt++)
      {
         int kp = ks * nt + kt;
-         const IntegrationPoint & ipt = irt.IntPoint(kt);
-         IntegrationPoint & ipp = PrismIntRules[Order]->IntPoint(kp);
+         const IntegrationPoint &ipt = irt.IntPoint(kt);
+         IntegrationPoint &ipp = PrismIntRules[Order]->IntPoint(kp);
         ipp.x = ipt.x;
         ipp.y = ipt.y;
         ipp.z = ips.x;
@@ -14,6 +14,9 @@

 #include "../config/config.hpp"
 #include "../general/array.hpp"
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+#include <omp.h>
+#endif

 #include <vector>
 #include <map>
@@ -428,14 +431,18 @@ private:
   Array<IntegrationRule *> PrismIntRules;
   Array<IntegrationRule *> CubeIntRules;

-   void AllocIntRule(Array<IntegrationRule *> &ir_array, int Order)
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   Array<omp_lock_t> IntRuleLocks;
+#endif
+
+   void AllocIntRule(Array<IntegrationRule *> &ir_array, int Order) const
   {
      if (ir_array.Size() <= Order)
      {
         ir_array.SetSize(Order + 1, NULL);
      }
   }
-   bool HaveIntRule(Array<IntegrationRule *> &ir_array, int Order)
+   bool HaveIntRule(Array<IntegrationRule *> &ir_array, int Order) const
   {
      return (ir_array.Size() > Order && ir_array[Order] != NULL);
   }
@@ -443,6 +450,7 @@ private:
   {
      return Order | 1; // valid for all quad_type's
   }
+   void DeleteIntRuleArray(Array<IntegrationRule *> &ir_array) const;

   /// The following methods allocate new IntegrationRule objects without
   /// checking if they already exist.  To avoid memory leaks use
@@ -457,12 +465,10 @@ private:
   IntegrationRule *PrismIntegrationRule(int Order);
   IntegrationRule *CubeIntegrationRule(int Order);

-   void DeleteIntRuleArray(Array<IntegrationRule *> &ir_array);
-
 public:
   /// Sets initial sizes for the integration rule arrays, but rules
   /// are defined the first time they are requested with the Get method.
-   explicit IntegrationRules(int Ref = 0,
+   explicit IntegrationRules(int ref = 0,
                             int type = Quadrature1D::GaussLegendre);

   /// Returns an integration rule for given GeomType and Order.
@@ -137,7 +137,7 @@ bool LinearForm::SupportsDevice() const
      // Make sure every boundary element corresponds to a boundary face
      for (int be = 0; be < fes->GetNBE(); ++be)
      {
-         const int f = mesh.GetBdrElementEdgeIndex(be);
+         const int f = mesh.GetBdrElementFaceIndex(be);
         const auto face_info = mesh.GetFaceInformation(f);
         if (!face_info.IsBoundary())
         {
@@ -148,7 +148,7 @@ void LinearFormExtension::Update()
      std::unordered_map<int,int> f_to_be;
      for (int i = 0; i < mesh.GetNBE(); ++i)
      {
-         const int f = mesh.GetBdrElementEdgeIndex(i);
+         const int f = mesh.GetBdrElementFaceIndex(i);
         f_to_be[f] = i;
      }
      MFEM_VERIFY(size_t(nf_bdr) == f_to_be.size(), "Incompatible sizes");
@@ -104,7 +104,7 @@ public:
 };


-/// Class for domain integration L(v) := (f, v)
+/// Class for domain integration \f$ L(v) := (f, v) \f$
 class DomainLFIntegrator : public DeltaLFIntegrator
 {
   Vector shape;
@@ -141,7 +141,7 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/// Class for domain integrator L(v) := (f, grad v)
+/// Class for domain integrator \f$ L(v) := (f, \nabla v) \f$
 class DomainLFGradIntegrator : public DeltaLFIntegrator
 {
 private:
@@ -150,7 +150,7 @@ private:
   DenseMatrix dshape;

 public:
-   /// Constructs the domain integrator (Q, grad v)
+   /// Constructs the domain integrator \f$ (Q, \nabla v) \f$
   DomainLFGradIntegrator(VectorCoefficient &QF)
      : DeltaLFIntegrator(QF), Q(QF) { }

@@ -175,7 +175,7 @@ public:
 };


-/// Class for boundary integration L(v) := (g, v)
+/// Class for boundary integration \f$ L(v) := (g, v) \f$
 class BoundaryLFIntegrator : public LinearFormIntegrator
 {
   Vector shape;
@@ -249,8 +249,8 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/** Class for domain integration of L(v) := (f, v), where
-    f=(f1,...,fn) and v=(v1,...,vn). */
+/** Class for domain integration of \f$ L(v) := (f, v) \f$, where
+    \f$ f = (f_1,\dots,f_n)\f$ and \f$ v = (v_1,\dots,v_n) \f$. */
 class VectorDomainLFIntegrator : public DeltaLFIntegrator
 {
 private:
@@ -282,8 +282,8 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/** Class for domain integrator L(v) := (f, grad v), where
-    f=(f1x,f1y,f1z,...,fnx,fny,fnz) and v=(v1,...,vn). */
+/** Class for domain integrator \f$ L(v) := (f, \nabla v) \f$, where
+    \f$ f = (f_{1x},f_{1y},f_{1z},\dots,f_{nx},f_{ny},f_{nz})\f$ and \f$v=(v_1,\dots,v_n)\f$. */
 class VectorDomainLFGradIntegrator : public DeltaLFIntegrator
 {
 private:
@@ -316,8 +316,8 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/** Class for boundary integration of L(v) := (g, v), where
-    f=(f1,...,fn) and v=(v1,...,vn). */
+/** Class for boundary integration of \f$ L(v) := (g, v) \f$, where
+    \f$f=(f_1,\dots,f_n)\f$ and \f$v=(v_1,\dots,v_n)\f$. */
 class VectorBoundaryLFIntegrator : public LinearFormIntegrator
 {
 private:
@@ -371,7 +371,7 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/// \f$ (Q, curl v)_{\Omega} \f$ for Nedelec Elements)
+/// \f$ (Q, \mathrm{curl}(v))_{\Omega} \f$ for Nedelec Elements
 class VectorFEDomainLFCurlIntegrator : public DeltaLFIntegrator
 {
 private:
@@ -380,7 +380,7 @@ private:
   Vector vec;

 public:
-   /// Constructs the domain integrator (Q, curl v)
+   /// Constructs the domain integrator \f$(Q, \mathrm{curl}(v))  \f$
   VectorFEDomainLFCurlIntegrator(VectorCoefficient &F)
      : DeltaLFIntegrator(F), QF(&F) { }

@@ -395,14 +395,14 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/// \f$ (Q, div v)_{\Omega} \f$ for RT Elements)
+/// \f$ (Q, \mathrm{div}(v))_{\Omega} \f$ for RT Elements
 class VectorFEDomainLFDivIntegrator : public DeltaLFIntegrator
 {
 private:
   Vector divshape;
   Coefficient &Q;
 public:
-   /// Constructs the domain integrator (Q, div v)
+   /// Constructs the domain integrator \f$ (Q, \mathrm{div}(v)) \f$
   VectorFEDomainLFDivIntegrator(Coefficient &QF)
      : DeltaLFIntegrator(QF), Q(QF) { }

@@ -420,7 +420,7 @@ public:
 };

 /** \f$ (f, v \cdot n)_{\partial\Omega} \f$ for vector test function
-    v=(v1,...,vn) where all vi are in the same scalar FE space and f is a
+    \f$v=(v_1,\dots,v_n)\f$ where all vi are in the same scalar FE space and \f$f\f$ is a
    scalar function. */
 class VectorBoundaryFluxLFIntegrator : public LinearFormIntegrator
 {
@@ -441,8 +441,8 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/** Class for boundary integration of (f, v.n) for scalar coefficient f and
-    RT vector test function v. This integrator works with RT spaces defined
+/** Class for boundary integration of \f$ (f, v \cdot n)\f$ for scalar coefficient \f$f\f$ and
+    RT vector test function \f$v\f$. This integrator works with RT spaces defined
    using the RT_FECollection class. */
 class VectorFEBoundaryFluxLFIntegrator : public LinearFormIntegrator
 {
@@ -491,9 +491,9 @@ public:


 /** Class for boundary integration of the linear form:
-    (alpha/2) < (u.n) f, w > - beta < |u.n| f, w >,
-    where f and u are given scalar and vector coefficients, respectively,
-    and w is the scalar test function. */
+    \f$ \frac{\alpha}{2} \langle (u \cdot n) f, w \rangle - \beta \langle |u \cdot n| f, w \rangle \f$
+    where \f$f\f$ and \f$u\f$ are given scalar and vector coefficients, respectively,
+    and \f$w\f$ is the scalar test function. */
 class BoundaryFlowIntegrator : public LinearFormIntegrator
 {
 private:
@@ -525,13 +525,13 @@ public:

 /** Boundary linear integrator for imposing non-zero Dirichlet boundary
    conditions, to be used in conjunction with DGDiffusionIntegrator.
-    Specifically, given the Dirichlet data u_D, the linear form assembles the
+    Specifically, given the Dirichlet data \f$u_D\f$, the linear form assembles the
    following integrals on the boundary:
-
-    sigma < u_D, (Q grad(v)).n > + kappa < {h^{-1} Q} u_D, v >,
-
+   \f[
+    \sigma \langle u_D, (Q \nabla v)) \cdot n \rangle + \kappa \langle {h^{-1} Q} u_D, v \rangle,
+   \f]
    where Q is a scalar or matrix diffusion coefficient and v is the test
-    function. The parameters sigma and kappa should be the same as the ones
+    function. The parameters \f$\sigma\f$ and \f$\kappa\f$ should be the same as the ones
    used in the DGDiffusionIntegrator. */
 class DGDirichletLFIntegrator : public LinearFormIntegrator
 {
@@ -568,12 +568,12 @@ public:
 /** Boundary linear form integrator for imposing non-zero Dirichlet boundary
    conditions, in a DG elasticity formulation. Specifically, the linear form is
    given by
-
-    alpha < u_D, (lambda div(v) I + mu (grad(v) + grad(v)^T)) . n > +
-      + kappa < h^{-1} (lambda + 2 mu) u_D, v >,
-
-    where u_D is the given Dirichlet data. The parameters alpha, kappa, lambda
-    and mu, should match the parameters with the same names used in the bilinear
+   \f[
+    \alpha \langle u_D, (\lambda \mathrm{div}(v) I + \mu (\nabla v + \nabla v^{\mathrm{T}})) \cdot n \rangle +
+      + \kappa \langle h^{-1} (\lambda + 2 \mu) u_D, v \rangle,
+   \f]
+    where u_D is the given Dirichlet data. The parameters \f$\alpha\f$, \f$\kappa\f$, \f$\lambda\f$
+    and \f$\mu\f$, should match the parameters with the same names used in the bilinear
    form integrator, DGElasticityIntegrator. */
 class DGElasticityDirichletLFIntegrator : public LinearFormIntegrator
 {
@@ -612,18 +612,18 @@ public:

 /** Class for spatial white Gaussian noise integration.

-    The target problem is the linear SPDE a(u,v) = F(v) with F(v) := <Ẇ,v>,
-    where Ẇ is spatial white Gaussian noise. When the Galerkin method is used to
-    discretize this problem into a linear system of equations Ax = b, the RHS is
-    a Gaussian random vector b~N(0,M) whose covariance matrix is the same as the
-    mass matrix M_ij = (v_i,v_j). This property can be ensured if b = H w, where
-    HHᵀ = M and each component w_i~N(0,1).
+    The target problem is the linear SPDE \f$ a(u,v) = F(v)\f$ with \f$F(v) := <\dot{W},v> \f$,
+    where \f$\dot{W}\f$ is spatial white Gaussian noise. When the Galerkin method is used to
+    discretize this problem into a linear system of equations \f$Ax = b\f$, the RHS is
+    a Gaussian random vector \f$b \sim N(0,M)\f$ whose covariance matrix is the same as the
+    mass matrix \f$M_{ij} = (v_i,v_j)\f$. This property can be ensured if \f$b = H w\f$, where
+    \f$HH^{\mathrm{T}} = M\f$ and each component \f$w_i\sim N(0,1)\f$.

-    There is much flexibility in how we may wish to define H. In this PR, we
-    define H = Pᵀ diag(L_e), where P is the local-to-global dof assembly matrix
-    and diag(L_e) is a block-diagonal matrix with L_e L_eᵀ = M_e, where M_e is
-    the element mass matrix for element e. A straightforward computation shows
-    that HHᵀ = Pᵀ diag(M_e) P = M, as necessary. */
+    There is much flexibility in how we may wish to define \f$H\f$. In this PR, we
+    define \f$H = P^{\mathrm{T}} diag(L_e)\f$, where \f$P\f$ is the local-to-global dof assembly matrix
+    and \f$\mathrm{diag}(L_e)\f$ is a block-diagonal matrix with \f$L_e L_e^{\mathrm{T}} = M_e\f$, where \f$M_e\f$ is
+    the element mass matrix for element \f$e\f$. A straightforward computation shows
+    that \f$HH^{\mathrm{T}} = P^{\mathrm{T}} diag(M_e) P = M\f$, as necessary. */
 class WhiteGaussianNoiseDomainLFIntegrator : public LinearFormIntegrator
 {
 #ifdef MFEM_USE_MPI
@@ -718,8 +718,8 @@ public:
 };


-/** Class for domain integration of L(v) := (f, v), where
-    f=(f1,...,fn) and v=(v1,...,vn). that makes use of
+/** Class for domain integration of \f$ L(v) := (f, v) \f$, where
+    \f$ f=(f_1,\dots,f_n)\f$ and \f$v=(v_1,\dots,v_n)\f$. that makes use of
    VectorQuadratureFunctionCoefficient*/
 class VectorQuadratureLFIntegrator : public LinearFormIntegrator
 {
@@ -751,7 +751,7 @@ public:
 };


-/** Class for domain integration L(v) := (f, v) that makes use
+/** Class for domain integration \f$ L(v) := (f, v) \f$ that makes use
    of QuadratureFunctionCoefficient. */
 class QuadratureLFIntegrator : public LinearFormIntegrator
 {
@@ -257,13 +257,13 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   // vertices of the LOR mesh. The vertex coordinates are already computed in
   // E-vector format and passed in in X_vert.
   //
-   // In this function, we need to convert X_vert (which has the shape (dim,
+   // In this function, we need to convert X_vert (which has the shape (sdim,
   // ndof_per_el, nel_ho)) to T-DOF format.
   //
-   // We place the results in the vector xyz_tvec, which has shape (ntdofs, dim)
+   // We place the results in the vector xyz_tvec, which has shape (ntdofs, sdim)
   // and then make the hypre vectors x, y, and z point to subvectors.
   //
-   // In 2D, z is NULL.
+   // When the space dimension is 2, z is NULL.

   // Create the H1 vertex space and get the element restriction
   ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
@@ -275,17 +275,17 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   const int nel_ho = vert_fes.GetNE();
   const int ndp1 = order + 1;
   const int ndof_per_el = static_cast<int>(pow(ndp1, dim));
-   const int sdim = dim;
+   const int sdim = vert_fes.GetMesh()->SpaceDimension();
   const int ntdofs = R->Height();

   const MemoryClass mc = GetHypreMemoryClass();
   bool dev = (mc == MemoryClass::DEVICE);

-   xyz_tvec = new Vector(ntdofs*dim);
+   xyz_tvec = new Vector(ntdofs*sdim);

-   auto xyz_tv = Reshape(HypreWrite(xyz_tvec->GetMemory()), ntdofs, dim);
+   auto xyz_tv = Reshape(HypreWrite(xyz_tvec->GetMemory()), ntdofs, sdim);
   const auto xyz_e =
-      Reshape(HypreRead(X_vert.GetMemory()), dim, ndof_per_el, nel_ho);
+      Reshape(HypreRead(X_vert.GetMemory()), sdim, ndof_per_el, nel_ho);
   const auto d_offsets = HypreRead(el_restr->Offsets().GetMemory());
   const auto d_indices = HypreRead(el_restr->Indices().GetMemory());
   const auto ltdof_ldof = HypreRead(R->GetMemoryJ());
@@ -309,7 +309,7 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   x = new HypreParVector(vert_fes.GetComm(), glob_size, d_x_ptr, cols, dev);
   double *d_y_ptr = xyz_tv + 1*ntdofs;
   y = new HypreParVector(vert_fes.GetComm(), glob_size, d_y_ptr, cols, dev);
-   if (dim == 3)
+   if (sdim == 3)
   {
      double *d_z_ptr = xyz_tv + 2*ntdofs;
      z = new HypreParVector(vert_fes.GetComm(), glob_size, d_z_ptr, cols, dev);
@@ -77,6 +77,7 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,

   // Get nodal points at the LOR vertices
   const int dim = mesh_ho.Dimension();
+   const int sdim = mesh_ho.SpaceDimension();
   const int nel_ho = mesh_ho.GetNE();
   const int order = fes_ho.GetMaxElementOrder();
   const int nd1d = order + 1;
@@ -94,7 +95,7 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
   IntegrationRule ir = GetCollocatedIntRule(fes_ho);

   // Map from nodal E-vector to Q-vector at the LOR vertex points
-   X_vert.SetSize(dim*ndof_per_el*nel_ho);
+   X_vert.SetSize(sdim*ndof_per_el*nel_ho);
   const QuadratureInterpolator *quad_interp =
      nodal_fes->GetQuadratureInterpolator(ir);
   quad_interp->SetOutputLayout(QVectorLayout::byVDIM);
@@ -380,44 +381,49 @@ void BatchedLORAssembly::SparseIJToCSR(OperatorHandle &A) const
   FillJAndData(*A_mat);
 }

+template <int ORDER, int SDIM, typename LOR_KERNEL>
+static void Assemble_(LOR_KERNEL &kernel, int dim)
+{
+   if (dim == 2) { kernel.template Assemble2D<ORDER,SDIM>(); }
+   else if (dim == 3) { kernel.template Assemble3D<ORDER>(); }
+   else { MFEM_ABORT("Unsupported dimension"); }
+}
+
+template <int ORDER, typename LOR_KERNEL>
+static void Assemble_(LOR_KERNEL &kernel, int dim, int sdim)
+{
+   if (sdim == 2) { Assemble_<ORDER,2>(kernel, dim); }
+   else if (sdim == 3) { Assemble_<ORDER,3>(kernel, dim); }
+   else { MFEM_ABORT("Unsupported space dimension."); }
+}
+
+template <typename LOR_KERNEL>
+static void Assemble_(LOR_KERNEL &kernel, int dim, int sdim, int order)
+{
+   switch (order)
+   {
+      case 1: Assemble_<1>(kernel, dim, sdim); break;
+      case 2: Assemble_<2>(kernel, dim, sdim); break;
+      case 3: Assemble_<3>(kernel, dim, sdim); break;
+      case 4: Assemble_<4>(kernel, dim, sdim); break;
+      case 5: Assemble_<5>(kernel, dim, sdim); break;
+      case 6: Assemble_<6>(kernel, dim, sdim); break;
+      case 7: Assemble_<7>(kernel, dim, sdim); break;
+      case 8: Assemble_<8>(kernel, dim, sdim); break;
+      default: MFEM_ABORT("No kernel order " << order << "!");
+   }
+}
+
 template <typename LOR_KERNEL>
 void BatchedLORAssembly::AssemblyKernel(BilinearForm &a)
 {
   LOR_KERNEL kernel(a, fes_ho, X_vert, sparse_ij, sparse_mapping);

   const int dim = fes_ho.GetMesh()->Dimension();
+   const int sdim = fes_ho.GetMesh()->SpaceDimension();
   const int order = fes_ho.GetMaxElementOrder();

-   if (dim == 2)
-   {
-      switch (order)
-      {
-         case 1: kernel.template Assemble2D<1>(); break;
-         case 2: kernel.template Assemble2D<2>(); break;
-         case 3: kernel.template Assemble2D<3>(); break;
-         case 4: kernel.template Assemble2D<4>(); break;
-         case 5: kernel.template Assemble2D<5>(); break;
-         case 6: kernel.template Assemble2D<6>(); break;
-         case 7: kernel.template Assemble2D<7>(); break;
-         case 8: kernel.template Assemble2D<8>(); break;
-         default: MFEM_ABORT("No kernel order " << order << "!");
-      }
-   }
-   else if (dim == 3)
-   {
-      switch (order)
-      {
-         case 1: kernel.template Assemble3D<1>(); break;
-         case 2: kernel.template Assemble3D<2>(); break;
-         case 3: kernel.template Assemble3D<3>(); break;
-         case 4: kernel.template Assemble3D<4>(); break;
-         case 5: kernel.template Assemble3D<5>(); break;
-         case 6: kernel.template Assemble3D<6>(); break;
-         case 7: kernel.template Assemble3D<7>(); break;
-         case 8: kernel.template Assemble3D<8>(); break;
-         default: MFEM_ABORT("No kernel order " << order << "!");
-      }
-   }
+   Assemble_(kernel, dim, sdim, order);
 }

 void BatchedLORAssembly::AssembleWithoutBC(BilinearForm &a, OperatorHandle &A)
@@ -22,15 +22,22 @@ namespace mfem
 class BatchedLOR_H1 : BatchedLORKernel
 {
 public:
-   template <int ORDER> void Assemble2D();
+   template <int ORDER, int SDIM> void Assemble2D();
   template <int ORDER> void Assemble3D();
   BatchedLOR_H1(BilinearForm &a,
                 FiniteElementSpace &fes_ho_,
                 Vector &X_vert_,
                 Vector &sparse_ij_,
-                 Array<int> &sparse_mapping_);
+                 Array<int> &sparse_mapping_)
+      : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
+   {
+      ProjectLORCoefficient<MassIntegrator>(a, c1);
+      ProjectLORCoefficient<DiffusionIntegrator>(a, c2);
+   }
 };

 }

+#include "lor_h1_impl.hpp"
+
 #endif
@@ -9,7 +9,6 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-#include "lor_h1.hpp"
 #include "lor_util.hpp"
 #include "../../linalg/dtensor.hpp"
 #include "../../general/forall.hpp"
@@ -17,7 +16,7 @@
 namespace mfem
 {

-template <int ORDER>
+template <int ORDER, int SDIM>
 void BatchedLOR_H1::Assemble2D()
 {
   const int nel_ho = fes_ho.GetNE();
@@ -74,31 +73,8 @@ void BatchedLOR_H1::Assemble2D()

            for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }

-            double vx[4], vy[4];
-            LORVertexCoordinates2D<ORDER>(X, iel_ho, kx, ky, vx, vy);
+            SetupLORQuadData2D<ORDER,SDIM,false,false>(X, iel_ho, kx, ky, Q, false);

-            for (int iqy=0; iqy<2; ++iqy)
-            {
-               for (int iqx=0; iqx<2; ++iqx)
-               {
-                  const double x = iqx;
-                  const double y = iqy;
-                  const double w = 1.0/4.0;
-
-                  double J_[2*2];
-                  DeviceTensor<2> J(J_, 2, 2);
-
-                  Jacobian2D(x, y, vx, vy, J);
-
-                  const double detJ = Det2D(J);
-                  const double w_detJ = w/detJ;
-
-                  Q(0,iqy,iqx) = w_detJ * (J(0,1)*J(0,1) + J(1,1)*J(1,1)); // 1,1
-                  Q(1,iqy,iqx) = -w_detJ * (J(0,1)*J(0,0) + J(1,1)*J(1,0)); // 1,2
-                  Q(2,iqy,iqx) = w_detJ * (J(0,0)*J(0,0) + J(1,0)*J(1,0)); // 2,2
-                  Q(3,iqy,iqx) = w*detJ;
-               }
-            }
            for (int iqx=0; iqx<2; ++iqx)
            {
               for (int iqy=0; iqy<2; ++iqy)
@@ -519,34 +495,4 @@ void BatchedLOR_H1::Assemble3D()
   }
 }

-// Explicit template instantiations
-template void BatchedLOR_H1::Assemble2D<1>();
-template void BatchedLOR_H1::Assemble2D<2>();
-template void BatchedLOR_H1::Assemble2D<3>();
-template void BatchedLOR_H1::Assemble2D<4>();
-template void BatchedLOR_H1::Assemble2D<5>();
-template void BatchedLOR_H1::Assemble2D<6>();
-template void BatchedLOR_H1::Assemble2D<7>();
-template void BatchedLOR_H1::Assemble2D<8>();
-
-template void BatchedLOR_H1::Assemble3D<1>();
-template void BatchedLOR_H1::Assemble3D<2>();
-template void BatchedLOR_H1::Assemble3D<3>();
-template void BatchedLOR_H1::Assemble3D<4>();
-template void BatchedLOR_H1::Assemble3D<5>();
-template void BatchedLOR_H1::Assemble3D<6>();
-template void BatchedLOR_H1::Assemble3D<7>();
-template void BatchedLOR_H1::Assemble3D<8>();
-
-BatchedLOR_H1::BatchedLOR_H1(BilinearForm &a,
-                             FiniteElementSpace &fes_ho_,
-                             Vector &X_vert_,
-                             Vector &sparse_ij_,
-                             Array<int> &sparse_mapping_)
-   : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
-{
-   ProjectLORCoefficient<MassIntegrator>(a, c1);
-   ProjectLORCoefficient<DiffusionIntegrator>(a, c2);
-}
-
 } // namespace mfem
@@ -22,15 +22,22 @@ namespace mfem
 class BatchedLOR_ND : BatchedLORKernel
 {
 public:
-   template <int ORDER> void Assemble2D();
+   template <int ORDER, int SDIM> void Assemble2D();
   template <int ORDER> void Assemble3D();
   BatchedLOR_ND(BilinearForm &a,
                 FiniteElementSpace &fes_ho_,
                 Vector &X_vert_,
                 Vector &sparse_ij_,
-                 Array<int> &sparse_mapping_);
+                 Array<int> &sparse_mapping_)
+      : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
+   {
+      ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
+      ProjectLORCoefficient<CurlCurlIntegrator>(a, c2);
+   }
 };

 }

+#include "lor_nd_impl.hpp"
+
 #endif
@@ -9,7 +9,6 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-#include "lor_nd.hpp"
 #include "lor_util.hpp"
 #include "../../linalg/dtensor.hpp"
 #include "../../general/forall.hpp"
@@ -17,7 +16,7 @@
 namespace mfem
 {

-template <int ORDER>
+template <int ORDER, int SDIM>
 void BatchedLOR_ND::Assemble2D()
 {
   const int nel_ho = fes_ho.GetNE();
@@ -83,31 +82,8 @@ void BatchedLOR_ND::Assemble2D()
            // local_mat is the local (dense) stiffness matrix
            for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }

-            double vx[4], vy[4];
-            LORVertexCoordinates2D<ORDER>(X, iel_ho, kx, ky, vx, vy);
+            SetupLORQuadData2D<ORDER,SDIM,false,true>(X, iel_ho, kx, ky, Q, true);

-            for (int iqx=0; iqx<2; ++iqx)
-            {
-               for (int iqy=0; iqy<2; ++iqy)
-               {
-                  const double x = iqx;
-                  const double y = iqy;
-                  const double w = 1.0/4.0;
-
-                  double J_[2*2];
-                  DeviceTensor<2> J(J_, 2, 2);
-
-                  Jacobian2D(x, y, vx, vy, J);
-
-                  const double detJ = Det2D(J);
-                  const double w_detJ = w/detJ;
-
-                  Q(0,iqy,iqx) = w_detJ * (J(0,1)*J(0,1) + J(1,1)*J(1,1)); // 1,1
-                  Q(1,iqy,iqx) = -w_detJ * (J(0,1)*J(0,0) + J(1,1)*J(1,0)); // 1,2
-                  Q(2,iqy,iqx) = w_detJ * (J(0,0)*J(0,0) + J(1,0)*J(1,0)); // 2,2
-                  Q(3,iqy,iqx) = w_detJ;
-               }
-            }
            for (int iqx=0; iqx<2; ++iqx)
            {
               for (int iqy=0; iqy<2; ++iqy)
@@ -563,34 +539,4 @@ void BatchedLOR_ND::Assemble3D()
   }
 }

-// Explicit template instantiations
-template void BatchedLOR_ND::Assemble2D<1>();
-template void BatchedLOR_ND::Assemble2D<2>();
-template void BatchedLOR_ND::Assemble2D<3>();
-template void BatchedLOR_ND::Assemble2D<4>();
-template void BatchedLOR_ND::Assemble2D<5>();
-template void BatchedLOR_ND::Assemble2D<6>();
-template void BatchedLOR_ND::Assemble2D<7>();
-template void BatchedLOR_ND::Assemble2D<8>();
-
-template void BatchedLOR_ND::Assemble3D<1>();
-template void BatchedLOR_ND::Assemble3D<2>();
-template void BatchedLOR_ND::Assemble3D<3>();
-template void BatchedLOR_ND::Assemble3D<4>();
-template void BatchedLOR_ND::Assemble3D<5>();
-template void BatchedLOR_ND::Assemble3D<6>();
-template void BatchedLOR_ND::Assemble3D<7>();
-template void BatchedLOR_ND::Assemble3D<8>();
-
-BatchedLOR_ND::BatchedLOR_ND(BilinearForm &a,
-                             FiniteElementSpace &fes_ho_,
-                             Vector &X_vert_,
-                             Vector &sparse_ij_,
-                             Array<int> &sparse_mapping_)
-   : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
-{
-   ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
-   ProjectLORCoefficient<CurlCurlIntegrator>(a, c2);
-}
-
 } // namespace mfem
@@ -22,15 +22,22 @@ namespace mfem
 class BatchedLOR_RT : BatchedLORKernel
 {
 public:
-   template <int ORDER> void Assemble2D();
+   template <int ORDER, int SDIM> void Assemble2D();
   template <int ORDER> void Assemble3D();
   BatchedLOR_RT(BilinearForm &a,
                 FiniteElementSpace &fes_ho_,
                 Vector &X_vert_,
                 Vector &sparse_ij_,
-                 Array<int> &sparse_mapping_);
+                 Array<int> &sparse_mapping_)
+      : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
+   {
+      ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
+      ProjectLORCoefficient<DivDivIntegrator>(a, c2);
+   }
 };

 }

+#include "lor_rt_impl.hpp"
+
 #endif
@@ -9,7 +9,6 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-#include "lor_rt.hpp"
 #include "lor_util.hpp"
 #include "../../linalg/dtensor.hpp"
 #include "../../general/forall.hpp"
@@ -17,7 +16,7 @@
 namespace mfem
 {

-template <int ORDER>
+template <int ORDER, int SDIM>
 void BatchedLOR_RT::Assemble2D()
 {
   const int nel_ho = fes_ho.GetNE();
@@ -79,31 +78,8 @@ void BatchedLOR_RT::Assemble2D()
            // local_mat is the local (dense) stiffness matrix
            for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }

-            double vx[4], vy[4];
-            LORVertexCoordinates2D<ORDER>(X, iel_ho, kx, ky, vx, vy);
+            SetupLORQuadData2D<ORDER,SDIM,true,false>(X, iel_ho, kx, ky, Q, true);

-            for (int iqx=0; iqx<2; ++iqx)
-            {
-               for (int iqy=0; iqy<2; ++iqy)
-               {
-                  const double x = iqx;
-                  const double y = iqy;
-                  const double w = 1.0/4.0;
-
-                  double J_[2*2];
-                  DeviceTensor<2> J(J_, 2, 2);
-
-                  Jacobian2D(x, y, vx, vy, J);
-
-                  const double detJ = Det2D(J);
-                  const double w_detJ = w/detJ;
-
-                  Q(0,iqy,iqx) = w_detJ * (J(0,0)*J(0,0) + J(1,0)*J(1,0)); // 1,1
-                  Q(1,iqy,iqx) = w_detJ * (J(0,0)*J(0,1) + J(1,0)*J(1,1)); // 1,2
-                  Q(2,iqy,iqx) = w_detJ * (J(0,1)*J(0,1) + J(1,1)*J(1,1)); // 2,2
-                  Q(3,iqy,iqx) = w_detJ;
-               }
-            }
            for (int iqx=0; iqx<2; ++iqx)
            {
               for (int iqy=0; iqy<2; ++iqy)
@@ -547,34 +523,4 @@ void BatchedLOR_RT::Assemble3D()
   }
 }

-// Explicit template instantiations
-template void BatchedLOR_RT::Assemble2D<1>();
-template void BatchedLOR_RT::Assemble2D<2>();
-template void BatchedLOR_RT::Assemble2D<3>();
-template void BatchedLOR_RT::Assemble2D<4>();
-template void BatchedLOR_RT::Assemble2D<5>();
-template void BatchedLOR_RT::Assemble2D<6>();
-template void BatchedLOR_RT::Assemble2D<7>();
-template void BatchedLOR_RT::Assemble2D<8>();
-
-template void BatchedLOR_RT::Assemble3D<1>();
-template void BatchedLOR_RT::Assemble3D<2>();
-template void BatchedLOR_RT::Assemble3D<3>();
-template void BatchedLOR_RT::Assemble3D<4>();
-template void BatchedLOR_RT::Assemble3D<5>();
-template void BatchedLOR_RT::Assemble3D<6>();
-template void BatchedLOR_RT::Assemble3D<7>();
-template void BatchedLOR_RT::Assemble3D<8>();
-
-BatchedLOR_RT::BatchedLOR_RT(BilinearForm &a,
-                             FiniteElementSpace &fes_ho_,
-                             Vector &X_vert_,
-                             Vector &sparse_ij_,
-                             Array<int> &sparse_mapping_)
-   : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
-{
-   ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
-   ProjectLORCoefficient<DivDivIntegrator>(a, c2);
-}
-
 } // namespace mfem
@@ -20,11 +20,22 @@
 namespace mfem
 {

-template <int ORDER>
-MFEM_HOST_DEVICE inline void LORVertexCoordinates2D(
-   const double *X, int iel_ho, int kx, int ky, double vx[4], double vy[4])
+MFEM_HOST_DEVICE inline double Det2D(DeviceMatrix &J)
+{
+   return J(0,0)*J(1,1) - J(1,0)*J(0,1);
+}
+
+MFEM_HOST_DEVICE inline double Det3D(DeviceMatrix &J)
+{
+   return J(0,0) * (J(1,1) * J(2,2) - J(2,1) * J(1,2)) -
+          J(1,0) * (J(0,1) * J(2,2) - J(2,1) * J(0,2)) +
+          J(2,0) * (J(0,1) * J(1,2) - J(1,1) * J(0,2));
+}
+
+template <int ORDER, int SDIM=2>
+MFEM_HOST_DEVICE inline void LORVertexCoordinates2D(
+   const double *X, int iel_ho, int kx, int ky, double **v)
 {
-   const int dim = 2;
   const int nd1d = ORDER + 1;
   const int nvert_per_el = nd1d*nd1d;

@@ -33,23 +44,31 @@ MFEM_HOST_DEVICE inline void LORVertexCoordinates2D(
   const int v2 = kx + 1 + nd1d*(ky + 1);
   const int v3 = kx + nd1d*(ky + 1);

-   const int e0 = dim*(v0 + nvert_per_el*iel_ho);
-   const int e1 = dim*(v1 + nvert_per_el*iel_ho);
-   const int e2 = dim*(v2 + nvert_per_el*iel_ho);
-   const int e3 = dim*(v3 + nvert_per_el*iel_ho);
+   const int e0 = SDIM*(v0 + nvert_per_el*iel_ho);
+   const int e1 = SDIM*(v1 + nvert_per_el*iel_ho);
+   const int e2 = SDIM*(v2 + nvert_per_el*iel_ho);
+   const int e3 = SDIM*(v3 + nvert_per_el*iel_ho);

   // Vertex coordinates
-   vx[0] = X[e0 + 0];
-   vy[0] = X[e0 + 1];
+   v[0][0] = X[e0 + 0];
+   v[1][0] = X[e0 + 1];

-   vx[1] = X[e1 + 0];
-   vy[1] = X[e1 + 1];
+   v[0][1] = X[e1 + 0];
+   v[1][1] = X[e1 + 1];

-   vx[2] = X[e2 + 0];
-   vy[2] = X[e2 + 1];
+   v[0][2] = X[e2 + 0];
+   v[1][2] = X[e2 + 1];

-   vx[3] = X[e3 + 0];
-   vy[3] = X[e3 + 1];
+   v[0][3] = X[e3 + 0];
+   v[1][3] = X[e3 + 1];
+
+   if (SDIM == 3)
+   {
+      v[2][0] = X[e0 + 2];
+      v[2][1] = X[e1 + 2];
+      v[2][2] = X[e2 + 2];
+      v[2][3] = X[e3 + 2];
+   }
 }

 template <int ORDER>
@@ -112,15 +131,80 @@ MFEM_HOST_DEVICE inline void LORVertexCoordinates3D(
   vz[7] = X[e7 + 2];
 }

+template <int SDIM=2>
 MFEM_HOST_DEVICE inline void Jacobian2D(
-   const double x, const double y, const double vx[4], const double vy[4],
-   DeviceMatrix &J)
-{
-   J(0,0) = -(1-y)*vx[0] + (1-y)*vx[1] + y*vx[2] - y*vx[3];
-   J(0,1) = -(1-x)*vx[0] - x*vx[1] + x*vx[2] + (1-x)*vx[3];
+   const double x, const double y, double **v, DeviceMatrix &J);

-   J(1,0) = -(1-y)*vy[0] + (1-y)*vy[1] + y*vy[2] - y*vy[3];
-   J(1,1) = -(1-x)*vy[0] - x*vy[1] + x*vy[2] + (1-x)*vy[3];
+template <> MFEM_HOST_DEVICE inline void Jacobian2D<2>(
+   const double x, const double y, double **v, DeviceMatrix &J)
+{
+   J(0,0) = -(1-y)*v[0][0] + (1-y)*v[0][1] + y*v[0][2] - y*v[0][3];
+   J(0,1) = -(1-x)*v[0][0] - x*v[0][1] + x*v[0][2] + (1-x)*v[0][3];
+
+   J(1,0) = -(1-y)*v[1][0] + (1-y)*v[1][1] + y*v[1][2] - y*v[1][3];
+   J(1,1) = -(1-x)*v[1][0] - x*v[1][1] + x*v[1][2] + (1-x)*v[1][3];
+}
+
+template <> MFEM_HOST_DEVICE inline void Jacobian2D<3>(
+   const double x, const double y, double **v, DeviceMatrix &J)
+{
+   J(0,0) = -(1-y)*v[0][0] + (1-y)*v[0][1] + y*v[0][2] - y*v[0][3];
+   J(0,1) = -(1-x)*v[0][0] - x*v[0][1] + x*v[0][2] + (1-x)*v[0][3];
+
+   J(1,0) = -(1-y)*v[1][0] + (1-y)*v[1][1] + y*v[1][2] - y*v[1][3];
+   J(1,1) = -(1-x)*v[1][0] - x*v[1][1] + x*v[1][2] + (1-x)*v[1][3];
+
+   J(2,0) = -(1-y)*v[2][0] + (1-y)*v[2][1] + y*v[2][2] - y*v[2][3];
+   J(2,1) = -(1-x)*v[2][0] - x*v[2][1] + x*v[2][2] + (1-x)*v[2][3];
+}
+
+template <int ORDER, int SDIM, bool RT, bool ND>
+MFEM_HOST_DEVICE inline void SetupLORQuadData2D(
+   const double *X, int iel_ho, int kx, int ky, DeviceTensor<3> &Q, bool piola)
+{
+   double vx[4], vy[4], vz[4];
+   double *v[] = {vx, vy, vz};
+   LORVertexCoordinates2D<ORDER,SDIM>(X, iel_ho, kx, ky, v);
+
+   for (int iqy=0; iqy<2; ++iqy)
+   {
+      for (int iqx=0; iqx<2; ++iqx)
+      {
+         const double x = iqx;
+         const double y = iqy;
+         const double w = 1.0/4.0;
+
+         double J_[SDIM*2];
+         DeviceTensor<2> J(J_, SDIM, 2);
+
+         Jacobian2D<SDIM>(x, y, v, J);
+
+         if (SDIM == 2)
+         {
+            const double detJ = Det2D(J);
+            const double w_detJ = w/detJ;
+            const double E = J(0,0)*J(0,0) + J(1,0)*J(1,0);
+            const double F = J(0,0)*J(0,1) + J(1,0)*J(1,1);
+            const double G = J(0,1)*J(0,1) + J(1,1)*J(1,1);
+            Q(0,iqy,iqx) = w_detJ * (RT ? E : G); // 1,1
+            Q(1,iqy,iqx) = w_detJ * (RT ? F : -F); // 1,2
+            Q(2,iqy,iqx) = w_detJ * (RT ? G : E); // 2,2
+            Q(3,iqy,iqx) = (ND || RT) ? w_detJ : w*detJ;
+         }
+         else
+         {
+            const double E = J(0,0)*J(0,0) + J(1,0)*J(1,0) + J(2,0)*J(2,0);
+            const double F = J(0,0)*J(0,1) + J(1,0)*J(1,1) + J(2,0)*J(2,1);
+            const double G = J(0,1)*J(0,1) + J(1,1)*J(1,1) + J(2,1)*J(2,1);
+            const double detJ = sqrt(E*G - F*F);
+            const double w_detJ = w/detJ;
+            Q(0,iqy,iqx) = w_detJ * (RT ? E : G); // 1,1
+            Q(1,iqy,iqx) = w_detJ * (RT ? F : -F); // 1,2
+            Q(2,iqy,iqx) = w_detJ * (RT ? G : E); // 2,2
+            Q(3,iqy,iqx) =  (ND || RT) ? w_detJ : w*detJ;
+         }
+      }
+   }
 }

 MFEM_HOST_DEVICE inline void Jacobian3D(
@@ -180,18 +264,6 @@ MFEM_HOST_DEVICE inline void Adjugate3D(const DeviceMatrix &J, DeviceMatrix &A)
   A(2,2) = (J(0,0) * J(1,1)) - (J(0,1) * J(1,0));
 }

-MFEM_HOST_DEVICE inline double Det2D(DeviceMatrix &J)
-{
-   return J(0,0)*J(1,1) - J(1,0)*J(0,1);
-}
-
-MFEM_HOST_DEVICE inline double Det3D(DeviceMatrix &J)
-{
-   return J(0,0) * (J(1,1) * J(2,2) - J(2,1) * J(1,2)) -
-          J(1,0) * (J(0,1) * J(2,2) - J(2,1) * J(0,2)) +
-          J(2,0) * (J(0,1) * J(1,2) - J(1,1) * J(0,2));
-}
-
 }

 #endif
@@ -97,12 +97,37 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
   const FiniteElement *fe;
   ElementTransformation *T;
   DofTransformation *doftrans;
+   Mesh *mesh = fes->GetMesh();
   double energy = 0.0;

   if (dnfi.Size())
   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes->GetNE(); i++)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         fe = fes->GetFE(i);
         doftrans = fes->GetElementVDofs(i, vdofs);
         T = fes->GetElementTransformation(i);
@@ -110,6 +135,9 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
         if (doftrans) {doftrans->InvTransformPrimal(el_x); }
         for (int k = 0; k < dnfi.Size(); k++)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            energy += dnfi[k]->GetElementEnergy(*fe, *T, el_x);
         }
      }
@@ -175,8 +203,32 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const

   if (dnfi.Size())
   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes->GetNE(); i++)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         fe = fes->GetFE(i);
         doftrans = fes->GetElementVDofs(i, vdofs);
         T = fes->GetElementTransformation(i);
@@ -184,6 +236,9 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const
         if (doftrans) {doftrans->InvTransformPrimal(el_x); }
         for (int k = 0; k < dnfi.Size(); k++)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            dnfi[k]->AssembleElementVector(*fe, *T, el_x, el_y);
            if (doftrans) {doftrans->TransformDual(el_y); }
            py.AddElementVector(vdofs, el_y);
@@ -322,8 +377,32 @@ Operator &NonlinearForm::GetGradient(const Vector &x) const

   if (dnfi.Size())
   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes->GetNE(); i++)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         fe = fes->GetFE(i);
         doftrans = fes->GetElementVDofs(i, vdofs);
         T = fes->GetElementTransformation(i);
@@ -331,6 +410,9 @@ Operator &NonlinearForm::GetGradient(const Vector &x) const
         if (doftrans) {doftrans->InvTransformPrimal(el_x); }
         for (int k = 0; k < dnfi.Size(); k++)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            dnfi[k]->AssembleElementGrad(*fe, *T, el_x, elmat);
            if (doftrans) { doftrans->TransformDual(elmat); }
            Grad->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
@@ -561,13 +643,6 @@ BlockNonlinearForm::BlockNonlinearForm(Array<FiniteElementSpace *> &f) :
   SetSpaces(f);
 }

-void BlockNonlinearForm::AddBdrFaceIntegrator(BlockNonlinearFormIntegrator *nfi,
-                                              Array<int> &bdr_attr_marker)
-{
-   bfnfi.Append(nfi);
-   bfnfi_marker.Append(&bdr_attr_marker);
-}
-
 void BlockNonlinearForm::SetEssentialBC(
   const Array<Array<int> *> &bdr_attr_is_ess, Array<Vector *> &rhs)
 {
@@ -592,6 +667,7 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
   Array<const FiniteElement *> fe(fes.Size());
   ElementTransformation *T;
   DofTransformation *doftrans;
+   Mesh *mesh = fes[0]->GetMesh();
   double energy = 0.0;

   for (int i=0; i<fes.Size(); ++i)
@@ -601,8 +677,33 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
   }

   if (dnfi.Size())
+   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes[0]->GetNE(); ++i)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         T = fes[0]->GetElementTransformation(i);
         for (int s=0; s<fes.Size(); ++s)
         {
@@ -614,9 +715,13 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const

         for (int k = 0; k < dnfi.Size(); ++k)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            energy += dnfi[k]->GetElementEnergy(fe, *T, el_x_const);
         }
      }
+   }

   // free the allocated memory
   for (int i = 0; i < fes.Size(); ++i)
@@ -656,6 +761,7 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
   Array<const FiniteElement *> fe2(fes.Size());
   ElementTransformation *T;
   Array<DofTransformation *> doftrans(fes.Size()); doftrans = nullptr;
+   Mesh *mesh = fes[0]->GetMesh();

   by.UseDevice(true);
   by = 0.0;
@@ -670,8 +776,32 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

   if (dnfi.Size())
   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes[0]->GetNE(); ++i)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         T = fes[0]->GetElementTransformation(i);
         for (int s = 0; s < fes.Size(); ++s)
         {
@@ -683,6 +813,9 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

         for (int k = 0; k < dnfi.Size(); ++k)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            dnfi[k]->AssembleElementVector(fe, *T,
                                           el_x_const, el_y);

@@ -698,7 +831,6 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

   if (fnfi.Size())
   {
-      Mesh *mesh = fes[0]->GetMesh();
      FaceElementTransformations *tr;

      for (int i = 0; i < mesh->GetNumFaces(); ++i)
@@ -736,8 +868,8 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

   if (bfnfi.Size())
   {
-      Mesh *mesh = fes[0]->GetMesh();
      FaceElementTransformations *tr;
+
      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
@@ -858,6 +990,7 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
   Array<const FiniteElement *>fe2(fes.Size());
   ElementTransformation * T;
   Array<DofTransformation *> doftrans(fes.Size()); doftrans = nullptr;
+   Mesh *mesh = fes[0]->GetMesh();

   for (int i=0; i<fes.Size(); ++i)
   {
@@ -888,8 +1021,32 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const

   if (dnfi.Size())
   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes[0]->GetNE(); ++i)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         T = fes[0]->GetElementTransformation(i);
         for (int s = 0; s < fes.Size(); ++s)
         {
@@ -901,6 +1058,9 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const

         for (int k = 0; k < dnfi.Size(); ++k)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            dnfi[k]->AssembleElementGrad(fe, *T, el_x_const, elmats);

            for (int j=0; j<fes.Size(); ++j)
@@ -923,7 +1083,6 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
   if (fnfi.Size())
   {
      FaceElementTransformations *tr;
-      Mesh *mesh = fes[0]->GetMesh();

      for (int i = 0; i < mesh->GetNumFaces(); ++i)
      {
@@ -960,7 +1119,6 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
   if (bfnfi.Size())
   {
      FaceElementTransformations *tr;
-      Mesh *mesh = fes[0]->GetMesh();

      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
@@ -37,6 +37,7 @@ protected:

   /// Set of Domain Integrators to be assembled (added).
   Array<NonlinearFormIntegrator*> dnfi; // owned
+   Array<Array<int>*>              dnfi_marker; // not owned

   /// Set of interior face Integrators to be assembled (added).
   Array<NonlinearFormIntegrator*> fnfi; // owned
@@ -108,7 +109,12 @@ public:

   /// Adds new Domain Integrator.
   void AddDomainIntegrator(NonlinearFormIntegrator *nlfi)
-   { dnfi.Append(nlfi); }
+   { dnfi.Append(nlfi); dnfi_marker.Append(NULL); }
+
+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(NonlinearFormIntegrator *nlfi,
+                            Array<int> &elem_marker)
+   { dnfi.Append(nlfi); dnfi_marker.Append(&elem_marker); }

   /// Access all integrators added with AddDomainIntegrator().
   Array<NonlinearFormIntegrator*> *GetDNFI() { return &dnfi; }
@@ -227,13 +233,14 @@ protected:

   /// Set of Domain Integrators to be assembled (added).
   Array<BlockNonlinearFormIntegrator*> dnfi;
+   Array<Array<int>*>                   dnfi_marker;

   /// Set of interior face Integrators to be assembled (added).
   Array<BlockNonlinearFormIntegrator*> fnfi;

   /// Set of Boundary Face Integrators to be assembled (added).
   Array<BlockNonlinearFormIntegrator*> bfnfi;
-   Array<Array<int>*>           bfnfi_marker;
+   Array<Array<int>*>                   bfnfi_marker;

   /** Auxiliary block-vectors for wrapping input and output vectors or holding
       GridFunction-like block-vector data (e.g. in parallel). */
@@ -298,7 +305,12 @@ public:

   /// Adds new Domain Integrator.
   void AddDomainIntegrator(BlockNonlinearFormIntegrator *nlfi)
-   { dnfi.Append(nlfi); }
+   { dnfi.Append(nlfi); dnfi_marker.Append(NULL); }
+
+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(BlockNonlinearFormIntegrator *nlfi,
+                            Array<int> &elem_marker)
+   { dnfi.Append(nlfi); dnfi_marker.Append(&elem_marker); }

   /// Adds new Interior Face Integrator.
   void AddInteriorFaceIntegrator(BlockNonlinearFormIntegrator *nlfi)
@@ -311,7 +323,8 @@ public:
   /** @brief Adds new Boundary Face Integrator, restricted to specific boundary
       attributes. */
   void AddBdrFaceIntegrator(BlockNonlinearFormIntegrator *nlfi,
-                             Array<int> &bdr_marker);
+                             Array<int> &bdr_marker)
+   { bfnfi.Append(nlfi); bfnfi_marker.Append(&bdr_marker); }

   virtual void SetEssentialBC(const Array<Array<int> *>&bdr_attr_is_ess,
                               Array<Vector *> &rhs);
@@ -0,0 +1,241 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// Implementation of class ParBlockForm
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+void ParBlockForm::FillEssTdofLists(const Array<int> & ess_tdof_list)
+{
+   int j;
+   for (int i = 0; i<ess_tdof_list.Size(); i++)
+   {
+      int tdof = ess_tdof_list[i];
+      for (j = 0; j < nblocks; j++)
+      {
+         if (tdof_offsets[j+1] > tdof) { break; }
+      }
+      ess_tdofs[j]->Append(tdof-tdof_offsets[j]);
+   }
+}
+
+void ParBlockForm::BuildProlongation()
+{
+   P = new BlockOperator(dof_offsets, tdof_offsets);
+   R = new BlockMatrix(tdof_offsets, dof_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+
+   for (int i = 0; i<nblocks; i++)
+   {
+      HypreParMatrix * P_ = pfes[i]->Dof_TrueDof_Matrix();
+      P->SetBlock(i,i,P_);
+      const SparseMatrix * R_ = pfes[i]->GetRestrictionMatrix();
+      R->SetBlock(i,i,const_cast<SparseMatrix*>(R_));
+   }
+}
+
+void ParBlockForm::ParallelAssemble(BlockMatrix *m)
+{
+   if (!P) { BuildProlongation(); }
+
+   p_mat = new BlockOperator(tdof_offsets);
+   p_mat_e = new BlockOperator(tdof_offsets);
+   p_mat->owns_blocks = 1;
+   p_mat_e->owns_blocks = 1;
+   HypreParMatrix * A = nullptr;
+   HypreParMatrix * PtAP = nullptr;
+   for (int i = 0; i<nblocks; i++)
+   {
+      HypreParMatrix * Pi = (HypreParMatrix*)(&P->GetBlock(i,i));
+      HypreParMatrix * Pit = Pi->Transpose();
+      for (int j = 0; j<nblocks; j++)
+      {
+         if (m->IsZeroBlock(i,j)) { continue; }
+         if (i == j)
+         {
+            // Make block diagonal square hypre matrix
+            A = new HypreParMatrix(pfes[i]->GetComm(), pfes[i]->GlobalVSize(),
+                                   pfes[i]->GetDofOffsets(),&m->GetBlock(i,i));
+            PtAP = RAP(A,Pi);
+            delete A;
+            p_mat_e->SetBlock(i,i,PtAP->EliminateRowsCols(*ess_tdofs[i]));
+         }
+         else
+         {
+            HypreParMatrix * Pj = (HypreParMatrix*)(&P->GetBlock(j,j));
+            A = new HypreParMatrix(pfes[i]->GetComm(), pfes[i]->GlobalVSize(),
+                                   pfes[j]->GlobalVSize(), pfes[i]->GetDofOffsets(),
+                                   pfes[j]->GetDofOffsets(), &m->GetBlock(i,j));
+            HypreParMatrix * APj = ParMult(A, Pj,true);
+            delete A;
+            PtAP = ParMult(Pit,APj,true);
+            delete APj;
+            p_mat_e->SetBlock(i,j,PtAP->EliminateCols(*ess_tdofs[j]));
+            PtAP->EliminateRows(*ess_tdofs[i]);
+         }
+         p_mat->SetBlock(i,j,PtAP);
+      }
+      delete Pit;
+   }
+}
+
+
+ParBlockForm::ParBlockForm(const Array<ParFiniteElementSpace*> pfes_ ): pfes(
+      pfes_)
+{
+   nblocks = pfes.Size();
+   bforms.SetSize(nblocks,nblocks);
+   mforms.SetSize(nblocks,nblocks);
+   ess_tdofs.SetSize(nblocks);
+   dof_offsets.Append(0);
+   tdof_offsets.Append(0);
+   for (int i = 0; i<nblocks; i++)
+   {
+      dof_offsets.Append(pfes[i]->GetVSize());
+      tdof_offsets.Append(pfes[i]->TrueVSize());
+      ess_tdofs[i] = new Array<int>();
+      for (int j = 0; j<nblocks; j++)
+      {
+         bforms(i,j) = nullptr;
+         mforms(i,j) = nullptr;
+      }
+   }
+   dof_offsets.PartialSum();
+   tdof_offsets.PartialSum();
+}
+
+void ParBlockForm::SetBlock(ParBilinearForm * bform, int row_idx, int col_idx)
+{
+   MFEM_VERIFY((row_idx >=0 && row_idx < nblocks), "row index out of bounds");
+   MFEM_VERIFY((col_idx >=0 && col_idx < nblocks), "col index out of bounds");
+   MFEM_VERIFY(!mforms(row_idx,col_idx), "Entry has already been set");
+   MFEM_VERIFY(!bforms(row_idx,col_idx), "Entry has already been set");
+   bforms(row_idx,col_idx) = bform;
+}
+void ParBlockForm::SetBlock(ParMixedBilinearForm * mform, int row_idx,
+                            int col_idx)
+{
+   MFEM_VERIFY((row_idx >=0 && row_idx < nblocks), "row index out of bounds");
+   MFEM_VERIFY((col_idx >=0 && col_idx < nblocks), "col index out of bounds");
+   MFEM_VERIFY(!mforms(row_idx,col_idx), "Entry has already been set");
+   MFEM_VERIFY(!bforms(row_idx,col_idx), "Entry has already been set");
+   mforms(row_idx,col_idx) = mform;
+}
+
+/// Assemble the local matrix
+void ParBlockForm::Assemble(int skip_zeros)
+{
+   sp_mat = new BlockMatrix(dof_offsets);
+   for (int i = 0; i<nblocks; i++)
+   {
+      int h = dof_offsets[i+1]-dof_offsets[i];
+      for (int j = 0; j<nblocks; j++)
+      {
+         int w = dof_offsets[j+1]-dof_offsets[j];
+         if (bforms(i,j))
+         {
+            bforms(i,j)->Assemble(skip_zeros);
+            MFEM_VERIFY(h = bforms(i,j)->Height(), "inconsistent height of bilinear form");
+            MFEM_VERIFY(w = bforms(i,j)->Width(), "inconsistent width of bilinear form");
+            sp_mat->SetBlock(i,j,&bforms(i,j)->SpMat());
+         }
+         else if (mforms(i,j))
+         {
+            mforms(i,j)->Assemble(skip_zeros);
+            MFEM_VERIFY(h = mforms(i,j)->Height(),
+                        "inconsistent height of MixedBilinear form");
+            MFEM_VERIFY(w = mforms(i,j)->Width(),
+                        "inconsistent width of Mixedbilinear form");
+            sp_mat->SetBlock(i,j,&mforms(i,j)->SpMat());
+         }
+         else
+         {
+            sp_mat->SetBlock(i,j,nullptr);
+         }
+      }
+   }
+}
+
+void ParBlockForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
+                                    Vector & b,
+                                    OperatorHandle &A, Vector &X,
+                                    Vector &B, int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+   B.SetSize(P->Width());
+   P->MultTranspose(b,B);
+   X.SetSize(R->Height());
+   R->Mult(x,X);
+
+   // eliminate tdof in RHS
+   // B -= Ae*X
+   Vector tmp(B.Size());
+   p_mat_e->Mult(X,tmp);
+   B-=tmp;
+
+   for (int j = 0; j<nblocks; j++)
+   {
+      if (!ess_tdofs[j]->Size()) { continue; }
+      for (int i = 0; i < ess_tdofs[j]->Size(); i++)
+      {
+         int tdof = (*ess_tdofs[j])[i];
+         int gdof = tdof + tdof_offsets[j];
+         B(gdof) = X(gdof); // diagonal policy in always one in parallel
+      }
+   }
+   if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+}
+
+void ParBlockForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                    OperatorHandle &A)
+{
+   FillEssTdofLists(ess_tdof_list);
+   if (sp_mat)
+   {
+      sp_mat->Finalize();
+      ParallelAssemble(sp_mat);
+      delete sp_mat;
+      sp_mat = nullptr;
+   }
+   A.Reset(p_mat,false);
+}
+
+void ParBlockForm::RecoverFEMSolution(const Vector &X, Vector &x)
+{
+   x.SetSize(P->Height());
+   P->Mult(X, x);
+}
+
+ParBlockForm::~ParBlockForm()
+{
+   delete p_mat_e;
+   p_mat_e = nullptr;
+   delete p_mat;
+   p_mat = nullptr;
+   for (int i = 0; i<nblocks; i++)
+   {
+      delete ess_tdofs[i];
+   }
+   delete P;
+   delete R;
+}
+
+};
+
+#endif
@@ -0,0 +1,82 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_PBLOCKFORM
+#define MFEM_PBLOCKFORM
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include <mpi.h>
+#include "pfespace.hpp"
+#include "pgridfunc.hpp"
+#include "pbilinearform.hpp"
+
+namespace mfem
+{
+// square block forms
+class ParBlockForm
+{
+
+private:
+   // BilinearForms
+   Array2D<ParBilinearForm * > bforms;
+   Array2D<ParMixedBilinearForm * > mforms;
+   Array<ParFiniteElementSpace *> pfes;
+   Array<int> dof_offsets;
+   Array<int> tdof_offsets;
+   int nblocks;
+
+   // ess_tdof list for each space
+   Array<Array<int> *> ess_tdofs;
+
+   // Block operator of HypreParMatrix
+   BlockOperator * P = nullptr; // Block Prolongation
+   BlockMatrix * R = nullptr; // Block Restriction
+
+   BlockMatrix * sp_mat = nullptr;
+   // Block operator of HypreParMatrix
+   BlockOperator * p_mat = nullptr;
+   BlockOperator * p_mat_e = nullptr;
+
+   void FillEssTdofLists(const Array<int> & ess_tdof_list);
+   void BuildProlongation();
+   void ParallelAssemble(BlockMatrix *m);
+
+public:
+
+   ParBlockForm(const Array<ParFiniteElementSpace*> pfes_ );
+
+   void SetBlock(ParBilinearForm * bform, int row_idx, int col_idx);
+   void SetBlock(ParMixedBilinearForm * mform, int row_idx, int col_idx);
+
+   /// Assemble the local matrix
+   void Assemble(int skip_zeros = 1);
+
+   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector & b,
+                         OperatorHandle &A, Vector &X,
+                         Vector &B, int copy_interior = 0);
+
+   void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                         OperatorHandle &A);
+
+   void RecoverFEMSolution(const Vector &X, Vector &x);
+
+   /// Destroys bilinear form.
+   ~ParBlockForm();
+};
+
+}
+
+#endif // MFEM_USE_MPI
+
+#endif
@@ -90,20 +90,19 @@ ParNURBSExtension *ParFiniteElementSpace::MakeLocalNURBSext(
 void ParFiniteElementSpace::ParInit(ParMesh *pm)
 {
   pmesh = pm;
-   pncmesh = NULL;
+   pncmesh = nullptr;

   MyComm = pmesh->GetComm();
   NRanks = pmesh->GetNRanks();
   MyRank = pmesh->GetMyRank();

-   gcomm = NULL;
+   gcomm = nullptr;

-   P = NULL;
-   Pconf = NULL;
+   P = nullptr;
+   Pconf = nullptr;
   nonconf_P = false;
-   Rconf = NULL;
-   R = NULL;
-
+   Rconf = nullptr;
+   R = nullptr;
   num_face_nbr_dofs = -1;

   if (NURBSext && !pNURBSext())
@@ -466,59 +465,60 @@ void ParFiniteElementSpace::ApplyLDofSigns(Table &el_dof) const
   ApplyLDofSigns(all_dofs);
 }

-DofTransformation *
-ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs) const
+void ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs,
+                                           DofTransformation &doftrans) const
 {
   if (elem_dof)
   {
      elem_dof->GetRow(i, dofs);

-      if (DoFTrans[mesh->GetElementBaseGeometry(i)])
+      if (DoFTransArray[mesh->GetElementBaseGeometry(i)])
      {
         Array<int> Fo;
         elem_fos->GetRow(i, Fo);
-         DoFTrans[mesh->GetElementBaseGeometry(i)]->SetFaceOrientations(Fo);
-         return DoFTrans[mesh->GetElementBaseGeometry(i)];
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetElementBaseGeometry(i)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
-      return NULL;
+      return;
   }
-   DofTransformation * doftrans = FiniteElementSpace::GetElementDofs(i, dofs);
+   FiniteElementSpace::GetElementDofs(i, dofs, doftrans);
   if (Conforming())
   {
      ApplyLDofSigns(dofs);
   }
-   return doftrans;
 }

-DofTransformation *
-ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs) const
+void ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs,
+                                              DofTransformation &doftrans) const
 {
   if (bdr_elem_dof)
   {
      bdr_elem_dof->GetRow(i, dofs);

-      if (DoFTrans[mesh->GetBdrElementBaseGeometry(i)])
+      if (DoFTransArray[mesh->GetBdrElementBaseGeometry(i)])
      {
         Array<int> Fo;
-         bdr_elem_fos -> GetRow (i, Fo);
-         DoFTrans[mesh->GetBdrElementBaseGeometry(i)]->SetFaceOrientations(Fo);
-         return DoFTrans[mesh->GetBdrElementBaseGeometry(i)];
+         bdr_elem_fos->GetRow(i, Fo);
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetBdrElementBaseGeometry(i)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
-      return NULL;
+      return;
   }
-   DofTransformation * doftrans =
-      FiniteElementSpace::GetBdrElementDofs(i, dofs);
+   FiniteElementSpace::GetBdrElementDofs(i, dofs, doftrans);
   if (Conforming())
   {
      ApplyLDofSigns(dofs);
   }
-   return doftrans;
 }

 int ParFiniteElementSpace::GetFaceDofs(int i, Array<int> &dofs,
                                       int variant) const
 {
-   if (face_dof && variant == 0)
+   if (face_dof != nullptr && variant == 0)
   {
      face_dof->GetRow(i, dofs);
      return fec->GetOrder();
@@ -939,8 +939,8 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
         }
         else if (i_offd[i+1] == i_offd[i] + 2)
         {
-            const double * T = ND_StatelessDofTransformation
-                               ::GetFaceTransform(ltori[i]).GetData();
+            const double *T =
+               ND_DofTransformation::GetFaceTransform(ltori[i]).GetData();
            j_offd[i_offd[i] + 1] = j_offd[i_offd[i]] + 1;
            d_offd[i_offd[i]] = T[0]; d_offd[i_offd[i] + 1] = T[2];
            i++;
@@ -1041,18 +1041,28 @@ void ParFiniteElementSpace::GetEssentialTrueDofs(const Array<int>
 #ifdef MFEM_DEBUG
   // Verify that in boolean arithmetic: P^T ess_dofs = R ess_dofs.
   Array<int> true_ess_dofs2(true_ess_dofs.Size());
-   HypreParMatrix *Pt = Dof_TrueDof_Matrix()->Transpose();
+   auto Pt = std::unique_ptr<HypreParMatrix>(Dof_TrueDof_Matrix()->Transpose());
+
   const int *ess_dofs_data = ess_dofs.HostRead();
   Pt->BooleanMult(1, ess_dofs_data, 0, true_ess_dofs2);
-   delete Pt;
   int counter = 0;
   const int *ted = true_ess_dofs.HostRead();
+   std::string error_msg = "failed dof: ";
   for (int i = 0; i < true_ess_dofs.Size(); i++)
   {
-      if (bool(ted[i]) != bool(true_ess_dofs2[i])) { counter++; }
+      if (bool(ted[i]) != bool(true_ess_dofs2[i]))
+      {
+         error_msg += std::to_string(i) += "(R ";
+         error_msg += std::to_string(bool(ted[i])) += " P^T ";
+         error_msg += std::to_string(bool(true_ess_dofs2[i])) += ") ";
+         ++counter;
+      }
   }
+   MFEM_ASSERT(R->Height() == P->Width(), "!");
+   MFEM_ASSERT(R->Width() == P->Height(), "!");
+   MFEM_ASSERT(R->Width() == ess_dofs.Size(), "!");
   MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter
-               << ", rank = " << MyRank);
+               << ", rank = " << MyRank << ", " << error_msg);
 #endif

   MarkerToList(true_ess_dofs, ess_tdof_list);
@@ -1454,31 +1464,30 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
   delete [] requests;
 }

-DofTransformation *ParFiniteElementSpace::GetFaceNbrElementVDofs(
-   int i, Array<int> &vdofs) const
+void ParFiniteElementSpace::GetFaceNbrElementVDofs(
+   int i, Array<int> &vdofs, DofTransformation &doftrans) const
 {
   face_nbr_element_dof.GetRow(i, vdofs);

-   DofTransformation *doftrans = NULL;
-   Geometry::Type geom = GetFaceNbrFE(i)->GetGeomType();
-   if (DoFTrans[geom])
+   if (DoFTransArray[GetFaceNbrFE(i)->GetGeomType()])
   {
      Array<int> F, Fo;
      pmesh->GetFaceNbrElementFaces(pmesh->GetNE() + i, F, Fo);
-      doftrans = DoFTrans[geom];
-      doftrans->SetFaceOrientations(Fo);
-   }
-   if (vdim == 1 || doftrans == NULL)
-   {
-      return doftrans;
-   }
-   else
-   {
-      VDoFTrans.SetDofTransformation(*doftrans);
-      return &VDoFTrans;
+      doftrans.SetDofTransformation(
+         *DoFTransArray[GetFaceNbrFE(i)->GetGeomType()]);
+      doftrans.SetFaceOrientations(Fo);
+      doftrans.SetVDim(vdim, ordering);
   }
 }

+DofTransformation *ParFiniteElementSpace::GetFaceNbrElementVDofs(
+   int i, Array<int> &vdofs) const
+{
+   DoFTrans.SetDofTransformation(NULL);
+   GetFaceNbrElementVDofs(i, vdofs, DoFTrans);
+   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
+}
+
 void ParFiniteElementSpace::GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const
 {
   // Works for NC mesh where 'i' is an index returned by
@@ -1948,8 +1957,7 @@ struct PMatrixRow
      elems.reserve(elems.size() + other.elems.size());
      for (const PMatrixElement &oei : other.elems)
      {
-         elems.push_back(
-            PMatrixElement(oei.column, oei.stride, coef * oei.value));
+         elems.emplace_back(oei.column, oei.stride, coef * oei.value);
      }
   }

@@ -2025,7 +2033,7 @@ public:
   void AddRow(int entity, int index, int edof, GroupId group,
               const PMatrixRow &row)
   {
-      rows.push_back(RowInfo(entity, index, edof, group, row));
+      rows.emplace_back(entity, index, edof, group, row);
   }

   const std::vector<RowInfo>& GetRows() const { return rows; }
@@ -2041,8 +2049,10 @@ protected:
   ParNCMesh *pncmesh;
   const FiniteElementCollection* fec;

-   virtual void Encode(int rank);
-   virtual void Decode(int);
+   /// Encode a NeighborRowMessage for sending via MPI.
+   void Encode(int rank) override;
+   /// Decode a NeighborRowMessage received via MPI.
+   void Decode(int rank) override;
 };

 void NeighborRowMessage::Encode(int rank)
@@ -2161,6 +2171,11 @@ void NeighborRowMessage::Decode(int rank)
            int fo = pncmesh->GetFaceOrientation(id.index);
            ind = fec->DofOrderForOrientation(geom, fo);
         }
+         // Tri faces with second order basis have dofs that must be processed
+         // in pairs, as the doftransformation is not diagonal.
+         const bool process_dof_pairs = (ent == 2 &&
+                                         fec->GetContType() == FiniteElementCollection::TANGENTIAL
+                                         && !Geometry::IsTensorProduct(geom));

 #ifdef MFEM_DEBUG_PMATRIX
         mfem::out << "Rank " << pncmesh->MyRank << " receiving from " << rank
@@ -2180,7 +2195,7 @@ void NeighborRowMessage::Decode(int rank)

         // Create a row for this entity, recording the index of the mesh
         // element
-         rows.push_back(RowInfo(ent, id.index, edof, group_ids[gi++]));
+         rows.emplace_back(ent, id.index, edof, group_ids[gi++]);
         rows.back().row.read(stream, s);

 #ifdef MFEM_DEBUG_PMATRIX
@@ -2190,8 +2205,7 @@ void NeighborRowMessage::Decode(int rank)
                   << std::endl;
 #endif

-         if (ent == 2 && fec->GetContType() == FiniteElementCollection::TANGENTIAL
-             && !Geometry::IsTensorProduct(geom))
+         if (process_dof_pairs)
         {
            // ND face dofs need to be processed together, as the transformation
            // is given by a 2x2 matrix, so we manually apply an extra increment
@@ -2229,25 +2243,38 @@ void NeighborRowMessage::Decode(int rank)
               edof = -1 - edof;
               s *= -1.0;
            }
-            rows.push_back(RowInfo(ent, next_id.index, edof, group_ids[gi++]));
+
+            rows.emplace_back(ent, next_id.index, edof, group_ids[gi++]);
            rows.back().row.read(stream, s);
            auto &second_row = rows.back().row;

            // This is the second "fundamental unit" used in the transformation.
            const auto initial_second_row = second_row;

-            const auto T = [&fo]()
-            {
-               auto T = ND_StatelessDofTransformation::GetFaceTransform(fo);
-               T(0,0) -= 1;
-               T(1,1) -= 1;
-               return T;
-            }();
+            // Transform the received dofs by the primal transform. This is
+            // because within mfem as a face is visited its orientation is
+            // assigned to match the element that visited it first. Thus on
+            // processor boundaries, the transform will always be identity going
+            // into the element. However, the sending processor also thought the
+            // face orientation was zero, so it has sent the information in a
+            // different orientation. To map onto the local orientation
+            // definition, extract the orientation of the sending rank (the
+            // lower rank face defines the orientation fo), then apply the
+            // transform to the dependencies. The action of this transform on
+            // the dependencies is performed by adding scaled versions of the
+            // original two rows (which by the mfem assumption of face
+            // orientation, represent the identity transform).
+            const double *T =
+               ND_DofTransformation::GetFaceTransform(fo).GetData();

-            first_row.AddRow(initial_first_row, T(0,0));
-            first_row.AddRow(initial_second_row, T(0,1));
-            second_row.AddRow(initial_first_row, T(1,0));
-            second_row.AddRow(initial_second_row, T(1,1));
+            MFEM_ASSERT(fo != 2 &&
+                        fo != 4, "This code branch is ambiguous for face orientations 2 and 4."
+                        " Please report this mesh for further testing.\n");
+
+            first_row.AddRow(initial_first_row, T[0] - 1.0);   // (0,0)
+            first_row.AddRow(initial_second_row, T[2]);        // (0,1)
+            second_row.AddRow(initial_first_row, T[1]);        // (1,0)
+            second_row.AddRow(initial_second_row, T[3] - 1.0); // (1,1)

            first_row.Collapse();
            second_row.Collapse();
@@ -2419,7 +2446,7 @@ int ParFiniteElementSpace

            if (master_dofs.Size() == 0) { continue; }

-            const FiniteElement* fe = fec->FiniteElementForGeometry(mf.Geom());
+            const FiniteElement * const fe = fec->FiniteElementForGeometry(mf.Geom());
            if (fe == nullptr) { continue; }

            switch (mf.Geom())
@@ -2448,7 +2475,6 @@ int ParFiniteElementSpace
            }
         }
      }
-
      deps.Finalize();
   }

@@ -2577,15 +2603,15 @@ int ParFiniteElementSpace

   // big container for all messages we send (the list is for iterations)
   std::list<NeighborRowMessage::Map> send_msg;
-   send_msg.push_back(NeighborRowMessage::Map());
+   send_msg.emplace_back();

   // put identity in P and R for true DOFs, set ldof_ltdof
   for (int dof = 0, tdof = 0; dof < ndofs; dof++)
   {
      if (finalized[dof])
      {
-         pmatrix[dof].elems.push_back(
-            PMatrixElement(my_tdof_offset + vdim_factor*tdof, tdof_stride, 1.));
+         pmatrix[dof].elems.emplace_back(
+            my_tdof_offset + vdim_factor*tdof, tdof_stride, 1.);

         // prepare messages to neighbors with identity rows
         if (dof_group[dof] != 0)
@@ -2629,7 +2655,7 @@ int ParFiniteElementSpace
      // prepare a new round of send buffers
      if (send_msg.back().size())
      {
-         send_msg.push_back(NeighborRowMessage::Map());
+         send_msg.emplace_back();
      }

      // check for incoming messages, receive PMatrixRows
@@ -124,8 +124,8 @@ private:
   void GetGhostVertexDofs(const MeshId &id, Array<int> &dofs) const;
   void GetGhostEdgeDofs(const MeshId &edge_id, Array<int> &dofs) const;
   void GetGhostFaceDofs(const MeshId &face_id, Array<int> &dofs) const;
-
   void GetGhostDofs(int entity, const MeshId &id, Array<int> &dofs) const;
+
   /// Return the dofs associated with the interior of the given mesh entity.
   void GetBareDofs(int entity, int index, Array<int> &dofs) const;

@@ -248,7 +248,11 @@ public:
       If the FiniteElementCollection, @a f, is NULL (default), the FE
       collection used by @a global_fes will be reused. If @a f is not NULL, it
       must be the same as, or a copy of, the FE collection used by
-       @a global_fes. */
+       @a global_fes.
+
+       @note Currently the @a partitioning array is not used by this
+       constructor, it is required for general parallel variable-order support.
+   */
   ParFiniteElementSpace(ParMesh *pm, const FiniteElementSpace *global_fes,
                         const int *partitioning,
                         const FiniteElementCollection *f = NULL);
@@ -284,11 +288,17 @@ public:
   /// Return the number of local vector true dofs.
   int GetTrueVSize() const override { return ltdof_size; }

-   /// Returns indexes of degrees of freedom in array dofs for i'th element.
-   DofTransformation *GetElementDofs(int i, Array<int> &dofs) const override;
+   /// Returns indexes of degrees of freedom in array dofs for i'th element and
+   /// returns the DofTransformation data in a user-provided object.
+   using FiniteElementSpace::GetElementDofs;
+   void GetElementDofs(int i, Array<int> &dofs,
+                       DofTransformation &doftrans) const override;

-   /// Returns indexes of degrees of freedom for i'th boundary element.
-   DofTransformation *GetBdrElementDofs(int i, Array<int> &dofs) const override;
+   /// Returns indexes of degrees of freedom for i'th boundary element and
+   /// returns the DofTransformation data in a user-provided object.
+   using FiniteElementSpace::GetBdrElementDofs;
+   void GetBdrElementDofs(int i, Array<int> &dofs,
+                          DofTransformation &doftrans) const override;

   /** Returns the indexes of the degrees of freedom for i'th face
       including the dofs for the edges and the vertices of the face. */
@@ -382,6 +392,8 @@ public:
   // Face-neighbor functions
   void ExchangeFaceNbrData();
   int GetFaceNbrVSize() const { return num_face_nbr_dofs; }
+   void GetFaceNbrElementVDofs(int i, Array<int> &vdofs,
+                               DofTransformation &doftrans) const;
   DofTransformation *GetFaceNbrElementVDofs(int i, Array<int> &vdofs) const;
   void GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const;
   const FiniteElement *GetFaceNbrFE(int i) const;
@@ -39,11 +39,9 @@ ParGridFunction::ParGridFunction(ParMesh *pmesh, const GridFunction *gf,
 {
   const FiniteElementSpace *glob_fes = gf->FESpace();
   // duplicate the FiniteElementCollection from 'gf'
-   fec.reset(FiniteElementCollection::New(glob_fes->FEColl()->Name()));
+   fec = FiniteElementCollection::New(glob_fes->FEColl()->Name());
   // create a local ParFiniteElementSpace from the global one:
-   fes = pfes = new ParFiniteElementSpace(pmesh, glob_fes, partitioning,
-                                          fec.get());
-   owned_fes.reset(pfes);
+   fes = pfes = new ParFiniteElementSpace(pmesh, glob_fes, partitioning, fec);
   SetSize(pfes->GetVSize());

   if (partitioning)
@@ -83,10 +81,10 @@ ParGridFunction::ParGridFunction(ParMesh *pmesh, std::istream &input)
   : GridFunction(pmesh, input)
 {
   // Convert the FiniteElementSpace, fes, to a ParFiniteElementSpace:
-   pfes = new ParFiniteElementSpace(pmesh, fec.get(), fes->GetVDim(),
+   pfes = new ParFiniteElementSpace(pmesh, fec, fes->GetVDim(),
                                    fes->GetOrdering());
+   delete fes;
   fes = pfes;
-   owned_fes.reset(pfes);
 }

 void ParGridFunction::Update()
@@ -695,7 +693,23 @@ void ParGridFunction::ProjectBdrCoefficient(

 #ifdef MFEM_DEBUG
   Array<int> ess_vdofs_marker;
-   pfes->GetEssentialVDofs(attr, ess_vdofs_marker);
+   if (vcoeff) { pfes->GetEssentialVDofs(attr, ess_vdofs_marker); }
+   else
+   {
+      ess_vdofs_marker.SetSize(Size());
+      ess_vdofs_marker = 0;
+      for (int i = 0; i < fes->GetVDim(); i++)
+      {
+         if (!coeff[i]) { continue; }
+         Array<int> component_dof_marker;
+         pfes->GetEssentialVDofs(attr, component_dof_marker,i);
+         for (int j = 0; j<Size(); j++)
+         {
+            ess_vdofs_marker[j] = bool(ess_vdofs_marker[j]) ||
+                                  bool(component_dof_marker[j]);
+         }
+      }
+   }
   for (int i = 0; i < values_counter.Size(); i++)
   {
      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
@@ -738,7 +752,8 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
   {
      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
                  bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
-                  "internal error");
+                  "internal error: " << pfes->GetLocalTDofNumber(i) << ' ' << bool(
+                     values_counter[i]));
   }
 #endif
 }
@@ -49,6 +49,10 @@ protected:
 public:
   ParGridFunction() { pfes = NULL; }

+   /// Copy constructor. The internal vector #face_nbr_data is not copied.
+   ParGridFunction(const ParGridFunction &orig)
+      : GridFunction(orig), pfes(orig.pfes) { }
+
   ParGridFunction(ParFiniteElementSpace *pf) : GridFunction(pf), pfes(pf) { }

   /// Construct a ParGridFunction using previously allocated array @a data.
@@ -89,6 +93,15 @@ public:
       constructed. The new ParGridFunction assumes ownership of both. */
   ParGridFunction(ParMesh *pmesh, std::istream &input);

+   /// Copy assignment. Only the data of the base class Vector is copied.
+   /** It is assumed that this object and @a rhs use ParFiniteElementSpace%s
+       that have the same size.
+
+       @note Defining this method overwrites the implicitly defined copy
+       assignment operator. */
+   ParGridFunction &operator=(const ParGridFunction &rhs)
+   { return operator=((const Vector &)rhs); }
+
   /// Assign constant values to the ParGridFunction data.
   ParGridFunction &operator=(double value)
   { GridFunction::operator=(value); return *this; }
@@ -99,12 +112,12 @@ public:

   ParFiniteElementSpace *ParFESpace() const { return pfes; }

-   virtual void Update();
+   void Update() override;

   /// Associate a new FiniteElementSpace with the ParGridFunction.
   /** The ParGridFunction is resized using the SetSize() method. The new space
       @a f is expected to be a ParFiniteElementSpace. */
-   virtual void SetSpace(FiniteElementSpace *f);
+   void SetSpace(FiniteElementSpace *f) override;

   /// Associate a new parallel space with the ParGridFunction.
   void SetSpace(ParFiniteElementSpace *f);
@@ -117,7 +130,7 @@ public:
       ParGridFunction and sets the pointer @a v as external data in the
       ParGridFunction. The new space @a f is expected to be a
       ParFiniteElementSpace. */
-   virtual void MakeRef(FiniteElementSpace *f, double *v);
+   void MakeRef(FiniteElementSpace *f, double *v) override;

   /** @brief Make the ParGridFunction reference external data on a new
       ParFiniteElementSpace. */
@@ -134,7 +147,7 @@ public:
       expected to be a ParFiniteElementSpace.
       @note This version of the method will also perform bounds checks when
       the build option MFEM_DEBUG is enabled. */
-   virtual void MakeRef(FiniteElementSpace *f, Vector &v, int v_offset);
+   void MakeRef(FiniteElementSpace *f, Vector &v, int v_offset) override;

   /** @brief Make the ParGridFunction reference external data on a new
       ParFiniteElementSpace. */
@@ -153,7 +166,7 @@ public:
   void AddDistribute(double a, const Vector &tv) { AddDistribute(a, &tv); }

   /// Set the GridFunction from the given true-dof vector.
-   virtual void SetFromTrueDofs(const Vector &tv) { Distribute(tv); }
+   void SetFromTrueDofs(const Vector &tv) override { Distribute(tv); }

   /// Short semantic for Distribute()
   ParGridFunction &operator=(const HypreParVector &tv)
@@ -196,26 +209,26 @@ public:
   const Vector &FaceNbrData() const { return face_nbr_data; }

   // Redefine to handle the case when i is a face-neighbor element
-   virtual double GetValue(int i, const IntegrationPoint &ip,
-                           int vdim = 1) const;
+   double GetValue(int i, const IntegrationPoint &ip,
+                   int vdim = 1) const override;
   double GetValue(ElementTransformation &T)
   { return GetValue(T, T.GetIntPoint()); }

   // Redefine to handle the case when T describes a face-neighbor element
-   virtual double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
-                           int comp = 0, Vector *tr = NULL) const;
+   double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
+                   int comp = 0, Vector *tr = NULL) const override;

-   virtual void GetVectorValue(int i, const IntegrationPoint &ip,
-                               Vector &val) const;
+   void GetVectorValue(int i, const IntegrationPoint &ip,
+                       Vector &val) const override;

   // Redefine to handle the case when T describes a face-neighbor element
-   virtual void GetVectorValue(ElementTransformation &T,
-                               const IntegrationPoint &ip,
-                               Vector &val, Vector *tr = NULL) const;
+   void GetVectorValue(ElementTransformation &T,
+                       const IntegrationPoint &ip,
+                       Vector &val, Vector *tr = NULL) const override;

   /** @brief For each vdof, counts how many elements contain the vdof,
       as containment is determined by FiniteElementSpace::GetElementVDofs(). */
-   virtual void CountElementsPerVDof(Array<int> &elem_per_vdof) const;
+   void CountElementsPerVDof(Array<int> &elem_per_vdof) const override;

   /// Parallel version of GridFunction::GetDerivative(); see its documentation.
   void GetDerivative(int comp, int der_comp, ParGridFunction &der);
@@ -224,112 +237,111 @@ public:
       freedom of element @a el. If @a el is greater than or equal to the number
       of local elements, it will be interpreted as a shifted index of a face
       neighbor element. */
-   virtual void GetElementDofValues(int el, Vector &dof_vals) const;
+   void GetElementDofValues(int el, Vector &dof_vals) const override;

   using GridFunction::ProjectCoefficient;
-   virtual void ProjectCoefficient(Coefficient &coeff);
+   void ProjectCoefficient(Coefficient &coeff) override;

   using GridFunction::ProjectDiscCoefficient;
   /** @brief Project a discontinuous vector coefficient as a grid function on
       a continuous finite element space. The values in shared dofs are
       determined from the element with maximal attribute. */
-   virtual void ProjectDiscCoefficient(VectorCoefficient &coeff);
+   void ProjectDiscCoefficient(VectorCoefficient &coeff) override;

-   virtual void ProjectDiscCoefficient(Coefficient &coeff, AvgType type);
+   void ProjectDiscCoefficient(Coefficient &coeff, AvgType type) override;

-   virtual void ProjectDiscCoefficient(VectorCoefficient &vcoeff, AvgType type);
+   void ProjectDiscCoefficient(VectorCoefficient &vcoeff, AvgType type) override;

   using GridFunction::ProjectBdrCoefficient;

   // Only the values in the master are guaranteed to be correct!
-   virtual void ProjectBdrCoefficient(VectorCoefficient &vcoeff,
-                                      Array<int> &attr)
+   void ProjectBdrCoefficient(VectorCoefficient &vcoeff,
+                              Array<int> &attr) override
   { ProjectBdrCoefficient(NULL, &vcoeff, attr); }

   // Only the values in the master are guaranteed to be correct!
-   virtual void ProjectBdrCoefficient(Coefficient *coeff[], Array<int> &attr)
+   void ProjectBdrCoefficient(Coefficient *coeff[], Array<int> &attr) override
   { ProjectBdrCoefficient(coeff, NULL, attr); }

   // Only the values in the master are guaranteed to be correct!
-   virtual void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
-                                             Array<int> &bdr_attr);
+   void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
+                                     Array<int> &bdr_attr) override;

-   virtual double ComputeL1Error(Coefficient *exsol[],
-                                 const IntegrationRule *irs[] = NULL) const
+   double ComputeL1Error(Coefficient *exsol[],
+                         const IntegrationRule *irs[] = NULL) const override
   {
-      return GlobalLpNorm(1.0, GridFunction::ComputeW11Error(
-                             *exsol, NULL, 1, NULL, irs), pfes->GetComm());
+      return GlobalLpNorm(1.0, GridFunction::ComputeL1Error(exsol, irs),
+                          pfes->GetComm());
   }

-   virtual double ComputeL1Error(Coefficient &exsol,
-                                 const IntegrationRule *irs[] = NULL) const
+   double ComputeL1Error(Coefficient &exsol,
+                         const IntegrationRule *irs[] = NULL) const override
   { return ComputeLpError(1.0, exsol, NULL, irs); }

-   virtual double ComputeL1Error(VectorCoefficient &exsol,
-                                 const IntegrationRule *irs[] = NULL) const
+   double ComputeL1Error(VectorCoefficient &exsol,
+                         const IntegrationRule *irs[] = NULL) const override
   { return ComputeLpError(1.0, exsol, NULL, NULL, irs); }

-   virtual double ComputeL2Error(Coefficient *exsol[],
-                                 const IntegrationRule *irs[] = NULL,
-                                 const Array<int> *elems = NULL) const
+   double ComputeL2Error(Coefficient *exsol[],
+                         const IntegrationRule *irs[] = NULL,
+                         const Array<int> *elems = NULL) const override
   {
      return GlobalLpNorm(2.0, GridFunction::ComputeL2Error(exsol, irs, elems),
                          pfes->GetComm());
   }

-   virtual double ComputeL2Error(Coefficient &exsol,
-                                 const IntegrationRule *irs[] = NULL,
-                                 const Array<int> *elems = NULL) const
+   double ComputeL2Error(Coefficient &exsol,
+                         const IntegrationRule *irs[] = NULL,
+                         const Array<int> *elems = NULL) const override
   {
      return GlobalLpNorm(2.0, GridFunction::ComputeL2Error(exsol, irs, elems),
                          pfes->GetComm());
   }


-   virtual double ComputeL2Error(VectorCoefficient &exsol,
-                                 const IntegrationRule *irs[] = NULL,
-                                 const Array<int> *elems = NULL) const
+   double ComputeL2Error(VectorCoefficient &exsol,
+                         const IntegrationRule *irs[] = NULL,
+                         const Array<int> *elems = NULL) const override
   {
      return GlobalLpNorm(2.0, GridFunction::ComputeL2Error(exsol, irs, elems),
                          pfes->GetComm());
   }

   /// Returns ||grad u_ex - grad u_h||_L2 for H1 or L2 elements
-   virtual double ComputeGradError(VectorCoefficient *exgrad,
-                                   const IntegrationRule *irs[] = NULL) const
+   double ComputeGradError(VectorCoefficient *exgrad,
+                           const IntegrationRule *irs[] = NULL) const override
   {
      return GlobalLpNorm(2.0, GridFunction::ComputeGradError(exgrad,irs),
                          pfes->GetComm());
   }

   /// Returns ||curl u_ex - curl u_h||_L2 for ND elements
-   virtual double ComputeCurlError(VectorCoefficient *excurl,
-                                   const IntegrationRule *irs[] = NULL) const
+   double ComputeCurlError(VectorCoefficient *excurl,
+                           const IntegrationRule *irs[] = NULL) const override
   {
      return GlobalLpNorm(2.0, GridFunction::ComputeCurlError(excurl,irs),
                          pfes->GetComm());
   }

   /// Returns ||div u_ex - div u_h||_L2 for RT elements
-   virtual double ComputeDivError(Coefficient *exdiv,
-                                  const IntegrationRule *irs[] = NULL) const
+   double ComputeDivError(Coefficient *exdiv,
+                          const IntegrationRule *irs[] = NULL) const override
   {
      return GlobalLpNorm(2.0, GridFunction::ComputeDivError(exdiv,irs),
                          pfes->GetComm());
   }

   /// Returns the Face Jumps error for L2 elements
-   virtual double ComputeDGFaceJumpError(Coefficient *exsol,
-                                         Coefficient *ell_coeff,
-                                         JumpScaling jump_scaling,
-                                         const IntegrationRule *irs[]=NULL)
-   const;
+   double ComputeDGFaceJumpError(Coefficient *exsol,
+                                 Coefficient *ell_coeff,
+                                 JumpScaling jump_scaling,
+                                 const IntegrationRule *irs[]=NULL) const override;

   /// Returns either the H1-seminorm or the DG Face Jumps error or both
   /// depending on norm_type = 1, 2, 3
-   virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
-                                 Coefficient *ell_coef, double Nu,
-                                 int norm_type) const
+   double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
+                         Coefficient *ell_coef, double Nu,
+                         int norm_type) const override
   {
      return GlobalLpNorm(2.0,
                          GridFunction::ComputeH1Error(exsol,exgrad,ell_coef,
@@ -339,56 +351,56 @@ public:

   /// Returns the error measured in H1-norm for H1 elements or in "broken"
   /// H1-norm for L2 elements
-   virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
-                                 const IntegrationRule *irs[] = NULL) const
+   double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
+                         const IntegrationRule *irs[] = NULL) const override
   {
      return GlobalLpNorm(2.0, GridFunction::ComputeH1Error(exsol,exgrad,irs),
                          pfes->GetComm());
   }

   /// Returns the error measured H(div)-norm for RT elements
-   virtual double ComputeHDivError(VectorCoefficient *exsol,
-                                   Coefficient *exdiv,
-                                   const IntegrationRule *irs[] = NULL) const
+   double ComputeHDivError(VectorCoefficient *exsol,
+                           Coefficient *exdiv,
+                           const IntegrationRule *irs[] = NULL) const override
   {
      return GlobalLpNorm(2.0, GridFunction::ComputeHDivError(exsol,exdiv,irs),
                          pfes->GetComm());
   }

   /// Returns the error measured H(curl)-norm for ND elements
-   virtual double ComputeHCurlError(VectorCoefficient *exsol,
-                                    VectorCoefficient *excurl,
-                                    const IntegrationRule *irs[] = NULL) const
+   double ComputeHCurlError(VectorCoefficient *exsol,
+                            VectorCoefficient *excurl,
+                            const IntegrationRule *irs[] = NULL) const override
   {
      return GlobalLpNorm(2.0,
                          GridFunction::ComputeHCurlError(exsol,excurl,irs),
                          pfes->GetComm());
   }

-   virtual double ComputeMaxError(Coefficient *exsol[],
-                                  const IntegrationRule *irs[] = NULL) const
+   double ComputeMaxError(Coefficient *exsol[],
+                          const IntegrationRule *irs[] = NULL) const override
   {
      return GlobalLpNorm(infinity(),
                          GridFunction::ComputeMaxError(exsol, irs),
                          pfes->GetComm());
   }

-   virtual double ComputeMaxError(Coefficient &exsol,
-                                  const IntegrationRule *irs[] = NULL) const
+   double ComputeMaxError(Coefficient &exsol,
+                          const IntegrationRule *irs[] = NULL) const override
   {
      return ComputeLpError(infinity(), exsol, NULL, irs);
   }

-   virtual double ComputeMaxError(VectorCoefficient &exsol,
-                                  const IntegrationRule *irs[] = NULL) const
+   double ComputeMaxError(VectorCoefficient &exsol,
+                          const IntegrationRule *irs[] = NULL) const override
   {
      return ComputeLpError(infinity(), exsol, NULL, NULL, irs);
   }

-   virtual double ComputeLpError(const double p, Coefficient &exsol,
-                                 Coefficient *weight = NULL,
-                                 const IntegrationRule *irs[] = NULL,
-                                 const Array<int> *elems = NULL) const
+   double ComputeLpError(const double p, Coefficient &exsol,
+                         Coefficient *weight = NULL,
+                         const IntegrationRule *irs[] = NULL,
+                         const Array<int> *elems = NULL) const override
   {
      return GlobalLpNorm(p, GridFunction::ComputeLpError(p, exsol, weight, irs,
                                                          elems), pfes->GetComm());
@@ -397,23 +409,23 @@ public:
   /** When given a vector weight, compute the pointwise (scalar) error as the
       dot product of the vector error with the vector weight. Otherwise, the
       scalar error is the l_2 norm of the vector error. */
-   virtual double ComputeLpError(const double p, VectorCoefficient &exsol,
-                                 Coefficient *weight = NULL,
-                                 VectorCoefficient *v_weight = NULL,
-                                 const IntegrationRule *irs[] = NULL) const
+   double ComputeLpError(const double p, VectorCoefficient &exsol,
+                         Coefficient *weight = NULL,
+                         VectorCoefficient *v_weight = NULL,
+                         const IntegrationRule *irs[] = NULL) const override
   {
      return GlobalLpNorm(p, GridFunction::ComputeLpError(
                             p, exsol, weight, v_weight, irs), pfes->GetComm());
   }

-   virtual void ComputeFlux(BilinearFormIntegrator &blfi,
-                            GridFunction &flux,
-                            bool wcoef = true, int subdomain = -1);
+   void ComputeFlux(BilinearFormIntegrator &blfi,
+                    GridFunction &flux,
+                    bool wcoef = true, int subdomain = -1) override;

   /** Save the local portion of the ParGridFunction. This differs from the
       serial GridFunction::Save in that it takes into account the signs of
       the local dofs. */
-   virtual void Save(std::ostream &out) const;
+   void Save(std::ostream &out) const override;

   /// Save the ParGridFunction to a single file (written using MPI rank 0). The
   /// given @a precision will be used for ASCII output.
@@ -422,7 +434,7 @@ public:
   /// Save the ParGridFunction to files (one for each MPI rank). The files will
   /// be given suffixes according to the MPI rank. The given @a precision will
   /// be used for ASCII output.
-   virtual void Save(const char *fname, int precision=16) const;
+   void Save(const char *fname, int precision=16) const override;

   /// Returns a GridFunction on MPI rank @a save_rank that does not have any
   /// duplication of vertices/nodes at processor boundaries.
@@ -439,13 +451,16 @@ public:
   /** Save the local portion of the ParGridFunction. This differs from the
       serial GridFunction::Save in that it takes into account the signs of
       the local dofs. */
-   virtual void Save(
+   void Save(
      adios2stream &out, const std::string &variable_name,
-      const adios2stream::data_type type = adios2stream::data_type::point_data) const;
+      const adios2stream::data_type type = adios2stream::data_type::point_data) const
+   override;
 #endif

   /// Merge the local grid functions
   void SaveAsOne(std::ostream &out = mfem::out) const;
+
+   virtual ~ParGridFunction() = default;
 };


@@ -177,7 +177,7 @@ int FaceQuadratureSpace::GetEntityIndex(const ElementTransformation &T) const
         return get_face_index(T.ElementNo);
      case ElementTransformation::BDR_ELEMENT:
      case ElementTransformation::BDR_FACE:
-         return get_face_index(mesh.GetBdrElementEdgeIndex(T.ElementNo));
+         return get_face_index(mesh.GetBdrElementFaceIndex(T.ElementNo));
      default:
         MFEM_ABORT("Invalid element type.");
         return -1;
@@ -462,21 +462,52 @@ void TMOP_Metric_009::AssembleH(const DenseMatrix &Jpt,
   ie.Assemble_ddI1b(weight, A.GetData());
 }

-// mu_14 = |T-I|^2
+double TMOP_Metric_014::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_14 = |J - I|^2.
+   DenseMatrix Mat(Jpt);
+   Mat(0,0) -= 1.0;
+   Mat(1,1) -= 1.0;
+   return Mat.FNorm2();
+}
+
 double TMOP_Metric_014::EvalW(const DenseMatrix &Jpt) const
 {
-   MFEM_VERIFY(Jtr != NULL,
-               "Requires a target Jacobian, use SetTargetJacobian().");
+   // mu_14 = |J - I|^2 = I1[J-I].
+   DenseMatrix Mat(Jpt);
+   Mat(0,0) -= 1.0;
+   Mat(1,1) -= 1.0;

-   DenseMatrix Id(2,2);
+   ie.SetJacobian(Mat.GetData());
+   return ie.Get_I1();
+}

-   Id(0,0) = 1; Id(0,1) = 0;
-   Id(1,0) = 0; Id(1,1) = 1;
+void TMOP_Metric_014::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+{
+   // P = dI1[J-I] d/dJ[J-I] = dI1[J-I].
+   DenseMatrix JptMinusId = Jpt;
+   for (int i = 0; i < Jpt.Size(); i++)
+   {
+      JptMinusId(i, i) -= 1.0;
+   }
+   ie.SetJacobian(JptMinusId.GetData());
+   P = ie.Get_dI1();
+}

-   DenseMatrix Mat(2,2);
-   Mat = Jpt;
-   Mat.Add(-1,Id);
-   return Mat.FNorm2();
+void TMOP_Metric_014::AssembleH(const DenseMatrix &Jpt,
+                                const DenseMatrix &DS,
+                                const double weight,
+                                DenseMatrix &A) const
+{
+   // dP = ddI1[J-I].
+   DenseMatrix JptMinusId = Jpt;
+   for (int i = 0; i < Jpt.Size(); i++)
+   {
+      JptMinusId(i, i) -= 1.0;
+   }
+   ie.SetJacobian(JptMinusId.GetData());
+   ie.SetDerivativeMatrix(DS.Height(), DS.GetData());
+   ie.Assemble_ddI1(weight, A.GetData());
 }

 double TMOP_Metric_022::EvalW(const DenseMatrix &Jpt) const
@@ -4347,6 +4378,8 @@ UpdateAfterMeshPositionChange(const Vector &x_new,
 {
   if (discr_tc) { PA.Jtr_needs_update = true; }

+   if (PA.enabled) { UpdateCoefficientsPA(x_new); }
+
   Ordering::Type ordering = x_fes.GetOrdering();

   // Update the finite difference delta if FD are used.
@@ -373,16 +373,20 @@ public:
 /// 2D non-barrier Shape+Size+Orientation (VOS) metric (polyconvex).
 class TMOP_Metric_014 : public TMOP_QualityMetric
 {
+protected:
+   mutable InvariantsEvaluator2D<double> ie;
+
 public:
-   // W = |T-I|^2.
+   // W = |J - I|^2.
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = I1[J-I].
   virtual double EvalW(const DenseMatrix &Jpt) const;

-   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
-   { MFEM_ABORT("Not implemented"); }
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;

   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
-                          const double weight, DenseMatrix &A) const
-   { MFEM_ABORT("Not implemented"); }
+                          const double weight, DenseMatrix &A) const;
 };

 /// 2D Shifted barrier form of shape metric (mu_2).
@@ -1817,17 +1821,27 @@ protected:

   // PA extension
   // ------------
+   // Jtr: all ref->target Jacobians, (dim x dim) Q-Vector as DenseTensor.
+   //      updated when needed, based on Jtr_needs_update.
+   //
   //  E: Q-vector for TMOP-energy
+   //     Used as temporary storage when the total energy is computed.
   //  O: Q-Vector of 1.0, used to compute sums using the dot product kernel.
   // X0: E-vector for initial nodal coordinates used for limiting.
+   //     Does not change during the TMOP iteration.
   //  H: Q-Vector for Hessian associated with the metric term.
+   //     Updated by every call to PANonlinearFormExtension::GetGradient().
   // C0: Q-Vector for spatial weight used for the limiting term.
+   //     Updated when the mesh nodes change.
   // LD: E-Vector constructed using limiting distance grid function (delta).
+   //     Does not change during the TMOP iteration.
   // H0: Q-Vector for Hessian associated with the limiting term.
+   //     Updated by every call to PANonlinearFormExtension::GetGradient().
+   // MC: Q-Vector for the metric Coefficient.
+   //     Updated when the mesh nodes change.
   //
-   // maps:     Dof2Quad map for fespace associate with nodal coordinates.
-   // maps_lim: Dof2Quad map for fespace associated with the limiting distance
-   //            grid function.
+   // maps:     Dof2Quad map for fes associated with the nodal coordinates.
+   // maps_lim: Dof2Quad map for fes associated with the limiting dist GridFunc.
   //
   // Jtr_debug_grad
   //     We keep track if Jtr was set by AssembleGradPA() in Jtr_debug_grad: it
@@ -1846,7 +1860,7 @@ protected:
      mutable DenseTensor Jtr;
      mutable bool Jtr_needs_update;
      mutable bool Jtr_debug_grad;
-      mutable Vector E, O, X0, H, C0, LD, H0;
+      mutable Vector E, O, X0, H, C0, LD, H0, MC;
      const DofToQuad *maps;
      const DofToQuad *maps_lim = nullptr;
      const GeometricFactors *geom;
@@ -1960,6 +1974,9 @@ protected:

   void AssemblePA_Limiting();
   void ComputeAllElementTargets(const Vector &xe = Vector()) const;
+   // Updates the Q-vectors for the metric_coeff and lim_coeff, based on the
+   // new physical positions of the quadrature points.
+   void UpdateCoefficientsPA(const Vector &x_loc);

   // Compute Min(Det(Jpt)) in the mesh, does not reduce over MPI.
   double ComputeMinDetT(const Vector &x, const FiniteElementSpace &fes);
@@ -176,6 +176,42 @@ void TMOP_Integrator::ComputeAllElementTargets(const Vector &xe) const
   targetC->ComputeAllElementTargets(*fes, ir, xe, PA.Jtr);
 }

+void TMOP_Integrator::UpdateCoefficientsPA(const Vector &x_loc)
+{
+   // Both are constant or not specified.
+   if (PA.MC.Size() == 1 && PA.C0.Size() == 1) { return; }
+
+   // Coefficients are always evaluated on the CPU for now.
+   PA.MC.HostWrite();
+   PA.C0.HostWrite();
+
+   const IntegrationRule &ir = *PA.ir;
+   auto T = new IsoparametricTransformation;
+   for (int e = 0; e < PA.ne; ++e)
+   {
+      // Uses the node positions in x_loc.
+      PA.fes->GetMesh()->GetElementTransformation(e, x_loc, T);
+
+      if (PA.MC.Size() > 1)
+      {
+         for (int q = 0; q < PA.nq; ++q)
+         {
+            PA.MC(q + e * PA.nq) = metric_coeff->Eval(*T, ir.IntPoint(q));
+         }
+      }
+
+      if (PA.C0.Size() > 1)
+      {
+         for (int q = 0; q < PA.nq; ++q)
+         {
+            PA.C0(q + e * PA.nq) = lim_coeff->Eval(*T, ir.IntPoint(q));
+         }
+      }
+   }
+
+   delete T;
+}
+
 void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
 {
   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
@@ -213,6 +249,35 @@ void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
   PA.O.SetSize(ne*nq, Device::GetDeviceMemoryType());
   PA.O = 1.0;

+   if (metric_coeff)
+   {
+      if (auto cc = dynamic_cast<ConstantCoefficient *>(metric_coeff))
+      {
+         PA.MC.SetSize(1, Device::GetMemoryType());
+         PA.MC.HostWrite();
+         PA.MC(0) = cc->constant;
+      }
+      else
+      {
+         PA.MC.SetSize(PA.nq * PA.ne, Device::GetMemoryType());
+         auto M0 = Reshape(PA.MC.HostWrite(), PA.nq, PA.ne);
+         for (int e = 0; e < PA.ne; ++e)
+         {
+            ElementTransformation& T = *PA.fes->GetElementTransformation(e);
+            for (int q = 0; q < ir.GetNPoints(); ++q)
+            {
+               M0(q,e) = metric_coeff->Eval(T, ir.IntPoint(q));
+            }
+         }
+      }
+   }
+   else
+   {
+      PA.MC.SetSize(1, Device::GetMemoryType());
+      PA.MC.HostWrite();
+      PA.MC(0) = 1.0;
+   }
+
   // Setup ref->target Jacobians, PA.Jtr, (dim x dim) Q-vector, DenseTensor
   PA.Jtr.SetSize(dim, dim, PA.ne*PA.nq, mt);
   PA.Jtr_needs_update = true;
@@ -258,6 +258,7 @@ void EvalH_094(const int e, const int qx, const int qy,
 MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
                           const Vector &x_,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -273,11 +274,16 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
               || mid == 80 || mid == 94,
               "2D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 2;
   constexpr int NBZ = 1;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, NE);
   const auto W = Reshape(w_.Read(), Q1D, Q1D);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
@@ -312,7 +318,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
         {
            const double *Jtr = &J(0,0,qx,qy,e);
            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = metric_normal * W(qx,qy) * detJtr;
+            const double m_coef = const_m0 ? MC(0,0,0) : MC(qx,qy,e);
+            const double weight = metric_normal * m_coef * W(qx,qy) * detJtr;

            // Jrt = Jtr^{-1}
            double Jrt[4];
@@ -347,6 +354,7 @@ void TMOP_Integrator::AssembleGradPA_2D(const Vector &X) const
   const int Q1D = PA.maps->nqpt;
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -359,7 +367,7 @@ void TMOP_Integrator::AssembleGradPA_2D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,mp,M,N,W,B,G,J,H);
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,MC,mp,M,N,W,B,G,J,H);
 }

 } // namespace mfem
@@ -312,6 +312,7 @@ void EvalH_338(const int e, const int qx, const int qy, const int qz,

 MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const Vector &x_,
@@ -328,10 +329,15 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
               mid == 321 || mid == 332 || mid == 338,
               "3D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, Q1D, NE);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
@@ -369,7 +375,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
            {
               const double *Jtr = &J(0,0,qx,qy,qz,e);
               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
+               const double m_coef = const_m0 ? MC(0,0,0,0) : MC(qx,qy,qz,e);
+               const double weight = metric_normal * m_coef *
+                                     W(qx,qy,qz) * detJtr;

               // Jrt = Jtr^{-1}
               double Jrt[9];
@@ -438,6 +446,7 @@ void TMOP_Integrator::AssembleGradPA_3D(const Vector &X) const
   const int M = metric->Id();
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -450,7 +459,7 @@ void TMOP_Integrator::AssembleGradPA_3D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,mp,M,X,N,W,B,G,J,H);
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,MC,mp,M,X,N,W,B,G,J,H);
 }

 } // namespace mfem
@@ -98,6 +98,7 @@ void EvalP_094(const double *Jpt, const double *w, double *P)

 MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -114,12 +115,17 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
               || mid == 80 || mid == 94,
               "2D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 2;
   constexpr int NBZ = 1;

   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
   const auto W = Reshape(w_.Read(), Q1D, Q1D);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
@@ -154,7 +160,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
         {
            const double *Jtr = &J(0,0,qx,qy,e);
            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = metric_normal * W(qx,qy) * detJtr;
+            const double m_coef = const_m0 ? MC(0,0,0) : MC(qx,qy,e);
+            const double weight = metric_normal * m_coef *
+                                  W(qx,qy) * detJtr;

            // Jrt = Jtr^{-1}
            double Jrt[4];
@@ -204,6 +212,7 @@ void TMOP_Integrator::AddMultPA_2D(const Vector &X, Vector &Y) const
   const Array<double> &B = PA.maps->B;
   const Array<double> &G = PA.maps->G;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;

   Array<double> mp;
   if (auto m = dynamic_cast<TMOP_Combo_QualityMetric *>(metric))
@@ -211,7 +220,7 @@ void TMOP_Integrator::AddMultPA_2D(const Vector &X, Vector &Y) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,mp,M,N,J,W,B,G,X,Y);
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,MC,mp,M,N,J,W,B,G,X,Y);
 }

 } // namespace mfem
@@ -131,6 +131,7 @@ void EvalP_338(const double *J, const double *w, double *P)

 MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -147,10 +148,15 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
               mid == 321 || mid == 332 || mid == 338,
               "3D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
@@ -188,7 +194,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
            {
               const double *Jtr = &J(0,0,qx,qy,qz,e);
               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
+               const double m_coef = const_m0 ? MC(0,0,0,0) : MC(qx,qy,qz,e);
+               const double weight = metric_normal * m_coef *
+                                     W(qx,qy,qz) * detJtr;

               // Jrt = Jtr^{-1}
               double Jrt[9];
@@ -240,6 +248,7 @@ void TMOP_Integrator::AddMultPA_3D(const Vector &X, Vector &Y) const
   const Array<double> &B = PA.maps->B;
   const Array<double> &G = PA.maps->G;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;

   Array<double> mp;
   if (auto m = dynamic_cast<TMOP_Combo_QualityMetric *>(metric))
@@ -247,7 +256,7 @@ void TMOP_Integrator::AddMultPA_3D(const Vector &X, Vector &Y) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,mp,M,N,J,W,B,G,X,Y);
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,MC,mp,M,N,J,W,B,G,X,Y);
 }

 } // namespace mfem
@@ -73,6 +73,7 @@ double EvalW_094(const double *Jpt, const double *w)

 MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -90,12 +91,17 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
               || mid == 80 || mid == 94,
               "2D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 2;
   constexpr int NBZ = 1;

   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
@@ -131,7 +137,8 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
         {
            const double *Jtr = &J(0,0,qx,qy,e);
            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = metric_normal * W(qx,qy) * detJtr;
+            const double m_coef = const_m0 ? MC(0,0,0) : MC(qx,qy,e);
+            const double weight = metric_normal * m_coef * W(qx,qy) * detJtr;

            // Jrt = Jtr^{-1}
            double Jrt[4];
@@ -169,6 +176,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_2D(const Vector &X) const
   const int Q1D = PA.maps->nqpt;
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -182,7 +190,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_2D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,mn,mp,M,N,J,W,B,G,X,O,E);
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,mn,MC,mp,M,N,J,W,B,G,X,O,E);
 }

 } // namespace mfem
@@ -82,6 +82,7 @@ double EvalW_338(const double *J, const double *w)

 MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -99,10 +100,15 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
               mid == 321 || mid == 332 || mid == 338,
               "3D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
@@ -141,7 +147,9 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
            {
               const double *Jtr = &J(0,0,qx,qy,qz,e);
               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
+               const double m_coef = const_m0 ? MC(0,0,0,0) : MC(qx,qy,qz,e);
+               const double weight = metric_normal * m_coef *
+                                     W(qx,qy,qz) * detJtr;

               // Jrt = Jtr^{-1}
               double Jrt[9];
@@ -181,6 +189,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_3D(const Vector &X) const
   const int Q1D = PA.maps->nqpt;
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -194,7 +203,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_3D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,mp,M,N,J,W,B,G,O,X,E);
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,MC,mp,M,N,J,W,B,G,O,X,E);
 }

 } // namespace mfem
@@ -781,7 +781,13 @@ std::unique_ptr<SparseMatrix>>
   int ndof_lor = fes_lor.GetNDofs();

   // If the local mesh is empty, skip all computations
-   if (nel_ho == 0) { return {nullptr, nullptr}; }
+   if (nel_ho == 0)
+   {
+      return std::make_pair(
+                std::unique_ptr<SparseMatrix>(new SparseMatrix),
+                std::unique_ptr<SparseMatrix>(new SparseMatrix)
+             );
+   }

   const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();

@@ -26,6 +26,10 @@
 #include "sort_pairs.hpp"
 #include "globals.hpp"

+#ifdef MFEM_USE_STRUMPACK
+#include <StrumpackConfig.hpp> // STRUMPACK_USE_PTSCOTCH, etc.
+#endif
+
 #include <iostream>
 #include <map>

@@ -34,6 +38,14 @@ using namespace std;
 namespace mfem
 {

+#if defined(MFEM_USE_STRUMPACK) && \
+    (defined(STRUMPACK_USE_PTSCOTCH) || defined(STRUMPACK_USE_SLATE_SCALAPACK))
+int Mpi::default_thread_required = MPI_THREAD_MULTIPLE;
+#else
+int Mpi::default_thread_required = MPI_THREAD_SINGLE;
+#endif
+
+
 GroupTopology::GroupTopology(const GroupTopology &gt)
   : MyComm(gt.MyComm),
     group_lproc(gt.group_lproc)
--- a/Show More
+++ b/Show More