serial/parallel builds can run serial example codes, serial build can run serial examples

possible for parallel builds to run serial example problem
merging in contact problems infastructure into the new problem/IPM framework. Serial contact has yet to be updated and the utilities for contact is separated into parallel/serial versions which is not consistent with how the IPM/problem framework no longer has separate files for parallel/serial versions and only makes use of the pre-compiler directive MFEM_USE_MPI
2024-01-22 18:37:05 -08:00 · 2024-01-11 18:53:46 -08:00 · 2024-01-06 12:45:35 -08:00 · 2023-12-28 13:23:56 -08:00 · 2023-12-27 13:32:32 -08:00 · 2023-12-21 10:46:03 -08:00
180 changed files with 20522 additions and 4062 deletions
@@ -128,6 +128,13 @@ examples/amgx/sol.gf
 examples/amgx/mesh.*
 examples/amgx/sol.*

+examples/caliper/ex1
+examples/caliper/ex1p
+examples/caliper/refined.mesh
+examples/caliper/sol.gf
+examples/caliper/mesh.*
+examples/caliper/sol.*
+
 examples/ginkgo/ex1
 examples/ginkgo/refined.mesh
 examples/ginkgo/sol.gf
@@ -276,6 +283,10 @@ miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
 miniapps/nurbs/sin-fit.mesh
 miniapps/nurbs/CurveInt
+miniapps/nurbs/nurbs_naca_cmesh
+miniapps/nurbs/naca-cmesh.mesh
+miniapps/nurbs/glvis_naca-cmesh.mesh
+miniapps/nurbs/Naca_cmesh

 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -300,7 +311,7 @@ miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/plor-transfer
 miniapps/tools/get-values
-miniapps/tools/check-tmop-metric
+miniapps/tools/tmop-check-metric
 miniapps/tools/tmop-metric-magnitude
 miniapps/tools/nodal-transfer
 miniapps/tools/ParaView
@@ -16,10 +16,21 @@ Discretization improvements
 - Introduced support for higher order non conformal Nedelec elements on
  simplices in ParMesh.

+- Added functionality for construction of cut-surface and cut-volume
+  IntegrationRules through a moment-fitting approach. The cut is specified by
+  the zero level set of a Coefficient. See fem/intrules_cut.hpp and Example 38.
+
 Miscellaneous
 -------------
 - The ReadCubit Genesis mesh importer has been rewritten to improve readability.

+- Updated the Doxygen documentation style, which now requires Doxygen version
+  1.9.8 or later. See the doc/ directory.
+
+- Improved thread safety for global variables in the library, for example
+  IntegrationRules IntRules, RefinedIntRules, GeometryRefiner
+  GlobGeometryRefiner, and FiniteElement::dof2quad_array.
+

 Version 4.6, released on September 27, 2023
 ===========================================
@@ -87,6 +98,8 @@ Linear and nonlinear solvers

 - Added HIP support to the PETSc and SUNDIALS interfaces.

+- Efficient GPU-accelerated LOR assembly now supports surface meshes.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new H(div) solver miniapp demonstrating the use of a matrix-free
@@ -139,10 +139,9 @@ if (MFEM_USE_CUDA)
      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
   endif()
   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
+   find_package(CUDAToolkit REQUIRED)
   set(CUSPARSE_FOUND TRUE)
-   set(CUSPARSE_LIBRARIES "cusparse")
-   set(CUBLAS_FOUND TRUE)
-   set(CUBLAS_LIBRARIES "cublas")
+   get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
 endif()

 if (XSDK_ENABLE_C)
@@ -531,7 +530,7 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
+    ADIOS2 CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
@@ -641,16 +640,34 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
  foreach(Header mfem.hpp mfem-performance.hpp)
    message(STATUS
      "Writing substitute header --> \"${Header}\"")
-    file(WRITE "${PROJECT_BINARY_DIR}/${Header}"
+    file(WRITE "${PROJECT_BINARY_DIR}/${Header}.tmp"
 "// Auto-generated file.
 #define MFEM_CONFIG_FILE \"${PROJECT_BINARY_DIR}/config/_config.hpp\"
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")
+
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
+      "${PROJECT_BINARY_DIR}/${Header}.tmp"
+      "${PROJECT_BINARY_DIR}/${Header}"
+    )
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
+      "${PROJECT_BINARY_DIR}/${Header}.tmp"
+    )
+
    # This version will be installed in the top include directory:
-    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
+    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
 "// Auto-generated file.
 #include \"mfem/${Header}\"
 ")
+
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
+      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
+      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
+    )
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
+      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
+    )
+
  endforeach()
 endif()

@@ -659,8 +659,7 @@ The specific libraries and their options are:
  requires the PT-Scotch and Scalapack libraries as well as ParMETIS, which
  includes METIS 5 in its distribution. Starting with STRUMPACK v2.2.0, ParMETIS
  and PT-Scotch are optional dependencies.
-  The support for STRUMPACK was added in MFEM v3.3.2 and it requires STRUMPACK
-  2.0.0 or later.
+  The support for STRUMPACK was added in MFEM v3.3.2.
  URL: http://portal.nersc.gov/project/sparse/strumpack
  Options: STRUMPACK_OPT, STRUMPACK_LIB.
  Versions: STRUMPACK >= 3.0.0.
@@ -797,7 +796,7 @@ The specific libraries and their options are:
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.10.
+  Versions: libCEED >= 0.12.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported.
@@ -14,9 +14,13 @@
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
+#   - HYPRE_USING_CUDA (internal)
 #   - HYPRE_USING_HIP (internal)

 if (HYPRE_FOUND)
+  if (HYPRE_USING_CUDA)
+    find_package(CUDAToolkit REQUIRED)
+  endif()
  if (HYPRE_USING_HIP)
    find_package(rocsparse REQUIRED)
    find_package(rocrand REQUIRED)
@@ -27,6 +31,20 @@ endif()
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
  "Paths to headers required by HYPRE." "Libraries required by HYPRE."
+  CHECK_BUILD HYPRE_USING_CUDA FALSE
+  "
+#undef HYPRE_USING_CUDA
+#include <HYPRE_config.h>
+
+#ifndef HYPRE_USING_CUDA
+#error HYPRE is built without CUDA.
+#endif
+
+int main()
+{
+   return 0;
+}
+"
  CHECK_BUILD HYPRE_USING_HIP FALSE
  "
 #undef HYPRE_USING_HIP
@@ -57,6 +75,16 @@ if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
  endif()
 endif()

+if (HYPRE_FOUND AND HYPRE_USING_CUDA)
+  find_package(CUDAToolkit REQUIRED)
+  get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
+  get_target_property(CURAND_LIBRARIES CUDA::curand LOCATION)
+  list(APPEND HYPRE_LIBRARIES ${CUSPARSE_LIBRARIES} ${CURAND_LIBRARIES})
+  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
+      "HYPRE libraries + dependencies." FORCE)
+  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
+endif()
+
 if (HYPRE_FOUND AND HYPRE_USING_HIP)
  find_package(rocsparse REQUIRED)
  find_package(rocrand REQUIRED)
@@ -106,12 +106,7 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-if (MFEM_USE_CUDA)
-   # This is only necessary when hypre is built with cuda:
-   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
-       "Libraries that HYPRE depends on.")
-endif()
-# HIP dependency for HYPRE is handled in FindHYPRE.cmake.
+# CUDA and HIP dependencies for HYPRE are handled in FindHYPRE.cmake.

 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")

@@ -157,7 +152,8 @@ set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
 # STRUMPACK may also depend on "OpenMP", depending on how it was compiled.
 # Starting with v2.2.0 of STRUMPACK, ParMETIS and Scotch are optional.
 set(STRUMPACK_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
-    "ScaLAPACK" "Scotch/ptscotch/ptscotcherr/scotch/scotcherr" CACHE STRING
+    "Scotch/ptscotch/ptscotcherr/scotch/scotcherr"
+    "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
    "Additional packages required by STRUMPACK.")
 # If the MPI package does not find all required Fortran libraries:
 # set(STRUMPACK_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
@@ -38,14 +38,14 @@ all: header config-mk
 MPI = $(MFEM_USE_MPI:NO=)
 GHV_CXX ?= $(MFEM_CXX)
 GHV = get_hypre_version
-GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
+GHV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
 SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
 SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
 MUMPS = $(MFEM_USE_MUMPS:NO=)
 GMV_CXX ?= $(MFEM_CXX)
 GMV = get_mumps_version
-GMV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))
+GMV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))

 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
@@ -110,3 +110,4 @@ config-mk:

 clean:
 	rm -f $(CONFIG_HPP) $(CONFIG_MK) sample-runs-build.log
+	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out
@@ -315,7 +315,7 @@ function extract_sample_runs()
   sruns=`grep -v "^//.*  mpirun .* ${app}" "${src}" |
          grep "^//.*  ${app}" |
          sed -e "s/.*  ${app}/${vg_app}/g"`
-   runs="${sruns}${pruns}"
+   runs="${sruns}"$'\n'"${pruns}"
   if [ "$skip_gen_meshes" == "yes" ]; then
      runs=`printf "%s" "$runs" | grep -v ".* -m .*\.gen"`
   fi
@@ -112,6 +112,7 @@ namespace mfem {
 * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
 * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: Topology optimization
 * - <a class="el" href="ex37p_8cpp_source.html">Example 37p</a>: parallel topology optimization
+ * - <a class="el" href="ex38_8cpp_source.html">Example 38</a>: cut-surface and cut-volume integration
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -14,7 +14,7 @@ If not already available, Doxygen can be downloaded from

   http://www.doxygen.org

-We recommend using version 1.8 or later.
+We recommend using version 1.9.8 or later.

 To build the documentation, simply type "make" in the doc/ directory. This will
 create the file CodeDocumentation.html, which can be viewed in any web browser.
@@ -0,0 +1,3 @@
+html {
+    --content-maxwidth: auto;
+}
@@ -0,0 +1,78 @@
+<!-- HTML header for doxygen 1.9.6-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="$langISO">
+<head>
+<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
+<meta http-equiv="X-UA-Compatible" content="IE=11"/>
+<meta name="generator" content="Doxygen $doxygenversion"/>
+<meta name="viewport" content="width=device-width, initial-scale=1"/>
+<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME-->
+<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME-->
+<link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/>
+<!--BEGIN DISABLE_INDEX-->
+  <!--BEGIN FULL_SIDEBAR-->
+<script type="text/javascript">var page_layout=1;</script>
+  <!--END FULL_SIDEBAR-->
+<!--END DISABLE_INDEX-->
+<script type="text/javascript" src="$relpath^jquery.js"></script>
+<script type="text/javascript" src="$relpath^dynsections.js"></script>
+$treeview
+$search
+$mathjax
+$darkmode
+<link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
+$extrastylesheet
+<script type="text/javascript" src="$relpath^doxygen-awesome-darkmode-toggle.js"></script>
+<script type="text/javascript">
+    DoxygenAwesomeDarkModeToggle.init()
+</script>
+</head>
+<body>
+<!--BEGIN DISABLE_INDEX-->
+  <!--BEGIN FULL_SIDEBAR-->
+<div id="side-nav" class="ui-resizable side-nav-resizable"><!-- do not remove this div, it is closed by doxygen! -->
+  <!--END FULL_SIDEBAR-->
+<!--END DISABLE_INDEX-->
+
+<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
+
+<!--BEGIN TITLEAREA-->
+<div id="titlearea">
+<table cellspacing="0" cellpadding="0">
+ <tbody>
+ <tr id="projectrow">
+  <!--BEGIN PROJECT_LOGO-->
+  <td id="projectlogo"><img alt="Logo" src="$relpath^$projectlogo"/></td>
+  <!--END PROJECT_LOGO-->
+  <!--BEGIN PROJECT_NAME-->
+  <td id="projectalign">
+   <div id="projectname">$projectname<!--BEGIN PROJECT_NUMBER--><span id="projectnumber">&#160;$projectnumber</span><!--END PROJECT_NUMBER-->
+   </div>
+   <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">$projectbrief</div><!--END PROJECT_BRIEF-->
+  </td>
+  <!--END PROJECT_NAME-->
+  <!--BEGIN !PROJECT_NAME-->
+   <!--BEGIN PROJECT_BRIEF-->
+    <td>
+    <div id="projectbrief">$projectbrief</div>
+    </td>
+   <!--END PROJECT_BRIEF-->
+  <!--END !PROJECT_NAME-->
+  <!--BEGIN DISABLE_INDEX-->
+   <!--BEGIN SEARCHENGINE-->
+     <!--BEGIN !FULL_SIDEBAR-->
+    <td>$searchbox</td>
+     <!--END !FULL_SIDEBAR-->
+   <!--END SEARCHENGINE-->
+  <!--END DISABLE_INDEX-->
+ </tr>
+  <!--BEGIN SEARCHENGINE-->
+   <!--BEGIN FULL_SIDEBAR-->
+   <tr><td colspan="2">$searchbox</td></tr>
+   <!--END FULL_SIDEBAR-->
+  <!--END SEARCHENGINE-->
+ </tbody>
+</table>
+</div>
+<!--END TITLEAREA-->
+<!-- end header part -->
@@ -0,0 +1,157 @@
+/**
+
+Doxygen Awesome
+https://github.com/jothepro/doxygen-awesome-css
+
+MIT License
+
+Copyright (c) 2021 - 2023 jothepro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+class DoxygenAwesomeDarkModeToggle extends HTMLElement {
+    // SVG icons from https://fonts.google.com/icons
+    // Licensed under the Apache 2.0 license:
+    // https://www.apache.org/licenses/LICENSE-2.0.html
+    static lightModeIcon = `<svg xmlns="http://www.w3.org/2000/svg" enable-background="new 0 0 24 24" height="24px" viewBox="0 0 24 24" width="24px" fill="#FCBF00"><rect fill="none" height="24" width="24"/><circle cx="12" cy="12" opacity=".3" r="3"/><path d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"/></svg>`
+    static darkModeIcon = `<svg xmlns="http://www.w3.org/2000/svg" enable-background="new 0 0 24 24" height="24px" viewBox="0 0 24 24" width="24px" fill="#FE9700"><rect fill="none" height="24" width="24"/><path d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27 C17.45,17.19,14.93,19,12,19c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z" opacity=".3"/><path d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"/></svg>`
+    static title = "Toggle Light/Dark Mode"
+
+    static prefersLightModeInDarkModeKey = "prefers-light-mode-in-dark-mode"
+    static prefersDarkModeInLightModeKey = "prefers-dark-mode-in-light-mode"
+
+    static _staticConstructor = function() {
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.userPreference)
+        // Update the color scheme when the browsers preference changes
+        // without user interaction on the website.
+        window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', event => {
+            DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
+        })
+        // Update the color scheme when the tab is made visible again.
+        // It is possible that the appearance was changed in another tab 
+        // while this tab was in the background.
+        document.addEventListener("visibilitychange", visibilityState => {
+            if (document.visibilityState === 'visible') {
+                DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
+            }
+        });
+    }()
+
+    static init() {
+        $(function() {
+            $(document).ready(function() {
+                const toggleButton = document.createElement('doxygen-awesome-dark-mode-toggle')
+                toggleButton.title = DoxygenAwesomeDarkModeToggle.title
+                toggleButton.updateIcon()
+
+                window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', event => {
+                    toggleButton.updateIcon()
+                })
+                document.addEventListener("visibilitychange", visibilityState => {
+                    if (document.visibilityState === 'visible') {
+                        toggleButton.updateIcon()
+                    }
+                });
+
+                $(document).ready(function(){
+                    document.getElementById("MSearchBox").parentNode.appendChild(toggleButton)
+                })
+                $(window).resize(function(){
+                    document.getElementById("MSearchBox").parentNode.appendChild(toggleButton)
+                })
+            })
+        })
+    }
+
+    constructor() {
+        super();
+        this.onclick=this.toggleDarkMode
+    }
+
+    /**
+     * @returns `true` for dark-mode, `false` for light-mode system preference
+     */
+    static get systemPreference() {
+        return window.matchMedia('(prefers-color-scheme: dark)').matches
+    }
+
+    /**
+     * @returns `true` for dark-mode, `false` for light-mode user preference
+     */
+    static get userPreference() {
+        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) || 
+        (DoxygenAwesomeDarkModeToggle.systemPreference && !localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey))
+    }
+
+    static set userPreference(userPreference) {
+        DoxygenAwesomeDarkModeToggle.darkModeEnabled = userPreference
+        if(!userPreference) {
+            if(DoxygenAwesomeDarkModeToggle.systemPreference) {
+                localStorage.setItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey, true)
+            } else {
+                localStorage.removeItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)
+            }
+        } else {
+            if(!DoxygenAwesomeDarkModeToggle.systemPreference) {
+                localStorage.setItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey, true)
+            } else {
+                localStorage.removeItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey)
+            }
+        }
+        DoxygenAwesomeDarkModeToggle.onUserPreferenceChanged()
+    }
+
+    static enableDarkMode(enable) {
+        if(enable) {
+            DoxygenAwesomeDarkModeToggle.darkModeEnabled = true
+            document.documentElement.classList.add("dark-mode")
+            document.documentElement.classList.remove("light-mode")
+        } else {
+            DoxygenAwesomeDarkModeToggle.darkModeEnabled = false
+            document.documentElement.classList.remove("dark-mode")
+            document.documentElement.classList.add("light-mode")
+        }
+    }
+
+    static onSystemPreferenceChanged() {
+        DoxygenAwesomeDarkModeToggle.darkModeEnabled = DoxygenAwesomeDarkModeToggle.userPreference
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.darkModeEnabled)
+    }
+
+    static onUserPreferenceChanged() {
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.darkModeEnabled)
+    }
+
+    toggleDarkMode() {
+        DoxygenAwesomeDarkModeToggle.userPreference = !DoxygenAwesomeDarkModeToggle.userPreference
+        this.updateIcon()
+    }
+
+    updateIcon() {
+        if(DoxygenAwesomeDarkModeToggle.darkModeEnabled) {
+            this.innerHTML = DoxygenAwesomeDarkModeToggle.darkModeIcon
+        } else {
+            this.innerHTML = DoxygenAwesomeDarkModeToggle.lightModeIcon
+        }
+    }
+}
+
+customElements.define("doxygen-awesome-dark-mode-toggle", DoxygenAwesomeDarkModeToggle);
@@ -16,7 +16,7 @@ DOXYGEN_CONF = CodeDocumentation.conf
 # doxygen uses: graphviz, latex
 html: $(DOXYGEN_CONF)
 	@# Generate the html documentation
-	@( cat $(DOXYGEN_CONF) ; echo "$(MFEM_DOXYGEN_FLAGS)" ) | doxygen -
+	@( cat $(DOXYGEN_CONF) ; printf "$(MFEM_DOXYGEN_FLAGS)\n" ) | doxygen -
 	@echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > CodeDocumentation.html
 	@cat warnings.log 1>&2
 	@# Generate the log of undocumented methods
@@ -45,6 +45,12 @@ list(APPEND ALL_EXE_SRCS
  ex37.cpp
  )

+if(MFEM_USE_LAPACK)
+list(APPEND ALL_EXE_SRCS
+  ex38.cpp
+  )
+endif()
+
 if (MFEM_USE_MPI)
  list(APPEND ALL_EXE_SRCS
    ex0p.cpp
@@ -262,12 +262,13 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
      if (sp_solver)
      {
-         STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+         STRUMPACKSolver * strumpack = new STRUMPACKSolver(MPI_COMM_WORLD, argc, argv);
         strumpack->SetPrintFactorStatistics(true);
         strumpack->SetPrintSolveStatistics(false);
         strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
         strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-         strumpack->DisableMatching();
+         strumpack->SetMatching(strumpack::MatchingJob::NONE);
+         strumpack->SetCompression(strumpack::CompressionType::NONE);
         strumpack->SetOperator(*Arow);
         strumpack->SetFromCommandLine();
         precond = strumpack;
@@ -26,13 +26,12 @@
 using namespace std;
 using namespace mfem;

-/** After spatial discretization, the conduction model can be written as:
+/** After spatial discretization, the wave model can be written as:
 *
 *     d^2u/dt^2 = M^{-1}(-Ku)
 *
- *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K is the diffusion operator with diffusivity depending on u:
- *  (\kappa + \alpha u).
+ *  where u is the vector representing the temperature, M is the mass,
+ *  and K is the stiffness matrix.
 *
 *  Class WaveOperator represents the right-hand side of the above ODE.
 */
@@ -301,7 +300,7 @@ int main(int argc, char *argv[])
   Vector dudt;
   dudt_gf.GetTrueDofs(dudt);

-   // 7. Initialize the conduction operator and the visualization.
+   // 7. Initialize the wave operator and the visualization.
   Array<int> ess_bdr;
   if (mesh->bdr_attributes.Size())
   {
@@ -356,7 +355,7 @@ int main(int argc, char *argv[])
      else
      {
         sout.precision(precision);
-         sout << "solution\n" << *mesh << dudt_gf;
+         sout << "solution\n" << *mesh << u_gf;
         sout << "pause\n";
         sout << flush;
         cout << "GLVis visualization paused."
@@ -170,6 +170,7 @@ int main(int argc, char *argv[])
   bool herm_conv = true;
   bool slu_solver  = false;
   bool mumps_solver = false;
+   bool strumpack_solver = false;
   bool visualization = 1;
   bool pa = false;
   const char *device_config = "cpu";
@@ -200,6 +201,11 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_MUMPS
   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
                  "--no-mumps-solver", "Use the MUMPS Solver.");
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&strumpack_solver, "-strumpack", "--strumpack-solver",
+                  "-no-strumpack", "--no-strumpack-solver",
+                  "Use the STRUMPACK Solver.");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -209,13 +215,14 @@ int main(int argc, char *argv[])
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.Parse();
-   if (slu_solver && mumps_solver)
+   if (slu_solver + mumps_solver + strumpack_solver > 1)
   {
      if (myid == 0)
-         cout << "WARNING: Both SuperLU and MUMPS have been selected,"
-              << " please choose either one." << endl
+         cout << "WARNING: More than one of SuperLU, MUMPS, and STRUMPACK have"
+              << " been selected, please choose only one." << endl
              << "         Defaulting to SuperLU." << endl;
      mumps_solver = false;
+      strumpack_solver = false;
   }

   if (iprob > 4) { iprob = 4; }
@@ -474,6 +481,24 @@ int main(int argc, char *argv[])
      delete A;
   }
 #endif
+#ifdef MFEM_USE_STRUMPACK
+   if (!pa && strumpack_solver)
+   {
+      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      STRUMPACKRowLocMatrix SA(*A);
+      STRUMPACKSolver strumpack(MPI_COMM_WORLD, argc, argv);
+      strumpack.SetPrintFactorStatistics(false);
+      strumpack.SetPrintSolveStatistics(false);
+      strumpack.SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack.SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      strumpack.SetMatching(strumpack::MatchingJob::NONE);
+      strumpack.SetCompression(strumpack::CompressionType::NONE);
+      strumpack.SetFromCommandLine();
+      strumpack.SetOperator(SA);
+      strumpack.Mult(B, X);
+      delete A;
+   }
+#endif
 #ifdef MFEM_USE_MUMPS
   if (!pa && mumps_solver)
   {
@@ -493,7 +518,7 @@ int main(int argc, char *argv[])
   //
   //    In PML:   1/mu (abs(1/det(J) J^T J) Curl E, Curl F)
   //              + omega^2 * epsilon (abs(det(J) * (J^T J)^-1) * E, F)
-   if (pa || (!slu_solver && !mumps_solver))
+   if (pa || (!slu_solver && !mumps_solver && !strumpack_solver))
   {
      ConstantCoefficient absomeg(pow(omega, 2) * epsilon);
      RestrictedCoefficient restr_absomeg(absomeg,attr);
@@ -0,0 +1,696 @@
+//                                MFEM Example 38
+//
+// Compile with: make ex38
+//
+// Sample runs:
+// (since all sample runs require LAPACK, the * symbol is used to exclude them
+// from the automatically generated internal MFEM tests).
+//              * ex38
+//              * ex38 -i volumetric1d
+//              * ex38 -i surface2d
+//              * ex38 -i surface2d -o 4 -r 5
+//              * ex38 -i volumetric2d
+//              * ex38 -i volumetric2d -o 4 -r 5
+//              * ex38 -i surface3d
+//              * ex38 -i surface3d -o 4 -r 5
+//              * ex38 -i volumetric3d
+//              * ex38 -i volumetric3d -o 4 -r 5
+//
+// Description: This example code demonstrates the use of MFEM to integrate
+//              functions over implicit interfaces and subdomains bounded by
+//              implicit interfaces.
+//
+//              The quadrature rules are constructed by means of moment-fitting.
+//              The interface is given by the zero isoline of a level-set
+//              function ϕ and the subdomain is given as the domain where ϕ>0
+//              holds. The algorithm for construction of the quadrature rules
+//              was introduced by Mueller, Kummer and Oberlack [1].
+//
+//              This example also showcases how to set up integrators using the
+//              integration rules on implicit surfaces and subdomains.
+//
+// [1] Mueller, B., Kummer, F. and Oberlack, M. (2013) Highly accurate surface
+//     and volume integration on implicit domains by means of moment-fitting.
+//     Int. J. Numer. Meth. Engr. (96) 512-528. DOI:10.1002/nme.4569
+
+#include "mfem.hpp"
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+/// @brief Integration rule the example should demonstrate
+enum class IntegrationType { Volumetric1D, Surface2D, Volumetric2D,
+                             Surface3D, Volumetric3D
+                           };
+IntegrationType itype;
+
+/// @brief Level-set function defining the implicit interface
+double lvlset(const Vector& X)
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return .55 - X(0);
+      case IntegrationType::Surface2D:
+         return 1. - (pow(X(0), 2.) + pow(X(1), 2.));
+      case IntegrationType::Volumetric2D:
+         return 1. - (pow(X(0) / 1.5, 2.) + pow(X(1) / .75, 2.));
+      case IntegrationType::Surface3D:
+         return 1. - (pow(X(0), 2.) + pow(X(1), 2.) + pow(X(2), 2.));
+      case IntegrationType::Volumetric3D:
+         return 1. - (pow(X(0) / 1.5, 2.) + pow(X(1) / .75, 2.) + pow(X(2) / .5, 2.));
+      default:
+         return 1.;
+   }
+}
+
+/// @brief Function that should be integrated
+double integrand(const Vector& X)
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return 1.;
+      case IntegrationType::Surface2D:
+         return 3. * pow(X(0), 2.) - pow(X(1), 2.);
+      case IntegrationType::Volumetric2D:
+         return 1.;
+      case IntegrationType::Surface3D:
+         return 4. - 3. * pow(X(0), 2.) + 2. * pow(X(1), 2.) - pow(X(2), 2.);
+      case IntegrationType::Volumetric3D:
+         return 1.;
+      default:
+         return 0.;
+   }
+}
+
+/// @brief Analytic surface integral
+double Surface()
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return 1.;
+      case IntegrationType::Surface2D:
+         return 2. * M_PI;
+      case IntegrationType::Volumetric2D:
+         return 7.26633616541076;
+      case IntegrationType::Surface3D:
+         return 40. / 3. * M_PI;
+      case IntegrationType::Volumetric3D:
+         return 9.90182151329315;
+      default:
+         return 0.;
+   }
+}
+
+/// @brief Analytic volume integral over subdomain with positive level-set
+double Volume()
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return .55;
+      case IntegrationType::Surface2D:
+         return NAN;
+      case IntegrationType::Volumetric2D:
+         return 9. / 8. * M_PI;
+      case IntegrationType::Surface3D:
+         return NAN;
+      case IntegrationType::Volumetric3D:
+         return 3. / 4. * M_PI;
+      default:
+         return 0.;
+   }
+}
+
+#ifdef MFEM_USE_LAPACK
+/**
+ @brief Class for surface IntegrationRule
+
+ This class demonstrates how IntegrationRules computed as CutIntegrationRules
+ can be saved to reduce the impact by computing them from scratch each time.
+ */
+class SIntegrationRule : public IntegrationRule
+{
+protected:
+   /// @brief Space Dimension of the IntegrationRule
+   int dim;
+   /// @brief Column-wise matrix of the quadtrature weights
+   DenseMatrix Weights;
+   /// @brief Column-wise matrix of the transformation weights of the normal
+   DenseMatrix SurfaceWeights;
+
+public:
+   /**
+    @brief Constructor of SIntegrationRule
+
+    The surface integrationRules are computed and saved in the constructor.
+
+    @param [in] Order Order of the IntegrationRule
+    @param [in] LvlSet Level-set defining the implicit interface
+    @param [in] lsOrder Polynomial degree for approx of level-set function
+    @param [in] mesh Pointer to the mesh that is used
+   */
+   SIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
+   {
+      dim = mesh->Dimension();
+
+      IsoparametricTransformation Tr;
+      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
+      mesh->GetElementTransformation(0, &Tr);
+      IntegrationRule ir;
+      MFIRs.GetSurfaceIntegrationRule(Tr, ir);
+      if (dim >1)
+      {
+         Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
+      }
+      else
+      {
+         Weights.SetSize(2, mesh->GetNE());
+      }
+      SurfaceWeights.SetSize(ir.GetNPoints(), mesh->GetNE());
+      Vector w;
+      MFIRs.GetSurfaceWeights(Tr, ir, w);
+      SurfaceWeights.SetCol(0, w);
+      SetSize(ir.GetNPoints());
+
+      for (int ip = 0; ip < GetNPoints(); ip++)
+      {
+         IntPoint(ip).index = ip;
+         IntegrationPoint &intp = IntPoint(ip);
+         intp.x = ir.IntPoint(ip).x;
+         intp.y = ir.IntPoint(ip).y;
+         intp.z = ir.IntPoint(ip).z;
+         if (dim > 1)
+         {
+            Weights(ip, 0) = ir.IntPoint(ip).weight;
+         }
+         else
+         {
+            Weights(0, 0) = ir.IntPoint(ip).x;
+            Weights(1, 0) = ir.IntPoint(ip).weight;
+         }
+      }
+
+
+      for (int elem = 1; elem < mesh->GetNE(); elem++)
+      {
+         mesh->GetElementTransformation(elem, &Tr);
+         MFIRs.GetSurfaceIntegrationRule(Tr, ir);
+         Vector w;
+         MFIRs.GetSurfaceWeights(Tr, ir, w);
+         SurfaceWeights.SetCol(elem, w);
+
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            if (dim > 1)
+            {
+               Weights(ip, elem) = ir.IntPoint(ip).weight;
+            }
+            else
+            {
+               Weights(0, elem) = ir.IntPoint(ip).x;
+               Weights(1, elem) = ir.IntPoint(ip).weight;
+            }
+         }
+      }
+   }
+
+   /**
+    @brief Set the weights for the given element and multiply them with the
+    transformation of the interface
+    */
+   void SetElementinclSurfaceWeight(int Element)
+   {
+      if (dim == 1)
+      {
+         IntegrationPoint &intp = IntPoint(0);
+         intp.x = Weights(0, Element);
+         intp.weight = Weights(1, Element);
+         cout << intp.x << " " << Element << endl;
+      }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element) * SurfaceWeights(ip, Element);
+         }
+   }
+
+   /// @brief Set the weights for the given element
+   void SetElement(int Element)
+   {
+      if (dim == 1)
+      {
+         IntegrationPoint &intp = IntPoint(0);
+         intp.x = Weights(0, Element);
+         intp.weight = Weights(1, Element);
+      }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element);
+         }
+   }
+
+   /// @brief Destructor of SIntegrationRule
+   ~SIntegrationRule() {}
+};
+
+/**
+ @brief Class for volume IntegrationRule
+
+ This class demonstrates how IntegrationRules computed as CutIntegrationRules
+ can be saved to reduce the impact by computing them from scratch each time.
+ */
+class CIntegrationRule : public IntegrationRule
+{
+protected:
+   /// @brief Space Dimension of the IntegrationRule
+   int dim;
+   /// @brief Column-wise matrix of the quadtrature weights
+   DenseMatrix Weights;
+
+public:
+   /**
+    @brief Constructor of CIntegrationRule
+
+    The volume integrationRules are computed and saved in the constructor.
+
+    @param [in] Order Order of the IntegrationRule
+    @param [in] LvlSet Level-set defining the implicit interface
+    @param [in] lsOrder Polynomial degree for approx of level-set function
+    @param [in] mesh Pointer to the mesh that is used
+   */
+   CIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
+   {
+      dim = mesh->Dimension();
+
+      IsoparametricTransformation Tr;
+      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
+      mesh->GetElementTransformation(0, &Tr);
+      IntegrationRule ir;
+      MFIRs.GetVolumeIntegrationRule(Tr, ir);
+      if (dim > 1)
+      {
+         Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
+      }
+      else
+      {
+         Weights.SetSize(2 * ir.GetNPoints(), mesh->GetNE());
+      }
+
+      SetSize(ir.GetNPoints());
+      for (int ip = 0; ip < GetNPoints(); ip++)
+      {
+         IntPoint(ip).index = ip;
+         IntegrationPoint &intp = IntPoint(ip);
+         intp.x = ir.IntPoint(ip).x;
+         intp.y = ir.IntPoint(ip).y;
+         intp.z = ir.IntPoint(ip).z;
+         if (dim > 1)
+         {
+            Weights(ip, 0) = ir.IntPoint(ip).weight;
+         }
+         else
+         {
+            Weights(2 * ip, 0) = ir.IntPoint(ip).x;
+            Weights(2 * ip + 1, 0) = ir.IntPoint(ip).weight;
+         }
+      }
+
+      for (int elem = 1; elem < mesh->GetNE(); elem++)
+      {
+         mesh->GetElementTransformation(elem, &Tr);
+         MFIRs.GetVolumeIntegrationRule(Tr, ir);
+
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            if (dim > 1)
+            {
+               Weights(ip, elem) = ir.IntPoint(ip).weight;
+            }
+            else
+            {
+               Weights(2 * ip, elem) = ir.IntPoint(ip).x;
+               Weights(2 * ip + 1, elem) = ir.IntPoint(ip).weight;
+            }
+         }
+      }
+   }
+
+   /// @brief Set the weights for the given element
+   void SetElement(int Element)
+   {
+      if (dim == 1)
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.x = Weights(2 * ip, Element);
+            intp.weight = Weights(2 * ip + 1, Element);
+         }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element);
+         }
+   }
+
+   /// @brief Destructor of CIntegrationRule
+   ~CIntegrationRule() {}
+};
+/**
+ @brief Class for surface linearform integrator
+
+ Integrator to demonstrate the use of the surface integration rule on an
+ implicit surface defined by a level-set.
+ */
+class SurfaceLFIntegrator : public LinearFormIntegrator
+{
+protected:
+   /// @brief vector to evaluate the basis functions
+   Vector shape;
+
+   /// @brief surface integration rule
+   SIntegrationRule* SIntRule;
+
+   /// @brief coefficient representing the level-set defining the interface
+   Coefficient &LevelSet;
+
+   /// @brief coefficient representing the integrand
+   Coefficient &Q;
+
+public:
+   /**
+    @brief Constructor for the surface linear form integrator
+
+    Constructor for the surface linear form integrator to demonstrate the use
+    of the surface integration rule by means of moment-fitting.
+
+    @param [in] q coefficient representing the inegrand
+    @param [in] levelset level-set defining the implicit interfac
+    @param [in] ir surface integrtion rule to be used
+    */
+   SurfaceLFIntegrator(Coefficient &q, Coefficient &levelset,
+                       SIntegrationRule* ir)
+      : LinearFormIntegrator(), SIntRule(ir), LevelSet(levelset), Q(q) { }
+
+   /**
+    @brief Assembly of the element vector
+
+    Assemble the element vector of for the right hand side on the element given
+    by the FiniteElement and ElementTransformation.
+
+    @param [in] el finite Element the vector belongs to
+    @param [in] Tr transformation of finite element
+    @param [out] elvect vector containing the
+   */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
+   {
+      int dof = el.GetDof();
+      shape.SetSize(dof);
+      elvect.SetSize(dof);
+      elvect = 0.;
+
+      // Update the surface integration rule for the current element
+      SIntRule->SetElementinclSurfaceWeight(Tr.ElementNo);
+
+      for (int ip = 0; ip < SIntRule->GetNPoints(); ip++)
+      {
+         Tr.SetIntPoint((&(SIntRule->IntPoint(ip))));
+         double val = Tr.Weight() * Q.Eval(Tr, SIntRule->IntPoint(ip));
+         el.CalcShape(SIntRule->IntPoint(ip), shape);
+         add(elvect, SIntRule->IntPoint(ip).weight * val, shape, elvect);
+      }
+   }
+};
+
+/**
+ @brief Class for subdomain linearform integrator
+
+ Integrator to demonstrate the use of the subdomain integration rule within
+ an area defined by an implicit surface defined by a level-set.
+ */
+class SubdomainLFIntegrator : public LinearFormIntegrator
+{
+protected:
+   /// @brief vector to evaluate the basis functions
+   Vector shape;
+
+   /// @brief surface integration rule
+   CIntegrationRule* CIntRule;
+
+   /// @brief coefficient representing the level-set defining the interface
+   Coefficient &LevelSet;
+
+   /// @brief coefficient representing the integrand
+   Coefficient &Q;
+
+public:
+   /**
+    @brief Constructor for the volumetric subdomain linear form integrator
+
+    Constructor for the subdomain linear form integrator to demonstrate the use
+    of the volumetric subdomain integration rule by means of moment-fitting.
+
+    @param [in] q coefficient representing the inegrand
+    @param [in] levelset level-set defining the implicit interfac
+    @param [in] ir subdomain integrtion rule to be used
+    */
+   SubdomainLFIntegrator(Coefficient &q, Coefficient &levelset,
+                         CIntegrationRule* ir)
+      : LinearFormIntegrator(), CIntRule(ir), LevelSet(levelset), Q(q) { }
+
+   /**
+    @brief Assembly of the element vector
+
+    Assemble the element vector of for the right hand side on the element given
+    by the FiniteElement and ElementTransformation.
+
+    @param [in] el finite Element the vector belongs to
+    @param [in] Tr transformation of finite element
+    @param [out] elvect vector containing the
+   */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
+   {
+      int dof = el.GetDof();
+      shape.SetSize(dof);
+      elvect.SetSize(dof);
+      elvect = 0.;
+
+      // Update the subdomain integration rule
+      CIntRule->SetElement(Tr.ElementNo);
+
+      for (int ip = 0; ip < CIntRule->GetNPoints(); ip++)
+      {
+         Tr.SetIntPoint((&(CIntRule->IntPoint(ip))));
+         double val = Tr.Weight()
+                      * Q.Eval(Tr, CIntRule->IntPoint(ip));
+         el.CalcPhysShape(Tr, shape);
+         add(elvect, CIntRule->IntPoint(ip).weight * val, shape, elvect);
+      }
+   }
+};
+#endif // MFEM_USE_LAPACK
+
+int main(int argc, char *argv[])
+{
+#ifndef MFEM_USE_LAPACK
+   cout << "MFEM must be build with LAPACK for this example." << endl;
+   return EXIT_FAILURE;
+#else
+   // 1. Parse he command-line options.
+   int ref_levels = 3;
+   int order = 2;
+   const char *inttype = "surface2d";
+   bool visualization = true;
+   itype = IntegrationType::Surface2D;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order", "Order of quadrature rule");
+   args.AddOption(&ref_levels, "-r", "--refine", "Number of meh refinements");
+   args.AddOption(&inttype, "-i", "--integrationtype",
+                  "IntegrationType to demonstrate");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.ParseCheck();
+
+   if (strcmp(inttype, "volumetric1d") == 0
+       || strcmp(inttype, "Volumetric1D") == 0)
+   {
+      itype = IntegrationType::Volumetric1D;
+   }
+   else if (strcmp(inttype, "surface2d") == 0
+            || strcmp(inttype, "Surface2D") == 0)
+   {
+      itype = IntegrationType::Surface2D;
+   }
+   else if (strcmp(inttype, "volumetric2d") == 0
+            || strcmp(inttype, "Volumetric2D") == 0)
+   {
+      itype = IntegrationType::Volumetric2D;
+   }
+   else if (strcmp(inttype, "surface3d") == 0
+            || strcmp(inttype, "Surface3d") == 0)
+   {
+      itype = IntegrationType::Surface3D;
+   }
+   else if (strcmp(inttype, "volumetric3d") == 0
+            || strcmp(inttype, "Volumetric3d") == 0)
+   {
+      itype = IntegrationType::Volumetric3D;
+   }
+
+   // 2. Construct and refine the mesh.
+   Mesh *mesh;
+   if (itype == IntegrationType::Volumetric1D)
+   {
+      mesh = new Mesh("../data/inline-segment.mesh");
+   }
+   if (itype == IntegrationType::Surface2D
+       || itype == IntegrationType::Volumetric2D)
+   {
+      mesh = new Mesh(2, 4, 1, 0, 2);
+      mesh->AddVertex(-1.6,-1.6);
+      mesh->AddVertex(1.6,-1.6);
+      mesh->AddVertex(1.6,1.6);
+      mesh->AddVertex(-1.6,1.6);
+      mesh->AddQuad(0,1,2,3);
+      mesh->FinalizeQuadMesh(1, 0, 1);
+   }
+   else if (itype == IntegrationType::Surface3D
+            || itype == IntegrationType::Volumetric3D)
+   {
+      mesh = new Mesh(3, 8, 1, 0, 3);
+      mesh->AddVertex(-1.6,-1.6,-1.6);
+      mesh->AddVertex(1.6,-1.6,-1.6);
+      mesh->AddVertex(1.6,1.6,-1.6);
+      mesh->AddVertex(-1.6,1.6,-1.6);
+      mesh->AddVertex(-1.6,-1.6,1.6);
+      mesh->AddVertex(1.6,-1.6,1.6);
+      mesh->AddVertex(1.6,1.6,1.6);
+      mesh->AddVertex(-1.6,1.6,1.6);
+      mesh->AddHex(0,1,2,3,4,5,6,7);
+      mesh->FinalizeHexMesh(1, 0, 1);
+   }
+
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 3. Define the necessary finite element space on the mesh.
+   H1_FECollection fe_coll(1, mesh->Dimension());
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, &fe_coll);
+
+   // 4. Construction Coefficients for the level set and the integrand.
+   FunctionCoefficient levelset(lvlset);
+   FunctionCoefficient u(integrand);
+
+   // 5. Define the necessary Integration rules on element 0.
+   IsoparametricTransformation Tr;
+   mesh->GetElementTransformation(0, &Tr);
+   SIntegrationRule* sir = new SIntegrationRule(order, levelset, 2, mesh);
+   CIntegrationRule* cir = NULL;
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      cir = new CIntegrationRule(order, levelset, 2, mesh);
+   }
+
+   // 6. Define and assemble the linear forms on the finite element space.
+   LinearForm surface(fespace);
+   LinearForm volume(fespace);
+
+   surface.AddDomainIntegrator(new SurfaceLFIntegrator(u, levelset, sir));
+   surface.Assemble();
+
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      volume.AddDomainIntegrator(new SubdomainLFIntegrator(u, levelset, cir));
+      volume.Assemble();
+   }
+
+   // 7. Print information, computed values and errors to the console.
+   int qorder = 0;
+   int nbasis = 2 * (order + 1) + (int)(order * (order + 1) / 2);
+   IntegrationRules irs(0, Quadrature1D::GaussLegendre);
+   IntegrationRule ir = irs.Get(Geometry::SQUARE, qorder);
+   for (; ir.GetNPoints() <= nbasis; qorder++)
+   {
+      ir = irs.Get(Geometry::SQUARE, qorder);
+   }
+   cout << "============================================" << endl;
+   cout << "Mesh size dx:                       ";
+   if (itype != IntegrationType::Volumetric1D)
+   {
+      cout << 3.2 / pow(2., (double)ref_levels) << endl;
+   }
+   else
+   {
+      cout << .25 / pow(2., (double)ref_levels) << endl;
+   }
+   if (itype == IntegrationType::Surface2D
+       || itype == IntegrationType::Volumetric2D)
+   {
+      cout << "Number of div free basis functions: " << nbasis << endl;
+      cout << "Number of quadrature points:        " << ir.GetNPoints() << endl;
+   }
+   cout << scientific << setprecision(2);
+   cout << "============================================" << endl;
+   cout << "Computed value of surface integral: " << surface.Sum() << endl;
+   cout << "True value of surface integral:     " << Surface() << endl;
+   cout << "Absolut Error (Surface):            ";
+   cout << abs(surface.Sum() - Surface()) << endl;
+   cout << "Relative Error (Surface):           ";
+   cout << abs(surface.Sum() - Surface()) / Surface() << endl;
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      cout << "--------------------------------------------" << endl;
+      cout << "Computed value of volume integral:  " << volume.Sum() << endl;
+      cout << "True value of volume integral:      " << Volume() << endl;
+      cout << "Absolut Error (Volume):             ";
+      cout << abs(volume.Sum() - Volume()) << endl;
+      cout << "Relative Error (Volume):            ";
+      cout << abs(volume.Sum() - Volume()) / Volume() << endl;
+   }
+   cout << "============================================" << endl;
+
+   // 8. Plot the level-set function on a high order finite element space.
+   if (visualization)
+   {
+      H1_FECollection fe_coll2(5, mesh->Dimension());
+      FiniteElementSpace fespace2(mesh, &fe_coll2);
+      FunctionCoefficient levelset_coeff(levelset);
+      GridFunction lgf(&fespace2);
+      lgf.ProjectCoefficient(levelset_coeff);
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << lgf << flush;
+      sol_sock << "keys pppppppppppppppppppppppppppcmmlRj\n";
+      sol_sock << "levellines " << 0. << " " << 0. << " " << 1 << "\n" << flush;
+   }
+
+   delete sir;
+   delete cir;
+   delete fespace;
+   delete mesh;
+   return EXIT_SUCCESS;
+#endif //MFEM_USE_LAPACK
+}
@@ -3,28 +3,28 @@
 //
 // Compile with: make ex1
 //
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/star-mixed.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/fichera-mixed.mesh
-//               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
+// Sample runs:  ex1 -m ../../data/square-disc.mesh
+//               ex1 -m ../../data/star.mesh
+//               ex1 -m ../../data/star-mixed.mesh
+//               ex1 -m ../../data/escher.mesh
+//               ex1 -m ../../data/fichera.mesh
+//               ex1 -m ../../data/fichera-mixed.mesh
+//               ex1 -m ../../data/toroid-wedge.mesh
+//               ex1 -m ../../data/square-disc-p2.vtk -o 2
+//               ex1 -m ../../data/square-disc-p3.mesh -o 3
+//               ex1 -m ../../data/square-disc-nurbs.mesh -o -1
+//               ex1 -m ../../data/star-mixed-p2.mesh -o 2
+//               ex1 -m ../../data/disc-nurbs.mesh -o -1
+//               ex1 -m ../../data/pipe-nurbs.mesh -o -1
+//               ex1 -m ../../data/fichera-mixed-p2.mesh -o 2
+//               ex1 -m ../../data/star-surf.mesh
+//               ex1 -m ../../data/square-disc-surf.mesh
+//               ex1 -m ../../data/inline-segment.mesh
+//               ex1 -m ../../data/amr-quad.mesh
+//               ex1 -m ../../data/amr-hex.mesh
+//               ex1 -m ../../data/fichera-amr.mesh
+//               ex1 -m ../../data/mobius-strip.mesh
+//               ex1 -m ../../data/mobius-strip.mesh -o -1 -sc
 //
 // Device sample runs:
 //               ex1 -pa -d cuda
@@ -32,7 +32,7 @@
 //               ex1 -pa -d occa-cuda
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
-//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
+//               ex1 -m ../../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -22,15 +22,19 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
-  ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-  ex31 ex33 ex34 ex36 ex37
+   ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
+   ex31 ex33 ex34 ex36 ex37
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
-  ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
-  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-	ex37p
+   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
+   ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
+   ex37p
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-  ex24p ex25p ex26p ex34p ex35p
+   ex24p ex25p ex26p ex34p ex35p
+
+ifeq ($(MFEM_USE_LAPACK),YES)
+   SEQ_EXAMPLES += ex38
+endif

 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
@@ -273,12 +273,13 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
         if (sp_solver)
         {
-            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            STRUMPACKSolver * strumpack = new STRUMPACKSolver(MPI_COMM_WORLD, argc, argv);
            strumpack->SetPrintFactorStatistics(true);
            strumpack->SetPrintSolveStatistics(false);
            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            strumpack->DisableMatching();
+            strumpack->SetMatching(strumpack::MatchingJob::NONE);
+            strumpack->SetCompression(strumpack::CompressionType::NONE);
            strumpack->SetOperator(*Arow);
            strumpack->SetFromCommandLine();
            precond = strumpack;
@@ -77,6 +77,7 @@ set(SRCS
  gridfunc.cpp
  hybridization.cpp
  intrules.cpp
+  intrules_cut.cpp
  ceed/interface/basis.cpp
  ceed/interface/restriction.cpp
  ceed/interface/operator.cpp
@@ -96,9 +97,6 @@ set(SRCS
  lor/lor_ads.cpp
  lor/lor_ams.cpp
  lor/lor_batched.cpp
-  lor/lor_h1.cpp
-  lor/lor_nd.cpp
-  lor/lor_rt.cpp
  multigrid.cpp
  nonlinearform.cpp
  nonlinearform_ext.cpp
@@ -186,6 +184,7 @@ set(HDRS
  gridfunc.hpp
  hybridization.hpp
  intrules.hpp
+  intrules_cut.hpp
  ceed/interface/basis.hpp
  ceed/interface/integrator.hpp
  ceed/interface/interface.hpp
@@ -101,6 +101,7 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)

   // Copy the pointers to the integrators
   domain_integs = bf->domain_integs;
+   domain_integs_marker = bf->domain_integs_marker;

   boundary_integs = bf->boundary_integs;
   boundary_integs_marker = bf->boundary_integs_marker;
@@ -433,6 +434,9 @@ void BilinearForm::Assemble(int skip_zeros)
      // Element-wise integration
      for (int i = 0; i < fes -> GetNE(); i++)
      {
+         // Set both doftrans (potentially needed to assemble the element
+         // matrix) and vdofs, which is also needed when the element matrices
+         // are pre-assembled.
         doftrans = fes->GetElementVDofs(i, vdofs);
         if (element_matrices)
         {
@@ -441,6 +445,8 @@ void BilinearForm::Assemble(int skip_zeros)
         else
         {
            const int elem_attr = fes->GetMesh()->GetAttribute(i);
+            eltrans = fes->GetElementTransformation(i);
+
            elmat.SetSize(0);
            for (int k = 0; k < domain_integs.Size(); k++)
            {
@@ -448,9 +454,8 @@ void BilinearForm::Assemble(int skip_zeros)
                    (*(domain_integs_marker[k]))[elem_attr-1] == 1)
                   && !domain_integs[k]->Patchwise())
               {
-                  const FiniteElement &fe = *fes->GetFE(i);
-                  eltrans = fes->GetElementTransformation(i);
-                  domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+                  domain_integs[k]->AssembleElementMatrix(*fes->GetFE(i),
+                                                          *eltrans, elemmat);
                  if (elmat.Size() == 0)
                  {
                     elmat = elemmat;
@@ -1222,11 +1227,14 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,

   // Copy the pointers to the integrators
   domain_integs = mbf->domain_integs;
-   boundary_integs = mbf->boundary_integs;
-   trace_face_integs = mbf->trace_face_integs;
-   boundary_trace_face_integs = mbf->boundary_trace_face_integs;
+   domain_integs_marker = mbf->domain_integs_marker;

+   boundary_integs = mbf->boundary_integs;
   boundary_integs_marker = mbf->boundary_integs_marker;
+
+   trace_face_integs = mbf->trace_face_integs;
+
+   boundary_trace_face_integs = mbf->boundary_trace_face_integs;
   boundary_trace_face_integs_marker = mbf->boundary_trace_face_integs_marker;

   assembly = AssemblyLevel::LEGACY;
@@ -1349,6 +1357,14 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const
 void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
   domain_integs.Append (bfi);
+   domain_integs_marker.Append(NULL); // NULL marker means apply everywhere
+}
+
+void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi,
+                                             Array<int> &elem_marker)
+{
+   domain_integs.Append (bfi);
+   domain_integs_marker.Append(&elem_marker);
 }

 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
@@ -1383,7 +1399,7 @@ void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
   boundary_trace_face_integs_marker.Append(&bdr_marker);
 }

-void MixedBilinearForm::Assemble (int skip_zeros)
+void MixedBilinearForm::Assemble(int skip_zeros)
 {
   if (ext)
   {
@@ -1405,8 +1421,20 @@ void MixedBilinearForm::Assemble (int skip_zeros)

   if (domain_integs.Size())
   {
+      for (int k = 0; k < domain_integs.Size(); k++)
+      {
+         if (domain_integs_marker[k] != NULL)
+         {
+            MFEM_VERIFY(domain_integs_marker[k]->Size() ==
+                        (mesh->attributes.Size() ? mesh->attributes.Max() : 0),
+                        "invalid element marker for domain integrator #"
+                        << k << ", counting from zero");
+         }
+      }
+
      for (int i = 0; i < test_fes -> GetNE(); i++)
      {
+         const int elem_attr = mesh->GetAttribute(i);
         dom_dof_trans = trial_fes -> GetElementVDofs (i, trial_vdofs);
         ran_dof_trans = test_fes  -> GetElementVDofs (i, test_vdofs);
         eltrans = test_fes -> GetElementTransformation (i);
@@ -1415,10 +1443,14 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         elmat = 0.0;
         for (int k = 0; k < domain_integs.Size(); k++)
         {
-            domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
-                                                        *test_fes  -> GetFE(i),
-                                                        *eltrans, elemmat);
-            elmat += elemmat;
+            if (domain_integs_marker[k] == NULL ||
+                (*(domain_integs_marker[k]))[elem_attr-1] == 1)
+            {
+               domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
+                                                           *test_fes  -> GetFE(i),
+                                                           *eltrans, elemmat);
+               elmat += elemmat;
+            }
         }
         if (ran_dof_trans || dom_dof_trans)
         {
@@ -1941,41 +1973,56 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      return;
   }

-   Array<int> dom_vdofs, ran_vdofs;
-   ElementTransformation *T;
+   ElementTransformation *eltrans;
   DofTransformation * dom_dof_trans;
   DofTransformation * ran_dof_trans;
-   const FiniteElement *dom_fe, *ran_fe;
-   DenseMatrix totelmat, elmat;
+   DenseMatrix elmat;
+
+   Mesh *mesh = test_fes->GetMesh();

   if (mat == NULL)
   {
      mat = new SparseMatrix(height, width);
   }

-   if (domain_integs.Size() > 0)
+   if (domain_integs.Size())
   {
+      for (int k = 0; k < domain_integs.Size(); k++)
+      {
+         if (domain_integs_marker[k] != NULL)
+         {
+            MFEM_VERIFY(domain_integs_marker[k]->Size() ==
+                        (mesh->attributes.Size() ? mesh->attributes.Max() : 0),
+                        "invalid element marker for domain integrator #"
+                        << k << ", counting from zero");
+         }
+      }
+
      for (int i = 0; i < test_fes->GetNE(); i++)
      {
-         dom_dof_trans = trial_fes->GetElementVDofs(i, dom_vdofs);
-         ran_dof_trans = test_fes->GetElementVDofs(i, ran_vdofs);
-         T = test_fes->GetElementTransformation(i);
-         dom_fe = trial_fes->GetFE(i);
-         ran_fe = test_fes->GetFE(i);
+         const int elem_attr = mesh->GetAttribute(i);
+         dom_dof_trans = trial_fes->GetElementVDofs(i, trial_vdofs);
+         ran_dof_trans = test_fes->GetElementVDofs(i, test_vdofs);
+         eltrans = test_fes->GetElementTransformation(i);

-         domain_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                  totelmat);
-         for (int j = 1; j < domain_integs.Size(); j++)
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
+         for (int k = 0; k < domain_integs.Size(); k++)
         {
-            domain_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                     elmat);
-            totelmat += elmat;
+            if (domain_integs_marker[k] == NULL ||
+                (*(domain_integs_marker[k]))[elem_attr-1] == 1)
+            {
+               domain_integs[k]->AssembleElementMatrix2(*trial_fes->GetFE(i),
+                                                        *test_fes->GetFE(i),
+                                                        *eltrans, elemmat);
+               elmat += elemmat;
+            }
         }
         if (ran_dof_trans || dom_dof_trans)
         {
-            TransformPrimal(ran_dof_trans, dom_dof_trans, totelmat);
+            TransformPrimal(ran_dof_trans, dom_dof_trans, elemmat);
         }
-         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
+         mat->SetSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
      }
   }

@@ -1984,21 +2031,20 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      const int nfaces = test_fes->GetMesh()->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
      {
-         trial_fes->GetFaceVDofs(i, dom_vdofs);
-         test_fes->GetFaceVDofs(i, ran_vdofs);
-         T = test_fes->GetMesh()->GetFaceTransformation(i);
-         dom_fe = trial_fes->GetFaceElement(i);
-         ran_fe = test_fes->GetFaceElement(i);
+         trial_fes->GetFaceVDofs(i, trial_vdofs);
+         test_fes->GetFaceVDofs(i, test_vdofs);
+         eltrans = test_fes->GetMesh()->GetFaceTransformation(i);

-         trace_face_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                      totelmat);
-         for (int j = 1; j < trace_face_integs.Size(); j++)
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
+         for (int k = 0; k < trace_face_integs.Size(); k++)
         {
-            trace_face_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                         elmat);
-            totelmat += elmat;
+            trace_face_integs[k]->AssembleElementMatrix2(*trial_fes->GetFaceElement(i),
+                                                         *test_fes->GetFaceElement(i),
+                                                         *eltrans, elemmat);
+            elmat += elemmat;
         }
-         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
+         mat->SetSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }
 }
@@ -100,7 +100,7 @@ protected:
   /// Includes all by default.
   /// 0 - ignore attribute
   /// 1 - include attribute
-   Array<Array<int>*>             domain_integs_marker;
+   Array<Array<int>*> domain_integs_marker; ///< Entries are not owned.

   /// Set of Boundary Integrators to be applied.
   Array<BilinearFormIntegrator*> boundary_integs;
@@ -722,10 +722,13 @@ protected:

   /// Domain integrators.
   Array<BilinearFormIntegrator*> domain_integs;
+   /// Entries are not owned.
+   Array<Array<int>*> domain_integs_marker;

   /// Boundary integrators.
   Array<BilinearFormIntegrator*> boundary_integs;
-   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.
+   /// Entries are not owned.
+   Array<Array<int>*> boundary_integs_marker;

   /// Trace face (skeleton) integrators.
   Array<BilinearFormIntegrator*> trace_face_integs;
@@ -805,12 +808,16 @@ public:
   /// Adds a domain integrator. Assumes ownership of @a bfi.
   void AddDomainIntegrator(BilinearFormIntegrator *bfi);

+   /// Adds a domain integrator. Assumes ownership of @a bfi.
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi,
+                            Array<int> &elem_marker);
+
   /// Adds a boundary integrator. Assumes ownership of @a bfi.
   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi);

   /// Adds a boundary integrator. Assumes ownership of @a bfi.
-   void AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
-                               Array<int> &bdr_marker);
+   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi,
+                              Array<int> &bdr_marker);

   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.

@@ -820,14 +827,18 @@ public:
   void AddTraceFaceIntegrator(BilinearFormIntegrator *bfi);

   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi);
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi);

   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi,
-                                   Array<int> &bdr_marker);
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi,
+                                  Array<int> &bdr_marker);

   /// Access all integrators added with AddDomainIntegrator().
   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }
+   /** @brief Access all domain markers added with AddDomainIntegrator().
+       If no marker was specified when the integrator was added, the
+       corresponding pointer (to Array<int>) will be NULL. */
+   Array<Array<int>*> *GetDBFI_Marker() { return &domain_integs_marker; }

   /// Access all integrators added with AddBoundaryIntegrator().
   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
@@ -1065,6 +1076,9 @@ public:
   /// Adds a domain interpolator. Assumes ownership of @a di.
   void AddDomainInterpolator(DiscreteInterpolator *di)
   { AddDomainIntegrator(di); }
+   void AddDomainInterpolator(DiscreteInterpolator *di,
+                              Array<int> &elem_marker)
+   { AddDomainIntegrator(di, elem_marker); }

   /// Adds a trace face interpolator. Assumes ownership of @a di.
   void AddTraceFaceInterpolator(DiscreteInterpolator *di)
@@ -1072,6 +1086,7 @@ public:

   /// Access all interpolators added with AddDomainInterpolator().
   Array<BilinearFormIntegrator*> *GetDI() { return &domain_integs; }
+   Array<Array<int>*> *GetDI_Marker() { return &domain_integs_marker; }

   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
   /** This method must be called before assembly. */
@@ -303,7 +303,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
      std::unordered_map<int,int> f_to_be;
      for (int i = 0; i < mesh.GetNBE(); ++i)
      {
-         const int f = mesh.GetBdrElementEdgeIndex(i);
+         const int f = mesh.GetBdrElementFaceIndex(i);
         f_to_be[f] = i;
      }
      const int nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary);
@@ -1340,10 +1340,11 @@ void MassIntegrator::AssembleElementMatrix2(
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
-      trial_fe.CalcShape(ip, shape);
-      test_fe.CalcShape(ip, te_shape);
-
      Trans.SetIntPoint (&ip);
+
+      trial_fe.CalcPhysShape(Trans, shape);
+      test_fe.CalcPhysShape(Trans, te_shape);
+
      w = Trans.Weight() * ip.weight;
      if (Q)
      {
@@ -64,7 +64,7 @@ public:
   /// Assemble diagonal and add it to Vector @a diag.
   virtual void AssembleDiagonalPA(Vector &diag);

-   /// Assemble diagonal of ADA^T (A is this integrator) and add it to @a diag.
+   /// Assemble diagonal of \f$ADA^{\mathrm{T}}\f$ (\f$A\f$ is this integrator) and add it to @a diag.
   virtual void AssembleDiagonalPA_ADAt(const Vector &D, Vector &diag);

   /// Method for partially assembled action.
@@ -137,8 +137,8 @@ public:
                                      DenseMatrix &elmat);

   /** Compute the local matrix representation of a bilinear form
-       a(u,v) defined on different trial (given by u) and test
-       (given by v) spaces. The rows in the local matrix correspond
+       \f$a(u,v)\f$ defined on different trial (given by \f$u\f$) and test
+       (given by \f$v\f$) spaces. The rows in the local matrix correspond
       to the test dofs and the columns -- to the trial dofs. */
   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
                                       const FiniteElement &test_fe,
@@ -706,9 +706,9 @@ private:

 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, v) in either 1D, 2D,
-    or 3D and where Q is an optional scalar coefficient, u and v are each in H1
-    or L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q u, v)\f$ in either 1D, 2D,
+    or 3D and where \f$Q\f$ is an optional scalar coefficient, \f$u\f$ and \f$v\f$ are each in \f$H^1\f$
+    or \f$L_2\f$. */
 class MixedScalarMassIntegrator : public MixedScalarIntegrator
 {
 public:
@@ -717,9 +717,9 @@ public:
      : MixedScalarIntegrator(q) { same_calc_shape = true; }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, v) in either 2D, or
-    3D and where Q is a vector coefficient, u is in H1 or L2 and v is in H(Curl)
-    or H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} u, v)\f$ in either 2D, or
+    3D and where \f$\vec{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H^1\f$ or \f$L_2\f$ and \f$v\f$ is in \f$H\f$(curl)
+    or \f$H\f$(div). */
 class MixedVectorProductIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -727,8 +727,8 @@ public:
      : MixedScalarVectorIntegrator(vq) {}
 };

-/** Class for integrating the bilinear form a(u,v) := (Q D u, v) in 1D where Q
-    is an optional scalar coefficient, u is in H1, and v is in L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla u, v)\f$ in 1D where Q
+    is an optional scalar coefficient, \f$u\f$ is in \f$H^1\f$, and \f$v\f$ is in \f$L_2\f$. */
 class MixedScalarDerivativeIntegrator : public MixedScalarIntegrator
 {
 public:
@@ -762,8 +762,8 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := -(Q u, D v) in 1D where Q
-    is an optional scalar coefficient, u is in L2, and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := -(Q u, \nabla v)\f$ in 1D where \f$Q\f$
+    is an optional scalar coefficient, \f$u\f$ is in \f$L_2\f$, and \f$v\f$ is in \f$H^1\f$. */
 class MixedScalarWeakDerivativeIntegrator : public MixedScalarIntegrator
 {
 public:
@@ -799,8 +799,8 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q div u, v) in either 2D
-    or 3D where Q is an optional scalar coefficient, u is in H(Div), and v is a
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla \cdot u, v)\f$ in either 2D
+    or 3D where \f$Q\f$ is an optional scalar coefficient, \f$u\f$ is in \f$H\f$(div), and \f$v\f$ is a
    scalar field. */
 class MixedScalarDivergenceIntegrator : public MixedScalarIntegrator
 {
@@ -821,7 +821,7 @@ protected:
   inline virtual const char * FiniteElementTypeFailureMessage() const
   {
      return "MixedScalarDivergenceIntegrator:  "
-             "Trial must be H(Div) and the test space must be a "
+             "Trial must be \f$H\f$(div) and the test space must be a "
             "scalar field";
   }

@@ -836,9 +836,8 @@ protected:
   { trial_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V div u, v) in either 2D
-    or 3D where V is a vector coefficient, u is in H(Div), and v is a vector
-    field. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \nabla \cdot u, v)\f$ in either 2D
+    or 3D where \f$\vec{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H\f$(div), and \f$v\f$ is in \f$H\f$(div). */
 class MixedVectorDivergenceIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -874,9 +873,9 @@ protected:
   { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := -(Q u, div v) in either 2D
-    or 3D where Q is an optional scalar coefficient, u is in L2 or H1, and v is
-    in H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := -(Q u, \nabla \cdot v)\f$ in either 2D
+    or 3D where \f$Q\f$ is an optional scalar coefficient, \f$u\f$ is in \f$L_2\f$ or \f$H^1\f$, and \f$v\f$ is
+    in \f$H\f$(div). */
 class MixedScalarWeakGradientIntegrator : public MixedScalarIntegrator
 {
 public:
@@ -914,9 +913,9 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) in 2D where
-    Q is an optional scalar coefficient, u is in H(Curl), and v is in L2 or
-    H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \mathrm{curl}(u), v)\f$ in 2D where
+    \f$Q\f$ is an optional scalar coefficient, \f$u\f$ is in \f$H\f$(curl), and \f$v\f$ is in \f$L_2\f$ or
+    \f$H^1\f$. */
 class MixedScalarCurlIntegrator : public MixedScalarIntegrator
 {
 public:
@@ -968,9 +967,9 @@ protected:
   int dim, ne, dofs1D, quad1D, dofs1Dtest;
 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, curl v) in 2D where
-    Q is an optional scalar coefficient, u is in L2 or H1, and v is in
-    H(Curl). Partial assembly (PA) is supported but could be further optimized
+/** Class for integrating the bilinear form \f$a(u,v) := (Q u, \mathrm{curl}(v))\f$ in 2D where
+    \f$Q\f$ is an optional scalar coefficient, \f$u\f$ is in \f$L_2\f$ or \f$H^1\f$, and \f$v\f$ is in
+    \f$H\f$(curl). Partial assembly (PA) is supported but could be further optimized
    by using more efficient threading and shared memory.
 */
 class MixedScalarWeakCurlIntegrator : public MixedScalarIntegrator
@@ -1006,9 +1005,9 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, v) in either 2D or
-    3D and where Q is an optional coefficient (of type scalar, matrix, or
-    diagonal matrix) u and v are each in H(Curl) or H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q u, v)\f$ in either 2D or
+    3D and where \f$Q\f$ is an optional coefficient (of type scalar, matrix, or
+    diagonal matrix) \f$u\f$ and \f$v\f$ are each in \f$H\f$(curl) or \f$H\f$(div). */
 class MixedVectorMassIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1021,8 +1020,8 @@ public:
      : MixedVectorIntegrator(mq) { same_calc_shape = true; }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x u, v) in 3D and where
-    V is a vector coefficient u and v are each in H(Curl) or H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times u, v)\f$ in 3D and where
+    \f$\vec{V}\f$ is a vector coefficient \f$u\f$ and \f$v\f$ are each in \f$H\f$(curl) or \f$H\f$(div). */
 class MixedCrossProductIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1030,9 +1029,9 @@ public:
      : MixedVectorIntegrator(vq, false) { same_calc_shape = true; }
 };

-/** Class for integrating the bilinear form a(u,v) := (V . u, v) in 2D or 3D and
-    where V is a vector coefficient u is in H(Curl) or H(Div) and v is in H1 or
-    L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \cdot u, v)\f$ in 2D or 3D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in \f$H^1\f$ or
+    \f$L_2\f$. */
 class MixedDotProductIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1055,9 +1054,9 @@ public:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (-V . u, Div v) in 2D or
-    3D and where V is a vector coefficient u is in H(Curl) or H(Div) and v is in
-    RT. */
+/** Class for integrating the bilinear form \f$a(u,v) := (-\vec{V} \cdot u, \nabla \cdot v)\f$ in 2D or
+    3D and where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in
+    \f$H\f$(div). */
 class MixedWeakGradDotIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1093,8 +1092,8 @@ public:
   { scalar_fe.CalcPhysDivShape(Trans, shape); shape *= -1.0; }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x u, Grad v) in 3D and
-    where V is a vector coefficient u is in H(Curl) or H(Div) and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (v \vec{V} \times u, \nabla v)\f$ in 3D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in \f$H^1\f$. */
 class MixedWeakDivCrossIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1127,9 +1126,9 @@ public:
   { test_fe.CalcPhysDShape(Trans, shape); shape *= -1.0; }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q Grad u, Grad v) in 3D
-    or in 2D and where Q is a scalar or matrix coefficient u and v are both in
-    H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla u, \nabla v)\f$ in 3D
+    or in 2D and where \f$Q\f$ is a scalar or matrix coefficient \f$u\f$ and \f$v\f$ are both in
+    \f$H^1\f$. */
 class MixedGradGradIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1185,8 +1184,8 @@ public:
   { test_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Grad u, Grad v) in 3D
-    or in 2D and where V is a vector coefficient u and v are both in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \nabla u, \nabla v)\f$ in 3D
+    or in 2D and where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ and \f$v\f$ are both in \f$H^1\f$. */
 class MixedCrossGradGradIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1227,9 +1226,9 @@ public:
   { test_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q Curl u, Curl v) in 3D
-    and where Q is a scalar or matrix coefficient u and v are both in
-    H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \mathrm{curl}(u), \mathrm{curl}(v))\f$ in 3D
+    and where \f$Q\f$ is a scalar or matrix coefficient \f$u\f$ and \f$v\f$ are both in
+    \f$H\f$(curl). */
 class MixedCurlCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1276,8 +1275,8 @@ public:
   { test_fe.CalcPhysCurlShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Curl u, Curl v) in 3D
-    and where V is a vector coefficient u and v are both in H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \mathrm{curl}(u), \mathrm{curl}(v))\f$ in 3D
+    and where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ and \f$v\f$ are both in \f$H\f$(curl). */
 class MixedCrossCurlCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1320,8 +1319,8 @@ public:
   { test_fe.CalcPhysCurlShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Curl u, Grad v) in 3D
-    and where V is a vector coefficient u is in H(Curl) and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \mathrm{curl}(u), \nabla \cdot v)\f$ in 3D
+    and where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) and \f$v\f$ is in \f$H^1\f$. */
 class MixedCrossCurlGradIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1363,8 +1362,8 @@ public:
   { test_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Grad u, Curl v) in 3D
-    and where V is a scalar coefficient u is in H1 and v is in H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (v \times \nabla \cdot u, \mathrm{curl}(v))\f$ in 3D
+    and where \f$v\f$ is a scalar coefficient \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H\f$(curl). */
 class MixedCrossGradCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1406,9 +1405,9 @@ public:
   { test_fe.CalcPhysCurlShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x u, Curl v) in 3D and
-    where V is a vector coefficient u is in H(Curl) or H(Div) and v is in
-    H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times u, \mathrm{curl}(v))\f$ in 3D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in
+    \f$H\f$(curl). */
 class MixedWeakCurlCrossIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1441,9 +1440,9 @@ public:
   { test_fe.CalcPhysCurlShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x u, Curl v) in 2D and
-    where V is a vector coefficient u is in H(Curl) or H(Div) and v is in
-    H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times u, \mathrm{curl}(v))\f$ in 2D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in
+    \f$H\f$(curl). */
 class MixedScalarWeakCurlCrossIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1476,9 +1475,9 @@ public:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Grad u, v) in 3D or
-    in 2D and where V is a vector coefficient u is in H1 and v is in H(Curl) or
-    H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \nabla \cdot u, v)\f$ in 3D or
+    in 2D and where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H\f$(curl) or
+    \f$H\f$(div). */
 class MixedCrossGradIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1516,9 +1515,9 @@ public:
   { test_fe.CalcVShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Curl u, v) in 3D and
-    where V is a vector coefficient u is in H(Curl) and v is in H(Curl) or
-    H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \mathrm{curl}(u), v)\f$ in 3D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) and \f$v\f$ is in \f$H\f$(curl) or
+    \f$H\f$(div). */
 class MixedCrossCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1551,9 +1550,9 @@ public:
   { trial_fe.CalcPhysCurlShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Curl u, v) in 2D and
-    where V is a vector coefficient u is in H(Curl) and v is in H(Curl) or
-    H(Div). */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \mathrm{curl}(u), v)\f$ in 2D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) and \f$v\f$ is in \f$H\f$(curl) or
+    \f$H\f$(div). */
 class MixedScalarCrossCurlIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1586,8 +1585,8 @@ public:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x Grad u, v) in 2D and
-    where V is a vector coefficient u is in H1 and v is in H1 or L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times \nabla \cdot u, v)\f$ in 2D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H^1\f$ or \f$L_2\f$. */
 class MixedScalarCrossGradIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1620,8 +1619,8 @@ public:
   { vector_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x u, v) in 2D and where
-    V is a vector coefficient u is in ND or RT and v is in H1 or L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times u, v)\f$ in 2D and where
+    \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H\f$(curl) or \f$H\f$(div) and \f$v\f$ is in \f$H^1\f$ or \f$L_2\f$. */
 class MixedScalarCrossProductIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1645,8 +1644,11 @@ public:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (V x z u, v) in 2D and
-    where V is a vector coefficient u is in H1 or L2 and v is in ND or RT. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \times u \hat{z}, v)\f$ in 2D and
+    where \f$\vec{V}\f$ is a vector coefficient \f$u\f$ is in \f$H^1\f$ or \f$L_2\f$ and \f$v\f$ is in \f$H\f$(curl) or \f$H\f$(div).
+
+    \todo Documentation what \f$\hat{z}\f$ is (also missing in https://mfem.org/bilininteg/).
+   */
 class MixedScalarWeakCrossProductIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1675,8 +1677,8 @@ public:
   { scalar_fe.CalcPhysShape(Trans, shape); shape *= -1.0; }
 };

-/** Class for integrating the bilinear form a(u,v) := (V . Grad u, v) in 2D or
-    3D and where V is a vector coefficient, u is in H1 and v is in H1 or L2. */
+/** Class for integrating the bilinear form \f$a(u,v) := (\vec{V} \cdot \nabla u, v)\f$ in 2D or
+    3D and where \f$\vec{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H^1\f$ or \f$L_2\f$. */
 class MixedDirectionalDerivativeIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1708,8 +1710,8 @@ public:
   { vector_fe.CalcPhysDShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (-V . Grad u, Div v) in 2D
-    or 3D and where V is a vector coefficient, u is in H1 and v is in RT. */
+/** Class for integrating the bilinear form \f$a(u,v) := (-\hat{V} \cdot  \nabla \cdot u, \nabla \cdot v)\f$ in 2D
+    or 3D and where \f$\hat{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H\f$(div). */
 class MixedGradDivIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1747,8 +1749,8 @@ public:
   { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (-V Div u, Grad v) in 2D
-    or 3D and where V is a vector coefficient, u is in RT and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (-\hat{V} \nabla \cdot u, \nabla \cdot v)\f$ in 2D
+    or 3D and where \f$\hat{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H\f$(div) and \f$v\f$ is in \f$H^1\f$. */
 class MixedDivGradIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1787,8 +1789,8 @@ public:
   { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };

-/** Class for integrating the bilinear form a(u,v) := (-V u, Grad v) in 2D or 3D
-    and where V is a vector coefficient, u is in H1 or L2 and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := (-\hat{V} u, \nabla \cdot v)\f$ in 2D or 3D
+    and where \f$\hat{V}\f$ is a vector coefficient, \f$u\f$ is in \f$H^1\f$ or \f$L_2\f$ and \f$v\f$ is in \f$H^1\f$. */
 class MixedScalarWeakDivergenceIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1820,9 +1822,9 @@ public:
   { vector_fe.CalcPhysDShape(Trans, shape); shape *= -1.0; }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q grad u, v) in either 2D
-    or 3D and where Q is an optional coefficient (of type scalar, matrix, or
-    diagonal matrix) u is in H1 and v is in H(Curl) or H(Div). Partial assembly
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla u, v)\f$ in either 2D
+    or 3D and where \f$Q\f$ is an optional coefficient (of type scalar, matrix, or
+    diagonal matrix) \f$u\f$ is in \f$H^1\f$ and \f$v\f$ is in \f$H\f$(curl) or \f$H\f$(div). Partial assembly
    (PA) is supported but could be further optimized by using more efficient
    threading and shared memory.
 */
@@ -1849,7 +1851,7 @@ protected:
   inline virtual const char * FiniteElementTypeFailureMessage() const
   {
      return "MixedVectorGradientIntegrator:  "
-             "Trial spaces must be H1 and the test space must be a "
+             "Trial spaces must be \f$H^1\f$ and the test space must be a "
             "vector field in 2D or 3D";
   }

@@ -1881,9 +1883,9 @@ private:
   int dim, ne, dofs1D, quad1D;
 };

-/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) in 3D and
-    where Q is an optional coefficient (of type scalar, matrix, or diagonal
-    matrix) u is in H(Curl) and v is in H(Div) or H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \mathrm{curl}(u), v)\f$ in 3D and
+    where \f$Q\f$ is an optional coefficient (of type scalar, matrix, or diagonal
+    matrix) \f$u\f$ is in \f$H\f$(curl) and \f$v\f$ is in \f$H\f$(div) or \f$H\f$(curl). */
 class MixedVectorCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1940,9 +1942,9 @@ private:
   int dim, ne, dofs1D, dofs1Dtest,quad1D, testType, trialType, coeffDim;
 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, curl v) in 3D and
-    where Q is an optional coefficient (of type scalar, matrix, or diagonal
-    matrix) u is in H(Div) or H(Curl) and v is in H(Curl). */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q u, \mathrm{curl}(v))\f$ in 3D and
+    where \f$Q\f$ is an optional coefficient (of type scalar, matrix, or diagonal
+    matrix) \f$u\f$ is in \f$H\f$(div) or \f$H\f$(curl) and \f$v\f$ is in \f$H\f$(curl). */
 class MixedVectorWeakCurlIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -1997,9 +1999,9 @@ private:
   int dim, ne, dofs1D, quad1D, testType, trialType, coeffDim;
 };

-/** Class for integrating the bilinear form a(u,v) := - (Q u, grad v) in either
-    2D or 3D and where Q is an optional coefficient (of type scalar, matrix, or
-    diagonal matrix) u is in H(Div) or H(Curl) and v is in H1. */
+/** Class for integrating the bilinear form \f$a(u,v) := - (Q u, \nabla v)\f$ in either
+    2D or 3D and where \f$Q\f$ is an optional coefficient (of type scalar, matrix, or
+    diagonal matrix) \f$u\f$ is in \f$H\f$(div) or \f$H\f$(curl) and \f$v\f$ is in \f$H^1\f$. */
 class MixedVectorWeakDivergenceIntegrator : public MixedVectorIntegrator
 {
 public:
@@ -2039,11 +2041,11 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q grad u, v) where Q is a
-    scalar coefficient, and v is a vector with components v_i in the same (H1) space
-    as u.
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla u, v)\f$ where \f$Q\f$ is a
+    scalar coefficient, and \f$v\f$ is a vector with components \f$v_i\f$ in the same (\f$H^1\f$) space
+    as \f$u\f$.

-    See also MixedVectorGradientIntegrator when v is in H(curl). */
+    See also MixedVectorGradientIntegrator when \f$v\f$ is in \f$H\f$(curl). */
 class GradientIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2090,7 +2092,7 @@ public:
                                         ElementTransformation &Trans);
 };

-/** Class for integrating the bilinear form a(u,v) := (Q grad u, grad v) where Q
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \nabla u, \nabla v)\f$ where \f$Q\f$
    can be a scalar or a matrix coefficient. */
 class DiffusionIntegrator: public BilinearFormIntegrator
 {
@@ -2257,7 +2259,7 @@ public:
   Coefficient *GetCoefficient() const { return Q; }
 };

-/** Class for local mass matrix assembling a(u,v) := (Q u, v) */
+/** Class for local mass matrix assembling \f$a(u,v) := (Q u, v)\f$ */
 class MassIntegrator: public BilinearFormIntegrator
 {
   friend class DGMassInverse;
@@ -2322,7 +2324,7 @@ public:
   const Coefficient *GetCoefficient() const { return Q; }
 };

-/** Mass integrator (u, v) restricted to the boundary of a domain */
+/** Mass integrator \f$(u, v)\f$ restricted to the boundary of a domain */
 class BoundaryMassIntegrator : public MassIntegrator
 {
 public:
@@ -2336,7 +2338,7 @@ public:
                                   DenseMatrix &elmat);
 };

-/// alpha (q . grad u, v)
+/// \f$ \alpha (Q \cdot \nabla u, v)\f$
 class ConvectionIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2394,7 +2396,7 @@ public:
 // Alias for @ConvectionIntegrator.
 using NonconservativeConvectionIntegrator = ConvectionIntegrator;

-/// -alpha (u, q . grad v), negative transpose of ConvectionIntegrator
+/// \f$-\alpha (u, q \cdot \nabla v)\f$, negative transpose of ConvectionIntegrator
 class ConservativeConvectionIntegrator : public TransposeIntegrator
 {
 public:
@@ -2402,7 +2404,7 @@ public:
      : TransposeIntegrator(new ConvectionIntegrator(q, -a)) { }
 };

-/// alpha (q . grad u, v) using the "group" FE discretization
+/// \f$ \alpha (Q \cdot \nabla u, v)\f$ using the "group" FE discretization
 class GroupConvectionIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2421,8 +2423,8 @@ public:
                                      DenseMatrix &);
 };

-/** Class for integrating the bilinear form a(u,v) := (Q u, v),
-    where u=(u1,...,un) and v=(v1,...,vn); ui and vi are defined
+/** Class for integrating the bilinear form \f$a(u,v) := (Q u, v)\f$,
+    where \f$ u=(u_1,\dots,u_n) \f$ and \f$ v=(v_1,\dots,v_n)\f$, \f$u_i\f$ and \f$v_i\f$ are defined
    by scalar FE through standard transformation. */
 class VectorMassIntegrator: public BilinearFormIntegrator
 {
@@ -2483,10 +2485,10 @@ public:
 };


-/** Class for integrating (div u, p) where u is a vector field given by
-    VectorFiniteElement through Piola transformation (for RT elements); p is
+/** Class for integrating \f$(\nabla \cdot u, p)\f$ where \f$u\f$ is a vector field given by
+    VectorFiniteElement through Piola transformation (for Raviart-Thomas elements); \f$p\f$ is
    scalar function given by FiniteElement through standard transformation.
-    Here, u is the trial function and p is the test function.
+    Here, \f$u\f$ is the trial function and \f$p\f$ is the test function.

    Note: if the test space does not have map type INTEGRAL, then the element
    matrix returned by AssembleElementMatrix2 will not depend on the
@@ -2530,8 +2532,8 @@ public:
 };


-/** Integrator for `(-Q u, grad v)` for Nedelec (`u`) and H1 (`v`) elements.
-    This is equivalent to a weak divergence of the Nedelec basis functions. */
+/** Integrator for \f$(-Q u, \nabla v)\f$ for Nedelec (\f$u\f$) and \f$H^1\f$ (\f$v\f$) elements.
+    This is equivalent to a weak divergence of the \f$H\f$(curl) basis functions. */
 class VectorFEWeakDivergenceIntegrator: public BilinearFormIntegrator
 {
 protected:
@@ -2557,8 +2559,8 @@ public:
                                       DenseMatrix &elmat);
 };

-/** Integrator for (curl u, v) for Nedelec and RT elements. If the trial and
-    test spaces are switched, assembles the form (u, curl v). */
+/** Integrator for \f$(\mathrm{curl}(u), v)\f$ for Nedelec and Raviart-Thomas elements. If the trial and
+    test spaces are switched, assembles the form \f$(u, \mathrm{curl}(v))\f$. */
 class VectorFECurlIntegrator: public BilinearFormIntegrator
 {
 protected:
@@ -2583,7 +2585,7 @@ public:
                                       DenseMatrix &elmat);
 };

-/// Class for integrating (Q D_i(u), v); u and v are scalars
+/// Class for integrating \f$ (Q \partial_i(u), v) \f$ where \f$u\f$ and \f$v\f$ are scalars
 class DerivativeIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2606,7 +2608,7 @@ public:
                                       DenseMatrix &elmat);
 };

-/// Integrator for (curl u, curl v) for Nedelec elements
+/// Integrator for \f$(\mathrm{curl}(u), \mathrm{curl}(v))\f$ for Nedelec elements
 class CurlCurlIntegrator: public BilinearFormIntegrator
 {
 private:
@@ -2671,7 +2673,7 @@ public:
   const Coefficient *GetCoefficient() const { return Q; }
 };

-/** Integrator for (curl u, curl v) for FE spaces defined by 'dim' copies of a
+/** Integrator for \f$(\mathrm{curl}(u), \mathrm{curl}(v))\f$ for FE spaces defined by 'dim' copies of a
    scalar FE space. */
 class VectorCurlCurlIntegrator: public BilinearFormIntegrator
 {
@@ -2692,20 +2694,21 @@ public:
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
-   /// Compute element energy: (1/2) (curl u, curl u)_E
+   /// Compute element energy: \f$ \frac{1}{2} (\mathrm{curl}(u), \mathrm{curl}(u))_E\f$
   virtual double GetElementEnergy(const FiniteElement &el,
                                   ElementTransformation &Tr,
                                   const Vector &elfun);
 };

-/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) where Q is
-    an optional scalar coefficient, and v is a vector with components v_i in
-    the L2 or H1 space. This integrator handles 3 cases:
-    (a) u ∈ H(curl) in 3D, v is a 3D vector with components v_i in L^2 or H^1
-    (b) u ∈ H(curl) in 2D, v is a scalar field in L^2 or H^1
-    (c) u is a scalar field in H^1, i.e, curl u := [0 1;-1 0]grad u and v is a
-        2D vector field with components v_i in L^2 or H^1 space.
-    Note: Case (b) can also be handled by MixedScalarCurlIntegrator  */
+/** Class for integrating the bilinear form \f$a(u,v) := (Q \mathrm{curl}(u), v)\f$ where \f$Q\f$ is
+    an optional scalar coefficient, and \f$v\f$ is a vector with components \f$v_i\f$ in
+    the \f$L_2\f$ or \f$H^1\f$ space. This integrator handles 3 cases:
+    1. u ∈ \f$H\f$(curl) in 3D, \f$v\f$ is a 3D vector with components \f$v_i\f$ in \f$L^2\f$ or \f$H^1\f$
+    2. u ∈ \f$H\f$(curl) in 2D, \f$v\f$ is a scalar field in \f$L^2\f$ or \f$H^1\f$
+    3. u is a scalar field in \f$H^1\f$, i.e, \f$\mathrm{curl}(u) := \begin{pmatrix} 0 & 1 \\ -1 & 0 \end{pmatrix}\f$, \f$\nabla u\f$ and \f$v\f$ is a
+        2D vector field with components \f$v_i\f$ in \f$L^2\f$ or \f$H^1\f$ space.
+
+    Note: Case 2 can also be handled by MixedScalarCurlIntegrator  */
 class MixedCurlIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2727,10 +2730,10 @@ public:
                                       DenseMatrix &elmat);
 };

-/** Integrator for (Q u, v), where Q is an optional coefficient (of type scalar,
-    vector (diagonal matrix), or matrix), trial function u is in H(Curl) or
-    H(Div), and test function v is in H(Curl), H(Div), or v=(v1,...,vn), where
-    vi are in H1. */
+/** Integrator for \f$(Q u, v)\f$, where \f$Q\f$ is an optional coefficient (of type scalar,
+    vector (diagonal matrix), or matrix), trial function \f$u\f$ is in \f$H\f$(curl) or
+    \f$H\f$(div), and test function \f$v\f$ is in \f$H\f$(curl), \f$H\f$(div), or \f$v=(v_1,\dots,v_n)\f$, where
+    \f$v_i\f$ are in \f$H^1\f$. */
 class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
@@ -2789,8 +2792,8 @@ public:
   const Coefficient *GetCoefficient() const { return Q; }
 };

-/** Integrator for (Q div u, p) where u=(v1,...,vn) and all vi are in the same
-    scalar FE space; p is also in a (different) scalar FE space.  */
+/** Integrator for \f$(Q \nabla \cdot u, v)\f$ where \f$u=(u_1,\cdots,u_n)\f$ and all \f$u_i\f$ are in the same
+    scalar FE space; \f$v\f$ is also in a (different) scalar FE space.  */
 class VectorDivergenceIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2837,7 +2840,7 @@ public:
                                         ElementTransformation &Trans);
 };

-/// (Q div u, div v) for RT elements
+/// \f$(Q \nabla \cdot u, \nabla \cdot v)\f$ for Raviart-Thomas elements
 class DivDivIntegrator: public BilinearFormIntegrator
 {
 protected:
@@ -2878,10 +2881,10 @@ public:
 };

 /** Integrator for
-
-      (Q grad u, grad v) = sum_i (Q grad u_i, grad v_i) e_i e_i^T
-
-    for vector FE spaces, where e_i is the unit vector in the i-th direction.
+    \f[
+      (Q \nabla u, \nabla v) = \sum_i (Q \nabla u_i, \nabla v_i) e_i e_i^{\mathrm{T}}
+    \f]
+    for vector FE spaces, where \f$e_i\f$ is the unit vector in the \f$i\f$-th direction.
    The resulting local element matrix is square, of size <tt> vdim*dof </tt>,
    where \c vdim is the vector dimension space and \c dof is the local degrees
    of freedom. The integrator is not aware of the true vector dimension and
@@ -2942,7 +2945,7 @@ public:
       \c Vector.

       The element matrix is block-diagonal and each block is integrated with
-       coefficient q_i.
+       coefficient \f$q_{i}\f$.

       If the vector dimension does not match the true dimension of the space,
       the resulting element matrix will be mathematically invalid. */
@@ -2954,7 +2957,7 @@ public:
       \c Matrix.

       The element matrix is populated in each block. Each block is integrated
-       with coefficient q_ij.
+       with coefficient \f$q_{ij}\f$.

       If the vector dimension does not match the true dimension of the space,
       the resulting element matrix will be mathematically invalid. */
@@ -2978,8 +2981,10 @@ public:
 };

 /** Integrator for the linear elasticity form:
-    a(u,v) = (lambda div(u), div(v)) + (2 mu e(u), e(v)),
-    where e(v) = (1/2) (grad(v) + grad(v)^T).
+    \f[
+      a(u,v) = (\lambda \mathrm{div}(u), \mathrm{div}(v)) + (2 \mu \varepsilon(u), \varepsilon(v)),
+    \f]
+    where \f$\varepsilon(v) = \frac{1}{2} (\mathrm{grad}(v) + \mathrm{grad}(v)^{\mathrm{T}})\f$.
    This is a 'Vector' integrator, i.e. defined for FE spaces
    using multiple copies of a scalar FE space. */
 class ElasticityIntegrator : public BilinearFormIntegrator
@@ -2998,8 +3003,8 @@ private:
 public:
   ElasticityIntegrator(Coefficient &l, Coefficient &m)
   { lambda = &l; mu = &m; }
-   /** With this constructor lambda = q_l * m and mu = q_m * m;
-       if dim * q_l + 2 * q_m = 0 then trace(sigma) = 0. */
+   /** With this constructor \f$\lambda = q_l * m\f$ and \f$\mu = q_m * m\f$
+       if \f$dim * q_l + 2 * q_m = 0\f$ then \f$\tr(\sigma) = 0\f$. */
   ElasticityIntegrator(Coefficient &m, double q_l, double q_m)
   { lambda = NULL; mu = &m; q_lambda = q_l; q_mu = q_m; }

@@ -3007,12 +3012,12 @@ public:
                                      ElementTransformation &,
                                      DenseMatrix &);

-   /** Compute the stress corresponding to the local displacement @a u and
+   /** Compute the stress corresponding to the local displacement @a \f$u\f$ and
       interpolate it at the nodes of the given @a fluxelem. Only the symmetric
       part of the stress is stored, so that the size of @a flux is equal to
       the number of DOFs in @a fluxelem times dim*(dim+1)/2. In 2D, the order
-       of the stress components is: s_xx, s_yy, s_xy. In 3D, it is: s_xx, s_yy,
-       s_zz, s_xy, s_xz, s_yz. In other words, @a flux is the local vector for
+       of the stress components is: \f$s_xx, s_yy, s_xy\f$. In 3D, it is: \f$s_xx, s_yy,
+       s_zz, s_xy, s_xz, s_yz\f$. In other words, @a flux is the local vector for
       a FE space with dim*(dim+1)/2 vector components, based on the finite
       element @a fluxelem. The integration rule is taken from @a fluxelem.
       @a ir exists to specific an alternative integration rule. */
@@ -3030,37 +3035,39 @@ public:
       dim*(dim+1)/2 vector components, based on the finite element @a fluxelem.
       The number of components, dim*(dim+1)/2 is such that it represents the
       symmetric part of the (symmetric) stress tensor. The order of the
-       components is: s_xx, s_yy, s_xy in 2D, and s_xx, s_yy, s_zz, s_xy, s_xz,
-       s_yz in 3D. */
+       components is: \f$s_xx, s_yy, s_xy\f$ in 2D, and \f$s_xx, s_yy, s_zz, s_xy, s_xz,
+       s_yz\f$ in 3D. */
   virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                    ElementTransformation &Trans,
                                    Vector &flux, Vector *d_energy = NULL);
 };

 /** Integrator for the DG form:
-    alpha < rho_u (u.n) {v},[w] > + beta < rho_u |u.n| [v],[w] >,
-    where v and w are the trial and test variables, respectively, and rho/u are
-    given scalar/vector coefficients. {v} represents the average value of v on
-    the face and [v] is the jump such that {v}=(v1+v2)/2 and [v]=(v1-v2) for the
-    face between elements 1 and 2. For boundary elements, v2=0. The vector
-    coefficient, u, is assumed to be continuous across the faces and when given
-    the scalar coefficient, rho, is assumed to be discontinuous. The integrator
-    uses the upwind value of rho, rho_u, which is value from the side into which
-    the vector coefficient, u, points.
+    \f[
+      \alpha \langle \rho_u (u \cdot n) \{v\},[w] \rangle + \beta \langle \rho_u |u \cdot n| [v],[w] \rangle,
+    \f]
+    where \f$v\f$ and \f$w\f$ are the trial and test variables, respectively, and \f$\rho\f$/\f$u\f$ are
+    given scalar/vector coefficients. \f$\{v\}\f$ represents the average value of \f$v\f$ on
+    the face and \f$[v]\f$ is the jump such that \f$\{v\}=(v_1+v_2)/2\f$ and \f$[v]=(v_1-v_2)\f$ for the
+    face between elements \f$1\f$ and \f$2\f$. For boundary elements, \f$v2=0\f$. The vector
+    coefficient, \f$u\f$, is assumed to be continuous across the faces and when given
+    the scalar coefficient, \f$\rho\f$, is assumed to be discontinuous. The integrator
+    uses the upwind value of \f$\rho\f$, denoted by \f$\rho_u\f$, which is value from the side into which
+    the vector coefficient, \f$u\f$, points.

-    One use case for this integrator is to discretize the operator -u.grad(v)
+    One use case for this integrator is to discretize the operator \f$-u \cdot \nabla v\f$
    with a DG formulation. The resulting formulation uses the
-    ConvectionIntegrator (with coefficient u, and parameter alpha = -1) and the
-    transpose of the DGTraceIntegrator (with coefficient u, and parameters alpha
-    = 1, beta = -1/2 to use the upwind face flux, see also
+    ConvectionIntegrator (with coefficient \f$u\f$, and parameter \f$\alpha = -1\f$) and the
+    transpose of the DGTraceIntegrator (with coefficient \f$u\f$, and parameters \f$\alpha
+    = 1\f$, \f$\beta = -1/2\f$ to use the upwind face flux, see also
    NonconservativeDGTraceIntegrator). This discretization and the handling of
    the inflow and outflow boundaries is illustrated in Example 9/9p.

-    Another use case for this integrator is to discretize the operator -div(u v)
+    Another use case for this integrator is to discretize the operator \f$-\mathrm{div}(u v)\f$
    with a DG formulation. The resulting formulation is conservative and
-    consists of the ConservativeConvectionIntegrator (with coefficient u, and
-    parameter alpha = -1) plus the DGTraceIntegrator (with coefficient u, and
-    parameters alpha = -1, beta = -1/2 to use the upwind face flux).
+    consists of the ConservativeConvectionIntegrator (with coefficient \f$u\f$, and
+    parameter \f$\alpha = -1\f$) plus the DGTraceIntegrator (with coefficient \f$u\f$, and
+    parameters \f$\alpha = -1\f$, \f$\beta = -1/2\f$ to use the upwind face flux).
    */
 class DGTraceIntegrator : public BilinearFormIntegrator
 {
@@ -3078,11 +3085,11 @@ private:
   Vector shape1, shape2;

 public:
-   /// Construct integrator with rho = 1, b = 0.5*a.
+   /// Construct integrator with \f$\rho = 1\f$, \f$\beta = \alpha/2\f$.
   DGTraceIntegrator(VectorCoefficient &u_, double a)
   { rho = NULL; u = &u_; alpha = a; beta = 0.5*a; }

-   /// Construct integrator with rho = 1.
+   /// Construct integrator with \f$\rho = 1\f$.
   DGTraceIntegrator(VectorCoefficient &u_, double a, double b)
   { rho = NULL; u = &u_; alpha = a; beta = b; }

@@ -3127,8 +3134,9 @@ using ConservativeDGTraceIntegrator = DGTraceIntegrator;

 /** Integrator that represents the face terms used for the non-conservative
    DG discretization of the convection equation:
-    -alpha < rho_u (u.n) {v},[w] > + beta < rho_u |u.n| [v],[w] >.
-
+    \f[
+      -\alpha \langle \rho_u (u \cdot n) \{v\},[w] \rangle + \beta \langle \rho_u |u \cdot n| [v],[w] \rangle.
+    \f]
    This integrator can be used with together with ConvectionIntegrator to
    implement an upwind DG discretization in non-conservative form, see ex9 and
    ex9p. */
@@ -3147,17 +3155,19 @@ public:
 };

 /** Integrator for the DG form:
-
-        - < {(Q grad(u)).n}, [v] > + sigma < [u], {(Q grad(v)).n} >
-        + kappa < {h^{-1} Q} [u], [v] >
-
-    where Q is a scalar or matrix diffusion coefficient and u, v are the trial
-    and test spaces, respectively. The parameters sigma and kappa determine the
+    \f[
+        - \langle \{(Q \nabla u) \cdot n\}, [v] \rangle + \sigma \langle [u], \{(Q \nabla v) \cdot n \} \rangle
+        + \kappa \langle \{h^{-1} Q\} [u], [v] \rangle
+    \f]
+    where \f$Q\f$ is a scalar or matrix diffusion coefficient and \f$u\f$, \f$v\f$ are the trial
+    and test spaces, respectively. The parameters \f$\sigma\f$ and \f$\kappa\f$ determine the
    DG method to be used (when this integrator is added to the "broken"
    DiffusionIntegrator):
-    * sigma = -1, kappa >= kappa0: symm. interior penalty (IP or SIPG) method,
-    * sigma = +1, kappa > 0: non-symmetric interior penalty (NIPG) method,
-    * sigma = +1, kappa = 0: the method of Baumann and Oden. */
+    - \f$\sigma = -1\f$, \f$\kappa \geq \kappa_0\f$: symm. interior penalty (IP or SIPG) method,
+    - \f$\sigma = +1\f$, \f$\kappa > 0\f$: non-symmetric interior penalty (NIPG) method,
+    - \f$\sigma = +1\f$, \f$\kappa = 0\f$: the method of Baumann and Oden.
+
+    \todo Clarify used notation. */
 class DGDiffusionIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -3184,11 +3194,11 @@ public:
 };

 /** Integrator for the "BR2" diffusion stabilization term
-
-    sum_e eta (r_e([u]), r_e([v]))
-
-    where r_e is the lifting operator defined on each edge e (potentially
-    weighted by a coefficient Q). The parameter eta can be chosen to be one to
+    \f[
+      \sum_e \eta (r_e([u]), r_e([v]))
+    \f]
+    where \f$r_e\f$ is the lifting operator defined on each edge \f$e\f$ (potentially
+    weighted by a coefficient \f$Q\f$). The parameter eta can be chosen to be one to
    obtain a stable discretization. The constructor for this integrator requires
    the finite element space because the lifting operator depends on the
    element-wise inverse mass matrix.
@@ -3275,8 +3285,8 @@ public:
    \begin{split}
    \sigma(u) &= \lambda \nabla \cdot u I + 2 \mu \varepsilon(u) \\
              &= \lambda \nabla \cdot u I + 2 \mu \frac{1}{2} (\nabla u + \nabla
-                 u^T) \\
-              &= \lambda \nabla \cdot u I + \mu (\nabla u + \nabla u^T)
+                 u^{\mathrm{T}}) \\
+              &= \lambda \nabla \cdot u I + \mu (\nabla u + \nabla u^{\mathrm{T}})
    \end{split}
    \f]

@@ -3353,8 +3363,8 @@ protected:
      DenseMatrix &elmat, DenseMatrix &jmat);
 };

-/** Integrator for the DPG form: < v, [w] > over all faces (the interface) where
-    the trial variable v is defined on the interface and the test variable w is
+/** Integrator for the DPG form:\f$ \langle v, [w] \rangle \f$ over all faces (the interface) where
+    the trial variable \f$v\f$ is defined on the interface and the test variable \f$w\f$ is
    defined inside the elements, generally in a DG space. */
 class TraceJumpIntegrator : public BilinearFormIntegrator
 {
@@ -3371,9 +3381,9 @@ public:
                                   DenseMatrix &elmat);
 };

-/** Integrator for the form: < v, [w.n] > over all faces (the interface) where
-    the trial variable v is defined on the interface and the test variable w is
-    in an H(div)-conforming space. */
+/** Integrator for the form:\f$ \langle v, [w \cdot n] \rangle \f$ over all faces (the interface) where
+    the trial variable \f$v\f$ is defined on the interface and the test variable \f$w\f$ is
+    in an \f$H\f$(div)-conforming space. */
 class NormalTraceJumpIntegrator : public BilinearFormIntegrator
 {
 private:
@@ -3390,10 +3400,10 @@ public:
                                   DenseMatrix &elmat);
 };

-/** Integrator for the DPG form: < v, w > over a face (the interface) where
-    the trial variable v is defined on the interface
-    (H^-1/2 i.e., v:=u⋅n normal trace of H(div))
-    and the test variable w is in an H1-conforming space. */
+/** Integrator for the DPG form:\f$ \langle v, w \rangle \f$ over a face (the interface) where
+    the trial variable \f$v\f$ is defined on the interface
+    (\f$H^{-1/2}\f$ i.e., \f$v := u \cdot n\f$ normal trace of \f$H\f$(div))
+    and the test variable \f$w\f$ is in an \f$H^1\f$-conforming space. */
 class TraceIntegrator : public BilinearFormIntegrator
 {
 private:
@@ -3407,9 +3417,9 @@ public:
                                DenseMatrix &elmat);
 };

-/** Integrator for the form: < v, w.n > over a face (the interface) where
-    the trial variable v is defined on the interface (H^1/2, i.e., trace of H1)
-    and the test variable w is in an H(div)-conforming space. */
+/** Integrator for the form: \f$ \langle v, w \cdot n \rangle \f$ over a face (the interface) where
+    the trial variable \f$v\f$ is defined on the interface (\f$H^{1/2}\f$, i.e., trace of \f$H^1\f$)
+    and the test variable \f$w\f$ is in an \f$H\f$(div)-conforming space. */
 class NormalTraceIntegrator : public BilinearFormIntegrator
 {
 private:
@@ -3426,10 +3436,10 @@ public:
 };


-/** Integrator for the form: < v, w × n > over a face (the interface)
- *  In 3D the trial variable v is defined on the interface (H^-1/2(curl), trace of H(curl))
- *  In 2D it's defined on the interface (H^1/2, trace of H1)
- *  The test variable w is in an H(curl)-conforming space. */
+/** Integrator for the form: \f$\langle v, w \times n \rangle\f$ over a face (the interface)
+ *  In 3D the trial variable \f$v\f$ is defined on the interface (\f$H^{-1/2}\f$(curl), trace of \f$H\f$(curl))
+ *  In 2D it's defined on the interface (\f$H^{1/2}\f$, trace of \f$H^1\f$)
+ *  The test variable \f$w\f$ is in an \f$H\f$(curl)-conforming space. */
 class TangentTraceIntegrator : public BilinearFormIntegrator
 {
 private:
@@ -3477,8 +3487,8 @@ class DiscreteInterpolator : public BilinearFormIntegrator { };


 /** Class for constructing the gradient as a DiscreteLinearOperator from an
-    H1-conforming space to an H(curl)-conforming space. The range space can be
-    vector L2 space as well. */
+    \f$H^1\f$-conforming space to an \f$H\f$(curl)-conforming space. The range space can be
+    vector \f$L_2\f$ space as well. */
 class GradientInterpolator : public DiscreteInterpolator
 {
 public:
@@ -3495,8 +3505,8 @@ public:

   /** @brief Setup method for PA data.

-       @param[in] trial_fes   H1 Lagrange space
-       @param[in] test_fes    H(curl) Nedelec space
+       @param[in] trial_fes   \f$H^1\f$ Lagrange space
+       @param[in] test_fes    \f$H\f$(curl) Nedelec space
    */
   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
                           const FiniteElementSpace &test_fes);
@@ -3570,7 +3580,7 @@ public:
    the global discrete divergence matrix.

    Note: Since the dofs in the L2_FECollection are nodal values, the local
-    discrete divergence matrix (with an RT-type domain space) will depend on
+    discrete divergence matrix (with an \f$H\f$(div)-type domain space) will depend on
    the transformation. On the other hand, the local matrix returned by
    VectorFEDivergenceIntegrator is independent of the transformation. */
 class DivergenceInterpolator : public DiscreteInterpolator
@@ -3585,8 +3595,8 @@ public:


 /** A trace face interpolator class for interpolating the normal component of
-    the domain space, e.g. vector H1, into the range space, e.g. the trace of
-    RT which uses FiniteElement::INTEGRAL map type. */
+    the domain space, e.g. vector \f$H^1\f$, into the range space, e.g. the trace of
+    \f$H\f$(div) which uses FiniteElement::INTEGRAL map type. */
 class NormalInterpolator : public DiscreteInterpolator
 {
 public:
@@ -3648,7 +3658,7 @@ protected:
 };

 /** Interpolator of the 2D cross product between a vector coefficient and an
-    H(curl)-conforming field onto an L2-conforming field. */
+    \f$H\f$(curl)-conforming field onto an \f$L_2\f$-conforming field. */
 class ScalarCrossProductInterpolator : public DiscreteInterpolator
 {
 public:
@@ -3664,8 +3674,8 @@ protected:
 };

 /** Interpolator of the cross product between a vector coefficient and an
-    H(curl)-conforming field onto an H(div)-conforming field. The range space
-    can also be vector L2. */
+    \f$H\f$(curl)-conforming field onto an \f$H\f$(div)-conforming field. The range space
+    can also be vector \f$L_2\f$. */
 class VectorCrossProductInterpolator : public DiscreteInterpolator
 {
 public:
@@ -3681,8 +3691,8 @@ protected:
 };

 /** Interpolator of the inner product between a vector coefficient and an
-    H(div)-conforming field onto an L2-conforming field. The range space can
-    also be H1. */
+    \f$H\f$(div)-conforming field onto an \f$L_2\f$-conforming field. The range space can
+    also be \f$H^1\f$. */
 class VectorInnerProductInterpolator : public DiscreteInterpolator
 {
 public:
@@ -12,10 +12,14 @@
 #ifndef MFEM_LIBCEED_CEED
 #define MFEM_LIBCEED_CEED

+#include "../../../config/config.hpp"
+
 #ifdef MFEM_USE_CEED
+
 #include <ceed.h>
-#if !CEED_VERSION_GE(0,10,0)
-#error MFEM requires a libCEED version >= 0.10.0
+
+#if !CEED_VERSION_GE(0,12,0)
+#error MFEM requires a libCEED version >= 0.12.0
 #endif

 namespace mfem
@@ -294,14 +294,14 @@ public:
                                nelem, nqpts, ncomp, strides,
                                &quadCoeff->restr);
         CeedOperatorSetField(build_oper, "coeff", quadCoeff->restr,
-                              CEED_BASIS_COLLOCATED, quadCoeff->coeffVector);
+                              CEED_BASIS_NONE, quadCoeff->coeffVector);
      }
      CeedOperatorSetField(build_oper, "dx", mesh_restr,
                           mesh_basis, CEED_VECTOR_ACTIVE);
      CeedOperatorSetField(build_oper, "weights", CEED_ELEMRESTRICTION_NONE,
                           mesh_basis, CEED_VECTOR_NONE);
      CeedOperatorSetField(build_oper, "qdata", restr_i,
-                           CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                           CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);

      // Compute the quadrature data for the operator.
      CeedOperatorApply(build_oper, node_coords, qdata, CEED_REQUEST_IMMEDIATE);
@@ -355,7 +355,7 @@ public:
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "u", trial_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "u", trial_restr, trial_basis, CEED_VECTOR_ACTIVE);
@@ -369,14 +369,14 @@ public:
            break;
      }
      // qdata
-      CeedOperatorSetField(oper, "qdata", restr_i, CEED_BASIS_COLLOCATED,
+      CeedOperatorSetField(oper, "qdata", restr_i, CEED_BASIS_NONE,
                           qdata);
      // output
      switch (op.test_op)
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "v", test_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "v", test_restr, test_basis, CEED_VECTOR_ACTIVE);
@@ -685,14 +685,14 @@ public:
                                nelem, nqpts, ncomp, strides,
                                &quadCoeff->restr);
         CeedOperatorSetField(oper, "coeff", quadCoeff->restr,
-                              CEED_BASIS_COLLOCATED, quadCoeff->coeffVector);
+                              CEED_BASIS_NONE, quadCoeff->coeffVector);
      }
      // input
      switch (op.trial_op)
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "u", trial_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "u", trial_restr, trial_basis,
@@ -718,7 +718,7 @@ public:
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "v", test_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "v", test_restr, test_basis,
@@ -132,24 +132,24 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedOperatorGetCeed(oper, &ceed); CeedChk(ierr);
+   ierr = CeedOperatorGetCeed(oper, &ceed); PCeedChk(ierr);

   CeedQFunction qf;
   bool isComposite;
-   ierr = CeedOperatorIsComposite(oper, &isComposite); CeedChk(ierr);
+   ierr = CeedOperatorIsComposite(oper, &isComposite); PCeedChk(ierr);
   CeedOperator *subops;
   if (isComposite)
   {
 #if CEED_VERSION_GE(0, 10, 2)
-      ierr = CeedCompositeOperatorGetSubList(oper, &subops); CeedChk(ierr);
+      ierr = CeedCompositeOperatorGetSubList(oper, &subops); PCeedChk(ierr);
 #else
-      ierr = CeedOperatorGetSubList(oper, &subops); CeedChk(ierr);
+      ierr = CeedOperatorGetSubList(oper, &subops); PCeedChk(ierr);
 #endif
-      ierr = CeedOperatorGetQFunction(subops[0], &qf); CeedChk(ierr);
+      ierr = CeedOperatorGetQFunction(subops[0], &qf); PCeedChk(ierr);
   }
   else
   {
-      ierr = CeedOperatorGetQFunction(oper, &qf); CeedChk(ierr);
+      ierr = CeedOperatorGetQFunction(oper, &qf); PCeedChk(ierr);
   }
   CeedInt numinputfields, numoutputfields;
   ierr = CeedQFunctionGetNumArgs(qf, &numinputfields, &numoutputfields);
@@ -157,12 +157,12 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
   if (isComposite)
   {
      ierr = CeedOperatorGetFields(subops[0], &numinputfields, &inputfields,
-                                   &numoutputfields, NULL); CeedChk(ierr);
+                                   &numoutputfields, NULL); PCeedChk(ierr);
   }
   else
   {
      ierr = CeedOperatorGetFields(oper, &numinputfields, &inputfields,
-                                   &numoutputfields, NULL); CeedChk(ierr);
+                                   &numoutputfields, NULL); PCeedChk(ierr);
   }

   CeedVector if_vector;
@@ -170,7 +170,7 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
   int found_index = -1;
   for (int i = 0; i < numinputfields; ++i)
   {
-      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector); CeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector); PCeedChk(ierr);
      if (if_vector == CEED_VECTOR_ACTIVE)
      {
         if (found)
@@ -13,12 +13,15 @@
 #define MFEM_LIBCEED_UTIL

 #include "../../../config/config.hpp"
+#include "../../../general/error.hpp"
+
 #include <functional>
 #include <string>
 #include <tuple>
 #include <unordered_map>

 #include "ceed.hpp"
+
 #ifdef MFEM_USE_CEED
 #include <ceed/backend.h>  // for CeedOperatorField
 #endif
@@ -170,10 +173,12 @@ namespace internal
 {

 #ifdef MFEM_USE_CEED
+
 /** @warning These maps have a tendency to create bugs when adding new "types"
    of CeedBasis and CeedElemRestriction. */
 extern ceed::BasisMap ceed_basis_map;
 extern ceed::RestrMap ceed_restr_map;
+
 #endif

 } // namespace internal
@@ -99,7 +99,7 @@ int CeedOperatorGetSize(CeedOperator oper, CeedInt * size)
 {
   CeedSize in_len, out_len;
   int ierr = CeedOperatorGetActiveVectorLengths(oper, &in_len, &out_len);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   *size = (CeedInt)in_len;
   MFEM_VERIFY(in_len == out_len, "not a square CeedOperator");
   MFEM_VERIFY(in_len == *size, "size overflow");
@@ -378,67 +378,68 @@ int AlgebraicInterpolation::Initialize(

   CeedSize height, width;
   ierr = CeedElemRestrictionGetLVectorSize(erestrictu_coarse, &width);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   ierr = CeedElemRestrictionGetLVectorSize(erestrictu_fine, &height);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // interpolation qfunction
   const int bp3_ncompu = 1;
   CeedQFunction l_qf_restrict, l_qf_prolong;
   ierr = CeedQFunctionCreateIdentity(ceed, bp3_ncompu, CEED_EVAL_NONE,
-                                      CEED_EVAL_INTERP, &l_qf_restrict); CeedChk(ierr);
+                                      CEED_EVAL_INTERP, &l_qf_restrict); PCeedChk(ierr);
   ierr = CeedQFunctionCreateIdentity(ceed, bp3_ncompu, CEED_EVAL_INTERP,
-                                      CEED_EVAL_NONE, &l_qf_prolong); CeedChk(ierr);
+                                      CEED_EVAL_NONE, &l_qf_prolong); PCeedChk(ierr);

   qf_restrict = l_qf_restrict;
   qf_prolong = l_qf_prolong;

   CeedVector c_fine_multiplicity;
-   ierr = CeedVectorCreate(ceed, height, &c_fine_multiplicity); CeedChk(ierr);
-   ierr = CeedVectorSetValue(c_fine_multiplicity, 0.0); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &c_fine_multiplicity); PCeedChk(ierr);
+   ierr = CeedVectorSetValue(c_fine_multiplicity, 0.0); PCeedChk(ierr);

   // Create the restriction operator
   // Restriction - Fine to coarse
   ierr = CeedOperatorCreate(ceed, qf_restrict, CEED_QFUNCTION_NONE,
-                             CEED_QFUNCTION_NONE, &op_restrict); CeedChk(ierr);
+                             CEED_QFUNCTION_NONE, &op_restrict); PCeedChk(ierr);
   ierr = CeedOperatorSetField(op_restrict, "input", erestrictu_fine,
-                               CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                               CEED_BASIS_NONE, CEED_VECTOR_ACTIVE); PCeedChk(ierr);
   ierr = CeedOperatorSetField(op_restrict, "output", erestrictu_coarse,
-                               basisctof, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                               basisctof, CEED_VECTOR_ACTIVE); PCeedChk(ierr);

   // Interpolation - Coarse to fine
   // Create the prolongation operator
   ierr =  CeedOperatorCreate(ceed, qf_prolong, CEED_QFUNCTION_NONE,
-                              CEED_QFUNCTION_NONE, &op_interp); CeedChk(ierr);
+                              CEED_QFUNCTION_NONE, &op_interp); PCeedChk(ierr);
   ierr =  CeedOperatorSetField(op_interp, "input", erestrictu_coarse,
-                                basisctof, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                                basisctof, CEED_VECTOR_ACTIVE); PCeedChk(ierr);
   ierr = CeedOperatorSetField(op_interp, "output", erestrictu_fine,
-                               CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                               CEED_BASIS_NONE, CEED_VECTOR_ACTIVE); PCeedChk(ierr);

   ierr = CeedElemRestrictionGetMultiplicity(erestrictu_fine,
-                                             c_fine_multiplicity); CeedChk(ierr);
-   ierr = CeedVectorCreate(ceed, height, &fine_multiplicity_r); CeedChk(ierr);
+                                             c_fine_multiplicity); PCeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &fine_multiplicity_r); PCeedChk(ierr);

   CeedScalar* fine_r_data;
   const CeedScalar* fine_data;
   ierr = CeedVectorGetArrayWrite(fine_multiplicity_r, CEED_MEM_HOST,
-                                  &fine_r_data); CeedChk(ierr);
+                                  &fine_r_data); PCeedChk(ierr);
   ierr = CeedVectorGetArrayRead(c_fine_multiplicity, CEED_MEM_HOST,
-                                 &fine_data); CeedChk(ierr);
+                                 &fine_data); PCeedChk(ierr);
   for (CeedSize i = 0; i < height; ++i)
   {
      fine_r_data[i] = 1.0 / fine_data[i];
   }

-   ierr = CeedVectorRestoreArray(fine_multiplicity_r, &fine_r_data); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(fine_multiplicity_r, &fine_r_data);
+   PCeedChk(ierr);
   ierr = CeedVectorRestoreArrayRead(c_fine_multiplicity, &fine_data);
-   CeedChk(ierr);
-   ierr = CeedVectorDestroy(&c_fine_multiplicity); CeedChk(ierr);
+   PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&c_fine_multiplicity); PCeedChk(ierr);

-   ierr = CeedVectorCreate(ceed, height, &fine_work); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &fine_work); PCeedChk(ierr);

-   ierr = CeedVectorCreate(ceed, height, &v_); CeedChk(ierr);
-   ierr = CeedVectorCreate(ceed, width, &u_); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &v_); PCeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, width, &u_); PCeedChk(ierr);

   return 0;
 }
@@ -447,12 +448,12 @@ int AlgebraicInterpolation::Finalize()
 {
   int ierr;

-   ierr = CeedQFunctionDestroy(&qf_restrict); CeedChk(ierr);
-   ierr = CeedQFunctionDestroy(&qf_prolong); CeedChk(ierr);
-   ierr = CeedOperatorDestroy(&op_interp); CeedChk(ierr);
-   ierr = CeedOperatorDestroy(&op_restrict); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&fine_multiplicity_r); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&fine_work); CeedChk(ierr);
+   ierr = CeedQFunctionDestroy(&qf_restrict); PCeedChk(ierr);
+   ierr = CeedQFunctionDestroy(&qf_prolong); PCeedChk(ierr);
+   ierr = CeedOperatorDestroy(&op_interp); PCeedChk(ierr);
+   ierr = CeedOperatorDestroy(&op_restrict); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&fine_multiplicity_r); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&fine_work); PCeedChk(ierr);

   return 0;
 }
@@ -498,8 +499,8 @@ int CeedVectorPointwiseMult(CeedVector a, const CeedVector b)
   CeedVectorGetCeed(a, &ceed);

   CeedSize length, length2;
-   ierr = CeedVectorGetLength(a, &length); CeedChk(ierr);
-   ierr = CeedVectorGetLength(b, &length2); CeedChk(ierr);
+   ierr = CeedVectorGetLength(a, &length); PCeedChk(ierr);
+   ierr = CeedVectorGetLength(b, &length2); PCeedChk(ierr);
   if (length != length2)
   {
      return CeedError(ceed, 1, "Vector sizes don't match");
@@ -516,14 +517,14 @@ int CeedVectorPointwiseMult(CeedVector a, const CeedVector b)
   }
   CeedScalar *a_data;
   const CeedScalar *b_data;
-   ierr = CeedVectorGetArray(a, mem, &a_data); CeedChk(ierr);
-   ierr = CeedVectorGetArrayRead(b, mem, &b_data); CeedChk(ierr);
+   ierr = CeedVectorGetArray(a, mem, &a_data); PCeedChk(ierr);
+   ierr = CeedVectorGetArrayRead(b, mem, &b_data); PCeedChk(ierr);
   MFEM_VERIFY(int(length) == length, "length overflow");
   mfem::forall(length, [=] MFEM_HOST_DEVICE (int i)
   {a_data[i] *= b_data[i];});

-   ierr = CeedVectorRestoreArray(a, &a_data); CeedChk(ierr);
-   ierr = CeedVectorRestoreArrayRead(b, &b_data); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(a, &a_data); PCeedChk(ierr);
+   ierr = CeedVectorRestoreArrayRead(b, &b_data); PCeedChk(ierr);

   return 0;
 }
@@ -11,12 +11,12 @@

 #include "full-assembly.hpp"

+#ifdef MFEM_USE_CEED
+
 #include "../../../linalg/sparsemat.hpp"
 #include "../interface/util.hpp"
 #include "../interface/ceed.hpp"

-#ifdef MFEM_USE_CEED
-
 namespace mfem
 {

@@ -45,32 +45,32 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
+   ierr = CeedOperatorGetCeed(op, &ceed); PCeedChk(ierr);

   // Assemble QFunction
   CeedQFunction qf;
-   ierr = CeedOperatorGetQFunction(op, &qf); CeedChk(ierr);
+   ierr = CeedOperatorGetQFunction(op, &qf); PCeedChk(ierr);
   CeedInt numinputfields, numoutputfields;
-   CeedChk(ierr);
+   PCeedChk(ierr);
   CeedVector assembledqf;
   CeedElemRestriction rstr_q;
   ierr = CeedOperatorLinearAssembleQFunction(
-             op, &assembledqf, &rstr_q, CEED_REQUEST_IMMEDIATE); CeedChk(ierr);
+             op, &assembledqf, &rstr_q, CEED_REQUEST_IMMEDIATE); PCeedChk(ierr);

   CeedSize qflength;
-   ierr = CeedVectorGetLength(assembledqf, &qflength); CeedChk(ierr);
+   ierr = CeedVectorGetLength(assembledqf, &qflength); PCeedChk(ierr);

   CeedOperatorField *input_fields;
   CeedOperatorField *output_fields;
   ierr = CeedOperatorGetFields(op, &numinputfields, &input_fields,
                                &numoutputfields, &output_fields);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // Determine active input basis
   CeedQFunctionField *qffields;
   ierr = CeedQFunctionGetFields(qf, &numinputfields, &qffields,
                                 &numoutputfields, NULL);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   CeedInt numemodein = 0, ncomp, dim = 1;
   CeedEvalMode *emodein = NULL;
   CeedBasis basisin = NULL;
@@ -78,28 +78,28 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
   for (CeedInt i=0; i<numinputfields; i++)
   {
      CeedVector vec;
-      ierr = CeedOperatorFieldGetVector(input_fields[i], &vec); CeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(input_fields[i], &vec); PCeedChk(ierr);
      if (vec == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorFieldGetBasis(input_fields[i], &basisin);
-         CeedChk(ierr);
-         ierr = CeedBasisGetNumComponents(basisin, &ncomp); CeedChk(ierr);
-         ierr = CeedBasisGetDimension(basisin, &dim); CeedChk(ierr);
+         PCeedChk(ierr);
+         ierr = CeedBasisGetNumComponents(basisin, &ncomp); PCeedChk(ierr);
+         ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
         ierr = CeedOperatorFieldGetElemRestriction(input_fields[i], &rstrin);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         CeedEvalMode emode;
         ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         switch (emode)
         {
            case CEED_EVAL_NONE:
            case CEED_EVAL_INTERP:
-               ierr = CeedHackRealloc(numemodein + 1, &emodein); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodein + 1, &emodein); PCeedChk(ierr);
               emodein[numemodein] = emode;
               numemodein += 1;
               break;
            case CEED_EVAL_GRAD:
-               ierr = CeedHackRealloc(numemodein + dim, &emodein); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodein + dim, &emodein); PCeedChk(ierr);
               for (CeedInt d=0; d<dim; d++)
               {
                  emodein[numemodein+d] = emode;
@@ -116,7 +116,7 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)

   // Determine active output basis
   ierr = CeedQFunctionGetFields(qf, &numinputfields, NULL, &numoutputfields,
-                                 &qffields); CeedChk(ierr);
+                                 &qffields); PCeedChk(ierr);
   CeedInt numemodeout = 0;
   CeedEvalMode *emodeout = NULL;
   CeedBasis basisout = NULL;
@@ -124,27 +124,27 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
   for (CeedInt i=0; i<numoutputfields; i++)
   {
      CeedVector vec;
-      ierr = CeedOperatorFieldGetVector(output_fields[i], &vec); CeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(output_fields[i], &vec); PCeedChk(ierr);
      if (vec == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorFieldGetBasis(output_fields[i], &basisout);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         ierr = CeedOperatorFieldGetElemRestriction(output_fields[i], &rstrout);
-         CeedChk(ierr);
-         CeedChk(ierr);
+         PCeedChk(ierr);
+         PCeedChk(ierr);
         CeedEvalMode emode;
         ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         switch (emode)
         {
            case CEED_EVAL_NONE:
            case CEED_EVAL_INTERP:
-               ierr = CeedHackRealloc(numemodeout + 1, &emodeout); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodeout + 1, &emodeout); PCeedChk(ierr);
               emodeout[numemodeout] = emode;
               numemodeout += 1;
               break;
            case CEED_EVAL_GRAD:
-               ierr = CeedHackRealloc(numemodeout + dim, &emodeout); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodeout + dim, &emodeout); PCeedChk(ierr);
               for (CeedInt d=0; d<dim; d++)
               {
                  emodeout[numemodeout+d] = emode;
@@ -161,47 +161,47 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)

   CeedInt nelem, elemsize, nqpts;
   CeedSize nnodes;
-   ierr = CeedElemRestrictionGetNumElements(rstrin, &nelem); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetElementSize(rstrin, &elemsize); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetLVectorSize(rstrin, &nnodes); CeedChk(ierr);
-   ierr = CeedBasisGetNumQuadraturePoints(basisin, &nqpts); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetNumElements(rstrin, &nelem); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetElementSize(rstrin, &elemsize); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetLVectorSize(rstrin, &nnodes); PCeedChk(ierr);
+   ierr = CeedBasisGetNumQuadraturePoints(basisin, &nqpts); PCeedChk(ierr);

   // Determine elem_dof relation
   CeedVector index_vec;
-   ierr = CeedVectorCreate(ceed, nnodes, &index_vec); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, nnodes, &index_vec); PCeedChk(ierr);
   CeedScalar *array;
   ierr = CeedVectorGetArrayWrite(index_vec, CEED_MEM_HOST, &array);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   for (CeedSize i = 0; i < nnodes; ++i)
   {
      array[i] = i;
   }
-   ierr = CeedVectorRestoreArray(index_vec, &array); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(index_vec, &array); PCeedChk(ierr);
   CeedVector elem_dof;
-   ierr = CeedVectorCreate(ceed, nelem * elemsize, &elem_dof); CeedChk(ierr);
-   ierr = CeedVectorSetValue(elem_dof, 0.0); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, nelem * elemsize, &elem_dof); PCeedChk(ierr);
+   ierr = CeedVectorSetValue(elem_dof, 0.0); PCeedChk(ierr);
   CeedElemRestrictionApply(rstrin, CEED_NOTRANSPOSE, index_vec,
-                            elem_dof, CEED_REQUEST_IMMEDIATE); CeedChk(ierr);
+                            elem_dof, CEED_REQUEST_IMMEDIATE); PCeedChk(ierr);
   const CeedScalar * elem_dof_a;
   ierr = CeedVectorGetArrayRead(elem_dof, CEED_MEM_HOST, &elem_dof_a);
-   CeedChk(ierr);
-   ierr = CeedVectorDestroy(&index_vec); CeedChk(ierr);
+   PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&index_vec); PCeedChk(ierr);

   // loop over elements and put in SparseMatrix
   // SparseMatrix * out = new SparseMatrix(nnodes, nnodes);
   MFEM_ASSERT(out->Height() == nnodes, "Sizes don't match!");
   MFEM_ASSERT(out->Width() == nnodes, "Sizes don't match!");
   const CeedScalar *interpin, *gradin;
-   ierr = CeedBasisGetInterp(basisin, &interpin); CeedChk(ierr);
-   ierr = CeedBasisGetGrad(basisin, &gradin); CeedChk(ierr);
+   ierr = CeedBasisGetInterp(basisin, &interpin); PCeedChk(ierr);
+   ierr = CeedBasisGetGrad(basisin, &gradin); PCeedChk(ierr);

   const CeedScalar * assembledqfarray;
   ierr = CeedVectorGetArrayRead(assembledqf, CEED_MEM_HOST, &assembledqfarray);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   CeedInt layout[3];
-   ierr = CeedElemRestrictionGetELayout(rstr_q, &layout); CeedChk(ierr);
-   ierr = CeedElemRestrictionDestroy(&rstr_q); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetELayout(rstr_q, &layout); PCeedChk(ierr);
+   ierr = CeedElemRestrictionDestroy(&rstr_q); PCeedChk(ierr);

   // enforce structurally symmetric for later elimination
   const int skip_zeros = 0;
@@ -280,13 +280,13 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
      out->AddSubMatrix(rows, rows, elem_mat, skip_zeros);
   }

-   ierr = CeedVectorRestoreArrayRead(elem_dof, &elem_dof_a); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&elem_dof); CeedChk(ierr);
+   ierr = CeedVectorRestoreArrayRead(elem_dof, &elem_dof_a); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&elem_dof); PCeedChk(ierr);
   ierr = CeedVectorRestoreArrayRead(assembledqf, &assembledqfarray);
-   CeedChk(ierr);
-   ierr = CeedVectorDestroy(&assembledqf); CeedChk(ierr);
-   ierr = CeedHackFree(&emodein); CeedChk(ierr);
-   ierr = CeedHackFree(&emodeout); CeedChk(ierr);
+   PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&assembledqf); PCeedChk(ierr);
+   ierr = CeedHackFree(&emodein); PCeedChk(ierr);
+   ierr = CeedHackFree(&emodeout); PCeedChk(ierr);

   return 0;
 }
@@ -297,7 +297,7 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)

   CeedSize in_len, out_len;
   ierr = CeedOperatorGetActiveVectorLengths(op, &in_len, &out_len);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   const int nnodes = in_len;
   MFEM_VERIFY(in_len == out_len, "not a square CeedOperator");
   MFEM_VERIFY(in_len == nnodes, "size overflow");
@@ -305,26 +305,26 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)
   SparseMatrix *out = new SparseMatrix(nnodes, nnodes);

   bool isComposite;
-   ierr = CeedOperatorIsComposite(op, &isComposite); CeedChk(ierr);
+   ierr = CeedOperatorIsComposite(op, &isComposite); PCeedChk(ierr);
   if (isComposite)
   {
      CeedInt numsub;
      CeedOperator *subops;
 #if CEED_VERSION_GE(0, 10, 2)
      CeedCompositeOperatorGetNumSub(op, &numsub);
-      ierr = CeedCompositeOperatorGetSubList(op, &subops); CeedChk(ierr);
+      ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr);
 #else
      CeedOperatorGetNumSub(op, &numsub);
-      ierr = CeedOperatorGetSubList(op, &subops); CeedChk(ierr);
+      ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr);
 #endif
      for (int i = 0; i < numsub; ++i)
      {
-         ierr = CeedSingleOperatorFullAssemble(subops[i], out); CeedChk(ierr);
+         ierr = CeedSingleOperatorFullAssemble(subops[i], out); PCeedChk(ierr);
      }
   }
   else
   {
-      ierr = CeedSingleOperatorFullAssemble(op, out); CeedChk(ierr);
+      ierr = CeedSingleOperatorFullAssemble(op, out); PCeedChk(ierr);
   }
   // enforce structurally symmetric for later elimination
   const int skip_zeros = 0;
@@ -338,4 +338,4 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)

 } // namespace mfem

-#endif
+#endif // MFEM_USE_CEED
@@ -19,6 +19,8 @@
 namespace mfem
 {

+class SparseMatrix;
+
 namespace ceed
 {

@@ -34,6 +36,6 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat);

 } // namespace mfem

-#endif
+#endif // MFEM_USE_CEED

-#endif
+#endif // MFEM_CEED_ASSEMBLE_HPP
@@ -15,8 +15,8 @@
 #include "../interface/util.hpp"

 #ifdef MFEM_USE_CEED
-#include <ceed/backend.h>

+#include <ceed/backend.h>
 #include <math.h>
 // todo: should probably use Ceed memory wrappers instead of calloc/free?
 #include <stdlib.h>
@@ -86,14 +86,14 @@ int CeedATPMGElemRestriction(int order,
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedElemRestrictionGetCeed(er_in, &ceed); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetCeed(er_in, &ceed); PCeedChk(ierr);

   CeedInt numelem, numcomp, elemsize;
   CeedSize numnodes;
-   ierr = CeedElemRestrictionGetNumElements(er_in, &numelem); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetLVectorSize(er_in, &numnodes); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetElementSize(er_in, &elemsize); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetNumComponents(er_in, &numcomp); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetNumElements(er_in, &numelem); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetLVectorSize(er_in, &numnodes); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetElementSize(er_in, &elemsize); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetNumComponents(er_in, &numcomp); PCeedChk(ierr);
   if (numcomp != 1)
   {
      // todo: multi-component will require more thought
@@ -107,31 +107,31 @@ int CeedATPMGElemRestriction(int order,

   CeedVector in_lvec, in_evec;
   ierr = CeedElemRestrictionCreateVector(er_in, &in_lvec, &in_evec);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // Create the elem_dof array from the given high-order ElemRestriction
   // by using it to map the L-vector indices to an E-vector
   CeedScalar * lvec_data;
   ierr = CeedVectorGetArrayWrite(in_lvec, CEED_MEM_HOST, &lvec_data);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   for (CeedSize i = 0; i < numnodes; ++i)
   {
      lvec_data[i] = (CeedScalar) i;
   }
-   ierr = CeedVectorRestoreArray(in_lvec, &lvec_data); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(in_lvec, &lvec_data); PCeedChk(ierr);
   CeedInt in_layout[3];
-   ierr = CeedElemRestrictionGetELayout(er_in, &in_layout); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetELayout(er_in, &in_layout); PCeedChk(ierr);
   if (in_layout[0] == 0 && in_layout[1] == 0 && in_layout[2] == 0)
   {
      return CeedError(ceed, 1, "Cannot interpret e-vector ordering of given"
                       "CeedElemRestriction!");
   }
   ierr = CeedElemRestrictionApply(er_in, CEED_NOTRANSPOSE, in_lvec, in_evec,
-                                   CEED_REQUEST_IMMEDIATE); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&in_lvec); CeedChk(ierr);
+                                   CEED_REQUEST_IMMEDIATE); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&in_lvec); PCeedChk(ierr);
   const CeedScalar * in_elem_dof;
   ierr = CeedVectorGetArrayRead(in_evec, CEED_MEM_HOST, &in_elem_dof);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // Create a map (dof_map) that maps high-order ldof indices to
   // low-order ldof indices, with -1 indicating no correspondence
@@ -469,13 +469,13 @@ int CeedATPMGElemRestriction(int order,
                       "CeedATPMGElemRestriction does not yet support this dimension.");
   }

-   ierr = CeedVectorRestoreArrayRead(in_evec, &in_elem_dof); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&in_evec); CeedChk(ierr);
+   ierr = CeedVectorRestoreArrayRead(in_evec, &in_elem_dof); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&in_evec); PCeedChk(ierr);

   ierr = CeedElemRestrictionCreate(ceed, numelem, coarse_elemsize, numcomp,
                                    0, running_out_ldof_count,
                                    CEED_MEM_HOST, CEED_COPY_VALUES, out_elem_dof,
-                                    er_out); CeedChk(ierr);
+                                    er_out); PCeedChk(ierr);

   delete [] out_elem_dof;

@@ -491,7 +491,7 @@ int CeedBasisATPMGCoarseToFine(Ceed ceed, int P1d, int dim, int order_reduction,
   // calling the following Ceed function)
   int ierr;
   ierr = CeedBasisCreateTensorH1Lagrange(ceed, dim, 1, P1d - order_reduction, P1d,
-                                          CEED_GAUSS_LOBATTO, basisc2f); CeedChk(ierr);
+                                          CEED_GAUSS_LOBATTO, basisc2f); PCeedChk(ierr);
   return 0;
 }

@@ -501,13 +501,13 @@ int CeedBasisATPMGCoarseToFine(CeedBasis basisin,
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedBasisGetCeed(basisin, &ceed); CeedChk(ierr);
+   ierr = CeedBasisGetCeed(basisin, &ceed); PCeedChk(ierr);

   CeedInt dim, P1d;
-   ierr = CeedBasisGetDimension(basisin, &dim); CeedChk(ierr);
-   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); CeedChk(ierr);
+   ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
+   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); PCeedChk(ierr);
   ierr = CeedBasisATPMGCoarseToFine(ceed, P1d, dim, order_reduction,
-                                     basisc2f); CeedChk(ierr);
+                                     basisc2f); PCeedChk(ierr);
   return 0;
 }

@@ -518,20 +518,20 @@ int CeedBasisATPMGCoarsen(CeedBasis basisin,
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedBasisGetCeed(basisin, &ceed); CeedChk(ierr);
+   ierr = CeedBasisGetCeed(basisin, &ceed); PCeedChk(ierr);

   CeedInt dim, ncomp, P1d, Q1d;
-   ierr = CeedBasisGetDimension(basisin, &dim); CeedChk(ierr);
-   ierr = CeedBasisGetNumComponents(basisin, &ncomp); CeedChk(ierr);
-   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); CeedChk(ierr);
-   ierr = CeedBasisGetNumQuadraturePoints1D(basisin, &Q1d); CeedChk(ierr);
+   ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
+   ierr = CeedBasisGetNumComponents(basisin, &ncomp); PCeedChk(ierr);
+   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); PCeedChk(ierr);
+   ierr = CeedBasisGetNumQuadraturePoints1D(basisin, &Q1d); PCeedChk(ierr);

   CeedInt coarse_P1d = P1d - order_reduction;

   const CeedScalar *interp1d;
-   ierr = CeedBasisGetInterp1D(basisin, &interp1d); CeedChk(ierr);
+   ierr = CeedBasisGetInterp1D(basisin, &interp1d); PCeedChk(ierr);
   const CeedScalar * grad1d;
-   ierr = CeedBasisGetGrad1D(basisin, &grad1d); CeedChk(ierr);
+   ierr = CeedBasisGetGrad1D(basisin, &grad1d); PCeedChk(ierr);

   CeedScalar * coarse_interp1d = new CeedScalar[coarse_P1d * Q1d];
   CeedScalar * coarse_grad1d = new CeedScalar[coarse_P1d * Q1d];
@@ -542,14 +542,14 @@ int CeedBasisATPMGCoarsen(CeedBasis basisin,
   /* one way you might be able to tell is to just run this algorithm
      with coarse_P1d = 2 (i.e., linear) and look for symmetry in the coarse
      basis matrix? */
-   ierr = CeedLobattoQuadrature(P1d, fine_nodal_points, NULL); CeedChk(ierr);
+   ierr = CeedLobattoQuadrature(P1d, fine_nodal_points, NULL); PCeedChk(ierr);
   for (int i = 0; i < P1d; ++i)
   {
      fine_nodal_points[i] = 0.5 * fine_nodal_points[i] + 0.5; // cheating
   }

   const CeedScalar *interp_ctof;
-   ierr = CeedBasisGetInterp1D(basisc2f, &interp_ctof); CeedChk(ierr);
+   ierr = CeedBasisGetInterp1D(basisc2f, &interp_ctof); PCeedChk(ierr);

   for (int i = 0; i < Q1d; ++i)
   {
@@ -568,12 +568,12 @@ int CeedBasisATPMGCoarsen(CeedBasis basisin,
   }

   const CeedScalar * qref1d;
-   ierr = CeedBasisGetQRef(basisin, &qref1d); CeedChk(ierr);
+   ierr = CeedBasisGetQRef(basisin, &qref1d); PCeedChk(ierr);
   const CeedScalar * qweight1d;
-   ierr = CeedBasisGetQWeights(basisin, &qweight1d); CeedChk(ierr);
+   ierr = CeedBasisGetQWeights(basisin, &qweight1d); PCeedChk(ierr);
   ierr = CeedBasisCreateTensorH1(ceed, dim, ncomp,
                                  coarse_P1d, Q1d, coarse_interp1d, coarse_grad1d,
-                                  qref1d, qweight1d, basisout); CeedChk(ierr);
+                                  qref1d, qweight1d, basisout); PCeedChk(ierr);

   delete [] fine_nodal_points;
   delete [] coarse_interp1d;
@@ -593,19 +593,19 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,

   int ierr;
   Ceed ceed;
-   ierr = CeedOperatorGetCeed(oper, &ceed); CeedChk(ierr);
+   ierr = CeedOperatorGetCeed(oper, &ceed); PCeedChk(ierr);

   CeedQFunction qf;
-   ierr = CeedOperatorGetQFunction(oper, &qf); CeedChk(ierr);
+   ierr = CeedOperatorGetQFunction(oper, &qf); PCeedChk(ierr);
   CeedInt numinputfields, numoutputfields;
   CeedQFunctionField *inputqfields, *outputqfields;
   ierr = CeedQFunctionGetFields(qf, &numinputfields, &inputqfields,
                                 &numoutputfields, &outputqfields);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   CeedOperatorField *inputfields, *outputfields;
   ierr = CeedOperatorGetFields(oper, &numinputfields, &inputfields,
                                &numoutputfields, &outputfields);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   CeedElemRestriction * er_input = new CeedElemRestriction[numinputfields];
   CeedElemRestriction * er_output = new CeedElemRestriction[numoutputfields];
@@ -619,10 +619,11 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
   for (int i = 0; i < numinputfields; ++i)
   {
      ierr = CeedOperatorFieldGetElemRestriction(inputfields[i],
-                                                 &er_input[i]); CeedChk(ierr);
-      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector[i]); CeedChk(ierr);
+                                                 &er_input[i]); PCeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector[i]);
+      PCeedChk(ierr);
      ierr = CeedOperatorFieldGetBasis(inputfields[i], &basis_input[i]);
-      CeedChk(ierr);
+      PCeedChk(ierr);
      if (if_vector[i] == CEED_VECTOR_ACTIVE)
      {
         if (active_input_basis < 0)
@@ -638,11 +639,11 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
   for (int i = 0; i < numoutputfields; ++i)
   {
      ierr = CeedOperatorFieldGetElemRestriction(outputfields[i],
-                                                 &er_output[i]); CeedChk(ierr);
+                                                 &er_output[i]); PCeedChk(ierr);
      ierr = CeedOperatorFieldGetVector(outputfields[i], &of_vector[i]);
-      CeedChk(ierr);
+      PCeedChk(ierr);
      ierr = CeedOperatorFieldGetBasis(outputfields[i], &basis_output[i]);
-      CeedChk(ierr);
+      PCeedChk(ierr);
      if (of_vector[i] == CEED_VECTOR_ACTIVE)
      {
         // should already be coarsened
@@ -659,36 +660,36 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,

   CeedOperator coper;
   ierr = CeedOperatorCreate(ceed, qf, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE,
-                             &coper); CeedChk(ierr);
+                             &coper); PCeedChk(ierr);

   for (int i = 0; i < numinputfields; ++i)
   {
      char * fieldname;
-      ierr = CeedQFunctionFieldGetName(inputqfields[i], &fieldname); CeedChk(ierr);
+      ierr = CeedQFunctionFieldGetName(inputqfields[i], &fieldname); PCeedChk(ierr);
      if (if_vector[i] == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorSetField(coper, fieldname, coarse_er, cbasis,
-                                     if_vector[i]); CeedChk(ierr);
+                                     if_vector[i]); PCeedChk(ierr);
      }
      else
      {
         ierr = CeedOperatorSetField(coper, fieldname, er_input[i], basis_input[i],
-                                     if_vector[i]); CeedChk(ierr);
+                                     if_vector[i]); PCeedChk(ierr);
      }
   }
   for (int i = 0; i < numoutputfields; ++i)
   {
      char * fieldname;
-      ierr = CeedQFunctionFieldGetName(outputqfields[i], &fieldname); CeedChk(ierr);
+      ierr = CeedQFunctionFieldGetName(outputqfields[i], &fieldname); PCeedChk(ierr);
      if (of_vector[i] == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorSetField(coper, fieldname, coarse_er, cbasis,
-                                     of_vector[i]); CeedChk(ierr);
+                                     of_vector[i]); PCeedChk(ierr);
      }
      else
      {
         ierr = CeedOperatorSetField(coper, fieldname, er_output[i], basis_output[i],
-                                     of_vector[i]); CeedChk(ierr);
+                                     of_vector[i]); PCeedChk(ierr);
      }
   }
   delete [] er_input;
@@ -711,21 +712,21 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
   int ierr;

   CeedQFunction qf;
-   ierr = CeedOperatorGetQFunction(oper, &qf); CeedChk(ierr);
+   ierr = CeedOperatorGetQFunction(oper, &qf); PCeedChk(ierr);
   CeedInt numinputfields, numoutputfields;
   CeedOperatorField *inputfields;
   ierr = CeedOperatorGetFields(oper, &numinputfields, &inputfields,
                                &numoutputfields, NULL);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   CeedBasis basis;
-   ierr = CeedOperatorGetActiveBasis(oper, &basis); CeedChk(ierr);
+   ierr = CeedOperatorGetActiveBasis(oper, &basis); PCeedChk(ierr);
   ierr = CeedBasisATPMGCoarseToFine(basis, basis_ctof_out, order_reduction);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   ierr = CeedBasisATPMGCoarsen(basis, *basis_ctof_out, coarse_basis_out,
-                                order_reduction); CeedChk(ierr);
+                                order_reduction); PCeedChk(ierr);
   ierr = CeedATPMGOperator(oper, order_reduction, coarse_er, *coarse_basis_out,
-                            *basis_ctof_out, out); CeedChk(ierr);
+                            *basis_ctof_out, out); PCeedChk(ierr);
   return 0;
 }

@@ -734,11 +735,11 @@ int CeedOperatorGetOrder(CeedOperator oper, CeedInt * order)
   int ierr;

   CeedOperatorField active_field;
-   ierr = CeedOperatorGetActiveField(oper, &active_field); CeedChk(ierr);
+   ierr = CeedOperatorGetActiveField(oper, &active_field); PCeedChk(ierr);
   CeedBasis basis;
-   ierr = CeedOperatorFieldGetBasis(active_field, &basis); CeedChk(ierr);
+   ierr = CeedOperatorFieldGetBasis(active_field, &basis); PCeedChk(ierr);
   int P1d;
-   ierr = CeedBasisGetNumNodes1D(basis, &P1d); CeedChk(ierr);
+   ierr = CeedBasisGetNumNodes1D(basis, &P1d); PCeedChk(ierr);
   *order = P1d - 1;

   return 0;
@@ -753,13 +754,13 @@ int CeedATPMGBundle(CeedOperator oper, int order_reduction,
 {
   int ierr;
   CeedInt order;
-   ierr = CeedOperatorGetOrder(oper, &order); CeedChk(ierr);
+   ierr = CeedOperatorGetOrder(oper, &order); PCeedChk(ierr);
   CeedElemRestriction ho_er;
-   ierr = CeedOperatorGetActiveElemRestriction(oper, &ho_er); CeedChk(ierr);
+   ierr = CeedOperatorGetActiveElemRestriction(oper, &ho_er); PCeedChk(ierr);
   ierr = CeedATPMGElemRestriction(order, order_reduction, ho_er, er_out, dof_map);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   ierr = CeedATPMGOperator(oper, order_reduction, *er_out, coarse_basis_out,
-                            basis_ctof_out, coarse_oper); CeedChk(ierr);
+                            basis_ctof_out, coarse_oper); PCeedChk(ierr);
   return 0;
 }

@@ -14,175 +14,166 @@
 namespace mfem
 {

+void DofTransformation::TransformPrimal(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Size();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->TransformPrimal(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->TransformPrimal(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void DofTransformation::InvTransformPrimal(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Height();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->InvTransformPrimal(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->InvTransformPrimal(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void DofTransformation::TransformDual(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Size();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->TransformDual(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->TransformDual(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void DofTransformation::InvTransformDual(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Size();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->InvTransformDual(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->InvTransformDual(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
 void TransformPrimal(const DofTransformation *ran_dof_trans,
                     const DofTransformation *dom_dof_trans,
                     DenseMatrix &elmat)
 {
-   if (ran_dof_trans && dom_dof_trans)
-   {
-      ran_dof_trans->TransformPrimalCols(elmat);
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else if (ran_dof_trans)
+   // No action if both transformations are NULL
+   if (ran_dof_trans)
   {
      ran_dof_trans->TransformPrimalCols(elmat);
   }
-   else if (dom_dof_trans)
+   if (dom_dof_trans)
   {
      dom_dof_trans->TransformDualRows(elmat);
   }
-   else
-   {
-      // If both transformations are NULL this function should not be called
-   }
 }

 void TransformDual(const DofTransformation *ran_dof_trans,
                   const DofTransformation *dom_dof_trans,
                   DenseMatrix &elmat)
 {
-   if (ran_dof_trans && dom_dof_trans)
-   {
-      ran_dof_trans->TransformDualCols(elmat);
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else if (ran_dof_trans)
+   // No action if both transformations are NULL
+   if (ran_dof_trans)
   {
      ran_dof_trans->TransformDualCols(elmat);
   }
-   else if (dom_dof_trans)
+   if (dom_dof_trans)
   {
      dom_dof_trans->TransformDualRows(elmat);
   }
-   else
-   {
-      // If both transformations are NULL this function should not be called
-   }
-}
-
-void StatelessVDofTransformation::TransformPrimal(const Array<int> & face_ori,
-                                                  double *v) const
-{
-   int size = sdoftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES || vdim_ == 1)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->TransformPrimal(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->TransformPrimal(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void StatelessVDofTransformation::InvTransformPrimal(
-   const Array<int> & face_ori,
-   double *v) const
-{
-   int size = sdoftrans_->Height();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->InvTransformPrimal(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->InvTransformPrimal(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void StatelessVDofTransformation::TransformDual(const Array<int> & face_ori,
-                                                double *v) const
-{
-   int size = sdoftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->TransformDual(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->TransformDual(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void StatelessVDofTransformation::InvTransformDual(const Array<int> & face_ori,
-                                                   double *v) const
-{
-   int size = sdoftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->InvTransformDual(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->InvTransformDual(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
 }

 // ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
-const double ND_StatelessDofTransformation::T_data[24] =
+const double ND_DofTransformation::T_data[24] =
 {
   1.0,  0.0,  0.0,  1.0,
   -1.0, -1.0,  0.0,  1.0,
@@ -192,11 +183,11 @@ const double ND_StatelessDofTransformation::T_data[24] =
   0.0,  1.0,  1.0,  0.0
 };

-const DenseTensor ND_StatelessDofTransformation
-::T(const_cast<double*>(ND_StatelessDofTransformation::T_data), 2, 2, 6);
+const DenseTensor ND_DofTransformation
+::T(const_cast<double *>(ND_DofTransformation::T_data), 2, 2, 6);

 // ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
-const double ND_StatelessDofTransformation::TInv_data[24] =
+const double ND_DofTransformation::TInv_data[24] =
 {
   1.0,  0.0,  0.0,  1.0,
   -1.0, -1.0,  0.0,  1.0,
@@ -206,12 +197,11 @@ const double ND_StatelessDofTransformation::TInv_data[24] =
   0.0,  1.0,  1.0,  0.0
 };

-const DenseTensor ND_StatelessDofTransformation
-::TInv(const_cast<double*>(TInv_data), 2, 2, 6);
+const DenseTensor ND_DofTransformation
+::TInv(const_cast<double *>(TInv_data), 2, 2, 6);

-ND_StatelessDofTransformation::ND_StatelessDofTransformation(int size, int p,
-                                                             int num_edges,
-                                                             int num_tri_faces)
+ND_DofTransformation::ND_DofTransformation(int size, int p, int num_edges,
+                                           int num_tri_faces)
   : StatelessDofTransformation(size)
   , order(p)
   , nedofs(p)
@@ -221,18 +211,19 @@ ND_StatelessDofTransformation::ND_StatelessDofTransformation(int size, int p,
 {
 }

-void ND_StatelessDofTransformation::TransformPrimal(const Array<int> & Fo,
-                                                    double *v) const
+void ND_DofTransformation::TransformPrimal(const Array<int> & Fo,
+                                           double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -240,23 +231,25 @@ void ND_StatelessDofTransformation::TransformPrimal(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2.UseExternalData(const_cast<double *>(T.GetData(Fo[f])), 2, 2);
+         T2.Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_StatelessDofTransformation::InvTransformPrimal(const Array<int> & Fo,
-                                                       double *v) const
+void ND_DofTransformation::InvTransformPrimal(const Array<int> & Fo,
+                                              double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2Inv;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -264,23 +257,24 @@ void ND_StatelessDofTransformation::InvTransformPrimal(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2Inv.UseExternalData(const_cast<double *>(TInv.GetData(Fo[f])), 2, 2);
+         T2Inv.Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_StatelessDofTransformation::TransformDual(const Array<int> & Fo,
-                                                  double *v) const
+void ND_DofTransformation::TransformDual(const Array<int> & Fo, double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2Inv;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -288,23 +282,25 @@ void ND_StatelessDofTransformation::TransformDual(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2Inv.UseExternalData(const_cast<double *>(TInv.GetData(Fo[f])), 2, 2);
+         T2Inv.MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_StatelessDofTransformation::InvTransformDual(const Array<int> & Fo,
-                                                     double *v) const
+void ND_DofTransformation::InvTransformDual(const Array<int> & Fo,
+                                            double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -312,7 +308,8 @@ void ND_StatelessDofTransformation::InvTransformDual(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2.UseExternalData(const_cast<double *>(T.GetData(Fo[f])), 2, 2);
+         T2.MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }
@@ -80,6 +80,9 @@ public:
   inline int Width() const { return size_; }
   inline int NumCols() const { return size_; }

+   /// If the DofTransformation performs no transformation
+   virtual bool IsIdentity() const = 0;
+
   /** Transform local DoFs to align with the global DoFs. For example, this
       transformation can be used to map the local vector computed by
       FiniteElement::Project() to the transformed vector stored within a
@@ -115,6 +118,8 @@ public:
   inline void InvTransformDual(const Array<int> & face_orientation,
                                Vector &v) const
   { InvTransformDual(face_orientation, v.GetData()); }
+
+   virtual ~StatelessDofTransformation() = default;
 };

 /** The DofTransformation class is an extension of the
@@ -133,35 +138,76 @@ public:
    transferring finite element degrees of freedom between different meshes.
    For examples of its use see the TransferMap used by the SubMesh class.
   */
-class DofTransformation : virtual public StatelessDofTransformation
+class DofTransformation
 {
 protected:
-   Array<int> Fo;
-
-   DofTransformation(int size)
-      : StatelessDofTransformation(size) {}
+   Array<int> Fo_;
+   const StatelessDofTransformation * dof_trans_;
+   int vdim_;
+   int ordering_;

 public:
+   /** @brief Default constructor which requires that SetDofTransformation be
+       called before use. */
+   DofTransformation(int vdim = 1, int ordering = 0)
+      : dof_trans_(NULL)
+      , vdim_(vdim)
+      , ordering_(ordering)
+   {}
+
+   /// Constructor with a known StatelessDofTransformation
+   DofTransformation(const StatelessDofTransformation & dof_trans,
+                     int vdim = 1, int ordering = 0)
+      : dof_trans_(&dof_trans)
+      , vdim_(vdim)
+      , ordering_(ordering)
+   {}

   /** @brief Configure the transformation using face orientations for the
       current element. */
   /// The face_orientation array can be obtained from Mesh::GetElementFaces.
-   inline void SetFaceOrientations(const Array<int> & face_orientation)
-   { Fo = face_orientation; }
+   inline void SetFaceOrientations(const Array<int> & Fo)
+   { Fo_ = Fo; }

-   inline const Array<int> & GetFaceOrientations() const { return Fo; }
+   /// Return the face orientations for the current element
+   inline const Array<int> & GetFaceOrientations() const { return Fo_; }

-   using StatelessDofTransformation::TransformPrimal;
-   using StatelessDofTransformation::InvTransformPrimal;
-   using StatelessDofTransformation::TransformDual;
-   using StatelessDofTransformation::InvTransformDual;
+   /// Set or change the nested StatelessDofTransformation object
+   inline void SetDofTransformation(const StatelessDofTransformation & dof_trans)
+   {
+      dof_trans_ = &dof_trans;
+   }
+   inline void SetDofTransformation(const StatelessDofTransformation * dof_trans)
+   {
+      dof_trans_ = dof_trans;
+   }
+
+   /// Return the nested StatelessDofTransformation object
+   inline const StatelessDofTransformation * GetDofTransformation() const
+   { return dof_trans_; }
+
+   /// Set or change the vdim and ordering parameter
+   inline void SetVDim(int vdim = 1, int ordering = 0)
+   {
+      vdim_ = vdim;
+      ordering_ = ordering;
+   }
+
+   /// Return the current vdim value
+   inline int GetVDim() const { return vdim_; }
+
+   inline int Size() const { return dof_trans_->Size(); }
+   inline int Height() const { return dof_trans_->Height(); }
+   inline int NumRows() const { return dof_trans_->NumRows(); }
+   inline int Width() const { return dof_trans_->Width(); }
+   inline int NumCols() const { return dof_trans_->NumCols(); }
+   inline bool IsIdentity() const { return dof_trans_->IsIdentity(); }

   /** Transform local DoFs to align with the global DoFs. For example, this
       transformation can be used to map the local vector computed by
       FiniteElement::Project() to the transformed vector stored within a
       GridFunction object. */
-   inline void TransformPrimal(double *v) const
-   { TransformPrimal(Fo, v); }
+   void TransformPrimal(double *v) const;
   inline void TransformPrimal(Vector &v) const
   { TransformPrimal(v.GetData()); }

@@ -179,21 +225,18 @@ public:
       transform the vector obtained using GridFunction::GetSubVector before it
       can be used to compute a local interpolation.
   */
-   inline void InvTransformPrimal(double *v) const
-   { InvTransformPrimal(Fo, v); }
+   void InvTransformPrimal(double *v) const;
   inline void InvTransformPrimal(Vector &v) const
   { InvTransformPrimal(v.GetData()); }

   /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
       into a LinearForm object. */
-   inline void TransformDual(double *v) const
-   { TransformDual(Fo, v); }
+   void TransformDual(double *v) const;
   inline void TransformDual(Vector &v) const
   { TransformDual(v.GetData()); }

   /** Inverse Transform dual DoFs */
-   inline void InvTransformDual(double *v) const
-   { InvTransformDual(Fo, v); }
+   void InvTransformDual(double *v) const;
   inline void InvTransformDual(Vector &v) const
   { InvTransformDual(v.GetData()); }

@@ -225,8 +268,6 @@ public:
         TransformDual(V.GetColumn(c));
      }
   }
-
-   virtual ~DofTransformation() = default;
 };

 /** Transform a matrix of DoFs entries from different finite element spaces as
@@ -245,145 +286,6 @@ void TransformDual(const DofTransformation *ran_dof_trans,
                   const DofTransformation *dom_dof_trans,
                   DenseMatrix &elmat);

-/** The StatelessVDofTransformation class implements a nested transformation
-    where an arbitrary StatelessDofTransformation is replicated with a
-    vdim >= 1.
-*/
-class StatelessVDofTransformation : virtual public StatelessDofTransformation
-{
-protected:
-   int vdim_;
-   int ordering_;
-   StatelessDofTransformation * sdoftrans_;
-
-public:
-   /** @brief Default constructor which requires that SetDofTransformation be
-       called before use. */
-   StatelessVDofTransformation(int vdim = 1, int ordering = 0)
-      : StatelessDofTransformation(0)
-      , vdim_(vdim)
-      , ordering_(ordering)
-      , sdoftrans_(NULL)
-   {}
-
-   /// Constructor with a known StatelessDofTransformation
-   StatelessVDofTransformation(StatelessDofTransformation & doftrans,
-                               int vdim = 1,
-                               int ordering = 0)
-      : StatelessDofTransformation(vdim * doftrans.Size())
-      , vdim_(vdim)
-      , ordering_(ordering)
-      , sdoftrans_(&doftrans)
-   {}
-
-   /// Set or change the vdim parameter
-   inline void SetVDim(int vdim)
-   {
-      vdim_ = vdim;
-      if (sdoftrans_)
-      {
-         size_ = vdim_ * sdoftrans_->Size();
-      }
-   }
-
-   /// Return the current vdim value
-   inline int GetVDim() const { return vdim_; }
-
-   /// Set or change the nested StatelessDofTransformation object
-   inline void SetDofTransformation(StatelessDofTransformation & doftrans)
-   {
-      size_ = vdim_ * doftrans.Size();
-      sdoftrans_ = &doftrans;
-   }
-
-   /// Return the nested StatelessDofTransformation object
-   inline StatelessDofTransformation * GetDofTransformation() const
-   { return sdoftrans_; }
-
-   using StatelessDofTransformation::TransformPrimal;
-   using StatelessDofTransformation::InvTransformPrimal;
-   using StatelessDofTransformation::TransformDual;
-   using StatelessDofTransformation::InvTransformDual;
-
-   /** Specializations of these base class methods which account for the vdim
-       and ordering of the full set of DoFs.
-   */
-   void TransformPrimal(const Array<int> & face_ori, double *v) const;
-   void InvTransformPrimal(const Array<int> & face_ori, double *v) const;
-   void TransformDual(const Array<int> & face_ori, double *v) const;
-   void InvTransformDual(const Array<int> & face_ori, double *v) const;
-};
-
-/** The VDofTransformation class implements a nested transformation where an
-    arbitrary DofTransformation is replicated with a vdim >= 1.
-*/
-class VDofTransformation : public StatelessVDofTransformation,
-   public DofTransformation
-{
-protected:
-   DofTransformation * doftrans_;
-
-public:
-   /** @brief Default constructor which requires that SetDofTransformation be
-       called before use. */
-   VDofTransformation(int vdim = 1, int ordering = 0)
-      : StatelessDofTransformation(0)
-      , StatelessVDofTransformation(vdim, ordering)
-      , DofTransformation(0)
-      , doftrans_(NULL)
-   {}
-
-   /// Constructor with a known DofTransformation
-   /// @note The face orientations in @a doftrans will be copied into the
-   /// new VDofTransformation object.
-   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
-                      int ordering = 0)
-      : StatelessDofTransformation(vdim * doftrans.Size())
-      , StatelessVDofTransformation(doftrans, vdim, ordering)
-      , DofTransformation(vdim * doftrans.Size())
-      , doftrans_(&doftrans)
-   {
-      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
-   }
-
-   using StatelessVDofTransformation::SetDofTransformation;
-
-   /// Set or change the nested DofTransformation object
-   /// @note The face orientations in @a doftrans will be copied into the
-   /// VDofTransformation object.
-   void SetDofTransformation(DofTransformation & doftrans)
-   {
-      doftrans_ = &doftrans;
-      StatelessVDofTransformation::SetDofTransformation(doftrans);
-      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
-   }
-
-   /// Return the nested DofTransformation object
-   inline DofTransformation * GetDofTransformation() const { return doftrans_; }
-
-   /// Set new face orientations in both the VDofTransformation and the
-   /// DofTransformation contained within (if there is one).
-   inline void SetFaceOrientations(const Array<int> & face_orientation)
-   {
-      DofTransformation::SetFaceOrientations(face_orientation);
-      if (doftrans_) { doftrans_->SetFaceOrientations(face_orientation); }
-   }
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   inline void TransformPrimal(double *v) const
-   { TransformPrimal(Fo, v); }
-   inline void InvTransformPrimal(double *v) const
-   { InvTransformPrimal(Fo, v); }
-   inline void TransformDual(double *v) const
-   { TransformDual(Fo, v); }
-   inline void InvTransformDual(double *v) const
-   { InvTransformDual(Fo, v); }
-};
-
 /** Abstract base class for high-order Nedelec spaces on elements with
    triangular faces.

@@ -396,7 +298,7 @@ public:
    be accessed as DenseMatrices using the GetFaceTransform() and
    GetFaceInverseTransform() methods.
 */
-class ND_StatelessDofTransformation : virtual public StatelessDofTransformation
+class ND_DofTransformation : public StatelessDofTransformation
 {
 private:
   static const double T_data[24];
@@ -410,8 +312,7 @@ protected:
   const int nedges; // number of edges per element
   const int nfaces; // number of triangular faces per element

-   ND_StatelessDofTransformation(int size, int order,
-                                 int num_edges, int num_tri_faces);
+   ND_DofTransformation(int size, int order, int num_edges, int num_tri_faces);

 public:
   // Return the 2x2 transformation operator for the given face orientation
@@ -421,116 +322,41 @@ public:
   static const DenseMatrix & GetFaceInverseTransform(int ori)
   { return TInv(ori); }

-   void TransformPrimal(const Array<int> & face_orientation,
-                        double *v) const;
+   bool IsIdentity() const override { return nfdofs < 2; }

-   void InvTransformPrimal(const Array<int> & face_orientation,
-                           double *v) const;
-
-   void TransformDual(const Array<int> & face_orientation,
-                      double *v) const;
-
-   void InvTransformDual(const Array<int> & face_orientation,
-                         double *v) const;
+   void TransformPrimal(const Array<int> & Fo, double *v) const override;
+   void InvTransformPrimal(const Array<int> & Fo, double *v) const override;
+   void TransformDual(const Array<int> & Fo, double *v) const override;
+   void InvTransformDual(const Array<int> & Fo, double *v) const override;
 };

 /// Stateless DoF transformation implementation for the Nedelec basis on
 /// triangles
-class ND_TriStatelessDofTransformation : public ND_StatelessDofTransformation
-{
-public:
-   ND_TriStatelessDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2))
-      , ND_StatelessDofTransformation(order*(order + 2), order, 3, 1)
-   {}
-};
-
-/// DoF transformation implementation for the Nedelec basis on triangles
-class ND_TriDofTransformation : public DofTransformation,
-   public ND_TriStatelessDofTransformation
+class ND_TriDofTransformation : public ND_DofTransformation
 {
 public:
   ND_TriDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2))
-      , DofTransformation(order*(order + 2))
-      , ND_TriStatelessDofTransformation(order)
-   {}
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   using ND_TriStatelessDofTransformation::TransformPrimal;
-   using ND_TriStatelessDofTransformation::InvTransformPrimal;
-   using ND_TriStatelessDofTransformation::TransformDual;
-   using ND_TriStatelessDofTransformation::InvTransformDual;
-};
-
-/// DoF transformation implementation for the Nedelec basis on tetrahedra
-class ND_TetStatelessDofTransformation : public ND_StatelessDofTransformation
-{
-public:
-   ND_TetStatelessDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
-      , ND_StatelessDofTransformation(order*(order + 2)*(order + 3)/2, order,
-                                      6, 4)
+      : ND_DofTransformation(order*(order + 2), order, 3, 1)
   {}
 };

 /// DoF transformation implementation for the Nedelec basis on tetrahedra
-class ND_TetDofTransformation : public DofTransformation,
-   public ND_TetStatelessDofTransformation
+class ND_TetDofTransformation : public ND_DofTransformation
 {
 public:
   ND_TetDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
-      , DofTransformation(order*(order + 2)*(order + 3)/2)
-      , ND_TetStatelessDofTransformation(order)
-   {}
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   using ND_TetStatelessDofTransformation::TransformPrimal;
-   using ND_TetStatelessDofTransformation::InvTransformPrimal;
-   using ND_TetStatelessDofTransformation::TransformDual;
-   using ND_TetStatelessDofTransformation::InvTransformDual;
-};
-
-/// DoF transformation implementation for the Nedelec basis on wedge elements
-class ND_WedgeStatelessDofTransformation : public ND_StatelessDofTransformation
-{
-public:
-   ND_WedgeStatelessDofTransformation(int order)
-      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
-      , ND_StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2,
-                                      order, 9, 2)
+      : ND_DofTransformation(order*(order + 2)*(order + 3)/2, order, 6, 4)
   {}
 };

 /// DoF transformation implementation for the Nedelec basis on wedge elements
-class ND_WedgeDofTransformation : public DofTransformation,
-   public ND_WedgeStatelessDofTransformation
+class ND_WedgeDofTransformation : public ND_DofTransformation
 {
 public:
   ND_WedgeDofTransformation(int order)
-      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
-      , DofTransformation(3 * order * ((order + 1) * (order + 2))/2)
-      , ND_WedgeStatelessDofTransformation(order)
+      : ND_DofTransformation(3 * order * ((order + 1) * (order + 2))/2,
+                             order, 9, 2)
   {}
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   using ND_WedgeStatelessDofTransformation::TransformPrimal;
-   using ND_WedgeStatelessDofTransformation::InvTransformPrimal;
-   using ND_WedgeStatelessDofTransformation::TransformDual;
-   using ND_WedgeStatelessDofTransformation::InvTransformDual;
 };

 } // namespace mfem
@@ -355,15 +355,11 @@ int InverseElementTransformation::Transform(const Vector &pt,
         }
         else
         {
-            const int old_type = GlobGeometryRefiner.GetType();
-            GlobGeometryRefiner.SetType(qpts_type);
-            RefinedGeometry &RefG =
-               *GlobGeometryRefiner.Refine(T->GetGeometryType(), order);
+            RefinedGeometry &RefG = *refiner.Refine(T->GetGeometryType(), order);
            int closest_idx = (init_guess_type == ClosestPhysNode) ?
                              FindClosestPhysPoint(pt, RefG.RefPts) :
                              FindClosestRefPoint(pt, RefG.RefPts);
            ip0 = &RefG.RefPts.IntPoint(closest_idx);
-            GlobGeometryRefiner.SetType(old_type);
         }
         break;
      }
@@ -233,7 +233,7 @@ protected:
   // Parameters of the inversion algorithms:
   const IntegrationPoint *ip0;
   int init_guess_type; // algorithm to use
-   int qpts_type; // Quadrature1D type for the initial guess type
+   GeometryRefiner refiner; // geometry refiner for initial guess
   int rel_qpts_order; // num_1D_qpts = max(trans_order+rel_qpts_order,0)+1
   int solver_type; // solution strategy to use
   int max_iter; // max. number of Newton iterations
@@ -276,7 +276,7 @@ public:
      : T(Trans),
        ip0(NULL),
        init_guess_type(Center),
-        qpts_type(Quadrature1D::OpenHalfUniform),
+        refiner(Quadrature1D::OpenHalfUniform),
        rel_qpts_order(-1),
        solver_type(NewtonElementProject),
        max_iter(16),
@@ -301,7 +301,7 @@ public:
   { ip0 = &init_ip; SetInitialGuessType(GivenPoint); }

   /// Set the Quadrature1D type used for the `Closest*` initial guess types.
-   void SetInitGuessPointsType(int q_type) { qpts_type = q_type; }
+   void SetInitGuessPointsType(int q_type) { refiner.SetType(q_type); }

   /// Set the relative order used for the `Closest*` initial guess types.
   /** The number of points in each spatial direction is given by the formula
@@ -361,7 +361,7 @@ public:
 class IsoparametricTransformation : public ElementTransformation
 {
 private:
-   DenseMatrix dshape,d2shape;
+   DenseMatrix dshape, d2shape;
   Vector shape;

   const FiniteElement *FElem;
@@ -74,11 +74,11 @@ public:
 /** @brief The ZienkiewiczZhuEstimator class implements the Zienkiewicz-Zhu
    error estimation procedure.

-    Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
+    [1] Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
    and a posteriori error estimates. Part 1: The recovery technique.
    Int. J. Num. Meth. Engng. 33, 1331-1364 (1992).

-    Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
+    [2] Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
    and a posteriori error estimates. Part 2: Error estimates and adaptivity.
    Int. J. Num. Meth. Engng. 33, 1365-1382 (1992).

@@ -220,8 +220,8 @@ public:
    The required BilinearFormIntegrator must implement the method
    ComputeElementFlux().

-   COMMENTS:
-   *  The present implementation ignores all single-element patches corresponding
+   @note
+   -  The present implementation ignores all single-element patches corresponding
      to boundary faces. This is appropriate for Dirichlet boundaries, but
      suboptimal for Neumann boundaries. Reference 3 shows that a constrained
      least-squares problem, where the reconstructed flux is constrained by the
@@ -229,13 +229,13 @@ public:
      NOTE THAT THIS CONSTRAINED LS PROBLEM IS NOT YET IMPLEMENTED, so it is
      possible that the local error estimates for elements on a Neumann boundary
      are suboptimal.
-   *  The global polynomial basis used for the flux reconstruction is, by default,
+   -  The global polynomial basis used for the flux reconstruction is, by default,
      aligned with the physical Cartesian axis. For patches with 2D elements, this
      has been improved on so that the basis is aligned with the physical patch
      orientation. Reorientation of the flux reconstruction basis is helpful to
      maintain symmetry in the refinement pattern and could be extended to 3D.
-   *  This estimator is ONLY implemented IN SERIAL.
-   *  Anisotropic refinement is NOT YET SUPPORTED.
+   -  This estimator is ONLY implemented IN SERIAL.
+   -  Anisotropic refinement is NOT YET SUPPORTED.

 */
 class LSZienkiewiczZhuEstimator : public ErrorEstimator
@@ -359,135 +359,148 @@ void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,

   // Hessian in physical coords
   lhm.Invert();
-   Mult( hess, lhm, Hessian);
+   Mult(hess, lhm, Hessian);
 }

 const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
                                             DofToQuad::Mode mode) const
 {
+   DofToQuad *d2q = nullptr;
   MFEM_VERIFY(mode == DofToQuad::FULL, "invalid mode requested");

-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-#ifdef MFEM_THREAD_SAFE
-   DenseMatrix vshape(dof, dim);
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
 #endif
-
-   DofToQuad *d2q = new DofToQuad;
-   const int nqpt = ir.GetNPoints();
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = dof;
-   d2q->nqpt = nqpt;
-   if (range_type == SCALAR)
   {
-      d2q->B.SetSize(nqpt*dof);
-      d2q->Bt.SetSize(dof*nqpt);
-
-      Vector shape;
-      vshape.GetColumnReference(0, shape);
-      for (int i = 0; i < nqpt; i++)
+      for (int i = 0; i < dof2quad_array.Size(); i++)
      {
-         const IntegrationPoint &ip = ir.IntPoint(i);
-         CalcShape(ip, shape);
-         for (int j = 0; j < dof; j++)
-         {
-            d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = shape(j);
-         }
+         d2q = dof2quad_array[i];
+         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
      }
-   }
-   else if (range_type == VECTOR)
-   {
-      d2q->B.SetSize(nqpt*dim*dof);
-      d2q->Bt.SetSize(dof*nqpt*dim);
-
-      for (int i = 0; i < nqpt; i++)
+      if (!d2q)
      {
-         const IntegrationPoint &ip = ir.IntPoint(i);
-         CalcVShape(ip, vshape);
-         for (int d = 0; d < dim; d++)
+#ifdef MFEM_THREAD_SAFE
+         DenseMatrix vshape(dof, dim);
+#endif
+         d2q = new DofToQuad;
+         const int nqpt = ir.GetNPoints();
+         d2q->FE = this;
+         d2q->IntRule = &ir;
+         d2q->mode = mode;
+         d2q->ndof = dof;
+         d2q->nqpt = nqpt;
+         switch (range_type)
         {
-            for (int j = 0; j < dof; j++)
+            case SCALAR:
            {
-               d2q->B[i+nqpt*(d+dim*j)] = d2q->Bt[j+dof*(i+nqpt*d)] = vshape(j, d);
-            }
-         }
-      }
-   }
-   else
-   {
-      // Skip B and Bt for unknown range type
-   }
-   switch (deriv_type)
-   {
-      case GRAD:
-      {
-         d2q->G.SetSize(nqpt*dim*dof);
-         d2q->Gt.SetSize(dof*nqpt*dim);
+               d2q->B.SetSize(nqpt*dof);
+               d2q->Bt.SetSize(dof*nqpt);

-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcDShape(ip, vshape);
-            for (int d = 0; d < dim; d++)
-            {
-               for (int j = 0; j < dof; j++)
+               Vector shape;
+               vshape.GetColumnReference(0, shape);
+               for (int i = 0; i < nqpt; i++)
               {
-                  d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j, d);
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcShape(ip, shape);
+                  for (int j = 0; j < dof; j++)
+                  {
+                     d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = shape(j);
+                  }
               }
+               break;
            }
-         }
-         break;
-      }
-      case DIV:
-      {
-         d2q->G.SetSize(nqpt*dof);
-         d2q->Gt.SetSize(dof*nqpt);
-
-         Vector divshape;
-         vshape.GetColumnReference(0, divshape);
-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcDivShape(ip, divshape);
-            for (int j = 0; j < dof; j++)
+            case VECTOR:
            {
-               d2q->G[i+nqpt*j] = d2q->Gt[j+dof*i] = divshape(j);
-            }
-         }
-         break;
-      }
-      case CURL:
-      {
-         d2q->G.SetSize(nqpt*cdim*dof);
-         d2q->Gt.SetSize(dof*nqpt*cdim);
+               d2q->B.SetSize(nqpt*dim*dof);
+               d2q->Bt.SetSize(dof*nqpt*dim);

-         DenseMatrix curlshape(vshape.GetData(), dof, cdim);  // cdim <= dim
-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcCurlShape(ip, curlshape);
-            for (int d = 0; d < cdim; d++)
-            {
-               for (int j = 0; j < dof; j++)
+               for (int i = 0; i < nqpt; i++)
               {
-                  d2q->G[i+nqpt*(d+cdim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcVShape(ip, vshape);
+                  for (int d = 0; d < dim; d++)
+                  {
+                     for (int j = 0; j < dof; j++)
+                     {
+                        d2q->B[i+nqpt*(d+dim*j)] =
+                           d2q->Bt[j+dof*(i+nqpt*d)] = vshape(j, d);
+                     }
+                  }
               }
+               break;
            }
+            case UNKNOWN_RANGE_TYPE:
+               // Skip B and Bt for unknown range type
+               break;
         }
-         break;
+         switch (deriv_type)
+         {
+            case GRAD:
+            {
+               d2q->G.SetSize(nqpt*dim*dof);
+               d2q->Gt.SetSize(dof*nqpt*dim);
+
+               for (int i = 0; i < nqpt; i++)
+               {
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcDShape(ip, vshape);
+                  for (int d = 0; d < dim; d++)
+                  {
+                     for (int j = 0; j < dof; j++)
+                     {
+                        d2q->G[i+nqpt*(d+dim*j)] =
+                           d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j, d);
+                     }
+                  }
+               }
+               break;
+            }
+            case DIV:
+            {
+               d2q->G.SetSize(nqpt*dof);
+               d2q->Gt.SetSize(dof*nqpt);
+
+               Vector divshape;
+               vshape.GetColumnReference(0, divshape);
+               for (int i = 0; i < nqpt; i++)
+               {
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcDivShape(ip, divshape);
+                  for (int j = 0; j < dof; j++)
+                  {
+                     d2q->G[i+nqpt*j] = d2q->Gt[j+dof*i] = divshape(j);
+                  }
+               }
+               break;
+            }
+            case CURL:
+            {
+               d2q->G.SetSize(nqpt*cdim*dof);
+               d2q->Gt.SetSize(dof*nqpt*cdim);
+
+               DenseMatrix curlshape(vshape.GetData(), dof, cdim);  // cdim <= dim
+               for (int i = 0; i < nqpt; i++)
+               {
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcCurlShape(ip, curlshape);
+                  for (int d = 0; d < cdim; d++)
+                  {
+                     for (int j = 0; j < dof; j++)
+                     {
+                        d2q->G[i+nqpt*(d+cdim*j)] =
+                           d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
+                     }
+                  }
+               }
+               break;
+            }
+            case NONE:
+               // Skip G and Gt for unknown derivative type
+               break;
+         }
+         dof2quad_array.Append(d2q);
      }
-      case NONE:
-      default:
-         // Skip G and Gt for unknown derivative type
-         break;
   }
-   dof2quad_array.Append(d2q);
   return *d2q;
 }

@@ -904,14 +917,14 @@ VectorFiniteElement::VectorFiniteElement(int D, Geometry::Type G,
 }

 void VectorFiniteElement::CalcShape(
-   const IntegrationPoint &ip, Vector &shape ) const
+   const IntegrationPoint &ip, Vector &shape) const
 {
   mfem_error("Error: Cannot use scalar CalcShape(...) function with\n"
              "   VectorFiniteElements!");
 }

 void VectorFiniteElement::CalcDShape(
-   const IntegrationPoint &ip, DenseMatrix &dshape ) const
+   const IntegrationPoint &ip, DenseMatrix &dshape) const
 {
   mfem_error("Error: Cannot use scalar CalcDShape(...) function with\n"
              "   VectorFiniteElements!");
@@ -2183,51 +2196,72 @@ void Poly_1D::CalcChebyshev(const int p, const double x, double *u, double *d,

 const double *Poly_1D::GetPoints(const int p, const int btype)
 {
+   Array<double*> *pts;
   BasisType::Check(btype);
   const int qtype = BasisType::GetQuadrature1D(btype);
-
   if (qtype == Quadrature1D::Invalid) { return NULL; }

-   if (points_container.find(btype) == points_container.end())
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (Poly1DGetPoints)
+#endif
   {
-      points_container[btype] = new Array<double*>(h_mt);
+      auto it = points_container.find(btype);
+      if (it != points_container.end())
+      {
+         pts = it->second;
+      }
+      else
+      {
+         pts = new Array<double*>(h_mt);
+         points_container[btype] = pts;
+      }
+      if (pts->Size() <= p)
+      {
+         pts->SetSize(p + 1, NULL);
+      }
+      if ((*pts)[p] == NULL)
+      {
+         (*pts)[p] = new double[p + 1];
+         quad_func.GivePolyPoints(p + 1, (*pts)[p], qtype);
+      }
   }
-   Array<double*> &pts = *points_container[btype];
-   if (pts.Size() <= p)
-   {
-      pts.SetSize(p + 1, NULL);
-   }
-   if (pts[p] == NULL)
-   {
-      pts[p] = new double[p + 1];
-      quad_func.GivePolyPoints(p+1, pts[p], qtype);
-   }
-   return pts[p];
+   return (*pts)[p];
 }

 Poly_1D::Basis &Poly_1D::GetBasis(const int p, const int btype)
 {
+   Array<Basis*> *bases;
   BasisType::Check(btype);

-   if ( bases_container.find(btype) == bases_container.end() )
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (Poly1DGetBasis)
+#endif
   {
-      // we haven't been asked for basis or points of this type yet
-      bases_container[btype] = new Array<Basis*>(h_mt);
+      auto it = bases_container.find(btype);
+      if (it != bases_container.end())
+      {
+         bases = it->second;
+      }
+      else
+      {
+         // we haven't been asked for basis or points of this type yet
+         bases = new Array<Basis*>(h_mt);
+         bases_container[btype] = bases;
+      }
+      if (bases->Size() <= p)
+      {
+         bases->SetSize(p + 1, NULL);
+      }
+      if ((*bases)[p] == NULL)
+      {
+         EvalType etype;
+         if (btype == BasisType::Positive) { etype = Positive; }
+         else if (btype == BasisType::IntegratedGLL) { etype = Integrated; }
+         else { etype = Barycentric; }
+         (*bases)[p] = new Basis(p, GetPoints(p, btype), etype);
+      }
   }
-   Array<Basis*> &bases = *bases_container[btype];
-   if (bases.Size() <= p)
-   {
-      bases.SetSize(p + 1, NULL);
-   }
-   if (bases[p] == NULL)
-   {
-      EvalType etype;
-      if (btype == BasisType::Positive) { etype = Positive; }
-      else if (btype == BasisType::IntegratedGLL) { etype = Integrated; }
-      else { etype = Barycentric; }
-      bases[p] = new Basis(p, GetPoints(p, btype), etype);
-   }
-   return *bases[p];
+   return *(*bases)[p];
 }

 Poly_1D::~Poly_1D()
@@ -2236,7 +2270,7 @@ Poly_1D::~Poly_1D()
        it != points_container.end() ; ++it)
   {
      Array<double*>& pts = *it->second;
-      for ( int i = 0 ; i < pts.Size() ; ++i )
+      for (int i = 0; i < pts.Size(); ++i)
      {
         delete [] pts[i];
      }
@@ -2247,7 +2281,7 @@ Poly_1D::~Poly_1D()
        it != bases_container.end() ; ++it)
   {
      Array<Basis*>& bases = *it->second;
-      for ( int i = 0 ; i < bases.Size() ; ++i )
+      for (int i = 0; i < bases.Size(); ++i)
      {
         delete bases[i];
      }
@@ -2461,39 +2495,47 @@ const DofToQuad &TensorBasisElement::GetTensorDofToQuad(
   DofToQuad::Mode mode, const Poly_1D::Basis &basis, bool closed,
   Array<DofToQuad*> &dof2quad_array)
 {
+   DofToQuad *d2q = nullptr;
   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");

-   for (int i = 0; i < dof2quad_array.Size(); i++)
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
+#endif
   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const int ndof = closed ? fe.GetOrder() + 1 : fe.GetOrder();
-   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
-   d2q->FE = &fe;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = ndof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*ndof);
-   d2q->Bt.SetSize(ndof*nqpt);
-   d2q->G.SetSize(nqpt*ndof);
-   d2q->Gt.SetSize(ndof*nqpt);
-   Vector val(ndof), grad(ndof);
-   for (int i = 0; i < nqpt; i++)
-   {
-      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
-      // of the 1D rule.
-      basis.Eval(ir.IntPoint(i).x, val, grad);
-      for (int j = 0; j < ndof; j++)
+      for (int i = 0; i < dof2quad_array.Size(); i++)
      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
-         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
+         d2q = dof2quad_array[i];
+         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
+      }
+      if (!d2q)
+      {
+         d2q = new DofToQuad;
+         const int ndof = closed ? fe.GetOrder() + 1 : fe.GetOrder();
+         const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
+         d2q->FE = &fe;
+         d2q->IntRule = &ir;
+         d2q->mode = mode;
+         d2q->ndof = ndof;
+         d2q->nqpt = nqpt;
+         d2q->B.SetSize(nqpt*ndof);
+         d2q->Bt.SetSize(ndof*nqpt);
+         d2q->G.SetSize(nqpt*ndof);
+         d2q->Gt.SetSize(ndof*nqpt);
+         Vector val(ndof), grad(ndof);
+         for (int i = 0; i < nqpt; i++)
+         {
+            // The first 'nqpt' points in 'ir' have the same x-coordinates as those
+            // of the 1D rule.
+            basis.Eval(ir.IntPoint(i).x, val, grad);
+            for (int j = 0; j < ndof; j++)
+            {
+               d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
+               d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
+            }
+         }
+         dof2quad_array.Append(d2q);
      }
   }
-   dof2quad_array.Append(d2q);
   return *d2q;
 }

@@ -250,7 +250,7 @@ protected:
   /// Container for all DofToQuad objects created by the FiniteElement.
   /** Multiple DofToQuad objects may be needed when different quadrature rules
       or different DofToQuad::Mode are used. */
-   mutable Array<DofToQuad*> dof2quad_array;
+   mutable Array<DofToQuad *> dof2quad_array;

 public:
   /// Enumeration for range_type and deriv_range_type
@@ -596,7 +596,7 @@ public:
   /** @brief Return a DoF transformation object for this particular type of
       basis.
   */
-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return NULL; }

   /// Deconstruct the FiniteElement
@@ -1026,8 +1026,8 @@ public:
   };

 private:
-   typedef std::map< int, Array<double*>* > PointsMap;
-   typedef std::map< int, Array<Basis*>* > BasisMap;
+   typedef std::map<int, Array<double*>*> PointsMap;
+   typedef std::map<int, Array<Basis*>*> BasisMap;

   MemoryType h_mt;
   PointsMap points_container;
@@ -6031,7 +6031,7 @@ void RT0PyrFiniteElement::CalcVShape(const IntegrationPoint &ip,
   shape(1,2) = z;

   shape(2,0) = x * (2.0 - z) * ozi;
-   shape(2,1) = - y * z * ozi;;
+   shape(2,1) = - y * z * ozi;
   shape(2,2) = z;

   shape(3,0) = - x * z * ozi;
@@ -179,7 +179,7 @@ class ND_TetrahedronElement : public VectorFiniteElement
   Array<int> dof2tk;
   DenseMatrixInverse Ti;

-   mutable ND_TetStatelessDofTransformation doftrans;
+   ND_TetDofTransformation doftrans;

 public:
   /// Construct the ND_TetrahedronElement of order @a p
@@ -201,7 +201,7 @@ public:
                                  ElementTransformation &Trans,
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return &doftrans; }
   using FiniteElement::Project;
   virtual void Project(VectorCoefficient &vc,
@@ -242,7 +242,7 @@ class ND_TriangleElement : public VectorFiniteElement
   Array<int> dof2tk;
   DenseMatrixInverse Ti;

-   mutable ND_TriStatelessDofTransformation doftrans;
+   ND_TriDofTransformation doftrans;

 public:
   /// Construct the ND_TriangleElement of order @a p
@@ -264,7 +264,7 @@ public:
                                  ElementTransformation &Trans,
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return &doftrans; }
   using FiniteElement::Project;
   virtual void Project(VectorCoefficient &vc,
@@ -346,7 +346,7 @@ private:
 #endif
   Array<int> dof2tk, t_dof, s_dof;

-   mutable ND_WedgeStatelessDofTransformation doftrans;
+   ND_WedgeDofTransformation doftrans;

   H1_TriangleElement H1TriangleFE;
   ND_TriangleElement NDTriangleFE;
@@ -379,7 +379,7 @@ public:
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }

-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return &doftrans; }

   using FiniteElement::Project;
@@ -59,7 +59,7 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
   int p = (this)->GetOrder();
   double x = ip.x, y = ip.y;

-   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
+   Poly_1D::Basis &edgeNodalBasis = poly1d.GetBasis(p, BasisType::GaussLobatto);
   Vector nodalX(p+1);
   Vector nodalY(p+1);

@@ -113,10 +113,9 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
   {
      double *legX = new double[p-1];
      double *legY = new double[p-1];
-      Poly_1D *storeLegendre = new Poly_1D();

-      storeLegendre->CalcLegendre(p-2, x, legX);
-      storeLegendre->CalcLegendre(p-2, y, legY);
+      Poly_1D::CalcLegendre(p-2, x, legX);
+      Poly_1D::CalcLegendre(p-2, y, legY);

      int interior_total = 0;
      for (int j = 4; j < p + 1; j++)
@@ -131,7 +130,6 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,

      delete[] legX;
      delete[] legY;
-      delete storeLegendre;
   }
 }

@@ -141,7 +139,7 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
   int p = (this)->GetOrder();
   double x = ip.x, y = ip.y;

-   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
+   Poly_1D::Basis &edgeNodalBasis = poly1d.GetBasis(p, BasisType::GaussLobatto);
   Vector nodalX(p+1);
   Vector DnodalX(p+1);
   Vector nodalY(p+1);
@@ -203,10 +201,9 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
      double *legY = new double[p-1];
      double *DlegX = new double[p-1];
      double *DlegY = new double[p-1];
-      Poly_1D *storeLegendre = new Poly_1D();

-      storeLegendre->CalcLegendre(p-2, x, legX, DlegX);
-      storeLegendre->CalcLegendre(p-2, y, legY, DlegY);
+      Poly_1D::CalcLegendre(p-2, x, legX, DlegX);
+      Poly_1D::CalcLegendre(p-2, y, legY, DlegY);

      int interior_total = 0;
      for (int j = 4; j < p + 1; j++)
@@ -224,7 +221,6 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
      delete[] legY;
      delete[] DlegX;
      delete[] DlegY;
-      delete storeLegendre;
   }
 }

@@ -2896,7 +2896,7 @@ ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
 }

-StatelessDofTransformation *
+const StatelessDofTransformation *
 ND_FECollection::DofTransformationForGeometry(Geometry::Type GeomType) const
 {
   if (!Geometry::IsTensorProduct(GeomType) && this->GetOrder() > 1)
@@ -63,7 +63,7 @@ public:
   /** @brief Returns a DoF transformation object compatible with this basis
       and geometry type.
   */
-   virtual StatelessDofTransformation *
+   virtual const StatelessDofTransformation *
   DofTransformationForGeometry(Geometry::Type GeomType) const
   { return NULL; }

@@ -483,7 +483,7 @@ public:
   int DofForGeometry(Geometry::Type GeomType) const override
   { return ND_dof[GeomType]; }

-   StatelessDofTransformation *
+   const StatelessDofTransformation *
   DofTransformationForGeometry(Geometry::Type GeomType) const override;

   const int *DofOrderForOrientation(Geometry::Type GeomType,
@@ -13,6 +13,7 @@
 #define MFEM_FEM_HPP

 #include "intrules.hpp"
+#include "intrules_cut.hpp"
 #include "geom.hpp"
 #include "fe.hpp"
 #include "fe_coll.hpp"
@@ -63,7 +63,6 @@ FiniteElementSpace::FiniteElementSpace()
     elem_dof(NULL), elem_fos(NULL), bdr_elem_dof(NULL), bdr_elem_fos(NULL),
     face_dof(NULL),
     NURBSext(NULL), own_ext(false),
-     DoFTrans(0), VDoFTrans(vdim, ordering),
     cP_is_set(false),
     Th(Operator::ANY_TYPE),
     sequence(0), mesh_sequence(0), orders_changed(false), relaxed_hp(false)
@@ -72,7 +71,6 @@ FiniteElementSpace::FiniteElementSpace()
 FiniteElementSpace::FiniteElementSpace(const FiniteElementSpace &orig,
                                       Mesh *mesh_,
                                       const FiniteElementCollection *fec_)
-   : VDoFTrans(orig.vdim, orig.ordering)
 {
   mesh_ = mesh_ ? mesh_ : orig.mesh;
   fec_ = fec_ ? fec_ : orig.fec;
@@ -212,7 +210,7 @@ void FiniteElementSpace::GetVDofs(int vd, Array<int>& dofs, int ndofs_) const
   }
 }

-void FiniteElementSpace::DofsToVDofs (Array<int> &dofs, int ndofs_) const
+void FiniteElementSpace::DofsToVDofs(Array<int> &dofs, int ndofs_) const
 {
   if (vdim == 1) { return; }
   if (ndofs_ < 0) { ndofs_ = this->ndofs; }
@@ -264,7 +262,7 @@ int FiniteElementSpace::DofToVDof(int dof, int vd, int ndofs_) const
 }

 // static function
-void FiniteElementSpace::AdjustVDofs (Array<int> &vdofs)
+void FiniteElementSpace::AdjustVDofs(Array<int> &vdofs)
 {
   int n = vdofs.Size(), *vdof = vdofs;
   for (int i = 0; i < n; i++)
@@ -277,36 +275,36 @@ void FiniteElementSpace::AdjustVDofs (Array<int> &vdofs)
   }
 }

+void FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs,
+                                         DofTransformation &doftrans) const
+{
+   GetElementDofs(i, vdofs, doftrans);
+   DofsToVDofs(vdofs);
+   doftrans.SetVDim(vdim, ordering);
+}
+
 DofTransformation *
 FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
 {
-   DofTransformation * doftrans = GetElementDofs(i, vdofs);
+   DoFTrans.SetDofTransformation(NULL);
+   GetElementVDofs(i, vdofs, DoFTrans);
+   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
+}
+
+void FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs,
+                                            DofTransformation &doftrans) const
+{
+   GetBdrElementDofs(i, vdofs, doftrans);
   DofsToVDofs(vdofs);
-   if (vdim == 1 || doftrans == NULL)
-   {
-      return doftrans;
-   }
-   else
-   {
-      VDoFTrans.SetDofTransformation(*doftrans);
-      return &VDoFTrans;
-   }
+   doftrans.SetVDim(vdim, ordering);
 }

 DofTransformation *
 FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
 {
-   DofTransformation * doftrans = GetBdrElementDofs(i, vdofs);
-   DofsToVDofs(vdofs);
-   if (vdim == 1 || doftrans == NULL)
-   {
-      return doftrans;
-   }
-   else
-   {
-      VDoFTrans.SetDofTransformation(*doftrans);
-      return &VDoFTrans;
-   }
+   DoFTrans.SetDofTransformation(NULL);
+   GetBdrElementVDofs(i, vdofs, DoFTrans);
+   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
 }

 void FiniteElementSpace::GetPatchVDofs(int i, Array<int> &vdofs) const
@@ -777,9 +775,9 @@ FiniteElementSpace::H2L_GlobalRestrictionMatrix (FiniteElementSpace *lfes)
   return R;
 }

-void FiniteElementSpace
-::AddDependencies(SparseMatrix& deps, Array<int>& master_dofs,
-                  Array<int>& slave_dofs, DenseMatrix& I, int skipfirst)
+void FiniteElementSpace::AddDependencies(
+   SparseMatrix& deps, Array<int>& master_dofs, Array<int>& slave_dofs,
+   DenseMatrix& I, int skipfirst)
 {
   for (int i = skipfirst; i < slave_dofs.Size(); i++)
   {
@@ -802,11 +800,9 @@ void FiniteElementSpace
   }
 }

-void FiniteElementSpace
-::AddEdgeFaceDependencies(SparseMatrix &deps, Array<int> &master_dofs,
-                          const FiniteElement *master_fe,
-                          Array<int> &slave_dofs, int slave_face,
-                          const DenseMatrix *pm) const
+void FiniteElementSpace::AddEdgeFaceDependencies(
+   SparseMatrix &deps, Array<int> &master_dofs, const FiniteElement *master_fe,
+   Array<int> &slave_dofs, int slave_face, const DenseMatrix *pm) const
 {
   // In variable-order spaces in 3D, we need to only constrain interior face
   // DOFs (this is done one level up), since edge dependencies can be more
@@ -1533,12 +1529,12 @@ SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
                                localP);
 }

-FiniteElementSpace::RefinementOperator::RefinementOperator
-(const FiniteElementSpace* fespace, Table* old_elem_dof, Table* old_elem_fos,
- int old_ndofs)
-   : fespace(fespace)
-   , old_elem_dof(old_elem_dof)
-   , old_elem_fos(old_elem_fos)
+FiniteElementSpace::RefinementOperator::RefinementOperator(
+   const FiniteElementSpace* fespace, Table* old_elem_dof, Table* old_elem_fos,
+   int old_ndofs)
+   : fespace(fespace),
+     old_elem_dof(old_elem_dof),
+     old_elem_fos(old_elem_fos)
 {
   MFEM_VERIFY(fespace->GetNE() >= old_elem_dof->Size(),
               "Previous mesh is not coarser.");
@@ -1553,7 +1549,7 @@ FiniteElementSpace::RefinementOperator::RefinementOperator
      fespace->GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
   }

-   ConstructDoFTrans();
+   ConstructDoFTransArray();
 }

 FiniteElementSpace::RefinementOperator::RefinementOperator(
@@ -1578,59 +1574,58 @@ FiniteElementSpace::RefinementOperator::RefinementOperator(
      old_elem_fos = new Table(*coarse_fes->GetElementToFaceOrientationTable());
   }

-   ConstructDoFTrans();
+   ConstructDoFTransArray();
 }

 FiniteElementSpace::RefinementOperator::~RefinementOperator()
 {
   delete old_elem_dof;
   delete old_elem_fos;
-   for (int i=0; i<old_DoFTrans.Size(); i++)
+   for (int i=0; i<old_DoFTransArray.Size(); i++)
   {
-      delete old_DoFTrans[i];
+      delete old_DoFTransArray[i];
   }
 }

-void FiniteElementSpace::RefinementOperator
-::ConstructDoFTrans()
+void FiniteElementSpace::RefinementOperator::ConstructDoFTransArray()
 {
-   old_DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
-   for (int i=0; i<old_DoFTrans.Size(); i++)
+   old_DoFTransArray.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<old_DoFTransArray.Size(); i++)
   {
-      old_DoFTrans[i] = NULL;
+      old_DoFTransArray[i] = NULL;
   }

   const FiniteElementCollection *fec_ref = fespace->FEColl();
   if (dynamic_cast<const ND_FECollection*>(fec_ref))
   {
-      const FiniteElement * nd_tri =
+      const FiniteElement *nd_tri =
         fec_ref->FiniteElementForGeometry(Geometry::TRIANGLE);
      if (nd_tri)
      {
-         old_DoFTrans[Geometry::TRIANGLE] =
+         old_DoFTransArray[Geometry::TRIANGLE] =
            new ND_TriDofTransformation(nd_tri->GetOrder());
      }

-      const FiniteElement * nd_tet =
+      const FiniteElement *nd_tet =
         fec_ref->FiniteElementForGeometry(Geometry::TETRAHEDRON);
      if (nd_tet)
      {
-         old_DoFTrans[Geometry::TETRAHEDRON] =
+         old_DoFTransArray[Geometry::TETRAHEDRON] =
            new ND_TetDofTransformation(nd_tet->GetOrder());
      }

-      const FiniteElement * nd_pri =
+      const FiniteElement *nd_pri =
         fec_ref->FiniteElementForGeometry(Geometry::PRISM);
      if (nd_pri)
      {
-         old_DoFTrans[Geometry::PRISM] =
+         old_DoFTransArray[Geometry::PRISM] =
            new ND_WedgeDofTransformation(nd_pri->GetOrder());
      }
   }
 }

-void FiniteElementSpace::RefinementOperator
-::Mult(const Vector &x, Vector &y) const
+void FiniteElementSpace::RefinementOperator::Mult(const Vector &x,
+                                                  Vector &y) const
 {
   Mesh* mesh_ref = fespace->GetMesh();
   const CoarseFineTransformations &trans_ref =
@@ -1662,6 +1657,7 @@ void FiniteElementSpace::RefinementOperator
            fespace->DofsToVDofs(vd, vdofs);
            old_dofs.Copy(old_vdofs);
            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+
            x.GetSubVector(old_vdofs, subX);
            lP.Mult(subX, subY);
            y.SetSubVector(vdofs, subY);
@@ -1670,40 +1666,30 @@ void FiniteElementSpace::RefinementOperator
      else
      {
         old_elem_fos->GetRow(emb.parent, old_Fo);
-         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
-
-         DofTransformation *new_doftrans = NULL;
-         VDofTransformation *vdoftrans =
-            dynamic_cast<VDofTransformation*>(doftrans);
-         if (vdoftrans)
-         {
-            new_doftrans = doftrans;
-            doftrans = vdoftrans->GetDofTransformation();
-         }
+         old_DoFTrans.SetDofTransformation(*old_DoFTransArray[geom]);
+         old_DoFTrans.SetFaceOrientations(old_Fo);

+         doftrans->SetVDim();
         for (int vd = 0; vd < rvdim; vd++)
         {
            dofs.Copy(vdofs);
            fespace->DofsToVDofs(vd, vdofs);
            old_dofs.Copy(old_vdofs);
            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+
            x.GetSubVector(old_vdofs, subX);
-            old_DoFTrans[geom]->InvTransformPrimal(subX);
+            old_DoFTrans.InvTransformPrimal(subX);
            lP.Mult(subX, subY);
            doftrans->TransformPrimal(subY);
            y.SetSubVector(vdofs, subY);
         }
-
-         if (vdoftrans)
-         {
-            doftrans = new_doftrans;
-         }
+         doftrans->SetVDim(rvdim, fespace->GetOrdering());
      }
   }
 }

-void FiniteElementSpace::RefinementOperator
-::MultTranspose(const Vector &x, Vector &y) const
+void FiniteElementSpace::RefinementOperator::MultTranspose(const Vector &x,
+                                                           Vector &y) const
 {
   y = 0.0;

@@ -1727,7 +1713,7 @@ void FiniteElementSpace::RefinementOperator
      const Geometry::Type geom = mesh_ref->GetElementBaseGeometry(k);
      const DenseMatrix &lP = localP[geom](emb.matrix);

-      DofTransformation * doftrans = fespace->GetElementDofs(k, f_dofs);
+      DofTransformation *doftrans = fespace->GetElementDofs(k, f_dofs);
      old_elem_dof->GetRow(emb.parent, c_dofs);

      if (!doftrans)
@@ -1742,7 +1728,6 @@ void FiniteElementSpace::RefinementOperator
            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);

            x.GetSubVector(f_vdofs, subX);
-
            for (int p = 0; p < f_dofs.Size(); ++p)
            {
               if (processed[DecodeDof(f_dofs[p])])
@@ -1750,7 +1735,6 @@ void FiniteElementSpace::RefinementOperator
                  subX[p] = 0.0;
               }
            }
-
            lP.MultTranspose(subX, subY);
            y.AddElementVector(c_vdofs, subY);
         }
@@ -1760,17 +1744,10 @@ void FiniteElementSpace::RefinementOperator
         subYt.SetSize(lP.Width());

         old_elem_fos->GetRow(emb.parent, old_Fo);
-         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
-
-         DofTransformation *new_doftrans = NULL;
-         VDofTransformation *vdoftrans =
-            dynamic_cast<VDofTransformation*>(doftrans);
-         if (vdoftrans)
-         {
-            new_doftrans = doftrans;
-            doftrans = vdoftrans->GetDofTransformation();
-         }
+         old_DoFTrans.SetDofTransformation(*old_DoFTransArray[geom]);
+         old_DoFTrans.SetFaceOrientations(old_Fo);

+         doftrans->SetVDim();
         for (int vd = 0; vd < rvdim; vd++)
         {
            f_dofs.Copy(f_vdofs);
@@ -1787,16 +1764,11 @@ void FiniteElementSpace::RefinementOperator
                  subX[p] = 0.0;
               }
            }
-
            lP.MultTranspose(subX, subYt);
-            old_DoFTrans[geom]->TransformDual(subYt);
+            old_DoFTrans.TransformDual(subYt);
            y.AddElementVector(c_vdofs, subYt);
         }
-
-         if (vdoftrans)
-         {
-            doftrans = new_doftrans;
-         }
+         doftrans->SetVDim(rvdim, fespace->GetOrdering());
      }

      for (int p = 0; p < f_dofs.Size(); ++p)
@@ -2024,8 +1996,8 @@ FiniteElementSpace::DerefinementOperator::~DerefinementOperator()
   delete coarse_elem_dof;
 }

-void FiniteElementSpace::DerefinementOperator
-::Mult(const Vector &x, Vector &y) const
+void FiniteElementSpace::DerefinementOperator::Mult(const Vector &x,
+                                                    Vector &y) const
 {
   Array<int> c_vdofs, f_vdofs;
   Vector loc_x, loc_y;
@@ -2227,7 +2199,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh_, NURBSExtension *NURBSext_,
      R_transpose.reset();
      cP_is_set = false;

-      ConstructDoFTrans();
+      ConstructDoFTransArray();
   }
   else
   {
@@ -2239,40 +2211,39 @@ void FiniteElementSpace::Constructor(Mesh *mesh_, NURBSExtension *NURBSext_,
   BuildElementToDofTable();
 }

-void FiniteElementSpace::ConstructDoFTrans()
+void FiniteElementSpace::ConstructDoFTransArray()
 {
-   DestroyDoFTrans();
+   DestroyDoFTransArray();

-   VDoFTrans.SetVDim(vdim);
-   DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
-   for (int i=0; i<DoFTrans.Size(); i++)
+   DoFTransArray.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<DoFTransArray.Size(); i++)
   {
-      DoFTrans[i] = NULL;
+      DoFTransArray[i] = NULL;
   }
   if (mesh->Dimension() < 3) { return; }
   if (dynamic_cast<const ND_FECollection*>(fec))
   {
-      const FiniteElement * nd_tri =
+      const FiniteElement *nd_tri =
         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
      if (nd_tri)
      {
-         DoFTrans[Geometry::TRIANGLE] =
+         DoFTransArray[Geometry::TRIANGLE] =
            new ND_TriDofTransformation(nd_tri->GetOrder());
      }

-      const FiniteElement * nd_tet =
+      const FiniteElement *nd_tet =
         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
      if (nd_tet)
      {
-         DoFTrans[Geometry::TETRAHEDRON] =
+         DoFTransArray[Geometry::TETRAHEDRON] =
            new ND_TetDofTransformation(nd_tet->GetOrder());
      }

-      const FiniteElement * nd_pri =
+      const FiniteElement *nd_pri =
         fec->FiniteElementForGeometry(Geometry::PRISM);
      if (nd_pri)
      {
-         DoFTrans[Geometry::PRISM] =
+         DoFTransArray[Geometry::PRISM] =
            new ND_WedgeDofTransformation(nd_pri->GetOrder());
      }
   }
@@ -2324,7 +2295,7 @@ void FiniteElementSpace::BuildNURBSFaceToDofTable() const
   face_to_be = -1;
   for (int b = 0; b < GetNBE(); b++)
   {
-      int f = mesh->GetBdrElementEdgeIndex(b);
+      int f = mesh->GetBdrElementFaceIndex(b);
      face_to_be[f] = b;
   }

@@ -2476,7 +2447,7 @@ void FiniteElementSpace::Construct()

   ndofs = nvdofs + nedofs + nfdofs + nbdofs;

-   ConstructDoFTrans();
+   ConstructDoFTransArray();

   // record the current mesh sequence number to detect refinement etc.
   mesh_sequence = mesh->GetSequence();
@@ -2501,9 +2472,8 @@ int FiniteElementSpace::MinOrder(VarOrderBits bits)
   return 0;
 }

-void FiniteElementSpace
-::CalcEdgeFaceVarOrders(Array<VarOrderBits> &edge_orders,
-                        Array<VarOrderBits> &face_orders) const
+void FiniteElementSpace::CalcEdgeFaceVarOrders(
+   Array<VarOrderBits> &edge_orders, Array<VarOrderBits> &face_orders) const
 {
   MFEM_ASSERT(IsVariableOrder(), "");
   MFEM_ASSERT(Nonconforming(), "");
@@ -2727,8 +2697,8 @@ int FiniteElementSpace::GetNVariants(int entity, int index) const
 static const char* msg_orders_changed =
   "Element orders changed, you need to Update() the space first.";

-DofTransformation *
-FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
+void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs,
+                                        DofTransformation &doftrans) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

@@ -2736,13 +2706,16 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
   {
      elem_dof->GetRow(elem, dofs);

-      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
+      if (DoFTransArray[mesh->GetElementBaseGeometry(elem)])
      {
         Array<int> Fo;
         elem_fos -> GetRow (elem, Fo);
-         DoFTrans[mesh->GetElementBaseGeometry(elem)]->SetFaceOrientations(Fo);
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetElementBaseGeometry(elem)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
-      return DoFTrans[mesh->GetElementBaseGeometry(elem)];
+      return;
   }

   Array<int> V, E, Eo, F, Fo; // TODO: LocalArray
@@ -2766,10 +2739,12 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
      {
         nfd += fec->GetNumDof(mesh->GetFaceGeometry(F[i]), order);
      }
-      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
+      if (DoFTransArray[mesh->GetElementBaseGeometry(elem)])
      {
-         DoFTrans[mesh->GetElementBaseGeometry(elem)]
-         -> SetFaceOrientations(Fo);
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetElementBaseGeometry(elem)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
   }

@@ -2828,54 +2803,18 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
         dofs.Append(bbase + j);
      }
   }
-   return DoFTrans[mesh->GetElementBaseGeometry(elem)];
 }

-void FiniteElementSpace::GetPatchDofs(int patch, Array<int> &dofs) const
+DofTransformation *FiniteElementSpace::GetElementDofs(int elem,
+                                                      Array<int> &dofs) const
 {
-   MFEM_ASSERT(NURBSext,
-               "FiniteElementSpace::GetPatchDofs needs a NURBSExtension");
-   NURBSext->GetPatchDofs(patch, dofs);
+   DoFTrans.SetDofTransformation(NULL);
+   GetElementDofs(elem, dofs, DoFTrans);
+   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
 }

-const FiniteElement *FiniteElementSpace::GetFE(int i) const
-{
-   if (i < 0 || i >= mesh->GetNE())
-   {
-      if (mesh->GetNE() == 0)
-      {
-         MFEM_ABORT("Empty MPI partitions are not permitted!");
-      }
-      MFEM_ABORT("Invalid element id:" << i << "; minimum allowed:" << 0 <<
-                 ", maximum allowed:" << mesh->GetNE()-1);
-   }
-
-   const FiniteElement *FE =
-      fec->GetFE(mesh->GetElementGeometry(i), GetElementOrderImpl(i));
-
-   if (NURBSext)
-   {
-      NURBSext->LoadFE(i, FE);
-   }
-   else
-   {
-#ifdef MFEM_DEBUG
-      // consistency check: fec->GetOrder() and FE->GetOrder() should return
-      // the same value (for standard, constant-order spaces)
-      if (!IsVariableOrder() && FE->GetDim() > 0)
-      {
-         MFEM_ASSERT(FE->GetOrder() == fec->GetOrder(),
-                     "internal error: " <<
-                     FE->GetOrder() << " != " << fec->GetOrder());
-      }
-#endif
-   }
-
-   return FE;
-}
-
-DofTransformation *
-FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
+void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs,
+                                           DofTransformation &doftrans) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

@@ -2883,17 +2822,19 @@ FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
   {
      bdr_elem_dof->GetRow(bel, dofs);

-      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
+      if (DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)])
      {
         Array<int> Fo;
         bdr_elem_fos -> GetRow (bel, Fo);
-         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
-         SetFaceOrientations(Fo);
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
-      return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
+      return;
   }

-   Array<int> V, E, Eo, Fo; // TODO: LocalArray
+   Array<int> V, E, Eo; // TODO: LocalArray
   int F, oF;

   int dim = mesh->Dimension();
@@ -2917,11 +2858,14 @@ FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
   {
      mesh->GetBdrElementFace(bel, &F, &oF);

-      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
+      if (DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)])
      {
-         Fo.Append(oF);
-         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
-         SetFaceOrientations(Fo);
+         mfem::Array<int> Fo(1);
+         Fo[0] = oF;
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
   }

@@ -2963,8 +2907,14 @@ FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
         dofs.Append(EncodeDof(nvdofs + nedofs + fbase, ind[j]));
      }
   }
+}

-   return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
+DofTransformation *FiniteElementSpace::GetBdrElementDofs(int bel,
+                                                         Array<int> &dofs) const
+{
+   DoFTrans.SetDofTransformation(NULL);
+   GetBdrElementDofs(bel, dofs, DoFTrans);
+   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
 }

 int FiniteElementSpace::GetFaceDofs(int face, Array<int> &dofs,
@@ -3134,18 +3084,6 @@ int FiniteElementSpace::GetNumElementInteriorDofs(int i) const
                         GetElementOrderImpl(i));
 }

-void FiniteElementSpace::GetEdgeInteriorDofs(int i, Array<int> &dofs) const
-{
-   MFEM_VERIFY(!IsVariableOrder(), "not implemented");
-
-   int ne = fec->DofForGeometry(Geometry::SEGMENT);
-   dofs.SetSize (ne);
-   for (int j = 0, k = nvdofs+i*ne; j < ne; j++, k++)
-   {
-      dofs[j] = k;
-   }
-}
-
 void FiniteElementSpace::GetFaceInteriorDofs(int i, Array<int> &dofs) const
 {
   MFEM_VERIFY(!IsVariableOrder(), "not implemented");
@@ -3170,6 +3108,61 @@ void FiniteElementSpace::GetFaceInteriorDofs(int i, Array<int> &dofs) const
   }
 }

+void FiniteElementSpace::GetEdgeInteriorDofs(int i, Array<int> &dofs) const
+{
+   MFEM_VERIFY(!IsVariableOrder(), "not implemented");
+
+   int ne = fec->DofForGeometry(Geometry::SEGMENT);
+   dofs.SetSize (ne);
+   for (int j = 0, k = nvdofs+i*ne; j < ne; j++, k++)
+   {
+      dofs[j] = k;
+   }
+}
+
+void FiniteElementSpace::GetPatchDofs(int patch, Array<int> &dofs) const
+{
+   MFEM_ASSERT(NURBSext,
+               "FiniteElementSpace::GetPatchDofs needs a NURBSExtension");
+   NURBSext->GetPatchDofs(patch, dofs);
+}
+
+const FiniteElement *FiniteElementSpace::GetFE(int i) const
+{
+   if (i < 0 || i >= mesh->GetNE())
+   {
+      if (mesh->GetNE() == 0)
+      {
+         MFEM_ABORT("Empty MPI partitions are not permitted!");
+      }
+      MFEM_ABORT("Invalid element id:" << i << "; minimum allowed:" << 0 <<
+                 ", maximum allowed:" << mesh->GetNE()-1);
+   }
+
+   const FiniteElement *FE =
+      fec->GetFE(mesh->GetElementGeometry(i), GetElementOrderImpl(i));
+
+   if (NURBSext)
+   {
+      NURBSext->LoadFE(i, FE);
+   }
+   else
+   {
+#ifdef MFEM_DEBUG
+      // consistency check: fec->GetOrder() and FE->GetOrder() should return
+      // the same value (for standard, constant-order spaces)
+      if (!IsVariableOrder() && FE->GetDim() > 0)
+      {
+         MFEM_ASSERT(FE->GetOrder() == fec->GetOrder(),
+                     "internal error: " <<
+                     FE->GetOrder() << " != " << fec->GetOrder());
+      }
+#endif
+   }
+
+   return FE;
+}
+
 const FiniteElement *FiniteElementSpace::GetBE(int i) const
 {
   int order = fec->GetOrder();
@@ -3242,8 +3235,8 @@ const FiniteElement *FiniteElementSpace::GetEdgeElement(int i,
   return fec->GetFE(Geometry::SEGMENT, eo);
 }

-const FiniteElement *FiniteElementSpace
-::GetTraceElement(int i, Geometry::Type geom_type) const
+const FiniteElement *FiniteElementSpace::GetTraceElement(
+   int i, Geometry::Type geom_type) const
 {
   return fec->TraceFiniteElementForGeometry(geom_type);
 }
@@ -3283,7 +3276,7 @@ void FiniteElementSpace::Destroy()
   }
   E2BFQ_array.SetSize(0);

-   DestroyDoFTrans();
+   DestroyDoFTransArray();

   dof_elem_array.DeleteAll();
   dof_ldof_array.DeleteAll();
@@ -3301,19 +3294,18 @@ void FiniteElementSpace::Destroy()
      delete bdr_elem_dof;
      delete bdr_elem_fos;
      delete face_dof;
-
      delete [] bdofs;
   }
   ceed::RemoveBasisAndRestriction(this);
 }

-void FiniteElementSpace::DestroyDoFTrans()
+void FiniteElementSpace::DestroyDoFTransArray()
 {
-   for (int i = 0; i < DoFTrans.Size(); i++)
+   for (int i = 0; i < DoFTransArray.Size(); i++)
   {
-      delete DoFTrans[i];
+      delete DoFTransArray[i];
   }
-   DoFTrans.SetSize(0);
+   DoFTransArray.SetSize(0);
 }

 void FiniteElementSpace::GetTransferOperator(
@@ -271,8 +271,8 @@ protected:
   int own_ext;
   mutable Array<int> face_to_be; // NURBS FE space only

-   Array<DofTransformation*> DoFTrans;
-   mutable VDofTransformation VDoFTrans;
+   Array<StatelessDofTransformation *> DoFTransArray;
+   mutable DofTransformation DoFTrans;

   /** Matrix representing the prolongation from the global conforming dofs to
       a set of intermediate partially conforming dofs, e.g. the dofs associated
@@ -328,8 +328,8 @@ protected:
   void Construct();
   void Destroy();

-   void ConstructDoFTrans();
-   void DestroyDoFTrans();
+   void ConstructDoFTransArray();
+   void DestroyDoFTransArray();

   void BuildElementToDofTable() const;
   void BuildBdrElementToDofTable() const;
@@ -416,10 +416,10 @@ protected:
      Table* old_elem_dof; // Owned.
      Table* old_elem_fos; // Owned.

-      Array<DofTransformation*> old_DoFTrans;
-      mutable VDofTransformation old_VDoFTrans;
+      Array<StatelessDofTransformation*> old_DoFTransArray;
+      mutable DofTransformation old_DoFTrans;

-      void ConstructDoFTrans();
+      void ConstructDoFTransArray();

   public:
      /** Construct the operator based on the elem_dof table of the original
@@ -803,7 +803,16 @@ public:
   /// with triangular faces.
   ///
   /// @note The returned object should NOT be deleted by the caller.
-   virtual DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;
+   DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;
+
+   /// @brief The same as GetElementDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   virtual void GetElementDofs(int elem, Array<int> &dofs,
+                               DofTransformation &doftrans) const;

   /// @brief Returns indices of degrees of freedom for boundary element 'bel'.
   /// The returned indices are offsets into an @ref ldof vector. See also
@@ -817,13 +826,16 @@ public:
   /// with triangular faces.
   ///
   /// @note The returned object should NOT be deleted by the caller.
-   virtual DofTransformation *GetBdrElementDofs(int bel,
-                                                Array<int> &dofs) const;
+   DofTransformation *GetBdrElementDofs(int bel, Array<int> &dofs) const;

-   /** @brief Returns indices of degrees of freedom for NURBS patch index
-    @a patch. Cartesian ordering is used, for the tensor-product degrees of
-    freedom. */
-   void GetPatchDofs(int patch, Array<int> &dofs) const;
+   /// @brief The same as GetBdrElementDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   virtual void GetBdrElementDofs(int bel, Array<int> &dofs,
+                                  DofTransformation &doftrans) const;

   /// @brief Returns the indices of the degrees of freedom for the specified
   /// face, including the DOFs for the edges and the vertices of the face.
@@ -870,6 +882,13 @@ public:
   /// GetElementInteriorVDofs().
   void GetElementInteriorDofs(int i, Array<int> &dofs) const;

+   /// @brief Returns the number of degrees of freedom associated with the
+   /// interior of the specified element.
+   ///
+   /// See GetElementInteriorDofs() for more information or to obtain the
+   /// relevant indices.
+   int GetNumElementInteriorDofs(int i) const;
+
   /// @brief Returns the indices of the degrees of freedom for the interior
   /// of the specified face.
   ///
@@ -882,13 +901,6 @@ public:
   /// GetFaceInteriorVDofs().
   void GetFaceInteriorDofs(int i, Array<int> &dofs) const;

-   /// @brief Returns the number of degrees of freedom associated with the
-   /// interior of the specified element.
-   ///
-   /// See GetElementInteriorDofs() for more information or to obtain the
-   /// relevant indices.
-   int GetNumElementInteriorDofs(int i) const;
-
   /// @brief Returns the indices of the degrees of freedom for the interior
   /// of the specified edge.
   ///
@@ -897,6 +909,11 @@ public:
   void GetEdgeInteriorDofs(int i, Array<int> &dofs) const;
   ///@}

+   /** @brief Returns indices of degrees of freedom for NURBS patch index
+    @a patch. Cartesian ordering is used, for the tensor-product degrees of
+    freedom. */
+   void GetPatchDofs(int patch, Array<int> &dofs) const;
+
   /// @anchor dof2vdof @name DoF To VDoF Conversion methods
   /// These methods convert between local dof and local vector dof using the
   /// appropriate relationship based on the Ordering::Type defined in this
@@ -1023,6 +1040,15 @@ public:
   /// @note The returned object should NOT be deleted by the caller.
   DofTransformation *GetElementVDofs(int i, Array<int> &vdofs) const;

+   /// @brief The same as GetElementVDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   void GetElementVDofs(int i, Array<int> &vdofs,
+                        DofTransformation &doftrans) const;
+
   /// @brief Returns indices of degrees of freedom for @a i'th boundary
   /// element.
   /// The returned indices are offsets into an @ref ldof vector with @b vdim
@@ -1038,6 +1064,15 @@ public:
   /// @note The returned object should NOT be deleted by the caller.
   DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;

+   /// @brief The same as GetBdrElementVDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   void GetBdrElementVDofs(int i, Array<int> &vdofs,
+                           DofTransformation &doftrans) const;
+
   /// Returns indices of degrees of freedom in @a vdofs for NURBS patch @a i.
   void GetPatchVDofs(int i, Array<int> &vdofs) const;

@@ -31,13 +31,13 @@ FmsBasisTypeToMfemBasis(FmsBasisType b)
   switch (b)
   {
      case FMS_NODAL_GAUSS_OPEN:
-         retval = mfem::BasisType::GaussLegendre;;
+         retval = mfem::BasisType::GaussLegendre;
         break;
      case FMS_NODAL_GAUSS_CLOSED:
-         retval = mfem::BasisType::GaussLobatto;;
+         retval = mfem::BasisType::GaussLobatto;
         break;
      case FMS_POSITIVE:
-         retval = mfem::BasisType::Positive;;
+         retval = mfem::BasisType::Positive;
         break;
      case FMS_NODAL_UNIFORM_OPEN:
         retval = mfem::BasisType::OpenUniform;
@@ -1812,22 +1812,22 @@ MeshToFmsMesh(const Mesh *mmesh, FmsMesh *fmesh, FmsComponent *volume)
      switch (betype)
      {
         case Element::POINT:
-            bdr_eles[FMS_VERTEX].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_VERTEX].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::SEGMENT:
-            bdr_eles[FMS_EDGE].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_EDGE].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::TRIANGLE:
-            bdr_eles[FMS_TRIANGLE].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_TRIANGLE].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::QUADRILATERAL:
-            bdr_eles[FMS_QUADRILATERAL].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_QUADRILATERAL].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::TETRAHEDRON:
-            bdr_eles[FMS_TETRAHEDRON].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_TETRAHEDRON].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::HEXAHEDRON:
-            bdr_eles[FMS_HEXAHEDRON].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_HEXAHEDRON].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         default:
            MFEM_WARNING("Unsupported boundary element " << betype << " at boundary index "
@@ -65,10 +65,10 @@ public:

   /** @brief Return an IntegrationRule consisting of all vertices of the given
       Geometry::Type, @a GeomType. */
-   const IntegrationRule *GetVertices(int GeomType);
+   const IntegrationRule *GetVertices(int GeomType) const;

   /// Return the center of the given Geometry::Type, @a GeomType.
-   const IntegrationPoint &GetCenter(int GeomType)
+   const IntegrationPoint &GetCenter(int GeomType) const
   { return GeomCenter[GeomType]; }

   /// Get a random point in the reference element specified by @a GeomType.
@@ -97,9 +97,9 @@ public:

   const DenseMatrix &GetGeomToPerfGeomJac(int GeomType) const
   { return *GeomToPerfGeomJac[GeomType]; }
-   DenseMatrix *GetPerfGeomToGeomJac(int GeomType)
+   const DenseMatrix *GetPerfGeomToGeomJac(int GeomType) const
   { return PerfGeomToGeomJac[GeomType]; }
-   void GetPerfPointMat(int GeomType, DenseMatrix &pm);
+   void GetPerfPointMat(int GeomType, DenseMatrix &pm) const;
   void JacToPerfJac(int GeomType, const DenseMatrix &J,
                     DenseMatrix &PJ) const;

@@ -123,7 +123,7 @@ public:
   }

   /// Return the number of boundary "faces" of a given Geometry::Type.
-   int NumBdr(int GeomType) { return NumBdrArray[GeomType]; }
+   int NumBdr(int GeomType) const { return NumBdrArray[GeomType]; }
 };

 template <> struct
@@ -317,27 +317,27 @@ public:
   int Type;

   RefinedGeometry(int NPts, int NRefG, int NRefE, int NBdrE = 0) :
-      RefPts(NPts), RefGeoms(NRefG), RefEdges(NRefE), NumBdrEdges(NBdrE) { }
+      RefPts(NPts), RefGeoms(NRefG), RefEdges(NRefE), NumBdrEdges(NBdrE) {}
 };

 class GeometryRefiner
 {
 private:
-   int type; // Quadrature1D type (ClosedUniform is default)
+   int Type; // Quadrature1D type (ClosedUniform is default)
   Array<RefinedGeometry *> RGeom[Geometry::NumGeom];
   Array<IntegrationRule *> IntPts[Geometry::NumGeom];

-   RefinedGeometry *FindInRGeom(Geometry::Type Geom, int Times, int ETimes,
-                                int Type);
-   IntegrationRule *FindInIntPts(Geometry::Type Geom, int NPts);
+   RefinedGeometry *FindInRGeom(Geometry::Type Geom, int Times,
+                                int ETimes) const;
+   IntegrationRule *FindInIntPts(Geometry::Type Geom, int NPts) const;

 public:
-   GeometryRefiner();
+   GeometryRefiner(int t = Quadrature1D::ClosedUniform) : Type(t) {}

   /// Set the Quadrature1D type of points to use for subdivision.
-   void SetType(const int t) { type = t; }
+   void SetType(int t) { Type = t; }
   /// Get the Quadrature1D type of points used for subdivision.
-   int GetType() const { return type; }
+   int GetType() const { return Type; }

   RefinedGeometry *Refine(Geometry::Type Geom, int Times, int ETimes = 1);

@@ -345,10 +345,10 @@ public:
   const IntegrationRule *RefineInterior(Geometry::Type Geom, int Times);

   /// Get the Refinement level based on number of points
-   virtual int GetRefinementLevelFromPoints(Geometry::Type Geom, int Npts);
+   static int GetRefinementLevelFromPoints(Geometry::Type Geom, int Npts);

   /// Get the Refinement level based on number of elements
-   virtual int GetRefinementLevelFromElems(Geometry::Type geom, int Npts);
+   static int GetRefinementLevelFromElems(Geometry::Type geom, int Npts);

   ~GeometryRefiner();
 };
@@ -67,6 +67,7 @@ FindPointsGSLIB::FindPointsGSLIB()

 FindPointsGSLIB::~FindPointsGSLIB()
 {
+   comm_free(gsl_comm);
   delete gsl_comm;
   delete cr;
   for (int i = 0; i < 4; i++)
@@ -737,7 +737,7 @@ void QuadratureFunctions1D::GivePolyPoints(const int np, double *pts,
         ClosedGL(np, &ir);
         break;
      }
-      default:
+      case Quadrature1D::Invalid:
      {
         MFEM_ABORT("Asking for an unknown type of 1D Quadrature points, "
                    "type = " << type);
@@ -831,7 +831,10 @@ void QuadratureFunctions1D::CalculateUniformWeights(IntegrationRule *ir,
         hinv = p+1;
         ihoffset = 1;
         break;
-      default:
+      case Quadrature1D::GaussLegendre:
+      case Quadrature1D::GaussLobatto:
+      case Quadrature1D::ClosedGL:
+      case Quadrature1D::Invalid:
         MFEM_ABORT("invalid Quadrature1D type: " << type);
   }
   // set w0 = (-1)^p*(p!)/(hinv^p)
@@ -940,10 +943,10 @@ IntegrationRules IntRules(0, Quadrature1D::GaussLegendre);

 IntegrationRules RefinedIntRules(1, Quadrature1D::GaussLegendre);

-IntegrationRules::IntegrationRules(int Ref, int type_):
-   quad_type(type_)
+IntegrationRules::IntegrationRules(int ref, int type)
+   : quad_type(type)
 {
-   refined = Ref;
+   refined = ref;

   if (refined < 0) { own_rules = 0; return; }

@@ -975,11 +978,19 @@ IntegrationRules::IntegrationRules(int Ref, int type_):

   CubeIntRules.SetSize(32, h_mt);
   CubeIntRules = NULL;
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   IntRuleLocks.SetSize(Geometry::NUM_GEOMETRIES, h_mt);
+   for (int i = 0; i < Geometry::NUM_GEOMETRIES; i++)
+   {
+      omp_init_lock(&IntRuleLocks[i]);
+   }
+#endif
 }

 const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
 {
-   Array<IntegrationRule *> *ir_array;
+   Array<IntegrationRule *> *ir_array = NULL;

   switch (GeomType)
   {
@@ -991,9 +1002,9 @@ const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
      case Geometry::CUBE:        ir_array = &CubeIntRules; break;
      case Geometry::PRISM:       ir_array = &PrismIntRules; break;
      case Geometry::PYRAMID:     ir_array = &PyramidIntRules; break;
-      default:
-         mfem_error("IntegrationRules::Get(...) : Unknown geometry type!");
-         ir_array = NULL;
+      case Geometry::INVALID:
+      case Geometry::NUM_GEOMETRIES:
+         MFEM_ABORT("Unknown type of reference element!");
   }

   if (Order < 0)
@@ -1001,36 +1012,35 @@ const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
      Order = 0;
   }

+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_set_lock(&IntRuleLocks[GeomType]);
+#endif
+
   if (!HaveIntRule(*ir_array, Order))
   {
-#ifdef MFEM_USE_LEGACY_OPENMP
-      #pragma omp critical
-#endif
-      {
-         if (!HaveIntRule(*ir_array, Order))
-         {
-            IntegrationRule *ir = GenerateIntegrationRule(GeomType, Order);
+      IntegrationRule *ir = GenerateIntegrationRule(GeomType, Order);
 #ifdef MFEM_DEBUG
-            int RealOrder = Order;
-            while (RealOrder+1 < ir_array->Size() &&
-                   (*ir_array)[RealOrder+1] == ir)
-            {
-               RealOrder++;
-            }
-            MFEM_VERIFY(RealOrder == ir->GetOrder(), "internal error");
-#else
-            MFEM_CONTRACT_VAR(ir);
-#endif
-         }
+      int RealOrder = Order;
+      while (RealOrder+1 < ir_array->Size() && (*ir_array)[RealOrder+1] == ir)
+      {
+         RealOrder++;
      }
+      MFEM_VERIFY(RealOrder == ir->GetOrder(), "internal error");
+#else
+      MFEM_CONTRACT_VAR(ir);
+#endif
   }

+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_unset_lock(&IntRuleLocks[GeomType]);
+#endif
+
   return *(*ir_array)[Order];
 }

 void IntegrationRules::Set(int GeomType, int Order, IntegrationRule &IntRule)
 {
-   Array<IntegrationRule *> *ir_array;
+   Array<IntegrationRule *> *ir_array = NULL;

   switch (GeomType)
   {
@@ -1042,11 +1052,15 @@ void IntegrationRules::Set(int GeomType, int Order, IntegrationRule &IntRule)
      case Geometry::CUBE:        ir_array = &CubeIntRules; break;
      case Geometry::PRISM:       ir_array = &PrismIntRules; break;
      case Geometry::PYRAMID:     ir_array = &PyramidIntRules; break;
-      default:
-         mfem_error("IntegrationRules::Set(...) : Unknown geometry type!");
-         ir_array = NULL;
+      case Geometry::INVALID:
+      case Geometry::NUM_GEOMETRIES:
+         MFEM_ABORT("Unknown type of reference element!");
   }

+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_set_lock(&IntRuleLocks[GeomType]);
+#endif
+
   if (HaveIntRule(*ir_array, Order))
   {
      MFEM_ABORT("Overwriting set rules is not supported!");
@@ -1055,16 +1069,19 @@ void IntegrationRules::Set(int GeomType, int Order, IntegrationRule &IntRule)
   AllocIntRule(*ir_array, Order);

   (*ir_array)[Order] = &IntRule;
+
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   omp_unset_lock(&IntRuleLocks[GeomType]);
+#endif
 }

-void IntegrationRules::DeleteIntRuleArray(Array<IntegrationRule *> &ir_array)
+void IntegrationRules::DeleteIntRuleArray(
+   Array<IntegrationRule *> &ir_array) const
 {
-   int i;
-   IntegrationRule *ir = NULL;
-
   // Many of the intrules have multiple contiguous copies in the ir_array
   // so we have to be careful to not delete them twice.
-   for (i = 0; i < ir_array.Size(); i++)
+   IntegrationRule *ir = NULL;
+   for (int i = 0; i < ir_array.Size(); i++)
   {
      if (ir_array[i] != NULL && ir_array[i] != ir)
      {
@@ -1076,6 +1093,13 @@ void IntegrationRules::DeleteIntRuleArray(Array<IntegrationRule *> &ir_array)

 IntegrationRules::~IntegrationRules()
 {
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   for (int i = 0; i < Geometry::NUM_GEOMETRIES; i++)
+   {
+      omp_destroy_lock(&IntRuleLocks[i]);
+   }
+#endif
+
   if (!own_rules) { return; }

   DeleteIntRuleArray(PointIntRules);
@@ -1110,10 +1134,11 @@ IntegrationRule *IntegrationRules::GenerateIntegrationRule(int GeomType,
         return PrismIntegrationRule(Order);
      case Geometry::PYRAMID:
         return PyramidIntegrationRule(Order);
-      default:
-         mfem_error("IntegrationRules::Set(...) : Unknown geometry type!");
-         return NULL;
+      case Geometry::INVALID:
+      case Geometry::NUM_GEOMETRIES:
+         MFEM_ABORT("Unknown type of reference element!");
   }
+   return NULL;
 }


@@ -1122,7 +1147,7 @@ IntegrationRule *IntegrationRules::PointIntegrationRule(int Order)
 {
   if (Order > 1)
   {
-      mfem_error("Point Integration Rule of Order > 1 not defined");
+      MFEM_ABORT("Point Integration Rule of Order > 1 not defined");
      return NULL;
   }

@@ -1185,7 +1210,7 @@ IntegrationRule *IntegrationRules::SegmentIntegrationRule(int Order)
         QuadratureFunctions1D::OpenHalfUniform(n, ir);
         break;
      }
-      default:
+      case Quadrature1D::Invalid:
      {
         MFEM_ABORT("unknown Quadrature1D type: " << quad_type);
      }
@@ -1762,8 +1787,8 @@ IntegrationRule *IntegrationRules::PyramidIntegrationRule(int Order)

   for (int k=0; k<npts; k++)
   {
-      const IntegrationPoint & ipc = irc.IntPoint(k);
-      IntegrationPoint & ipp = PyramidIntRules[Order]->IntPoint(k);
+      const IntegrationPoint &ipc = irc.IntPoint(k);
+      IntegrationPoint &ipp = PyramidIntRules[Order]->IntPoint(k);
      ipp.x = ipc.x * (1.0 - ipc.z);
      ipp.y = ipc.y * (1.0 - ipc.z);
      ipp.z = ipc.z;
@@ -1775,8 +1800,8 @@ IntegrationRule *IntegrationRules::PyramidIntegrationRule(int Order)
 // Integration rules for reference prism
 IntegrationRule *IntegrationRules::PrismIntegrationRule(int Order)
 {
-   const IntegrationRule & irt = Get(Geometry::TRIANGLE, Order);
-   const IntegrationRule & irs = Get(Geometry::SEGMENT, Order);
+   const IntegrationRule &irt = Get(Geometry::TRIANGLE, Order);
+   const IntegrationRule &irs = Get(Geometry::SEGMENT, Order);
   int nt = irt.GetNPoints();
   int ns = irs.GetNPoints();
   AllocIntRule(PrismIntRules, Order);
@@ -1790,12 +1815,12 @@ IntegrationRule *IntegrationRules::PrismIntegrationRule(int Order)

   for (int ks=0; ks<ns; ks++)
   {
-      const IntegrationPoint & ips = irs.IntPoint(ks);
+      const IntegrationPoint &ips = irs.IntPoint(ks);
      for (int kt=0; kt<nt; kt++)
      {
         int kp = ks * nt + kt;
-         const IntegrationPoint & ipt = irt.IntPoint(kt);
-         IntegrationPoint & ipp = PrismIntRules[Order]->IntPoint(kp);
+         const IntegrationPoint &ipt = irt.IntPoint(kt);
+         IntegrationPoint &ipp = PrismIntRules[Order]->IntPoint(kp);
         ipp.x = ipt.x;
         ipp.y = ipt.y;
         ipp.z = ips.x;
@@ -14,6 +14,9 @@

 #include "../config/config.hpp"
 #include "../general/array.hpp"
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+#include <omp.h>
+#endif

 #include <vector>
 #include <map>
@@ -428,14 +431,18 @@ private:
   Array<IntegrationRule *> PrismIntRules;
   Array<IntegrationRule *> CubeIntRules;

-   void AllocIntRule(Array<IntegrationRule *> &ir_array, int Order)
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   Array<omp_lock_t> IntRuleLocks;
+#endif
+
+   void AllocIntRule(Array<IntegrationRule *> &ir_array, int Order) const
   {
      if (ir_array.Size() <= Order)
      {
         ir_array.SetSize(Order + 1, NULL);
      }
   }
-   bool HaveIntRule(Array<IntegrationRule *> &ir_array, int Order)
+   bool HaveIntRule(Array<IntegrationRule *> &ir_array, int Order) const
   {
      return (ir_array.Size() > Order && ir_array[Order] != NULL);
   }
@@ -443,6 +450,7 @@ private:
   {
      return Order | 1; // valid for all quad_type's
   }
+   void DeleteIntRuleArray(Array<IntegrationRule *> &ir_array) const;

   /// The following methods allocate new IntegrationRule objects without
   /// checking if they already exist.  To avoid memory leaks use
@@ -457,12 +465,10 @@ private:
   IntegrationRule *PrismIntegrationRule(int Order);
   IntegrationRule *CubeIntegrationRule(int Order);

-   void DeleteIntRuleArray(Array<IntegrationRule *> &ir_array);
-
 public:
   /// Sets initial sizes for the integration rule arrays, but rules
   /// are defined the first time they are requested with the Get method.
-   explicit IntegrationRules(int Ref = 0,
+   explicit IntegrationRules(int ref = 0,
                             int type = Quadrature1D::GaussLegendre);

   /// Returns an integration rule for given GeomType and Order.
@@ -137,7 +137,7 @@ bool LinearForm::SupportsDevice() const
      // Make sure every boundary element corresponds to a boundary face
      for (int be = 0; be < fes->GetNBE(); ++be)
      {
-         const int f = mesh.GetBdrElementEdgeIndex(be);
+         const int f = mesh.GetBdrElementFaceIndex(be);
         const auto face_info = mesh.GetFaceInformation(f);
         if (!face_info.IsBoundary())
         {
@@ -148,7 +148,7 @@ void LinearFormExtension::Update()
      std::unordered_map<int,int> f_to_be;
      for (int i = 0; i < mesh.GetNBE(); ++i)
      {
-         const int f = mesh.GetBdrElementEdgeIndex(i);
+         const int f = mesh.GetBdrElementFaceIndex(i);
         f_to_be[f] = i;
      }
      MFEM_VERIFY(size_t(nf_bdr) == f_to_be.size(), "Incompatible sizes");
@@ -104,7 +104,7 @@ public:
 };


-/// Class for domain integration L(v) := (f, v)
+/// Class for domain integration \f$ L(v) := (f, v) \f$
 class DomainLFIntegrator : public DeltaLFIntegrator
 {
   Vector shape;
@@ -141,7 +141,7 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/// Class for domain integrator L(v) := (f, grad v)
+/// Class for domain integrator \f$ L(v) := (f, \nabla v) \f$
 class DomainLFGradIntegrator : public DeltaLFIntegrator
 {
 private:
@@ -150,7 +150,7 @@ private:
   DenseMatrix dshape;

 public:
-   /// Constructs the domain integrator (Q, grad v)
+   /// Constructs the domain integrator \f$ (Q, \nabla v) \f$
   DomainLFGradIntegrator(VectorCoefficient &QF)
      : DeltaLFIntegrator(QF), Q(QF) { }

@@ -175,7 +175,7 @@ public:
 };


-/// Class for boundary integration L(v) := (g, v)
+/// Class for boundary integration \f$ L(v) := (g, v) \f$
 class BoundaryLFIntegrator : public LinearFormIntegrator
 {
   Vector shape;
@@ -249,8 +249,8 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/** Class for domain integration of L(v) := (f, v), where
-    f=(f1,...,fn) and v=(v1,...,vn). */
+/** Class for domain integration of \f$ L(v) := (f, v) \f$, where
+    \f$ f = (f_1,\dots,f_n)\f$ and \f$ v = (v_1,\dots,v_n) \f$. */
 class VectorDomainLFIntegrator : public DeltaLFIntegrator
 {
 private:
@@ -282,8 +282,8 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/** Class for domain integrator L(v) := (f, grad v), where
-    f=(f1x,f1y,f1z,...,fnx,fny,fnz) and v=(v1,...,vn). */
+/** Class for domain integrator \f$ L(v) := (f, \nabla v) \f$, where
+    \f$ f = (f_{1x},f_{1y},f_{1z},\dots,f_{nx},f_{ny},f_{nz})\f$ and \f$v=(v_1,\dots,v_n)\f$. */
 class VectorDomainLFGradIntegrator : public DeltaLFIntegrator
 {
 private:
@@ -316,8 +316,8 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/** Class for boundary integration of L(v) := (g, v), where
-    f=(f1,...,fn) and v=(v1,...,vn). */
+/** Class for boundary integration of \f$ L(v) := (g, v) \f$, where
+    \f$f=(f_1,\dots,f_n)\f$ and \f$v=(v_1,\dots,v_n)\f$. */
 class VectorBoundaryLFIntegrator : public LinearFormIntegrator
 {
 private:
@@ -371,7 +371,7 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/// \f$ (Q, curl v)_{\Omega} \f$ for Nedelec Elements)
+/// \f$ (Q, \mathrm{curl}(v))_{\Omega} \f$ for Nedelec Elements
 class VectorFEDomainLFCurlIntegrator : public DeltaLFIntegrator
 {
 private:
@@ -380,7 +380,7 @@ private:
   Vector vec;

 public:
-   /// Constructs the domain integrator (Q, curl v)
+   /// Constructs the domain integrator \f$(Q, \mathrm{curl}(v))  \f$
   VectorFEDomainLFCurlIntegrator(VectorCoefficient &F)
      : DeltaLFIntegrator(F), QF(&F) { }

@@ -395,14 +395,14 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/// \f$ (Q, div v)_{\Omega} \f$ for RT Elements)
+/// \f$ (Q, \mathrm{div}(v))_{\Omega} \f$ for RT Elements
 class VectorFEDomainLFDivIntegrator : public DeltaLFIntegrator
 {
 private:
   Vector divshape;
   Coefficient &Q;
 public:
-   /// Constructs the domain integrator (Q, div v)
+   /// Constructs the domain integrator \f$ (Q, \mathrm{div}(v)) \f$
   VectorFEDomainLFDivIntegrator(Coefficient &QF)
      : DeltaLFIntegrator(QF), Q(QF) { }

@@ -420,7 +420,7 @@ public:
 };

 /** \f$ (f, v \cdot n)_{\partial\Omega} \f$ for vector test function
-    v=(v1,...,vn) where all vi are in the same scalar FE space and f is a
+    \f$v=(v_1,\dots,v_n)\f$ where all vi are in the same scalar FE space and \f$f\f$ is a
    scalar function. */
 class VectorBoundaryFluxLFIntegrator : public LinearFormIntegrator
 {
@@ -441,8 +441,8 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

-/** Class for boundary integration of (f, v.n) for scalar coefficient f and
-    RT vector test function v. This integrator works with RT spaces defined
+/** Class for boundary integration of \f$ (f, v \cdot n)\f$ for scalar coefficient \f$f\f$ and
+    RT vector test function \f$v\f$. This integrator works with RT spaces defined
    using the RT_FECollection class. */
 class VectorFEBoundaryFluxLFIntegrator : public LinearFormIntegrator
 {
@@ -491,9 +491,9 @@ public:


 /** Class for boundary integration of the linear form:
-    (alpha/2) < (u.n) f, w > - beta < |u.n| f, w >,
-    where f and u are given scalar and vector coefficients, respectively,
-    and w is the scalar test function. */
+    \f$ \frac{\alpha}{2} \langle (u \cdot n) f, w \rangle - \beta \langle |u \cdot n| f, w \rangle \f$
+    where \f$f\f$ and \f$u\f$ are given scalar and vector coefficients, respectively,
+    and \f$w\f$ is the scalar test function. */
 class BoundaryFlowIntegrator : public LinearFormIntegrator
 {
 private:
@@ -525,13 +525,13 @@ public:

 /** Boundary linear integrator for imposing non-zero Dirichlet boundary
    conditions, to be used in conjunction with DGDiffusionIntegrator.
-    Specifically, given the Dirichlet data u_D, the linear form assembles the
+    Specifically, given the Dirichlet data \f$u_D\f$, the linear form assembles the
    following integrals on the boundary:
-
-    sigma < u_D, (Q grad(v)).n > + kappa < {h^{-1} Q} u_D, v >,
-
+   \f[
+    \sigma \langle u_D, (Q \nabla v)) \cdot n \rangle + \kappa \langle {h^{-1} Q} u_D, v \rangle,
+   \f]
    where Q is a scalar or matrix diffusion coefficient and v is the test
-    function. The parameters sigma and kappa should be the same as the ones
+    function. The parameters \f$\sigma\f$ and \f$\kappa\f$ should be the same as the ones
    used in the DGDiffusionIntegrator. */
 class DGDirichletLFIntegrator : public LinearFormIntegrator
 {
@@ -568,12 +568,12 @@ public:
 /** Boundary linear form integrator for imposing non-zero Dirichlet boundary
    conditions, in a DG elasticity formulation. Specifically, the linear form is
    given by
-
-    alpha < u_D, (lambda div(v) I + mu (grad(v) + grad(v)^T)) . n > +
-      + kappa < h^{-1} (lambda + 2 mu) u_D, v >,
-
-    where u_D is the given Dirichlet data. The parameters alpha, kappa, lambda
-    and mu, should match the parameters with the same names used in the bilinear
+   \f[
+    \alpha \langle u_D, (\lambda \mathrm{div}(v) I + \mu (\nabla v + \nabla v^{\mathrm{T}})) \cdot n \rangle +
+      + \kappa \langle h^{-1} (\lambda + 2 \mu) u_D, v \rangle,
+   \f]
+    where u_D is the given Dirichlet data. The parameters \f$\alpha\f$, \f$\kappa\f$, \f$\lambda\f$
+    and \f$\mu\f$, should match the parameters with the same names used in the bilinear
    form integrator, DGElasticityIntegrator. */
 class DGElasticityDirichletLFIntegrator : public LinearFormIntegrator
 {
@@ -612,18 +612,18 @@ public:

 /** Class for spatial white Gaussian noise integration.

-    The target problem is the linear SPDE a(u,v) = F(v) with F(v) := <Ẇ,v>,
-    where Ẇ is spatial white Gaussian noise. When the Galerkin method is used to
-    discretize this problem into a linear system of equations Ax = b, the RHS is
-    a Gaussian random vector b~N(0,M) whose covariance matrix is the same as the
-    mass matrix M_ij = (v_i,v_j). This property can be ensured if b = H w, where
-    HHᵀ = M and each component w_i~N(0,1).
+    The target problem is the linear SPDE \f$ a(u,v) = F(v)\f$ with \f$F(v) := <\dot{W},v> \f$,
+    where \f$\dot{W}\f$ is spatial white Gaussian noise. When the Galerkin method is used to
+    discretize this problem into a linear system of equations \f$Ax = b\f$, the RHS is
+    a Gaussian random vector \f$b \sim N(0,M)\f$ whose covariance matrix is the same as the
+    mass matrix \f$M_{ij} = (v_i,v_j)\f$. This property can be ensured if \f$b = H w\f$, where
+    \f$HH^{\mathrm{T}} = M\f$ and each component \f$w_i\sim N(0,1)\f$.

-    There is much flexibility in how we may wish to define H. In this PR, we
-    define H = Pᵀ diag(L_e), where P is the local-to-global dof assembly matrix
-    and diag(L_e) is a block-diagonal matrix with L_e L_eᵀ = M_e, where M_e is
-    the element mass matrix for element e. A straightforward computation shows
-    that HHᵀ = Pᵀ diag(M_e) P = M, as necessary. */
+    There is much flexibility in how we may wish to define \f$H\f$. In this PR, we
+    define \f$H = P^{\mathrm{T}} diag(L_e)\f$, where \f$P\f$ is the local-to-global dof assembly matrix
+    and \f$\mathrm{diag}(L_e)\f$ is a block-diagonal matrix with \f$L_e L_e^{\mathrm{T}} = M_e\f$, where \f$M_e\f$ is
+    the element mass matrix for element \f$e\f$. A straightforward computation shows
+    that \f$HH^{\mathrm{T}} = P^{\mathrm{T}} diag(M_e) P = M\f$, as necessary. */
 class WhiteGaussianNoiseDomainLFIntegrator : public LinearFormIntegrator
 {
 #ifdef MFEM_USE_MPI
@@ -718,8 +718,8 @@ public:
 };


-/** Class for domain integration of L(v) := (f, v), where
-    f=(f1,...,fn) and v=(v1,...,vn). that makes use of
+/** Class for domain integration of \f$ L(v) := (f, v) \f$, where
+    \f$ f=(f_1,\dots,f_n)\f$ and \f$v=(v_1,\dots,v_n)\f$. that makes use of
    VectorQuadratureFunctionCoefficient*/
 class VectorQuadratureLFIntegrator : public LinearFormIntegrator
 {
@@ -751,7 +751,7 @@ public:
 };


-/** Class for domain integration L(v) := (f, v) that makes use
+/** Class for domain integration \f$ L(v) := (f, v) \f$ that makes use
    of QuadratureFunctionCoefficient. */
 class QuadratureLFIntegrator : public LinearFormIntegrator
 {
@@ -257,13 +257,13 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   // vertices of the LOR mesh. The vertex coordinates are already computed in
   // E-vector format and passed in in X_vert.
   //
-   // In this function, we need to convert X_vert (which has the shape (dim,
+   // In this function, we need to convert X_vert (which has the shape (sdim,
   // ndof_per_el, nel_ho)) to T-DOF format.
   //
-   // We place the results in the vector xyz_tvec, which has shape (ntdofs, dim)
+   // We place the results in the vector xyz_tvec, which has shape (ntdofs, sdim)
   // and then make the hypre vectors x, y, and z point to subvectors.
   //
-   // In 2D, z is NULL.
+   // When the space dimension is 2, z is NULL.

   // Create the H1 vertex space and get the element restriction
   ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
@@ -275,17 +275,17 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   const int nel_ho = vert_fes.GetNE();
   const int ndp1 = order + 1;
   const int ndof_per_el = static_cast<int>(pow(ndp1, dim));
-   const int sdim = dim;
+   const int sdim = vert_fes.GetMesh()->SpaceDimension();
   const int ntdofs = R->Height();

   const MemoryClass mc = GetHypreMemoryClass();
   bool dev = (mc == MemoryClass::DEVICE);

-   xyz_tvec = new Vector(ntdofs*dim);
+   xyz_tvec = new Vector(ntdofs*sdim);

-   auto xyz_tv = Reshape(HypreWrite(xyz_tvec->GetMemory()), ntdofs, dim);
+   auto xyz_tv = Reshape(HypreWrite(xyz_tvec->GetMemory()), ntdofs, sdim);
   const auto xyz_e =
-      Reshape(HypreRead(X_vert.GetMemory()), dim, ndof_per_el, nel_ho);
+      Reshape(HypreRead(X_vert.GetMemory()), sdim, ndof_per_el, nel_ho);
   const auto d_offsets = HypreRead(el_restr->Offsets().GetMemory());
   const auto d_indices = HypreRead(el_restr->Indices().GetMemory());
   const auto ltdof_ldof = HypreRead(R->GetMemoryJ());
@@ -309,7 +309,7 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   x = new HypreParVector(vert_fes.GetComm(), glob_size, d_x_ptr, cols, dev);
   double *d_y_ptr = xyz_tv + 1*ntdofs;
   y = new HypreParVector(vert_fes.GetComm(), glob_size, d_y_ptr, cols, dev);
-   if (dim == 3)
+   if (sdim == 3)
   {
      double *d_z_ptr = xyz_tv + 2*ntdofs;
      z = new HypreParVector(vert_fes.GetComm(), glob_size, d_z_ptr, cols, dev);
@@ -77,6 +77,7 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,

   // Get nodal points at the LOR vertices
   const int dim = mesh_ho.Dimension();
+   const int sdim = mesh_ho.SpaceDimension();
   const int nel_ho = mesh_ho.GetNE();
   const int order = fes_ho.GetMaxElementOrder();
   const int nd1d = order + 1;
@@ -94,7 +95,7 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
   IntegrationRule ir = GetCollocatedIntRule(fes_ho);

   // Map from nodal E-vector to Q-vector at the LOR vertex points
-   X_vert.SetSize(dim*ndof_per_el*nel_ho);
+   X_vert.SetSize(sdim*ndof_per_el*nel_ho);
   const QuadratureInterpolator *quad_interp =
      nodal_fes->GetQuadratureInterpolator(ir);
   quad_interp->SetOutputLayout(QVectorLayout::byVDIM);
@@ -380,44 +381,49 @@ void BatchedLORAssembly::SparseIJToCSR(OperatorHandle &A) const
   FillJAndData(*A_mat);
 }

+template <int ORDER, int SDIM, typename LOR_KERNEL>
+static void Assemble_(LOR_KERNEL &kernel, int dim)
+{
+   if (dim == 2) { kernel.template Assemble2D<ORDER,SDIM>(); }
+   else if (dim == 3) { kernel.template Assemble3D<ORDER>(); }
+   else { MFEM_ABORT("Unsupported dimension"); }
+}
+
+template <int ORDER, typename LOR_KERNEL>
+static void Assemble_(LOR_KERNEL &kernel, int dim, int sdim)
+{
+   if (sdim == 2) { Assemble_<ORDER,2>(kernel, dim); }
+   else if (sdim == 3) { Assemble_<ORDER,3>(kernel, dim); }
+   else { MFEM_ABORT("Unsupported space dimension."); }
+}
+
+template <typename LOR_KERNEL>
+static void Assemble_(LOR_KERNEL &kernel, int dim, int sdim, int order)
+{
+   switch (order)
+   {
+      case 1: Assemble_<1>(kernel, dim, sdim); break;
+      case 2: Assemble_<2>(kernel, dim, sdim); break;
+      case 3: Assemble_<3>(kernel, dim, sdim); break;
+      case 4: Assemble_<4>(kernel, dim, sdim); break;
+      case 5: Assemble_<5>(kernel, dim, sdim); break;
+      case 6: Assemble_<6>(kernel, dim, sdim); break;
+      case 7: Assemble_<7>(kernel, dim, sdim); break;
+      case 8: Assemble_<8>(kernel, dim, sdim); break;
+      default: MFEM_ABORT("No kernel order " << order << "!");
+   }
+}
+
 template <typename LOR_KERNEL>
 void BatchedLORAssembly::AssemblyKernel(BilinearForm &a)
 {
   LOR_KERNEL kernel(a, fes_ho, X_vert, sparse_ij, sparse_mapping);

   const int dim = fes_ho.GetMesh()->Dimension();
+   const int sdim = fes_ho.GetMesh()->SpaceDimension();
   const int order = fes_ho.GetMaxElementOrder();

-   if (dim == 2)
-   {
-      switch (order)
-      {
-         case 1: kernel.template Assemble2D<1>(); break;
-         case 2: kernel.template Assemble2D<2>(); break;
-         case 3: kernel.template Assemble2D<3>(); break;
-         case 4: kernel.template Assemble2D<4>(); break;
-         case 5: kernel.template Assemble2D<5>(); break;
-         case 6: kernel.template Assemble2D<6>(); break;
-         case 7: kernel.template Assemble2D<7>(); break;
-         case 8: kernel.template Assemble2D<8>(); break;
-         default: MFEM_ABORT("No kernel order " << order << "!");
-      }
-   }
-   else if (dim == 3)
-   {
-      switch (order)
-      {
-         case 1: kernel.template Assemble3D<1>(); break;
-         case 2: kernel.template Assemble3D<2>(); break;
-         case 3: kernel.template Assemble3D<3>(); break;
-         case 4: kernel.template Assemble3D<4>(); break;
-         case 5: kernel.template Assemble3D<5>(); break;
-         case 6: kernel.template Assemble3D<6>(); break;
-         case 7: kernel.template Assemble3D<7>(); break;
-         case 8: kernel.template Assemble3D<8>(); break;
-         default: MFEM_ABORT("No kernel order " << order << "!");
-      }
-   }
+   Assemble_(kernel, dim, sdim, order);
 }

 void BatchedLORAssembly::AssembleWithoutBC(BilinearForm &a, OperatorHandle &A)
@@ -22,15 +22,22 @@ namespace mfem
 class BatchedLOR_H1 : BatchedLORKernel
 {
 public:
-   template <int ORDER> void Assemble2D();
+   template <int ORDER, int SDIM> void Assemble2D();
   template <int ORDER> void Assemble3D();
   BatchedLOR_H1(BilinearForm &a,
                 FiniteElementSpace &fes_ho_,
                 Vector &X_vert_,
                 Vector &sparse_ij_,
-                 Array<int> &sparse_mapping_);
+                 Array<int> &sparse_mapping_)
+      : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
+   {
+      ProjectLORCoefficient<MassIntegrator>(a, c1);
+      ProjectLORCoefficient<DiffusionIntegrator>(a, c2);
+   }
 };

 }

+#include "lor_h1_impl.hpp"
+
 #endif
@@ -9,7 +9,6 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-#include "lor_h1.hpp"
 #include "lor_util.hpp"
 #include "../../linalg/dtensor.hpp"
 #include "../../general/forall.hpp"
@@ -17,7 +16,7 @@
 namespace mfem
 {

-template <int ORDER>
+template <int ORDER, int SDIM>
 void BatchedLOR_H1::Assemble2D()
 {
   const int nel_ho = fes_ho.GetNE();
@@ -74,31 +73,8 @@ void BatchedLOR_H1::Assemble2D()

            for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }

-            double vx[4], vy[4];
-            LORVertexCoordinates2D<ORDER>(X, iel_ho, kx, ky, vx, vy);
+            SetupLORQuadData2D<ORDER,SDIM,false,false>(X, iel_ho, kx, ky, Q, false);

-            for (int iqy=0; iqy<2; ++iqy)
-            {
-               for (int iqx=0; iqx<2; ++iqx)
-               {
-                  const double x = iqx;
-                  const double y = iqy;
-                  const double w = 1.0/4.0;
-
-                  double J_[2*2];
-                  DeviceTensor<2> J(J_, 2, 2);
-
-                  Jacobian2D(x, y, vx, vy, J);
-
-                  const double detJ = Det2D(J);
-                  const double w_detJ = w/detJ;
-
-                  Q(0,iqy,iqx) = w_detJ * (J(0,1)*J(0,1) + J(1,1)*J(1,1)); // 1,1
-                  Q(1,iqy,iqx) = -w_detJ * (J(0,1)*J(0,0) + J(1,1)*J(1,0)); // 1,2
-                  Q(2,iqy,iqx) = w_detJ * (J(0,0)*J(0,0) + J(1,0)*J(1,0)); // 2,2
-                  Q(3,iqy,iqx) = w*detJ;
-               }
-            }
            for (int iqx=0; iqx<2; ++iqx)
            {
               for (int iqy=0; iqy<2; ++iqy)
@@ -519,34 +495,4 @@ void BatchedLOR_H1::Assemble3D()
   }
 }

-// Explicit template instantiations
-template void BatchedLOR_H1::Assemble2D<1>();
-template void BatchedLOR_H1::Assemble2D<2>();
-template void BatchedLOR_H1::Assemble2D<3>();
-template void BatchedLOR_H1::Assemble2D<4>();
-template void BatchedLOR_H1::Assemble2D<5>();
-template void BatchedLOR_H1::Assemble2D<6>();
-template void BatchedLOR_H1::Assemble2D<7>();
-template void BatchedLOR_H1::Assemble2D<8>();
-
-template void BatchedLOR_H1::Assemble3D<1>();
-template void BatchedLOR_H1::Assemble3D<2>();
-template void BatchedLOR_H1::Assemble3D<3>();
-template void BatchedLOR_H1::Assemble3D<4>();
-template void BatchedLOR_H1::Assemble3D<5>();
-template void BatchedLOR_H1::Assemble3D<6>();
-template void BatchedLOR_H1::Assemble3D<7>();
-template void BatchedLOR_H1::Assemble3D<8>();
-
-BatchedLOR_H1::BatchedLOR_H1(BilinearForm &a,
-                             FiniteElementSpace &fes_ho_,
-                             Vector &X_vert_,
-                             Vector &sparse_ij_,
-                             Array<int> &sparse_mapping_)
-   : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
-{
-   ProjectLORCoefficient<MassIntegrator>(a, c1);
-   ProjectLORCoefficient<DiffusionIntegrator>(a, c2);
-}
-
 } // namespace mfem
@@ -22,15 +22,22 @@ namespace mfem
 class BatchedLOR_ND : BatchedLORKernel
 {
 public:
-   template <int ORDER> void Assemble2D();
+   template <int ORDER, int SDIM> void Assemble2D();
   template <int ORDER> void Assemble3D();
   BatchedLOR_ND(BilinearForm &a,
                 FiniteElementSpace &fes_ho_,
                 Vector &X_vert_,
                 Vector &sparse_ij_,
-                 Array<int> &sparse_mapping_);
+                 Array<int> &sparse_mapping_)
+      : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
+   {
+      ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
+      ProjectLORCoefficient<CurlCurlIntegrator>(a, c2);
+   }
 };

 }

+#include "lor_nd_impl.hpp"
+
 #endif
@@ -9,7 +9,6 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-#include "lor_nd.hpp"
 #include "lor_util.hpp"
 #include "../../linalg/dtensor.hpp"
 #include "../../general/forall.hpp"
@@ -17,7 +16,7 @@
 namespace mfem
 {

-template <int ORDER>
+template <int ORDER, int SDIM>
 void BatchedLOR_ND::Assemble2D()
 {
   const int nel_ho = fes_ho.GetNE();
@@ -83,31 +82,8 @@ void BatchedLOR_ND::Assemble2D()
            // local_mat is the local (dense) stiffness matrix
            for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }

-            double vx[4], vy[4];
-            LORVertexCoordinates2D<ORDER>(X, iel_ho, kx, ky, vx, vy);
+            SetupLORQuadData2D<ORDER,SDIM,false,true>(X, iel_ho, kx, ky, Q, true);

-            for (int iqx=0; iqx<2; ++iqx)
-            {
-               for (int iqy=0; iqy<2; ++iqy)
-               {
-                  const double x = iqx;
-                  const double y = iqy;
-                  const double w = 1.0/4.0;
-
-                  double J_[2*2];
-                  DeviceTensor<2> J(J_, 2, 2);
-
-                  Jacobian2D(x, y, vx, vy, J);
-
-                  const double detJ = Det2D(J);
-                  const double w_detJ = w/detJ;
-
-                  Q(0,iqy,iqx) = w_detJ * (J(0,1)*J(0,1) + J(1,1)*J(1,1)); // 1,1
-                  Q(1,iqy,iqx) = -w_detJ * (J(0,1)*J(0,0) + J(1,1)*J(1,0)); // 1,2
-                  Q(2,iqy,iqx) = w_detJ * (J(0,0)*J(0,0) + J(1,0)*J(1,0)); // 2,2
-                  Q(3,iqy,iqx) = w_detJ;
-               }
-            }
            for (int iqx=0; iqx<2; ++iqx)
            {
               for (int iqy=0; iqy<2; ++iqy)
@@ -563,34 +539,4 @@ void BatchedLOR_ND::Assemble3D()
   }
 }

-// Explicit template instantiations
-template void BatchedLOR_ND::Assemble2D<1>();
-template void BatchedLOR_ND::Assemble2D<2>();
-template void BatchedLOR_ND::Assemble2D<3>();
-template void BatchedLOR_ND::Assemble2D<4>();
-template void BatchedLOR_ND::Assemble2D<5>();
-template void BatchedLOR_ND::Assemble2D<6>();
-template void BatchedLOR_ND::Assemble2D<7>();
-template void BatchedLOR_ND::Assemble2D<8>();
-
-template void BatchedLOR_ND::Assemble3D<1>();
-template void BatchedLOR_ND::Assemble3D<2>();
-template void BatchedLOR_ND::Assemble3D<3>();
-template void BatchedLOR_ND::Assemble3D<4>();
-template void BatchedLOR_ND::Assemble3D<5>();
-template void BatchedLOR_ND::Assemble3D<6>();
-template void BatchedLOR_ND::Assemble3D<7>();
-template void BatchedLOR_ND::Assemble3D<8>();
-
-BatchedLOR_ND::BatchedLOR_ND(BilinearForm &a,
-                             FiniteElementSpace &fes_ho_,
-                             Vector &X_vert_,
-                             Vector &sparse_ij_,
-                             Array<int> &sparse_mapping_)
-   : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
-{
-   ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
-   ProjectLORCoefficient<CurlCurlIntegrator>(a, c2);
-}
-
 } // namespace mfem
@@ -22,15 +22,22 @@ namespace mfem
 class BatchedLOR_RT : BatchedLORKernel
 {
 public:
-   template <int ORDER> void Assemble2D();
+   template <int ORDER, int SDIM> void Assemble2D();
   template <int ORDER> void Assemble3D();
   BatchedLOR_RT(BilinearForm &a,
                 FiniteElementSpace &fes_ho_,
                 Vector &X_vert_,
                 Vector &sparse_ij_,
-                 Array<int> &sparse_mapping_);
+                 Array<int> &sparse_mapping_)
+      : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
+   {
+      ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
+      ProjectLORCoefficient<DivDivIntegrator>(a, c2);
+   }
 };

 }

+#include "lor_rt_impl.hpp"
+
 #endif
@@ -9,7 +9,6 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-#include "lor_rt.hpp"
 #include "lor_util.hpp"
 #include "../../linalg/dtensor.hpp"
 #include "../../general/forall.hpp"
@@ -17,7 +16,7 @@
 namespace mfem
 {

-template <int ORDER>
+template <int ORDER, int SDIM>
 void BatchedLOR_RT::Assemble2D()
 {
   const int nel_ho = fes_ho.GetNE();
@@ -79,31 +78,8 @@ void BatchedLOR_RT::Assemble2D()
            // local_mat is the local (dense) stiffness matrix
            for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }

-            double vx[4], vy[4];
-            LORVertexCoordinates2D<ORDER>(X, iel_ho, kx, ky, vx, vy);
+            SetupLORQuadData2D<ORDER,SDIM,true,false>(X, iel_ho, kx, ky, Q, true);

-            for (int iqx=0; iqx<2; ++iqx)
-            {
-               for (int iqy=0; iqy<2; ++iqy)
-               {
-                  const double x = iqx;
-                  const double y = iqy;
-                  const double w = 1.0/4.0;
-
-                  double J_[2*2];
-                  DeviceTensor<2> J(J_, 2, 2);
-
-                  Jacobian2D(x, y, vx, vy, J);
-
-                  const double detJ = Det2D(J);
-                  const double w_detJ = w/detJ;
-
-                  Q(0,iqy,iqx) = w_detJ * (J(0,0)*J(0,0) + J(1,0)*J(1,0)); // 1,1
-                  Q(1,iqy,iqx) = w_detJ * (J(0,0)*J(0,1) + J(1,0)*J(1,1)); // 1,2
-                  Q(2,iqy,iqx) = w_detJ * (J(0,1)*J(0,1) + J(1,1)*J(1,1)); // 2,2
-                  Q(3,iqy,iqx) = w_detJ;
-               }
-            }
            for (int iqx=0; iqx<2; ++iqx)
            {
               for (int iqy=0; iqy<2; ++iqy)
@@ -547,34 +523,4 @@ void BatchedLOR_RT::Assemble3D()
   }
 }

-// Explicit template instantiations
-template void BatchedLOR_RT::Assemble2D<1>();
-template void BatchedLOR_RT::Assemble2D<2>();
-template void BatchedLOR_RT::Assemble2D<3>();
-template void BatchedLOR_RT::Assemble2D<4>();
-template void BatchedLOR_RT::Assemble2D<5>();
-template void BatchedLOR_RT::Assemble2D<6>();
-template void BatchedLOR_RT::Assemble2D<7>();
-template void BatchedLOR_RT::Assemble2D<8>();
-
-template void BatchedLOR_RT::Assemble3D<1>();
-template void BatchedLOR_RT::Assemble3D<2>();
-template void BatchedLOR_RT::Assemble3D<3>();
-template void BatchedLOR_RT::Assemble3D<4>();
-template void BatchedLOR_RT::Assemble3D<5>();
-template void BatchedLOR_RT::Assemble3D<6>();
-template void BatchedLOR_RT::Assemble3D<7>();
-template void BatchedLOR_RT::Assemble3D<8>();
-
-BatchedLOR_RT::BatchedLOR_RT(BilinearForm &a,
-                             FiniteElementSpace &fes_ho_,
-                             Vector &X_vert_,
-                             Vector &sparse_ij_,
-                             Array<int> &sparse_mapping_)
-   : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
-{
-   ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
-   ProjectLORCoefficient<DivDivIntegrator>(a, c2);
-}
-
 } // namespace mfem
@@ -20,11 +20,22 @@
 namespace mfem
 {

-template <int ORDER>
-MFEM_HOST_DEVICE inline void LORVertexCoordinates2D(
-   const double *X, int iel_ho, int kx, int ky, double vx[4], double vy[4])
+MFEM_HOST_DEVICE inline double Det2D(DeviceMatrix &J)
+{
+   return J(0,0)*J(1,1) - J(1,0)*J(0,1);
+}
+
+MFEM_HOST_DEVICE inline double Det3D(DeviceMatrix &J)
+{
+   return J(0,0) * (J(1,1) * J(2,2) - J(2,1) * J(1,2)) -
+          J(1,0) * (J(0,1) * J(2,2) - J(2,1) * J(0,2)) +
+          J(2,0) * (J(0,1) * J(1,2) - J(1,1) * J(0,2));
+}
+
+template <int ORDER, int SDIM=2>
+MFEM_HOST_DEVICE inline void LORVertexCoordinates2D(
+   const double *X, int iel_ho, int kx, int ky, double **v)
 {
-   const int dim = 2;
   const int nd1d = ORDER + 1;
   const int nvert_per_el = nd1d*nd1d;

@@ -33,23 +44,31 @@ MFEM_HOST_DEVICE inline void LORVertexCoordinates2D(
   const int v2 = kx + 1 + nd1d*(ky + 1);
   const int v3 = kx + nd1d*(ky + 1);

-   const int e0 = dim*(v0 + nvert_per_el*iel_ho);
-   const int e1 = dim*(v1 + nvert_per_el*iel_ho);
-   const int e2 = dim*(v2 + nvert_per_el*iel_ho);
-   const int e3 = dim*(v3 + nvert_per_el*iel_ho);
+   const int e0 = SDIM*(v0 + nvert_per_el*iel_ho);
+   const int e1 = SDIM*(v1 + nvert_per_el*iel_ho);
+   const int e2 = SDIM*(v2 + nvert_per_el*iel_ho);
+   const int e3 = SDIM*(v3 + nvert_per_el*iel_ho);

   // Vertex coordinates
-   vx[0] = X[e0 + 0];
-   vy[0] = X[e0 + 1];
+   v[0][0] = X[e0 + 0];
+   v[1][0] = X[e0 + 1];

-   vx[1] = X[e1 + 0];
-   vy[1] = X[e1 + 1];
+   v[0][1] = X[e1 + 0];
+   v[1][1] = X[e1 + 1];

-   vx[2] = X[e2 + 0];
-   vy[2] = X[e2 + 1];
+   v[0][2] = X[e2 + 0];
+   v[1][2] = X[e2 + 1];

-   vx[3] = X[e3 + 0];
-   vy[3] = X[e3 + 1];
+   v[0][3] = X[e3 + 0];
+   v[1][3] = X[e3 + 1];
+
+   if (SDIM == 3)
+   {
+      v[2][0] = X[e0 + 2];
+      v[2][1] = X[e1 + 2];
+      v[2][2] = X[e2 + 2];
+      v[2][3] = X[e3 + 2];
+   }
 }

 template <int ORDER>
@@ -112,15 +131,80 @@ MFEM_HOST_DEVICE inline void LORVertexCoordinates3D(
   vz[7] = X[e7 + 2];
 }

+template <int SDIM=2>
 MFEM_HOST_DEVICE inline void Jacobian2D(
-   const double x, const double y, const double vx[4], const double vy[4],
-   DeviceMatrix &J)
-{
-   J(0,0) = -(1-y)*vx[0] + (1-y)*vx[1] + y*vx[2] - y*vx[3];
-   J(0,1) = -(1-x)*vx[0] - x*vx[1] + x*vx[2] + (1-x)*vx[3];
+   const double x, const double y, double **v, DeviceMatrix &J);

-   J(1,0) = -(1-y)*vy[0] + (1-y)*vy[1] + y*vy[2] - y*vy[3];
-   J(1,1) = -(1-x)*vy[0] - x*vy[1] + x*vy[2] + (1-x)*vy[3];
+template <> MFEM_HOST_DEVICE inline void Jacobian2D<2>(
+   const double x, const double y, double **v, DeviceMatrix &J)
+{
+   J(0,0) = -(1-y)*v[0][0] + (1-y)*v[0][1] + y*v[0][2] - y*v[0][3];
+   J(0,1) = -(1-x)*v[0][0] - x*v[0][1] + x*v[0][2] + (1-x)*v[0][3];
+
+   J(1,0) = -(1-y)*v[1][0] + (1-y)*v[1][1] + y*v[1][2] - y*v[1][3];
+   J(1,1) = -(1-x)*v[1][0] - x*v[1][1] + x*v[1][2] + (1-x)*v[1][3];
+}
+
+template <> MFEM_HOST_DEVICE inline void Jacobian2D<3>(
+   const double x, const double y, double **v, DeviceMatrix &J)
+{
+   J(0,0) = -(1-y)*v[0][0] + (1-y)*v[0][1] + y*v[0][2] - y*v[0][3];
+   J(0,1) = -(1-x)*v[0][0] - x*v[0][1] + x*v[0][2] + (1-x)*v[0][3];
+
+   J(1,0) = -(1-y)*v[1][0] + (1-y)*v[1][1] + y*v[1][2] - y*v[1][3];
+   J(1,1) = -(1-x)*v[1][0] - x*v[1][1] + x*v[1][2] + (1-x)*v[1][3];
+
+   J(2,0) = -(1-y)*v[2][0] + (1-y)*v[2][1] + y*v[2][2] - y*v[2][3];
+   J(2,1) = -(1-x)*v[2][0] - x*v[2][1] + x*v[2][2] + (1-x)*v[2][3];
+}
+
+template <int ORDER, int SDIM, bool RT, bool ND>
+MFEM_HOST_DEVICE inline void SetupLORQuadData2D(
+   const double *X, int iel_ho, int kx, int ky, DeviceTensor<3> &Q, bool piola)
+{
+   double vx[4], vy[4], vz[4];
+   double *v[] = {vx, vy, vz};
+   LORVertexCoordinates2D<ORDER,SDIM>(X, iel_ho, kx, ky, v);
+
+   for (int iqy=0; iqy<2; ++iqy)
+   {
+      for (int iqx=0; iqx<2; ++iqx)
+      {
+         const double x = iqx;
+         const double y = iqy;
+         const double w = 1.0/4.0;
+
+         double J_[SDIM*2];
+         DeviceTensor<2> J(J_, SDIM, 2);
+
+         Jacobian2D<SDIM>(x, y, v, J);
+
+         if (SDIM == 2)
+         {
+            const double detJ = Det2D(J);
+            const double w_detJ = w/detJ;
+            const double E = J(0,0)*J(0,0) + J(1,0)*J(1,0);
+            const double F = J(0,0)*J(0,1) + J(1,0)*J(1,1);
+            const double G = J(0,1)*J(0,1) + J(1,1)*J(1,1);
+            Q(0,iqy,iqx) = w_detJ * (RT ? E : G); // 1,1
+            Q(1,iqy,iqx) = w_detJ * (RT ? F : -F); // 1,2
+            Q(2,iqy,iqx) = w_detJ * (RT ? G : E); // 2,2
+            Q(3,iqy,iqx) = (ND || RT) ? w_detJ : w*detJ;
+         }
+         else
+         {
+            const double E = J(0,0)*J(0,0) + J(1,0)*J(1,0) + J(2,0)*J(2,0);
+            const double F = J(0,0)*J(0,1) + J(1,0)*J(1,1) + J(2,0)*J(2,1);
+            const double G = J(0,1)*J(0,1) + J(1,1)*J(1,1) + J(2,1)*J(2,1);
+            const double detJ = sqrt(E*G - F*F);
+            const double w_detJ = w/detJ;
+            Q(0,iqy,iqx) = w_detJ * (RT ? E : G); // 1,1
+            Q(1,iqy,iqx) = w_detJ * (RT ? F : -F); // 1,2
+            Q(2,iqy,iqx) = w_detJ * (RT ? G : E); // 2,2
+            Q(3,iqy,iqx) =  (ND || RT) ? w_detJ : w*detJ;
+         }
+      }
+   }
 }

 MFEM_HOST_DEVICE inline void Jacobian3D(
@@ -180,18 +264,6 @@ MFEM_HOST_DEVICE inline void Adjugate3D(const DeviceMatrix &J, DeviceMatrix &A)
   A(2,2) = (J(0,0) * J(1,1)) - (J(0,1) * J(1,0));
 }

-MFEM_HOST_DEVICE inline double Det2D(DeviceMatrix &J)
-{
-   return J(0,0)*J(1,1) - J(1,0)*J(0,1);
-}
-
-MFEM_HOST_DEVICE inline double Det3D(DeviceMatrix &J)
-{
-   return J(0,0) * (J(1,1) * J(2,2) - J(2,1) * J(1,2)) -
-          J(1,0) * (J(0,1) * J(2,2) - J(2,1) * J(0,2)) +
-          J(2,0) * (J(0,1) * J(1,2) - J(1,1) * J(0,2));
-}
-
 }

 #endif
@@ -97,12 +97,37 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
   const FiniteElement *fe;
   ElementTransformation *T;
   DofTransformation *doftrans;
+   Mesh *mesh = fes->GetMesh();
   double energy = 0.0;

   if (dnfi.Size())
   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes->GetNE(); i++)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         fe = fes->GetFE(i);
         doftrans = fes->GetElementVDofs(i, vdofs);
         T = fes->GetElementTransformation(i);
@@ -110,6 +135,9 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
         if (doftrans) {doftrans->InvTransformPrimal(el_x); }
         for (int k = 0; k < dnfi.Size(); k++)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            energy += dnfi[k]->GetElementEnergy(*fe, *T, el_x);
         }
      }
@@ -175,8 +203,32 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const

   if (dnfi.Size())
   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes->GetNE(); i++)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         fe = fes->GetFE(i);
         doftrans = fes->GetElementVDofs(i, vdofs);
         T = fes->GetElementTransformation(i);
@@ -184,6 +236,9 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const
         if (doftrans) {doftrans->InvTransformPrimal(el_x); }
         for (int k = 0; k < dnfi.Size(); k++)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            dnfi[k]->AssembleElementVector(*fe, *T, el_x, el_y);
            if (doftrans) {doftrans->TransformDual(el_y); }
            py.AddElementVector(vdofs, el_y);
@@ -322,8 +377,32 @@ Operator &NonlinearForm::GetGradient(const Vector &x) const

   if (dnfi.Size())
   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes->GetNE(); i++)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         fe = fes->GetFE(i);
         doftrans = fes->GetElementVDofs(i, vdofs);
         T = fes->GetElementTransformation(i);
@@ -331,6 +410,9 @@ Operator &NonlinearForm::GetGradient(const Vector &x) const
         if (doftrans) {doftrans->InvTransformPrimal(el_x); }
         for (int k = 0; k < dnfi.Size(); k++)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            dnfi[k]->AssembleElementGrad(*fe, *T, el_x, elmat);
            if (doftrans) { doftrans->TransformDual(elmat); }
            Grad->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
@@ -561,13 +643,6 @@ BlockNonlinearForm::BlockNonlinearForm(Array<FiniteElementSpace *> &f) :
   SetSpaces(f);
 }

-void BlockNonlinearForm::AddBdrFaceIntegrator(BlockNonlinearFormIntegrator *nfi,
-                                              Array<int> &bdr_attr_marker)
-{
-   bfnfi.Append(nfi);
-   bfnfi_marker.Append(&bdr_attr_marker);
-}
-
 void BlockNonlinearForm::SetEssentialBC(
   const Array<Array<int> *> &bdr_attr_is_ess, Array<Vector *> &rhs)
 {
@@ -592,6 +667,7 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
   Array<const FiniteElement *> fe(fes.Size());
   ElementTransformation *T;
   DofTransformation *doftrans;
+   Mesh *mesh = fes[0]->GetMesh();
   double energy = 0.0;

   for (int i=0; i<fes.Size(); ++i)
@@ -601,8 +677,33 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
   }

   if (dnfi.Size())
+   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes[0]->GetNE(); ++i)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         T = fes[0]->GetElementTransformation(i);
         for (int s=0; s<fes.Size(); ++s)
         {
@@ -614,9 +715,13 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const

         for (int k = 0; k < dnfi.Size(); ++k)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            energy += dnfi[k]->GetElementEnergy(fe, *T, el_x_const);
         }
      }
+   }

   // free the allocated memory
   for (int i = 0; i < fes.Size(); ++i)
@@ -656,6 +761,7 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
   Array<const FiniteElement *> fe2(fes.Size());
   ElementTransformation *T;
   Array<DofTransformation *> doftrans(fes.Size()); doftrans = nullptr;
+   Mesh *mesh = fes[0]->GetMesh();

   by.UseDevice(true);
   by = 0.0;
@@ -670,8 +776,32 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

   if (dnfi.Size())
   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes[0]->GetNE(); ++i)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         T = fes[0]->GetElementTransformation(i);
         for (int s = 0; s < fes.Size(); ++s)
         {
@@ -683,6 +813,9 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

         for (int k = 0; k < dnfi.Size(); ++k)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            dnfi[k]->AssembleElementVector(fe, *T,
                                           el_x_const, el_y);

@@ -698,7 +831,6 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

   if (fnfi.Size())
   {
-      Mesh *mesh = fes[0]->GetMesh();
      FaceElementTransformations *tr;

      for (int i = 0; i < mesh->GetNumFaces(); ++i)
@@ -736,8 +868,8 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

   if (bfnfi.Size())
   {
-      Mesh *mesh = fes[0]->GetMesh();
      FaceElementTransformations *tr;
+
      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
@@ -858,6 +990,7 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
   Array<const FiniteElement *>fe2(fes.Size());
   ElementTransformation * T;
   Array<DofTransformation *> doftrans(fes.Size()); doftrans = nullptr;
+   Mesh *mesh = fes[0]->GetMesh();

   for (int i=0; i<fes.Size(); ++i)
   {
@@ -888,8 +1021,32 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const

   if (dnfi.Size())
   {
+      // Which attributes need to be processed?
+      Array<int> attr_marker(mesh->attributes.Size() ?
+                             mesh->attributes.Max() : 0);
+      attr_marker = 0;
+      for (int k = 0; k < dnfi.Size(); k++)
+      {
+         if (dnfi_marker[k] == NULL)
+         {
+            attr_marker = 1;
+            break;
+         }
+         Array<int> &marker = *dnfi_marker[k];
+         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
+                     "invalid marker for domain integrator #"
+                     << k << ", counting from zero");
+         for (int i = 0; i < attr_marker.Size(); i++)
+         {
+            attr_marker[i] |= marker[i];
+         }
+      }
+
      for (int i = 0; i < fes[0]->GetNE(); ++i)
      {
+         const int attr = mesh->GetAttribute(i);
+         if (attr_marker[attr-1] == 0) { continue; }
+
         T = fes[0]->GetElementTransformation(i);
         for (int s = 0; s < fes.Size(); ++s)
         {
@@ -901,6 +1058,9 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const

         for (int k = 0; k < dnfi.Size(); ++k)
         {
+            if (dnfi_marker[k] &&
+                (*dnfi_marker[k])[attr-1] == 0) { continue; }
+
            dnfi[k]->AssembleElementGrad(fe, *T, el_x_const, elmats);

            for (int j=0; j<fes.Size(); ++j)
@@ -923,7 +1083,6 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
   if (fnfi.Size())
   {
      FaceElementTransformations *tr;
-      Mesh *mesh = fes[0]->GetMesh();

      for (int i = 0; i < mesh->GetNumFaces(); ++i)
      {
@@ -960,7 +1119,6 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
   if (bfnfi.Size())
   {
      FaceElementTransformations *tr;
-      Mesh *mesh = fes[0]->GetMesh();

      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
@@ -37,6 +37,7 @@ protected:

   /// Set of Domain Integrators to be assembled (added).
   Array<NonlinearFormIntegrator*> dnfi; // owned
+   Array<Array<int>*>              dnfi_marker; // not owned

   /// Set of interior face Integrators to be assembled (added).
   Array<NonlinearFormIntegrator*> fnfi; // owned
@@ -108,7 +109,12 @@ public:

   /// Adds new Domain Integrator.
   void AddDomainIntegrator(NonlinearFormIntegrator *nlfi)
-   { dnfi.Append(nlfi); }
+   { dnfi.Append(nlfi); dnfi_marker.Append(NULL); }
+
+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(NonlinearFormIntegrator *nlfi,
+                            Array<int> &elem_marker)
+   { dnfi.Append(nlfi); dnfi_marker.Append(&elem_marker); }

   /// Access all integrators added with AddDomainIntegrator().
   Array<NonlinearFormIntegrator*> *GetDNFI() { return &dnfi; }
@@ -227,13 +233,14 @@ protected:

   /// Set of Domain Integrators to be assembled (added).
   Array<BlockNonlinearFormIntegrator*> dnfi;
+   Array<Array<int>*>                   dnfi_marker;

   /// Set of interior face Integrators to be assembled (added).
   Array<BlockNonlinearFormIntegrator*> fnfi;

   /// Set of Boundary Face Integrators to be assembled (added).
   Array<BlockNonlinearFormIntegrator*> bfnfi;
-   Array<Array<int>*>           bfnfi_marker;
+   Array<Array<int>*>                   bfnfi_marker;

   /** Auxiliary block-vectors for wrapping input and output vectors or holding
       GridFunction-like block-vector data (e.g. in parallel). */
@@ -298,7 +305,12 @@ public:

   /// Adds new Domain Integrator.
   void AddDomainIntegrator(BlockNonlinearFormIntegrator *nlfi)
-   { dnfi.Append(nlfi); }
+   { dnfi.Append(nlfi); dnfi_marker.Append(NULL); }
+
+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(BlockNonlinearFormIntegrator *nlfi,
+                            Array<int> &elem_marker)
+   { dnfi.Append(nlfi); dnfi_marker.Append(&elem_marker); }

   /// Adds new Interior Face Integrator.
   void AddInteriorFaceIntegrator(BlockNonlinearFormIntegrator *nlfi)
@@ -311,7 +323,8 @@ public:
   /** @brief Adds new Boundary Face Integrator, restricted to specific boundary
       attributes. */
   void AddBdrFaceIntegrator(BlockNonlinearFormIntegrator *nlfi,
-                             Array<int> &bdr_marker);
+                             Array<int> &bdr_marker)
+   { bfnfi.Append(nlfi); bfnfi_marker.Append(&bdr_marker); }

   virtual void SetEssentialBC(const Array<Array<int> *>&bdr_attr_is_ess,
                               Array<Vector *> &rhs);
@@ -466,53 +466,54 @@ void ParFiniteElementSpace::ApplyLDofSigns(Table &el_dof) const
   ApplyLDofSigns(all_dofs);
 }

-DofTransformation *
-ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs) const
+void ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs,
+                                           DofTransformation &doftrans) const
 {
   if (elem_dof)
   {
      elem_dof->GetRow(i, dofs);

-      if (DoFTrans[mesh->GetElementBaseGeometry(i)])
+      if (DoFTransArray[mesh->GetElementBaseGeometry(i)])
      {
         Array<int> Fo;
         elem_fos->GetRow(i, Fo);
-         DoFTrans[mesh->GetElementBaseGeometry(i)]->SetFaceOrientations(Fo);
-         return DoFTrans[mesh->GetElementBaseGeometry(i)];
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetElementBaseGeometry(i)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
-      return NULL;
+      return;
   }
-   DofTransformation * doftrans = FiniteElementSpace::GetElementDofs(i, dofs);
+   FiniteElementSpace::GetElementDofs(i, dofs, doftrans);
   if (Conforming())
   {
      ApplyLDofSigns(dofs);
   }
-   return doftrans;
 }

-DofTransformation *
-ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs) const
+void ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs,
+                                              DofTransformation &doftrans) const
 {
   if (bdr_elem_dof)
   {
      bdr_elem_dof->GetRow(i, dofs);

-      if (DoFTrans[mesh->GetBdrElementBaseGeometry(i)])
+      if (DoFTransArray[mesh->GetBdrElementBaseGeometry(i)])
      {
         Array<int> Fo;
-         bdr_elem_fos -> GetRow (i, Fo);
-         DoFTrans[mesh->GetBdrElementBaseGeometry(i)]->SetFaceOrientations(Fo);
-         return DoFTrans[mesh->GetBdrElementBaseGeometry(i)];
+         bdr_elem_fos->GetRow(i, Fo);
+         doftrans.SetDofTransformation(
+            *DoFTransArray[mesh->GetBdrElementBaseGeometry(i)]);
+         doftrans.SetFaceOrientations(Fo);
+         doftrans.SetVDim();
      }
-      return NULL;
+      return;
   }
-   DofTransformation * doftrans =
-      FiniteElementSpace::GetBdrElementDofs(i, dofs);
+   FiniteElementSpace::GetBdrElementDofs(i, dofs, doftrans);
   if (Conforming())
   {
      ApplyLDofSigns(dofs);
   }
-   return doftrans;
 }

 int ParFiniteElementSpace::GetFaceDofs(int i, Array<int> &dofs,
@@ -939,8 +940,8 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
         }
         else if (i_offd[i+1] == i_offd[i] + 2)
         {
-            const double * T = ND_StatelessDofTransformation
-                               ::GetFaceTransform(ltori[i]).GetData();
+            const double *T =
+               ND_DofTransformation::GetFaceTransform(ltori[i]).GetData();
            j_offd[i_offd[i] + 1] = j_offd[i_offd[i]] + 1;
            d_offd[i_offd[i]] = T[0]; d_offd[i_offd[i] + 1] = T[2];
            i++;
@@ -1454,31 +1455,30 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
   delete [] requests;
 }

-DofTransformation *ParFiniteElementSpace::GetFaceNbrElementVDofs(
-   int i, Array<int> &vdofs) const
+void ParFiniteElementSpace::GetFaceNbrElementVDofs(
+   int i, Array<int> &vdofs, DofTransformation &doftrans) const
 {
   face_nbr_element_dof.GetRow(i, vdofs);

-   DofTransformation *doftrans = NULL;
-   Geometry::Type geom = GetFaceNbrFE(i)->GetGeomType();
-   if (DoFTrans[geom])
+   if (DoFTransArray[GetFaceNbrFE(i)->GetGeomType()])
   {
      Array<int> F, Fo;
      pmesh->GetFaceNbrElementFaces(pmesh->GetNE() + i, F, Fo);
-      doftrans = DoFTrans[geom];
-      doftrans->SetFaceOrientations(Fo);
-   }
-   if (vdim == 1 || doftrans == NULL)
-   {
-      return doftrans;
-   }
-   else
-   {
-      VDoFTrans.SetDofTransformation(*doftrans);
-      return &VDoFTrans;
+      doftrans.SetDofTransformation(
+         *DoFTransArray[GetFaceNbrFE(i)->GetGeomType()]);
+      doftrans.SetFaceOrientations(Fo);
+      doftrans.SetVDim(vdim, ordering);
   }
 }

+DofTransformation *ParFiniteElementSpace::GetFaceNbrElementVDofs(
+   int i, Array<int> &vdofs) const
+{
+   DoFTrans.SetDofTransformation(NULL);
+   GetFaceNbrElementVDofs(i, vdofs, DoFTrans);
+   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
+}
+
 void ParFiniteElementSpace::GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const
 {
   // Works for NC mesh where 'i' is an index returned by
@@ -2235,19 +2235,13 @@ void NeighborRowMessage::Decode(int rank)

            // This is the second "fundamental unit" used in the transformation.
            const auto initial_second_row = second_row;
+            const double *T =
+               ND_DofTransformation::GetFaceTransform(fo).GetData();

-            const auto T = [&fo]()
-            {
-               auto T = ND_StatelessDofTransformation::GetFaceTransform(fo);
-               T(0,0) -= 1;
-               T(1,1) -= 1;
-               return T;
-            }();
-
-            first_row.AddRow(initial_first_row, T(0,0));
-            first_row.AddRow(initial_second_row, T(0,1));
-            second_row.AddRow(initial_first_row, T(1,0));
-            second_row.AddRow(initial_second_row, T(1,1));
+            first_row.AddRow(initial_first_row, T[0] - 1.0);
+            first_row.AddRow(initial_second_row, T[2]);
+            second_row.AddRow(initial_first_row, T[1]);
+            second_row.AddRow(initial_second_row, T[3] - 1.0);

            first_row.Collapse();
            second_row.Collapse();
@@ -248,7 +248,11 @@ public:
       If the FiniteElementCollection, @a f, is NULL (default), the FE
       collection used by @a global_fes will be reused. If @a f is not NULL, it
       must be the same as, or a copy of, the FE collection used by
-       @a global_fes. */
+       @a global_fes.
+
+       @note Currently the @a partitioning array is not used by this
+       constructor, it is required for general parallel variable-order support.
+   */
   ParFiniteElementSpace(ParMesh *pm, const FiniteElementSpace *global_fes,
                         const int *partitioning,
                         const FiniteElementCollection *f = NULL);
@@ -284,11 +288,17 @@ public:
   /// Return the number of local vector true dofs.
   int GetTrueVSize() const override { return ltdof_size; }

-   /// Returns indexes of degrees of freedom in array dofs for i'th element.
-   DofTransformation *GetElementDofs(int i, Array<int> &dofs) const override;
+   /// Returns indexes of degrees of freedom in array dofs for i'th element and
+   /// returns the DofTransformation data in a user-provided object.
+   using FiniteElementSpace::GetElementDofs;
+   void GetElementDofs(int i, Array<int> &dofs,
+                       DofTransformation &doftrans) const override;

-   /// Returns indexes of degrees of freedom for i'th boundary element.
-   DofTransformation *GetBdrElementDofs(int i, Array<int> &dofs) const override;
+   /// Returns indexes of degrees of freedom for i'th boundary element and
+   /// returns the DofTransformation data in a user-provided object.
+   using FiniteElementSpace::GetBdrElementDofs;
+   void GetBdrElementDofs(int i, Array<int> &dofs,
+                          DofTransformation &doftrans) const override;

   /** Returns the indexes of the degrees of freedom for i'th face
       including the dofs for the edges and the vertices of the face. */
@@ -382,6 +392,8 @@ public:
   // Face-neighbor functions
   void ExchangeFaceNbrData();
   int GetFaceNbrVSize() const { return num_face_nbr_dofs; }
+   void GetFaceNbrElementVDofs(int i, Array<int> &vdofs,
+                               DofTransformation &doftrans) const;
   DofTransformation *GetFaceNbrElementVDofs(int i, Array<int> &vdofs) const;
   void GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const;
   const FiniteElement *GetFaceNbrFE(int i) const;
@@ -693,7 +693,23 @@ void ParGridFunction::ProjectBdrCoefficient(

 #ifdef MFEM_DEBUG
   Array<int> ess_vdofs_marker;
-   pfes->GetEssentialVDofs(attr, ess_vdofs_marker);
+   if (vcoeff) { pfes->GetEssentialVDofs(attr, ess_vdofs_marker); }
+   else
+   {
+      ess_vdofs_marker.SetSize(Size());
+      ess_vdofs_marker = 0;
+      for (int i = 0; i < fes->GetVDim(); i++)
+      {
+         if (!coeff[i]) { continue; }
+         Array<int> component_dof_marker;
+         pfes->GetEssentialVDofs(attr, component_dof_marker,i);
+         for (int j = 0; j<Size(); j++)
+         {
+            ess_vdofs_marker[j] = bool(ess_vdofs_marker[j]) ||
+                                  bool(component_dof_marker[j]);
+         }
+      }
+   }
   for (int i = 0; i < values_counter.Size(); i++)
   {
      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
@@ -177,7 +177,7 @@ int FaceQuadratureSpace::GetEntityIndex(const ElementTransformation &T) const
         return get_face_index(T.ElementNo);
      case ElementTransformation::BDR_ELEMENT:
      case ElementTransformation::BDR_FACE:
-         return get_face_index(mesh.GetBdrElementEdgeIndex(T.ElementNo));
+         return get_face_index(mesh.GetBdrElementFaceIndex(T.ElementNo));
      default:
         MFEM_ABORT("Invalid element type.");
         return -1;
@@ -462,21 +462,52 @@ void TMOP_Metric_009::AssembleH(const DenseMatrix &Jpt,
   ie.Assemble_ddI1b(weight, A.GetData());
 }

-// mu_14 = |T-I|^2
+double TMOP_Metric_014::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_14 = |J - I|^2.
+   DenseMatrix Mat(Jpt);
+   Mat(0,0) -= 1.0;
+   Mat(1,1) -= 1.0;
+   return Mat.FNorm2();
+}
+
 double TMOP_Metric_014::EvalW(const DenseMatrix &Jpt) const
 {
-   MFEM_VERIFY(Jtr != NULL,
-               "Requires a target Jacobian, use SetTargetJacobian().");
+   // mu_14 = |J - I|^2 = I1[J-I].
+   DenseMatrix Mat(Jpt);
+   Mat(0,0) -= 1.0;
+   Mat(1,1) -= 1.0;

-   DenseMatrix Id(2,2);
+   ie.SetJacobian(Mat.GetData());
+   return ie.Get_I1();
+}

-   Id(0,0) = 1; Id(0,1) = 0;
-   Id(1,0) = 0; Id(1,1) = 1;
+void TMOP_Metric_014::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+{
+   // P = dI1[J-I] d/dJ[J-I] = dI1[J-I].
+   DenseMatrix JptMinusId = Jpt;
+   for (int i = 0; i < Jpt.Size(); i++)
+   {
+      JptMinusId(i, i) -= 1.0;
+   }
+   ie.SetJacobian(JptMinusId.GetData());
+   P = ie.Get_dI1();
+}

-   DenseMatrix Mat(2,2);
-   Mat = Jpt;
-   Mat.Add(-1,Id);
-   return Mat.FNorm2();
+void TMOP_Metric_014::AssembleH(const DenseMatrix &Jpt,
+                                const DenseMatrix &DS,
+                                const double weight,
+                                DenseMatrix &A) const
+{
+   // dP = ddI1[J-I].
+   DenseMatrix JptMinusId = Jpt;
+   for (int i = 0; i < Jpt.Size(); i++)
+   {
+      JptMinusId(i, i) -= 1.0;
+   }
+   ie.SetJacobian(JptMinusId.GetData());
+   ie.SetDerivativeMatrix(DS.Height(), DS.GetData());
+   ie.Assemble_ddI1(weight, A.GetData());
 }

 double TMOP_Metric_022::EvalW(const DenseMatrix &Jpt) const
@@ -4347,6 +4378,8 @@ UpdateAfterMeshPositionChange(const Vector &x_new,
 {
   if (discr_tc) { PA.Jtr_needs_update = true; }

+   if (PA.enabled) { UpdateCoefficientsPA(x_new); }
+
   Ordering::Type ordering = x_fes.GetOrdering();

   // Update the finite difference delta if FD are used.
@@ -373,16 +373,20 @@ public:
 /// 2D non-barrier Shape+Size+Orientation (VOS) metric (polyconvex).
 class TMOP_Metric_014 : public TMOP_QualityMetric
 {
+protected:
+   mutable InvariantsEvaluator2D<double> ie;
+
 public:
-   // W = |T-I|^2.
+   // W = |J - I|^2.
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = I1[J-I].
   virtual double EvalW(const DenseMatrix &Jpt) const;

-   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
-   { MFEM_ABORT("Not implemented"); }
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;

   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
-                          const double weight, DenseMatrix &A) const
-   { MFEM_ABORT("Not implemented"); }
+                          const double weight, DenseMatrix &A) const;
 };

 /// 2D Shifted barrier form of shape metric (mu_2).
@@ -1817,17 +1821,27 @@ protected:

   // PA extension
   // ------------
+   // Jtr: all ref->target Jacobians, (dim x dim) Q-Vector as DenseTensor.
+   //      updated when needed, based on Jtr_needs_update.
+   //
   //  E: Q-vector for TMOP-energy
+   //     Used as temporary storage when the total energy is computed.
   //  O: Q-Vector of 1.0, used to compute sums using the dot product kernel.
   // X0: E-vector for initial nodal coordinates used for limiting.
+   //     Does not change during the TMOP iteration.
   //  H: Q-Vector for Hessian associated with the metric term.
+   //     Updated by every call to PANonlinearFormExtension::GetGradient().
   // C0: Q-Vector for spatial weight used for the limiting term.
+   //     Updated when the mesh nodes change.
   // LD: E-Vector constructed using limiting distance grid function (delta).
+   //     Does not change during the TMOP iteration.
   // H0: Q-Vector for Hessian associated with the limiting term.
+   //     Updated by every call to PANonlinearFormExtension::GetGradient().
+   // MC: Q-Vector for the metric Coefficient.
+   //     Updated when the mesh nodes change.
   //
-   // maps:     Dof2Quad map for fespace associate with nodal coordinates.
-   // maps_lim: Dof2Quad map for fespace associated with the limiting distance
-   //            grid function.
+   // maps:     Dof2Quad map for fes associated with the nodal coordinates.
+   // maps_lim: Dof2Quad map for fes associated with the limiting dist GridFunc.
   //
   // Jtr_debug_grad
   //     We keep track if Jtr was set by AssembleGradPA() in Jtr_debug_grad: it
@@ -1846,7 +1860,7 @@ protected:
      mutable DenseTensor Jtr;
      mutable bool Jtr_needs_update;
      mutable bool Jtr_debug_grad;
-      mutable Vector E, O, X0, H, C0, LD, H0;
+      mutable Vector E, O, X0, H, C0, LD, H0, MC;
      const DofToQuad *maps;
      const DofToQuad *maps_lim = nullptr;
      const GeometricFactors *geom;
@@ -1960,6 +1974,9 @@ protected:

   void AssemblePA_Limiting();
   void ComputeAllElementTargets(const Vector &xe = Vector()) const;
+   // Updates the Q-vectors for the metric_coeff and lim_coeff, based on the
+   // new physical positions of the quadrature points.
+   void UpdateCoefficientsPA(const Vector &x_loc);

   // Compute Min(Det(Jpt)) in the mesh, does not reduce over MPI.
   double ComputeMinDetT(const Vector &x, const FiniteElementSpace &fes);
@@ -176,6 +176,42 @@ void TMOP_Integrator::ComputeAllElementTargets(const Vector &xe) const
   targetC->ComputeAllElementTargets(*fes, ir, xe, PA.Jtr);
 }

+void TMOP_Integrator::UpdateCoefficientsPA(const Vector &x_loc)
+{
+   // Both are constant or not specified.
+   if (PA.MC.Size() == 1 && PA.C0.Size() == 1) { return; }
+
+   // Coefficients are always evaluated on the CPU for now.
+   PA.MC.HostWrite();
+   PA.C0.HostWrite();
+
+   const IntegrationRule &ir = *PA.ir;
+   auto T = new IsoparametricTransformation;
+   for (int e = 0; e < PA.ne; ++e)
+   {
+      // Uses the node positions in x_loc.
+      PA.fes->GetMesh()->GetElementTransformation(e, x_loc, T);
+
+      if (PA.MC.Size() > 1)
+      {
+         for (int q = 0; q < PA.nq; ++q)
+         {
+            PA.MC(q + e * PA.nq) = metric_coeff->Eval(*T, ir.IntPoint(q));
+         }
+      }
+
+      if (PA.C0.Size() > 1)
+      {
+         for (int q = 0; q < PA.nq; ++q)
+         {
+            PA.C0(q + e * PA.nq) = lim_coeff->Eval(*T, ir.IntPoint(q));
+         }
+      }
+   }
+
+   delete T;
+}
+
 void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
 {
   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
@@ -213,6 +249,35 @@ void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
   PA.O.SetSize(ne*nq, Device::GetDeviceMemoryType());
   PA.O = 1.0;

+   if (metric_coeff)
+   {
+      if (auto cc = dynamic_cast<ConstantCoefficient *>(metric_coeff))
+      {
+         PA.MC.SetSize(1, Device::GetMemoryType());
+         PA.MC.HostWrite();
+         PA.MC(0) = cc->constant;
+      }
+      else
+      {
+         PA.MC.SetSize(PA.nq * PA.ne, Device::GetMemoryType());
+         auto M0 = Reshape(PA.MC.HostWrite(), PA.nq, PA.ne);
+         for (int e = 0; e < PA.ne; ++e)
+         {
+            ElementTransformation& T = *PA.fes->GetElementTransformation(e);
+            for (int q = 0; q < ir.GetNPoints(); ++q)
+            {
+               M0(q,e) = metric_coeff->Eval(T, ir.IntPoint(q));
+            }
+         }
+      }
+   }
+   else
+   {
+      PA.MC.SetSize(1, Device::GetMemoryType());
+      PA.MC.HostWrite();
+      PA.MC(0) = 1.0;
+   }
+
   // Setup ref->target Jacobians, PA.Jtr, (dim x dim) Q-vector, DenseTensor
   PA.Jtr.SetSize(dim, dim, PA.ne*PA.nq, mt);
   PA.Jtr_needs_update = true;
@@ -258,6 +258,7 @@ void EvalH_094(const int e, const int qx, const int qy,
 MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
                           const Vector &x_,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -273,11 +274,16 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
               || mid == 80 || mid == 94,
               "2D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 2;
   constexpr int NBZ = 1;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, NE);
   const auto W = Reshape(w_.Read(), Q1D, Q1D);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
@@ -312,7 +318,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
         {
            const double *Jtr = &J(0,0,qx,qy,e);
            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = metric_normal * W(qx,qy) * detJtr;
+            const double m_coef = const_m0 ? MC(0,0,0) : MC(qx,qy,e);
+            const double weight = metric_normal * m_coef * W(qx,qy) * detJtr;

            // Jrt = Jtr^{-1}
            double Jrt[4];
@@ -347,6 +354,7 @@ void TMOP_Integrator::AssembleGradPA_2D(const Vector &X) const
   const int Q1D = PA.maps->nqpt;
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -359,7 +367,7 @@ void TMOP_Integrator::AssembleGradPA_2D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,mp,M,N,W,B,G,J,H);
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,MC,mp,M,N,W,B,G,J,H);
 }

 } // namespace mfem
@@ -312,6 +312,7 @@ void EvalH_338(const int e, const int qx, const int qy, const int qz,

 MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const Vector &x_,
@@ -328,10 +329,15 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
               mid == 321 || mid == 332 || mid == 338,
               "3D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, Q1D, NE);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
@@ -369,7 +375,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
            {
               const double *Jtr = &J(0,0,qx,qy,qz,e);
               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
+               const double m_coef = const_m0 ? MC(0,0,0,0) : MC(qx,qy,qz,e);
+               const double weight = metric_normal * m_coef *
+                                     W(qx,qy,qz) * detJtr;

               // Jrt = Jtr^{-1}
               double Jrt[9];
@@ -438,6 +446,7 @@ void TMOP_Integrator::AssembleGradPA_3D(const Vector &X) const
   const int M = metric->Id();
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -450,7 +459,7 @@ void TMOP_Integrator::AssembleGradPA_3D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,mp,M,X,N,W,B,G,J,H);
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,MC,mp,M,X,N,W,B,G,J,H);
 }

 } // namespace mfem
@@ -98,6 +98,7 @@ void EvalP_094(const double *Jpt, const double *w, double *P)

 MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -114,12 +115,17 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
               || mid == 80 || mid == 94,
               "2D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 2;
   constexpr int NBZ = 1;

   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
   const auto W = Reshape(w_.Read(), Q1D, Q1D);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
@@ -154,7 +160,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
         {
            const double *Jtr = &J(0,0,qx,qy,e);
            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = metric_normal * W(qx,qy) * detJtr;
+            const double m_coef = const_m0 ? MC(0,0,0) : MC(qx,qy,e);
+            const double weight = metric_normal * m_coef *
+                                  W(qx,qy) * detJtr;

            // Jrt = Jtr^{-1}
            double Jrt[4];
@@ -204,6 +212,7 @@ void TMOP_Integrator::AddMultPA_2D(const Vector &X, Vector &Y) const
   const Array<double> &B = PA.maps->B;
   const Array<double> &G = PA.maps->G;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;

   Array<double> mp;
   if (auto m = dynamic_cast<TMOP_Combo_QualityMetric *>(metric))
@@ -211,7 +220,7 @@ void TMOP_Integrator::AddMultPA_2D(const Vector &X, Vector &Y) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,mp,M,N,J,W,B,G,X,Y);
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,MC,mp,M,N,J,W,B,G,X,Y);
 }

 } // namespace mfem
@@ -131,6 +131,7 @@ void EvalP_338(const double *J, const double *w, double *P)

 MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -147,10 +148,15 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
               mid == 321 || mid == 332 || mid == 338,
               "3D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
@@ -188,7 +194,9 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
            {
               const double *Jtr = &J(0,0,qx,qy,qz,e);
               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
+               const double m_coef = const_m0 ? MC(0,0,0,0) : MC(qx,qy,qz,e);
+               const double weight = metric_normal * m_coef *
+                                     W(qx,qy,qz) * detJtr;

               // Jrt = Jtr^{-1}
               double Jrt[9];
@@ -240,6 +248,7 @@ void TMOP_Integrator::AddMultPA_3D(const Vector &X, Vector &Y) const
   const Array<double> &B = PA.maps->B;
   const Array<double> &G = PA.maps->G;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;

   Array<double> mp;
   if (auto m = dynamic_cast<TMOP_Combo_QualityMetric *>(metric))
@@ -247,7 +256,7 @@ void TMOP_Integrator::AddMultPA_3D(const Vector &X, Vector &Y) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,mp,M,N,J,W,B,G,X,Y);
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,MC,mp,M,N,J,W,B,G,X,Y);
 }

 } // namespace mfem
@@ -73,6 +73,7 @@ double EvalW_094(const double *Jpt, const double *w)

 MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -90,12 +91,17 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
               || mid == 80 || mid == 94,
               "2D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 2;
   constexpr int NBZ = 1;

   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
@@ -131,7 +137,8 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
         {
            const double *Jtr = &J(0,0,qx,qy,e);
            const double detJtr = kernels::Det<2>(Jtr);
-            const double weight = metric_normal * W(qx,qy) * detJtr;
+            const double m_coef = const_m0 ? MC(0,0,0) : MC(qx,qy,e);
+            const double weight = metric_normal * m_coef * W(qx,qy) * detJtr;

            // Jrt = Jtr^{-1}
            double Jrt[4];
@@ -169,6 +176,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_2D(const Vector &X) const
   const int Q1D = PA.maps->nqpt;
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -182,7 +190,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_2D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,mn,mp,M,N,J,W,B,G,X,O,E);
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,mn,MC,mp,M,N,J,W,B,G,X,O,E);
 }

 } // namespace mfem
@@ -82,6 +82,7 @@ double EvalW_338(const double *J, const double *w)

 MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
                           const double metric_normal,
+                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -99,10 +100,15 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
               mid == 321 || mid == 332 || mid == 338,
               "3D metric not yet implemented!");

+   const bool const_m0 = mc_.Size() == 1;
+
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

+   const auto MC = const_m0 ?
+                   Reshape(mc_.Read(), 1, 1, 1, 1) :
+                   Reshape(mc_.Read(), Q1D, Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
@@ -141,7 +147,9 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
            {
               const double *Jtr = &J(0,0,qx,qy,qz,e);
               const double detJtr = kernels::Det<3>(Jtr);
-               const double weight = metric_normal * W(qx,qy,qz) * detJtr;
+               const double m_coef = const_m0 ? MC(0,0,0,0) : MC(qx,qy,qz,e);
+               const double weight = metric_normal * m_coef *
+                                     W(qx,qy,qz) * detJtr;

               // Jrt = Jtr^{-1}
               double Jrt[9];
@@ -181,6 +189,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_3D(const Vector &X) const
   const int Q1D = PA.maps->nqpt;
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
+   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -194,7 +203,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_3D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,mp,M,N,J,W,B,G,O,X,E);
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,MC,mp,M,N,J,W,B,G,O,X,E);
 }

 } // namespace mfem
@@ -781,7 +781,13 @@ std::unique_ptr<SparseMatrix>>
   int ndof_lor = fes_lor.GetNDofs();

   // If the local mesh is empty, skip all computations
-   if (nel_ho == 0) { return {nullptr, nullptr}; }
+   if (nel_ho == 0)
+   {
+      return std::make_pair(
+                std::unique_ptr<SparseMatrix>(new SparseMatrix),
+                std::unique_ptr<SparseMatrix>(new SparseMatrix)
+             );
+   }

   const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();

@@ -26,6 +26,10 @@
 #include "sort_pairs.hpp"
 #include "globals.hpp"

+#ifdef MFEM_USE_STRUMPACK
+#include <StrumpackConfig.hpp> // STRUMPACK_USE_PTSCOTCH, etc.
+#endif
+
 #include <iostream>
 #include <map>

@@ -34,6 +38,14 @@ using namespace std;
 namespace mfem
 {

+#if defined(MFEM_USE_STRUMPACK) && \
+    (defined(STRUMPACK_USE_PTSCOTCH) || defined(STRUMPACK_USE_SLATE_SCALAPACK))
+int Mpi::default_thread_required = MPI_THREAD_MULTIPLE;
+#else
+int Mpi::default_thread_required = MPI_THREAD_SINGLE;
+#endif
+
+
 GroupTopology::GroupTopology(const GroupTopology &gt)
   : MyComm(gt.MyComm),
     group_lproc(gt.group_lproc)
@@ -22,7 +22,6 @@
 #include "globals.hpp"
 #include <mpi.h>

-
 namespace mfem
 {

@@ -32,10 +31,34 @@ namespace mfem
 class Mpi
 {
 public:
-   /// Singleton creation with Mpi::Init();
-   static void Init() { Init_(NULL, NULL); }
-   /// Singleton creation with Mpi::Init(argc,argv);
-   static void Init(int &argc, char **&argv) { Init_(&argc, &argv); }
+   /// Singleton creation with Mpi::Init(argc, argv).
+   static void Init(int &argc, char **&argv,
+                    int required = default_thread_required,
+                    int *provided = nullptr)
+   { Init(&argc, &argv, required, provided); }
+   /// Singleton creation with Mpi::Init().
+   static void Init(int *argc = nullptr, char ***argv = nullptr,
+                    int required = default_thread_required,
+                    int *provided = nullptr)
+   {
+      MFEM_VERIFY(!IsInitialized(), "MPI already initialized!");
+      if (required == MPI_THREAD_SINGLE)
+      {
+         int mpi_err = MPI_Init(argc, argv);
+         MFEM_VERIFY(!mpi_err, "error in MPI_Init()!");
+         if (provided) { *provided = MPI_THREAD_SINGLE; }
+      }
+      else
+      {
+         int mpi_provided;
+         int mpi_err = MPI_Init_thread(argc, argv, required, &mpi_provided);
+         MFEM_VERIFY(!mpi_err, "error in MPI_Init()!");
+         if (provided) { *provided = mpi_provided; }
+      }
+      // The Mpi singleton object below needs to be created after MPI_Init() for
+      // some MPI implementations.
+      Singleton();
+   }
   /// Finalize MPI (if it has been initialized and not yet already finalized).
   static void Finalize()
   {
@@ -71,20 +94,19 @@ public:
   }
   /// Return true if the rank in MPI_COMM_WORLD is zero.
   static bool Root() { return WorldRank() == 0; }
+   /// Default level of thread support for MPI_Init_thread.
+   static MFEM_EXPORT int default_thread_required;
 private:
-   /// Initialize MPI
-   static void Init_(int *argc, char ***argv)
+   /// Initialize the Mpi singleton.
+   static Mpi &Singleton()
   {
-      MFEM_VERIFY(!IsInitialized(), "MPI already initialized!")
-      MPI_Init(argc, argv);
-      // The "mpi" object below needs to be created after MPI_Init() for some
-      // MPI implementations
      static Mpi mpi;
+      return mpi;
   }
-   /// Finalize MPI
+   /// Finalize MPI.
   ~Mpi() { Finalize(); }
-   /// Prevent direct construction of objects of this class
-   Mpi() { }
+   /// Prevent direct construction of objects of this class.
+   Mpi() {}
 };

 /** @brief A simple convenience class based on the Mpi singleton class above.
@@ -56,7 +56,7 @@ void mfem_backtrace(int mode = 0, int depth = -1);

 /** @brief Function called when an error is encountered. Used by the macros
    MFEM_ABORT, MFEM_ASSERT, MFEM_VERIFY. */
-void mfem_error(const char *msg = NULL);
+[[noreturn]] void mfem_error(const char *msg = NULL);

 /// Function called by the macro MFEM_WARNING.
 void mfem_warning(const char *msg = NULL);
@@ -16,13 +16,13 @@
 #include <cstdlib>
 #include <errno.h>
 #ifndef _WIN32
-#include <netinet/in.h>
 #include <netdb.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <unistd.h>
 #else
-#include <winsock.h>
+#include <winsock2.h>
+#include <ws2tcpip.h>
 #ifdef _MSC_VER
 typedef int ssize_t;
 // Link with ws2_32.lib
@@ -51,47 +51,66 @@ int isockstream::establish()
 {
   // char myname[129];
   char   myname[] = "localhost";
-   int    port;
-   struct sockaddr_in sa;
-   struct hostent *hp;
+   int    sfd;
+   struct addrinfo hints, *res, *rp;

-   memset(&sa, 0, sizeof(struct sockaddr_in));
-   // gethostname(myname, 128);
-   hp= gethostbyname(myname);
+   memset(&hints, 0, sizeof(hints));
+   hints.ai_family = AF_UNSPEC;
+   hints.ai_socktype = SOCK_STREAM;
+   hints.ai_protocol = 0;

-   if (hp == NULL)
+   int s = getaddrinfo(myname, NULL, &hints, &res);
+   if (s != 0)
   {
-      mfem::err << "isockstream::establish(): gethostbyname() failed!\n"
-                << "isockstream::establish(): gethostname() returned: '"
+      mfem::err << "isockstream::establish(): getaddrinfo() failed!\n"
+                << "isockstream::establish(): getaddrinfo() returned: '"
                << myname << "'" << endl;
      error = 1;
      return (-1);
   }

-   sa.sin_family= hp->h_addrtype;
-   sa.sin_port= htons(portnum);
-
-   if ((port = socket(AF_INET, SOCK_STREAM, 0)) < 0)
+   // loop the list of address structures returned by getaddrinfo()
+   for (rp = res; rp != NULL; rp = rp->ai_next)
   {
-      mfem::err << "isockstream::establish(): socket() failed!" << endl;
-      error = 2;
+      if ((sfd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol)) < 0)
+      {
+         mfem::err << "isockstream::establish(): socket() failed!" << endl;
+         error = 2;
+         return (-1);
+      }
+
+      int on = 1;
+      if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on)) < 0)
+      {
+         mfem::err << "isockstream::establish(): setsockopt() failed!" << endl;
+         return (-1);
+      }
+
+#if defined(__APPLE__)
+      if (bind(sfd, (const struct sockaddr *)rp->ai_addr, rp->ai_addrlen) < 0)
+#else
+      if (bind(sfd, rp->ai_addr, rp->ai_addrlen) < 0)
+#endif
+      {
+         mfem::err << "isockstream::establish(): bind() failed!" << endl;
+         close(sfd);
+         error = 3;
+         continue;
+      }
+
+      break;
+   }
+
+   // No address succeeded
+   if (rp == NULL)
+   {
+      mfem::err << "Could not bind\n";
      return (-1);
   }

-   int on=1;
-   setsockopt(port, SOL_SOCKET, SO_REUSEADDR, (char *)(&on), sizeof(on));
-
-   if (bind(port,(const sockaddr*)&sa,(socklen_t)sizeof(struct sockaddr_in)) < 0)
-   {
-      mfem::err << "isockstream::establish(): bind() failed!" << endl;
-      close(port);
-      error = 3;
-      return (-1);
-   }
-
-   listen(port, 4);
-   error = 0;
-   return (port);
+   freeaddrinfo(res);
+   listen(sfd, 4);
+   return (sfd);
 }

 int isockstream::read_data(int s, char *buf, int n)
@@ -30,7 +30,7 @@ namespace KDTreeNorms
 template <typename Tfloat, int ndim>
 struct Norm_l1
 {
-   Tfloat operator() (const Tfloat* xx)
+   Tfloat operator()(const Tfloat* xx) const
   {
      Tfloat tm=abs(xx[0]);
      for (int i=1; i<ndim; i++)
@@ -45,7 +45,7 @@ struct Norm_l1
 template<typename Tfloat,int ndim>
 struct Norm_l2
 {
-   Tfloat operator() (const Tfloat* xx)
+   Tfloat operator()(const Tfloat* xx) const
   {
      Tfloat tm;
      tm=xx[0]*xx[0];
@@ -61,7 +61,7 @@ struct Norm_l2
 template<typename Tfloat,int ndim>
 struct Norm_li
 {
-   Tfloat operator() (const Tfloat* xx)
+   Tfloat operator()(const Tfloat* xx) const
   {
      Tfloat tm;
      if (xx[0]<Tfloat(0.0)) { tm=-xx[0];}
@@ -83,6 +83,23 @@ struct Norm_li

 }

+/// @brief Abstract base class for KDTree. Can be used when the dimension of the
+/// space is known dynamically.
+template <typename Tindex, typename Tfloat>
+class KDTreeBase
+{
+public:
+   /// Adds a point to the tree. See KDTree::AddPoint().
+   virtual void AddPoint(const Tfloat *xx, Tindex ii) = 0;
+   /// @brief Sorts the tree. Should be performed after adding points and before
+   /// performing queries. See KDTree::Sort().
+   virtual void Sort() = 0;
+   /// Returns the index of the closest point to @a xx.
+   virtual Tindex FindClosestPoint(const Tfloat *xx) const = 0;
+   /// Virtual destructor.
+   virtual ~KDTreeBase() { }
+};
+
 /// Template class for build KDTree with template parameters Tindex
 /// specifying the type utilized for indexing the points, Tfloat
 /// specifying a float type for representing the coordinates of the
@@ -96,19 +113,20 @@ struct Norm_li
 /// computed with n or 1 rank(s).
 template <typename Tindex, typename Tfloat, size_t ndim=3,
          typename Tnorm=KDTreeNorms::Norm_l2<Tfloat,ndim> >
-class KDTree
+class KDTree : public KDTreeBase<Tindex, Tfloat>
 {
 public:

-   /// Structure defining a geometric point in the ndim-dimensional
-   /// space. The coordinate type (Tfloat) can be any floating or
-   /// integer type. It can be even a character if necessary. For
-   /// such types users should redefine the norms.
+   /// Structure defining a geometric point in the ndim-dimensional space. The
+   /// coordinate type (Tfloat) can be any floating or integer type. It can be
+   /// even a character if necessary. For such types users should redefine the
+   /// norms.
   struct PointND
   {
-      /// Geometric point constructor
-      PointND() { std::fill(xx,xx+ndim,Tfloat(0.0));}
-
+      /// Default constructor: fill with zeros
+      PointND() { std::fill(xx,xx+ndim,Tfloat(0.0)); }
+      /// Copy coordinates from pointer/array @a xx_
+      PointND(const Tfloat *xx_) { std::copy(xx_,xx_+ndim,xx); }
      /// Coordinates of the point
      Tfloat xx[ndim];
   };
@@ -118,16 +136,19 @@ public:
   {
      /// Defines a point in the ndim-dimensional space
      PointND pt;
-
      /// Defines the attached index
-      Tindex  ind;
+      Tindex  ind = 0;
+      /// Default constructor: fill with zeros
+      NodeND() = default;
+      /// Create from given point and index
+      NodeND(PointND pt_, Tindex ind_ = 0) : pt(pt_), ind(ind_) { }
   };

   /// Default constructor
   KDTree() = default;

   /// Returns the spatial dimension of the points
-   int SpaceDimension()
+   int SpaceDimension() const
   {
      return ndim;
   }
@@ -148,7 +169,7 @@ public:
   }

   /// Returns the size of the point cloud
-   size_t size()
+   size_t size() const
   {
      return data.size();
   }
@@ -161,34 +182,25 @@ public:

   /// Builds the KDTree. If the point cloud is modified the tree
   /// needs to be rebuild by a new call to Sort().
-   void Sort()
+   void Sort() override
   {
      SortInPlace(data.begin(),data.end(),0);
   }

   /// Adds a new node to the point cloud
-   void AddPoint(PointND& pt, Tindex ii)
+   void AddPoint(const PointND &pt, Tindex ii)
   {
-      NodeND nd;
-      nd.pt=pt;
-      nd.ind=ii;
-      data.push_back(nd);
+      data.emplace_back(pt, ii);
   }

   /// Adds a new node by coordinates and an associated index
-   void AddPoint(Tfloat* xx,Tindex ii)
+   void AddPoint(const Tfloat *xx,Tindex ii) override
   {
-      NodeND nd;
-      for (size_t i=0; i<ndim; i++)
-      {
-         nd.pt.xx[i]=xx[i];
-      }
-      nd.ind=ii;
-      data.push_back(nd);
+      data.emplace_back(xx, ii);
   }

   /// Finds the nearest neighbour index
-   Tindex FindClosestPoint(PointND& pt)
+   Tindex FindClosestPoint(const PointND &pt) const
   {
      PointS best_candidate;
      best_candidate.sp=pt;
@@ -200,8 +212,13 @@ public:
      return data[best_candidate.pos].ind;
   }

-   /// Finds the nearest neighbour index and return the clossest poitn in clp
-   Tindex FindClosestPoint(PointND& pt, PointND& clp)
+   Tindex FindClosestPoint(const Tfloat *xx) const override
+   {
+      return FindClosestPoint(PointND(xx));
+   }
+
+   /// Finds the nearest neighbour index and return the clossest point in clp
+   Tindex FindClosestPoint(const PointND &pt, const PointND &clp) const
   {
      PointS best_candidate;
      best_candidate.sp=pt;
@@ -216,15 +233,15 @@ public:
   }

   /// Returns the closest point and the distance to the input point pt.
-   void FindClosestPoint(PointND& pt, Tindex& ind, Tfloat& dist)
+   void FindClosestPoint(const PointND &pt, Tindex &ind, Tfloat &dist) const
   {
      PointND clp;
      FindClosestPoint(pt,ind,dist,clp);
-
   }

   /// Returns the closest point and the distance to the input point pt.
-   void FindClosestPoint(PointND& pt, Tindex& ind, Tfloat& dist,  PointND& clp)
+   void FindClosestPoint(const PointND &pt, Tindex &ind, Tfloat &dist,
+                         PointND &clp) const
   {
      PointS best_candidate;
      best_candidate.sp=pt;
@@ -241,7 +258,7 @@ public:


   /// Brute force search - please, use it only for debuging purposes
-   void FindClosestPointSlow(PointND& pt, Tindex& ind, Tfloat& dist)
+   void FindClosestPointSlow(const PointND &pt, Tindex &ind, Tfloat &dist) const
   {
      PointS best_candidate;
      best_candidate.sp=pt;
@@ -265,7 +282,7 @@ public:

   /// Finds all points within a distance R from point pt. The indices are
   /// returned in the vector res and the correponding distances in vector dist.
-   void FindNeighborPoints(PointND& pt,Tfloat R, std::vector<Tindex> & res,
+   void FindNeighborPoints(const PointND &pt,Tfloat R, std::vector<Tindex> & res,
                           std::vector<Tfloat> & dist)
   {
      FindNeighborPoints(pt,R,data.begin(),data.end(),0,res,dist);
@@ -273,14 +290,15 @@ public:

   /// Finds all points within a distance R from point pt. The indices are
   /// returned in the vector res and the correponding distances in vector dist.
-   void FindNeighborPoints(PointND& pt,Tfloat R, std::vector<Tindex> & res)
+   void FindNeighborPoints(const PointND &pt,Tfloat R, std::vector<Tindex> & res)
   {
      FindNeighborPoints(pt,R,data.begin(),data.end(),0,res);
   }

   /// Brute force search - please, use it only for debuging purposes
-   void FindNeighborPointsSlow(PointND& pt,Tfloat R, std::vector<Tindex> & res,
-                               std::vector<Tfloat> & dist)
+   void FindNeighborPointsSlow(const PointND &pt,Tfloat R,
+                               std::vector<Tindex> &res,
+                               std::vector<Tfloat> &dist)
   {
      Tfloat dd;
      for (auto iti=data.begin(); iti!=data.end(); iti++)
@@ -295,7 +313,8 @@ public:
   }

   /// Brute force search - please, use it only for debuging purposes
-   void FindNeighborPointsSlow(PointND& pt,Tfloat R, std::vector<Tindex> & res)
+   void FindNeighborPointsSlow(const PointND &pt,Tfloat R,
+                               std::vector<Tindex> &res)
   {
      Tfloat dd;
      for (auto iti=data.begin(); iti!=data.end(); iti++)
@@ -333,12 +352,11 @@ private:
      }
   };

-   /// Point for storing tmp data
-   PointND tp;
+   mutable PointND tp; ///< Point for storing tmp data
   Tnorm fnorm;

   /// Computes the distance between two nodes
-   Tfloat Dist(const PointND& pt1,const  PointND& pt2)
+   Tfloat Dist(const PointND &pt1, const PointND &pt2) const
   {
      for (size_t i=0; i<ndim; i++)
      {
@@ -388,13 +406,13 @@ private:

   /// Finds the closest point to bc.sp in the point cloud
   /// bounded between [itb,ite).
-   void PSearch(typename std::vector<NodeND>::iterator itb,
-                typename std::vector<NodeND>::iterator ite,
-                size_t level, PointS& bc)
+   void PSearch(typename std::vector<NodeND>::const_iterator itb,
+                typename std::vector<NodeND>::const_iterator ite,
+                size_t level, PointS& bc) const
   {
      std::uint8_t dim=(std::uint8_t) (level%ndim);
      size_t siz=ite-itb;
-      typename std::vector<NodeND>::iterator mtb=itb+siz/2;
+      typename std::vector<NodeND>::const_iterator mtb=itb+siz/2;
      if (siz>2)
      {
         // median is at itb+siz/2
@@ -469,7 +487,7 @@ private:
            typename std::vector<NodeND>::iterator itb,
            typename std::vector<NodeND>::iterator ite,
            size_t level,
-            std::vector< std::tuple<Tfloat,Tindex> > & res)
+            std::vector< std::tuple<Tfloat,Tindex> > & res) const
   {
      std::uint8_t dim=(std::uint8_t) (level%ndim);
      size_t siz=ite-itb;
@@ -521,7 +539,7 @@ private:
                           typename std::vector<NodeND>::iterator itb,
                           typename std::vector<NodeND>::iterator ite,
                           size_t level,
-                           std::vector<Tindex> & res)
+                           std::vector<Tindex> & res) const
   {
      std::uint8_t dim=(std::uint8_t) (level%ndim);
      size_t siz=ite-itb;
@@ -570,7 +588,7 @@ private:
                           typename std::vector<NodeND>::iterator itb,
                           typename std::vector<NodeND>::iterator ite,
                           size_t level,
-                           std::vector<Tindex> & res, std::vector<Tfloat> & dist)
+                           std::vector<Tindex> & res, std::vector<Tfloat> & dist) const
   {
      std::uint8_t dim=(std::uint8_t) (level%ndim);
      size_t siz=ite-itb;
@@ -19,15 +19,15 @@
 #include <cstring>      // memset, memcpy, strerror
 #include <cerrno>       // errno
 #ifndef _WIN32
-#include <netdb.h>      // gethostbyname
+#include <netdb.h>      // getaddrinfo
 #include <arpa/inet.h>  // htons
 #include <sys/types.h>  // socket, setsockopt, connect, recv, send
 #include <sys/socket.h> // socket, setsockopt, connect, recv, send
 #include <unistd.h>     // close
-#include <netinet/in.h> // sockaddr_in
 #define closesocket (::close)
 #else
-#include <winsock.h>
+#include <winsock2.h>
+#include <ws2tcpip.h>
 #ifdef _MSC_VER
 typedef int ssize_t;
 // Link with ws2_32.lib
@@ -93,8 +93,7 @@ int socketbuf::attach(int sd)

 int socketbuf::open(const char hostname[], int port)
 {
-   struct sockaddr_in  sa;
-   struct hostent     *hp;
+   struct addrinfo     hints, *res, *rp;

   if (!wsInit_.Initialized())
   {
@@ -105,42 +104,50 @@ int socketbuf::open(const char hostname[], int port)
   setg(NULL, NULL, NULL);
   setp(obuf, obuf + buflen);

-   hp = gethostbyname(hostname);
-   if (hp == NULL)
+   hints.ai_family = AF_UNSPEC;
+   hints.ai_socktype = SOCK_STREAM;
+   hints.ai_flags = 0;
+   hints.ai_protocol = 0;
+
+   std::string portStr = std::to_string(port);
+   int s = getaddrinfo(hostname, portStr.c_str(), &hints, &res);
+   if (s != 0)
   {
      socket_descriptor = -3;
      return -1;
   }
-   memset(&sa, 0, sizeof(sa));
-   memcpy((char *)&sa.sin_addr, hp->h_addr, hp->h_length);
-   sa.sin_family = hp->h_addrtype;
-   sa.sin_port = htons(port);
-   socket_descriptor = socket(hp->h_addrtype, SOCK_STREAM, 0);
-   if (socket_descriptor < 0)
+
+   for (rp = res; rp != NULL; rp = rp->ai_next)
   {
-      return -1;
-   }
+      socket_descriptor = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
+      if (socket_descriptor < 0)
+      {
+         continue;
+      }

 #if defined __APPLE__
-   // OS X does not support the MSG_NOSIGNAL option of send().
-   // Instead we can use the SO_NOSIGPIPE socket option.
-   int on = 1;
-   if (setsockopt(socket_descriptor, SOL_SOCKET, SO_NOSIGPIPE,
-                  (char *)(&on), sizeof(on)) < 0)
-   {
-      closesocket(socket_descriptor);
-      socket_descriptor = -2;
-      return -1;
-   }
+      // OS X does not support the MSG_NOSIGNAL option of send().
+      // Instead we can use the SO_NOSIGPIPE socket option.
+      int on = 1;
+      if (setsockopt(socket_descriptor, SOL_SOCKET, SO_NOSIGPIPE,
+                     &on, sizeof(on)) < 0)
+      {
+         closesocket(socket_descriptor);
+         socket_descriptor = -2;
+         return -1;
+      }
 #endif

-   if (connect(socket_descriptor,
-               (const struct sockaddr *)&sa, sizeof(sa)) < 0)
-   {
-      closesocket(socket_descriptor);
-      socket_descriptor = -2;
-      return -1;
+      if (connect(socket_descriptor, rp->ai_addr, rp->ai_addrlen) < 0)
+      {
+         closesocket(socket_descriptor);
+         socket_descriptor = -2;
+         continue;
+      }
+      break;
   }
+
+   freeaddrinfo(res);
   return 0;
 }

@@ -23,14 +23,12 @@ BlockOperator::BlockOperator(const Array<int> & offsets)
     owns_blocks(0),
     nRowBlocks(offsets.Size() - 1),
     nColBlocks(offsets.Size() - 1),
-     row_offsets(0),
-     col_offsets(0),
+     row_offsets(offsets),
+     col_offsets(offsets),
     op(nRowBlocks, nRowBlocks),
     coef(nRowBlocks, nColBlocks)
 {
   op = static_cast<Operator *>(NULL);
-   row_offsets.MakeRef(offsets);
-   col_offsets.MakeRef(offsets);
 }

 BlockOperator::BlockOperator(const Array<int> & row_offsets_,
@@ -39,14 +37,12 @@ BlockOperator::BlockOperator(const Array<int> & row_offsets_,
     owns_blocks(0),
     nRowBlocks(row_offsets_.Size()-1),
     nColBlocks(col_offsets_.Size()-1),
-     row_offsets(0),
-     col_offsets(0),
+     row_offsets(row_offsets_),
+     col_offsets(col_offsets_),
     op(nRowBlocks, nColBlocks),
     coef(nRowBlocks, nColBlocks)
 {
   op = static_cast<Operator *>(NULL);
-   row_offsets.MakeRef(row_offsets_);
-   col_offsets.MakeRef(col_offsets_);
 }

 void BlockOperator::SetDiagonalBlock(int iblock, Operator *opt, double c)
@@ -38,16 +38,14 @@ public:
   //! columns.
   /**
    *  offsets: offsets that mark the start of each row/column block (size
-    *  nRowBlocks+1).  Note: BlockOperator will not own/copy the data contained
-    *  in offsets.
+    *  nRowBlocks+1).
    */
   BlockOperator(const Array<int> & offsets);
   //! Constructor for general BlockOperators.
   /**
    *  row_offsets: offsets that mark the start of each row block (size
    *  nRowBlocks+1).  col_offsets: offsets that mark the start of each column
-    *  block (size nColBlocks+1).  Note: BlockOperator will not own/copy the
-    *  data contained in offsets.
+    *  block (size nColBlocks+1).
    */
   BlockOperator(const Array<int> & row_offsets, const Array<int> & col_offsets);

@@ -937,11 +937,14 @@ GMRESSolver::GMRESSolver(GinkgoExecutor &exec,
 void GMRESSolver::SetKDim(int dim)
 {
   m = dim;
-   using gmres_type = gko::solver::Gmres<double>;
-   gko::as<gmres_type::Factory>(solver_gen)->get_parameters().krylov_dim = m;
+   using gmres = gko::solver::Gmres<double>;
+   // Create new solver factory with other parameters the same, but new value for krylov_dim
+   auto current_params = gko::as<gmres::Factory>(solver_gen)->get_parameters();
+   this->solver_gen = current_params.with_krylov_dim(static_cast<unsigned long>(m))
+                      .on(this->executor);
   if (solver)
   {
-      gko::as<gmres_type>(solver)->set_krylov_dim(static_cast<unsigned long>(m));
+      gko::as<gmres>(solver)->set_krylov_dim(static_cast<unsigned long>(m));
   }
 }

@@ -1036,11 +1039,14 @@ CBGMRESSolver::CBGMRESSolver(GinkgoExecutor &exec,
 void CBGMRESSolver::SetKDim(int dim)
 {
   m = dim;
-   using gmres_type = gko::solver::CbGmres<double>;
-   gko::as<gmres_type::Factory>(solver_gen)->get_parameters().krylov_dim = m;
+   using gmres = gko::solver::CbGmres<double>;
+   // Create new solver factory with other parameters the same, but new value for krylov_dim
+   auto current_params = gko::as<gmres::Factory>(solver_gen)->get_parameters();
+   this->solver_gen = current_params.with_krylov_dim(static_cast<unsigned long>(m))
+                      .on(this->executor);
   if (solver)
   {
-      gko::as<gmres_type>(solver)->set_krylov_dim(static_cast<unsigned long>(m));
+      gko::as<gmres>(solver)->set_krylov_dim(static_cast<unsigned long>(m));
   }
 }

@@ -1205,7 +1211,11 @@ IluPreconditioner::IluPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ilu<>::build()
+#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
+#else
+                    .with_factorization(fact_factory)
+#endif
                    .on(executor);
   }
   else
@@ -1217,7 +1227,11 @@ IluPreconditioner::IluPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ilu<>::build()
+#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
+#else
+                    .with_factorization(fact_factory)
+#endif
                    .on(executor);
   }

@@ -1255,9 +1269,15 @@ IluIsaiPreconditioner::IluIsaiPreconditioner(
         .on(executor);
      precond_gen = gko::preconditioner::Ilu<l_solver_type,
      u_solver_type>::build()
+#if MFEM_GINKGO_VERSION < 10700
      .with_factorization_factory(fact_factory)
      .with_l_solver_factory(l_solver_factory)
      .with_u_solver_factory(u_solver_factory)
+#else
+      .with_factorization(fact_factory)
+      .with_l_solver(l_solver_factory)
+      .with_u_solver(u_solver_factory)
+#endif
      .on(executor);

   }
@@ -1271,9 +1291,15 @@ IluIsaiPreconditioner::IluIsaiPreconditioner(
         .on(executor);
      precond_gen = gko::preconditioner::Ilu<l_solver_type,
      u_solver_type>::build()
+#if MFEM_GINKGO_VERSION < 10700
      .with_factorization_factory(fact_factory)
      .with_l_solver_factory(l_solver_factory)
      .with_u_solver_factory(u_solver_factory)
+#else
+      .with_factorization(fact_factory)
+      .with_l_solver(l_solver_factory)
+      .with_u_solver(u_solver_factory)
+#endif
      .on(executor);
   }
 }
@@ -1298,7 +1324,11 @@ IcPreconditioner::IcPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ic<>::build()
+#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
+#else
+                    .with_factorization(fact_factory)
+#endif
                    .on(executor);
   }
   else
@@ -1311,7 +1341,11 @@ IcPreconditioner::IcPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ic<>::build()
+#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
+#else
+                    .with_factorization(fact_factory)
+#endif
                    .on(executor);
   }
 }
@@ -1340,8 +1374,13 @@ IcIsaiPreconditioner::IcIsaiPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ic<l_solver_type>::build()
+#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
                    .with_l_solver_factory(l_solver_factory)
+#else
+                    .with_factorization(fact_factory)
+                    .with_l_solver(l_solver_factory)
+#endif
                    .on(executor);
   }
   else
@@ -1354,8 +1393,13 @@ IcIsaiPreconditioner::IcIsaiPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ic<l_solver_type>::build()
+#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
                    .with_l_solver_factory(l_solver_factory)
+#else
+                    .with_factorization(fact_factory)
+                    .with_l_solver(l_solver_factory)
+#endif
                    .on(executor);
   }
 }
@@ -862,8 +862,10 @@ public:
   {
      rel_tol = rtol;
      this->update_stop_factory();
-      gko::as<typename SolverType::Factory>(solver_gen)->get_parameters().criteria =
-      { combined_factory };
+      auto current_params = gko::as<typename SolverType::Factory>
+                            (solver_gen)->get_parameters();
+      this->solver_gen = current_params.with_criteria(this->combined_factory)
+                         .on(this->executor);
      if (solver)
      {
         gko::as<SolverType>(solver)->set_stop_criterion_factory(combined_factory);
@@ -874,8 +876,10 @@ public:
   {
      abs_tol = atol;
      this->update_stop_factory();
-      gko::as<typename SolverType::Factory>(solver_gen)->get_parameters().criteria =
-      { combined_factory };
+      auto current_params = gko::as<typename SolverType::Factory>
+                            (solver_gen)->get_parameters();
+      this->solver_gen = current_params.with_criteria(this->combined_factory)
+                         .on(this->executor);
      if (solver)
      {
         gko::as<SolverType>(solver)->set_stop_criterion_factory(combined_factory);
@@ -886,8 +890,10 @@ public:
   {
      max_iter = max_it;
      this->update_stop_factory();
-      gko::as<typename SolverType::Factory>(solver_gen)->get_parameters().criteria =
-      { combined_factory };
+      auto current_params = gko::as<typename SolverType::Factory>
+                            (solver_gen)->get_parameters();
+      this->solver_gen = current_params.with_criteria(this->combined_factory)
+                         .on(this->executor);
      if (solver)
      {
         gko::as<SolverType>(solver)->set_stop_criterion_factory(combined_factory);
--- a/Show More
+++ b/Show More