Merge remote-tracking branch 'origin/hypre-runtime-compute-policy' into apps-dev

Merge remote-tracking branch 'origin/feature/askinner/hdiv-saddle-point-solver' into apps-dev
Merge remote-tracking branch 'origin/master' into hypre-runtime-compute-policy
2023-10-27 09:22:16 -07:00 · 2023-10-27 09:22:12 -07:00 · 2023-10-27 09:03:44 -07:00 · 2023-08-29 21:16:35 -07:00 · 2023-08-25 14:36:14 -07:00
183 changed files with 5027 additions and 13800 deletions
@@ -128,13 +128,6 @@ examples/amgx/sol.gf
 examples/amgx/mesh.*
 examples/amgx/sol.*

-examples/caliper/ex1
-examples/caliper/ex1p
-examples/caliper/refined.mesh
-examples/caliper/sol.gf
-examples/caliper/mesh.*
-examples/caliper/sol.*
-
 examples/ginkgo/ex1
 examples/ginkgo/refined.mesh
 examples/ginkgo/sol.gf
@@ -273,7 +266,6 @@ miniapps/navier/*_output
 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
 miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/nurbs_printfunc
 miniapps/nurbs/nurbs_patch_ex1
 miniapps/nurbs/nurbs_curveint
 miniapps/nurbs/refined.mesh
@@ -307,7 +299,7 @@ miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/plor-transfer
 miniapps/tools/get-values
-miniapps/tools/tmop-check-metric
+miniapps/tools/check-tmop-metric
 miniapps/tools/tmop-metric-magnitude
 miniapps/tools/nodal-transfer
 miniapps/tools/ParaView
@@ -11,23 +11,6 @@
 Version 4.6.1 (development)
 ===========================

-Discretization improvements
---------------------------
- Introduced support for higher order non conformal Nedelec elements on
-  simplices in ParMesh.
-
- Added functionality for construction of cut-surface and cut-volume
-  IntegrationRules through a moment-fitting approach. The cut is specified by
-  the zero level set of a Coefficient. See fem/intrules_cut.hpp and Example 38.
-
-Miscellaneous
-------------
- The ReadCubit Genesis mesh importer has been rewritten to improve readability.
-
- Improved thread safety for global variables in the library, for example
-  IntegrationRules IntRules, RefinedIntRules, GeometryRefiner
-  GlobGeometryRefiner, and FiniteElement::dof2quad_array.
-

 Version 4.6, released on September 27, 2023
 ===========================================
@@ -48,7 +31,6 @@ Meshing improvements
  * The edge to knot map for NURBS meshes can be determined automatically. It is
    no longer needed to specify this in the NURBS mesh.
  * Added curve interpolation method for NURBS.
-  * Added new small miniapp for printing of shape functions of a KnotVector
  * See miniapps/nurbs for example meshes and miniapps.

 Discretization improvements
@@ -95,15 +77,12 @@ Linear and nonlinear solvers

 - Added HIP support to the PETSc and SUNDIALS interfaces.

- Efficient GPU-accelerated LOR assembly now supports surface meshes.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new H(div) solver miniapp demonstrating the use of a matrix-free
  saddle-point solver methodology, suitable for high-order discretizations and
  for GPU acceleration. Examples illustrating the solution of Darcy and grad-div
  problems are included. See miniapps/hdiv-linear-solver.
->>>>>>> master

 - Added new Discontinuous Petrov-Galerkin (DPG) miniapp which includes serial
  and parallel examples for diffusion, convection-diffusion, acoustics and
@@ -139,9 +139,10 @@ if (MFEM_USE_CUDA)
      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
   endif()
   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
-   find_package(CUDAToolkit REQUIRED)
   set(CUSPARSE_FOUND TRUE)
-   get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
+   set(CUSPARSE_LIBRARIES "cusparse")
+   set(CUBLAS_FOUND TRUE)
+   set(CUBLAS_LIBRARIES "cublas")
 endif()

 if (XSDK_ENABLE_C)
@@ -530,7 +531,7 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
+    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
@@ -640,34 +641,16 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
  foreach(Header mfem.hpp mfem-performance.hpp)
    message(STATUS
      "Writing substitute header --> \"${Header}\"")
-    file(WRITE "${PROJECT_BINARY_DIR}/${Header}.tmp"
+    file(WRITE "${PROJECT_BINARY_DIR}/${Header}"
 "// Auto-generated file.
 #define MFEM_CONFIG_FILE \"${PROJECT_BINARY_DIR}/config/_config.hpp\"
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")
-
-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
-      "${PROJECT_BINARY_DIR}/${Header}.tmp"
-      "${PROJECT_BINARY_DIR}/${Header}"
-    )
-    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
-      "${PROJECT_BINARY_DIR}/${Header}.tmp"
-    )
-
    # This version will be installed in the top include directory:
-    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
+    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
 "// Auto-generated file.
 #include \"mfem/${Header}\"
 ")
-
-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
-      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
-      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
-    )
-    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
-      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
-    )
-
  endforeach()
 endif()

@@ -659,7 +659,8 @@ The specific libraries and their options are:
  requires the PT-Scotch and Scalapack libraries as well as ParMETIS, which
  includes METIS 5 in its distribution. Starting with STRUMPACK v2.2.0, ParMETIS
  and PT-Scotch are optional dependencies.
-  The support for STRUMPACK was added in MFEM v3.3.2.
+  The support for STRUMPACK was added in MFEM v3.3.2 and it requires STRUMPACK
+  2.0.0 or later.
  URL: http://portal.nersc.gov/project/sparse/strumpack
  Options: STRUMPACK_OPT, STRUMPACK_LIB.
  Versions: STRUMPACK >= 3.0.0.
@@ -14,13 +14,9 @@
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
-#   - HYPRE_USING_CUDA (internal)
 #   - HYPRE_USING_HIP (internal)

 if (HYPRE_FOUND)
-  if (HYPRE_USING_CUDA)
-    find_package(CUDAToolkit REQUIRED)
-  endif()
  if (HYPRE_USING_HIP)
    find_package(rocsparse REQUIRED)
    find_package(rocrand REQUIRED)
@@ -31,20 +27,6 @@ endif()
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
  "Paths to headers required by HYPRE." "Libraries required by HYPRE."
-  CHECK_BUILD HYPRE_USING_CUDA FALSE
-  "
-#undef HYPRE_USING_CUDA
-#include <HYPRE_config.h>
-
-#ifndef HYPRE_USING_CUDA
-#error HYPRE is built without CUDA.
-#endif
-
-int main()
-{
-   return 0;
-}
-"
  CHECK_BUILD HYPRE_USING_HIP FALSE
  "
 #undef HYPRE_USING_HIP
@@ -75,16 +57,6 @@ if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
  endif()
 endif()

-if (HYPRE_FOUND AND HYPRE_USING_CUDA)
-  find_package(CUDAToolkit REQUIRED)
-  get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
-  get_target_property(CURAND_LIBRARIES CUDA::curand LOCATION)
-  list(APPEND HYPRE_LIBRARIES ${CUSPARSE_LIBRARIES} ${CURAND_LIBRARIES})
-  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
-      "HYPRE libraries + dependencies." FORCE)
-  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
-endif()
-
 if (HYPRE_FOUND AND HYPRE_USING_HIP)
  find_package(rocsparse REQUIRED)
  find_package(rocrand REQUIRED)
@@ -106,7 +106,12 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-# CUDA and HIP dependencies for HYPRE are handled in FindHYPRE.cmake.
+if (MFEM_USE_CUDA)
+   # This is only necessary when hypre is built with cuda:
+   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
+       "Libraries that HYPRE depends on.")
+endif()
+# HIP dependency for HYPRE is handled in FindHYPRE.cmake.

 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")

@@ -152,8 +157,7 @@ set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
 # STRUMPACK may also depend on "OpenMP", depending on how it was compiled.
 # Starting with v2.2.0 of STRUMPACK, ParMETIS and Scotch are optional.
 set(STRUMPACK_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
-    "Scotch/ptscotch/ptscotcherr/scotch/scotcherr"
-    "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
+    "ScaLAPACK" "Scotch/ptscotch/ptscotcherr/scotch/scotcherr" CACHE STRING
    "Additional packages required by STRUMPACK.")
 # If the MPI package does not find all required Fortran libraries:
 # set(STRUMPACK_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
@@ -331,30 +331,16 @@ STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 $(SCOTCH_LIB) $(SCALAPACK_LIB)

-# Ginkgo library configuration
+# Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
-GINKGO_SEARCH_DIR = $(subst @MFEM_DIR@,$(MFEM_DIR),$(GINKGO_DIR))
 GINKGO_BUILD_TYPE=Release
 ifeq ($(MFEM_USE_GINKGO),YES)
   BASE_FLAGS = -std=c++14
 endif
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB_DIR = $(sort $(dir $(wildcard\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.a\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.so\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dylib\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dll)))
-GINKGO_LINK_LIB_DIR = $(GINKGO_DIR)$(subst $(GINKGO_SEARCH_DIR),,$(GINKGO_LIB_DIR))
-ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.a\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.so\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.dylib\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.dll)))
-ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.a\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.so\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dylib\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dll)))
+GINKGO_LIB_DIR = $(sort $(dir $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
+ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*d.a  $(GINKGO_DIR)/lib*/libginkgo*d.so $(GINKGO_DIR)/lib*/libginkgo*d.dylib $(GINKGO_DIR)/lib*/libginkgo*d.dll)))
+ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
 ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS))
 GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE))
 ifeq ($(GINKGO_BUILD_TYPE),Debug)
@@ -363,8 +349,7 @@ ifeq ($(GINKGO_BUILD_TYPE),Debug)
  endif
 else
 endif
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LINK_LIB_DIR) -L$(GINKGO_LINK_LIB_DIR)\
- $(GINKGO_LINK)
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LIB_DIR) -L$(GINKGO_LIB_DIR) $(GINKGO_LINK)

 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
@@ -38,14 +38,14 @@ all: header config-mk
 MPI = $(MFEM_USE_MPI:NO=)
 GHV_CXX ?= $(MFEM_CXX)
 GHV = get_hypre_version
-GHV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
+GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
 SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
 SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
 MUMPS = $(MFEM_USE_MUMPS:NO=)
 GMV_CXX ?= $(MFEM_CXX)
 GMV = get_mumps_version
-GMV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))
+GMV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))

 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
@@ -110,4 +110,3 @@ config-mk:

 clean:
 	rm -f $(CONFIG_HPP) $(CONFIG_MK) sample-runs-build.log
-	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out
@@ -315,7 +315,7 @@ function extract_sample_runs()
   sruns=`grep -v "^//.*  mpirun .* ${app}" "${src}" |
          grep "^//.*  ${app}" |
          sed -e "s/.*  ${app}/${vg_app}/g"`
-   runs="${sruns}"$'\n'"${pruns}"
+   runs="${sruns}${pruns}"
   if [ "$skip_gen_meshes" == "yes" ]; then
      runs=`printf "%s" "$runs" | grep -v ".* -m .*\.gen"`
   fi
@@ -112,7 +112,6 @@ namespace mfem {
 * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
 * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: Topology optimization
 * - <a class="el" href="ex37p_8cpp_source.html">Example 37p</a>: parallel topology optimization
- * - <a class="el" href="ex38_8cpp_source.html">Example 38</a>: cut-surface and cut-volume integration
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -45,12 +45,6 @@ list(APPEND ALL_EXE_SRCS
  ex37.cpp
  )

-if(MFEM_USE_LAPACK)
-list(APPEND ALL_EXE_SRCS
-  ex38.cpp
-  )
-endif()
-
 if (MFEM_USE_MPI)
  list(APPEND ALL_EXE_SRCS
    ex0p.cpp
@@ -262,13 +262,12 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
      if (sp_solver)
      {
-         STRUMPACKSolver * strumpack = new STRUMPACKSolver(MPI_COMM_WORLD, argc, argv);
+         STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
         strumpack->SetPrintFactorStatistics(true);
         strumpack->SetPrintSolveStatistics(false);
         strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
         strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-         strumpack->SetMatching(strumpack::MatchingJob::NONE);
-         strumpack->SetCompression(strumpack::CompressionType::NONE);
+         strumpack->DisableMatching();
         strumpack->SetOperator(*Arow);
         strumpack->SetFromCommandLine();
         precond = strumpack;
@@ -5,7 +5,6 @@
 // Sample runs:  mpirun -np 4 ex13p -m ../data/star.mesh
 //               mpirun -np 4 ex13p -m ../data/square-disc.mesh -o 2 -n 4
 //               mpirun -np 4 ex13p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex13p -m ../data/beam-tet.mesh -nc -o 2 -rs 1
 //               mpirun -np 4 ex13p -m ../data/beam-hex.mesh
 //               mpirun -np 4 ex13p -m ../data/escher.mesh
 //               mpirun -np 4 ex13p -m ../data/fichera.mesh
@@ -55,7 +54,6 @@ int main(int argc, char *argv[])
   int par_ref_levels = 1;
   int order = 1;
   int nev = 5;
-   bool nc = false;
   bool visualization = 1;
   const char *device_config = "cpu";

@@ -71,9 +69,6 @@ int main(int argc, char *argv[])
                  " isoparametric space.");
   args.AddOption(&nev, "-n", "--num-eigs",
                  "Number of desired eigenmodes.");
-   args.AddOption(&nc, "-nc", "--non-conforming", "-c",
-                  "--conforming",
-                  "Mark the mesh as nonconforming before partitioning.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -103,10 +98,6 @@ int main(int argc, char *argv[])
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();
-   if (nc)
-   {
-      mesh->EnsureNCMesh(true);
-   }

   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
@@ -13,7 +13,6 @@
 //               mpirun -np 4 ex15p -m ../data/square-disc-nurbs.mesh
 //               mpirun -np 4 ex15p -m ../data/disc-nurbs.mesh
 //               mpirun -np 4 ex15p -m ../data/fichera.mesh -tf 0.5
-//               mpirun -np 4 ex15p -m ../data/fichera-mixed.mesh -tf 0.5
 //               mpirun -np 4 ex15p -m ../data/ball-nurbs.mesh -tf 0.5
 //               mpirun -np 4 ex15p -m ../data/mobius-strip.mesh
 //               mpirun -np 4 ex15p -m ../data/amr-quad.mesh
@@ -26,12 +26,13 @@
 using namespace std;
 using namespace mfem;

-/** After spatial discretization, the wave model can be written as:
+/** After spatial discretization, the conduction model can be written as:
 *
 *     d^2u/dt^2 = M^{-1}(-Ku)
 *
- *  where u is the vector representing the temperature, M is the mass,
- *  and K is the stiffness matrix.
+ *  where u is the vector representing the temperature, M is the mass matrix,
+ *  and K is the diffusion operator with diffusivity depending on u:
+ *  (\kappa + \alpha u).
 *
 *  Class WaveOperator represents the right-hand side of the above ODE.
 */
@@ -300,7 +301,7 @@ int main(int argc, char *argv[])
   Vector dudt;
   dudt_gf.GetTrueDofs(dudt);

-   // 7. Initialize the wave operator and the visualization.
+   // 7. Initialize the conduction operator and the visualization.
   Array<int> ess_bdr;
   if (mesh->bdr_attributes.Size())
   {
@@ -355,7 +356,7 @@ int main(int argc, char *argv[])
      else
      {
         sout.precision(precision);
-         sout << "solution\n" << *mesh << u_gf;
+         sout << "solution\n" << *mesh << dudt_gf;
         sout << "pause\n";
         sout << flush;
         cout << "GLVis visualization paused."
@@ -170,7 +170,6 @@ int main(int argc, char *argv[])
   bool herm_conv = true;
   bool slu_solver  = false;
   bool mumps_solver = false;
-   bool strumpack_solver = false;
   bool visualization = 1;
   bool pa = false;
   const char *device_config = "cpu";
@@ -201,11 +200,6 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_MUMPS
   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
                  "--no-mumps-solver", "Use the MUMPS Solver.");
-#endif
-#ifdef MFEM_USE_STRUMPACK
-   args.AddOption(&strumpack_solver, "-strumpack", "--strumpack-solver",
-                  "-no-strumpack", "--no-strumpack-solver",
-                  "Use the STRUMPACK Solver.");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -215,14 +209,13 @@ int main(int argc, char *argv[])
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.Parse();
-   if (slu_solver + mumps_solver + strumpack_solver > 1)
+   if (slu_solver && mumps_solver)
   {
      if (myid == 0)
-         cout << "WARNING: More than one of SuperLU, MUMPS, and STRUMPACK have"
-              << " been selected, please choose only one." << endl
+         cout << "WARNING: Both SuperLU and MUMPS have been selected,"
+              << " please choose either one." << endl
              << "         Defaulting to SuperLU." << endl;
      mumps_solver = false;
-      strumpack_solver = false;
   }

   if (iprob > 4) { iprob = 4; }
@@ -481,24 +474,6 @@ int main(int argc, char *argv[])
      delete A;
   }
 #endif
-#ifdef MFEM_USE_STRUMPACK
-   if (!pa && strumpack_solver)
-   {
-      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
-      STRUMPACKRowLocMatrix SA(*A);
-      STRUMPACKSolver strumpack(MPI_COMM_WORLD, argc, argv);
-      strumpack.SetPrintFactorStatistics(false);
-      strumpack.SetPrintSolveStatistics(false);
-      strumpack.SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack.SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      strumpack.SetMatching(strumpack::MatchingJob::NONE);
-      strumpack.SetCompression(strumpack::CompressionType::NONE);
-      strumpack.SetFromCommandLine();
-      strumpack.SetOperator(SA);
-      strumpack.Mult(B, X);
-      delete A;
-   }
-#endif
 #ifdef MFEM_USE_MUMPS
   if (!pa && mumps_solver)
   {
@@ -518,7 +493,7 @@ int main(int argc, char *argv[])
   //
   //    In PML:   1/mu (abs(1/det(J) J^T J) Curl E, Curl F)
   //              + omega^2 * epsilon (abs(det(J) * (J^T J)^-1) * E, F)
-   if (pa || (!slu_solver && !mumps_solver && !strumpack_solver))
+   if (pa || (!slu_solver && !mumps_solver))
   {
      ConstantCoefficient absomeg(pow(omega, 2) * epsilon);
      RestrictedCoefficient restr_absomeg(absomeg,attr);
@@ -100,6 +100,21 @@ int main(int argc, char *argv[])
   Device device(device_config);
   if (myid == 0) { device.Print(); }

+   if (mfem::Device::Allows(mfem::Backend::DEVICE_MASK))
+   {
+      HYPRE_SetMemoryLocation(HYPRE_MEMORY_DEVICE);
+      HYPRE_SetExecutionPolicy(HYPRE_EXEC_DEVICE);
+      HYPRE_DeviceInitialize();
+   }
+   else
+   {
+      HYPRE_SetMemoryLocation(HYPRE_MEMORY_HOST);
+      HYPRE_SetExecutionPolicy(HYPRE_EXEC_HOST);
+   }
+
+   auto loc = mfem::GetHypreMemoryLocation();
+   auto exec = mfem::GetHypreExecutionPolicy();
+
   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
@@ -63,7 +63,6 @@ int main(int argc, char *argv[])
   int order = 1;
   bool static_cond = false;
   bool pa = false;
-   bool nc = false;
   const char *device_config = "cpu";
   bool visualization = 1;

@@ -78,9 +77,6 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&nc, "-nc", "--non-conforming", "-c",
-                  "--conforming",
-                  "Mark the mesh as nonconforming before partitioning.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -106,11 +102,6 @@ int main(int argc, char *argv[])
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   dim = mesh->Dimension();
   int sdim = mesh->SpaceDimension();
-   if (nc)
-   {
-      // Can set to false to use conformal refinement for simplices.
-      mesh->EnsureNCMesh(true);
-   }

   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
@@ -1,696 +0,0 @@
-//                                MFEM Example 38
-//
-// Compile with: make ex38
-//
-// Sample runs:
-// (since all sample runs require LAPACK, the * symbol is used to exclude them
-// from the automatically generated internal MFEM tests).
-//              * ex38
-//              * ex38 -i volumetric1d
-//              * ex38 -i surface2d
-//              * ex38 -i surface2d -o 4 -r 5
-//              * ex38 -i volumetric2d
-//              * ex38 -i volumetric2d -o 4 -r 5
-//              * ex38 -i surface3d
-//              * ex38 -i surface3d -o 4 -r 5
-//              * ex38 -i volumetric3d
-//              * ex38 -i volumetric3d -o 4 -r 5
-//
-// Description: This example code demonstrates the use of MFEM to integrate
-//              functions over implicit interfaces and subdomains bounded by
-//              implicit interfaces.
-//
-//              The quadrature rules are constructed by means of moment-fitting.
-//              The interface is given by the zero isoline of a level-set
-//              function ϕ and the subdomain is given as the domain where ϕ>0
-//              holds. The algorithm for construction of the quadrature rules
-//              was introduced by Mueller, Kummer and Oberlack [1].
-//
-//              This example also showcases how to set up integrators using the
-//              integration rules on implicit surfaces and subdomains.
-//
-// [1] Mueller, B., Kummer, F. and Oberlack, M. (2013) Highly accurate surface
-//     and volume integration on implicit domains by means of moment-fitting.
-//     Int. J. Numer. Meth. Engr. (96) 512-528. DOI:10.1002/nme.4569
-
-#include "mfem.hpp"
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-/// @brief Integration rule the example should demonstrate
-enum class IntegrationType { Volumetric1D, Surface2D, Volumetric2D,
-                             Surface3D, Volumetric3D
-                           };
-IntegrationType itype;
-
-/// @brief Level-set function defining the implicit interface
-double lvlset(const Vector& X)
-{
-   switch (itype)
-   {
-      case IntegrationType::Volumetric1D:
-         return .55 - X(0);
-      case IntegrationType::Surface2D:
-         return 1. - (pow(X(0), 2.) + pow(X(1), 2.));
-      case IntegrationType::Volumetric2D:
-         return 1. - (pow(X(0) / 1.5, 2.) + pow(X(1) / .75, 2.));
-      case IntegrationType::Surface3D:
-         return 1. - (pow(X(0), 2.) + pow(X(1), 2.) + pow(X(2), 2.));
-      case IntegrationType::Volumetric3D:
-         return 1. - (pow(X(0) / 1.5, 2.) + pow(X(1) / .75, 2.) + pow(X(2) / .5, 2.));
-      default:
-         return 1.;
-   }
-}
-
-/// @brief Function that should be integrated
-double integrand(const Vector& X)
-{
-   switch (itype)
-   {
-      case IntegrationType::Volumetric1D:
-         return 1.;
-      case IntegrationType::Surface2D:
-         return 3. * pow(X(0), 2.) - pow(X(1), 2.);
-      case IntegrationType::Volumetric2D:
-         return 1.;
-      case IntegrationType::Surface3D:
-         return 4. - 3. * pow(X(0), 2.) + 2. * pow(X(1), 2.) - pow(X(2), 2.);
-      case IntegrationType::Volumetric3D:
-         return 1.;
-      default:
-         return 0.;
-   }
-}
-
-/// @brief Analytic surface integral
-double Surface()
-{
-   switch (itype)
-   {
-      case IntegrationType::Volumetric1D:
-         return 1.;
-      case IntegrationType::Surface2D:
-         return 2. * M_PI;
-      case IntegrationType::Volumetric2D:
-         return 7.26633616541076;
-      case IntegrationType::Surface3D:
-         return 40. / 3. * M_PI;
-      case IntegrationType::Volumetric3D:
-         return 9.90182151329315;
-      default:
-         return 0.;
-   }
-}
-
-/// @brief Analytic volume integral over subdomain with positive level-set
-double Volume()
-{
-   switch (itype)
-   {
-      case IntegrationType::Volumetric1D:
-         return .55;
-      case IntegrationType::Surface2D:
-         return NAN;
-      case IntegrationType::Volumetric2D:
-         return 9. / 8. * M_PI;
-      case IntegrationType::Surface3D:
-         return NAN;
-      case IntegrationType::Volumetric3D:
-         return 3. / 4. * M_PI;
-      default:
-         return 0.;
-   }
-}
-
-#ifdef MFEM_USE_LAPACK
-/**
- @brief Class for surface IntegrationRule
-
- This class demonstrates how IntegrationRules computed as CutIntegrationRules
- can be saved to reduce the impact by computing them from scratch each time.
- */
-class SIntegrationRule : public IntegrationRule
-{
-protected:
-   /// @brief Space Dimension of the IntegrationRule
-   int dim;
-   /// @brief Column-wise matrix of the quadtrature weights
-   DenseMatrix Weights;
-   /// @brief Column-wise matrix of the transformation weights of the normal
-   DenseMatrix SurfaceWeights;
-
-public:
-   /**
-    @brief Constructor of SIntegrationRule
-
-    The surface integrationRules are computed and saved in the constructor.
-
-    @param [in] Order Order of the IntegrationRule
-    @param [in] LvlSet Level-set defining the implicit interface
-    @param [in] lsOrder Polynomial degree for approx of level-set function
-    @param [in] mesh Pointer to the mesh that is used
-   */
-   SIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
-   {
-      dim = mesh->Dimension();
-
-      IsoparametricTransformation Tr;
-      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
-      mesh->GetElementTransformation(0, &Tr);
-      IntegrationRule ir;
-      MFIRs.GetSurfaceIntegrationRule(Tr, ir);
-      if (dim >1)
-      {
-         Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
-      }
-      else
-      {
-         Weights.SetSize(2, mesh->GetNE());
-      }
-      SurfaceWeights.SetSize(ir.GetNPoints(), mesh->GetNE());
-      Vector w;
-      MFIRs.GetSurfaceWeights(Tr, ir, w);
-      SurfaceWeights.SetCol(0, w);
-      SetSize(ir.GetNPoints());
-
-      for (int ip = 0; ip < GetNPoints(); ip++)
-      {
-         IntPoint(ip).index = ip;
-         IntegrationPoint &intp = IntPoint(ip);
-         intp.x = ir.IntPoint(ip).x;
-         intp.y = ir.IntPoint(ip).y;
-         intp.z = ir.IntPoint(ip).z;
-         if (dim > 1)
-         {
-            Weights(ip, 0) = ir.IntPoint(ip).weight;
-         }
-         else
-         {
-            Weights(0, 0) = ir.IntPoint(ip).x;
-            Weights(1, 0) = ir.IntPoint(ip).weight;
-         }
-      }
-
-
-      for (int elem = 1; elem < mesh->GetNE(); elem++)
-      {
-         mesh->GetElementTransformation(elem, &Tr);
-         MFIRs.GetSurfaceIntegrationRule(Tr, ir);
-         Vector w;
-         MFIRs.GetSurfaceWeights(Tr, ir, w);
-         SurfaceWeights.SetCol(elem, w);
-
-         for (int ip = 0; ip < GetNPoints(); ip++)
-         {
-            if (dim > 1)
-            {
-               Weights(ip, elem) = ir.IntPoint(ip).weight;
-            }
-            else
-            {
-               Weights(0, elem) = ir.IntPoint(ip).x;
-               Weights(1, elem) = ir.IntPoint(ip).weight;
-            }
-         }
-      }
-   }
-
-   /**
-    @brief Set the weights for the given element and multiply them with the
-    transformation of the interface
-    */
-   void SetElementinclSurfaceWeight(int Element)
-   {
-      if (dim == 1)
-      {
-         IntegrationPoint &intp = IntPoint(0);
-         intp.x = Weights(0, Element);
-         intp.weight = Weights(1, Element);
-         cout << intp.x << " " << Element << endl;
-      }
-      else
-         for (int ip = 0; ip < GetNPoints(); ip++)
-         {
-            IntegrationPoint &intp = IntPoint(ip);
-            intp.weight = Weights(ip, Element) * SurfaceWeights(ip, Element);
-         }
-   }
-
-   /// @brief Set the weights for the given element
-   void SetElement(int Element)
-   {
-      if (dim == 1)
-      {
-         IntegrationPoint &intp = IntPoint(0);
-         intp.x = Weights(0, Element);
-         intp.weight = Weights(1, Element);
-      }
-      else
-         for (int ip = 0; ip < GetNPoints(); ip++)
-         {
-            IntegrationPoint &intp = IntPoint(ip);
-            intp.weight = Weights(ip, Element);
-         }
-   }
-
-   /// @brief Destructor of SIntegrationRule
-   ~SIntegrationRule() {}
-};
-
-/**
- @brief Class for volume IntegrationRule
-
- This class demonstrates how IntegrationRules computed as CutIntegrationRules
- can be saved to reduce the impact by computing them from scratch each time.
- */
-class CIntegrationRule : public IntegrationRule
-{
-protected:
-   /// @brief Space Dimension of the IntegrationRule
-   int dim;
-   /// @brief Column-wise matrix of the quadtrature weights
-   DenseMatrix Weights;
-
-public:
-   /**
-    @brief Constructor of CIntegrationRule
-
-    The volume integrationRules are computed and saved in the constructor.
-
-    @param [in] Order Order of the IntegrationRule
-    @param [in] LvlSet Level-set defining the implicit interface
-    @param [in] lsOrder Polynomial degree for approx of level-set function
-    @param [in] mesh Pointer to the mesh that is used
-   */
-   CIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
-   {
-      dim = mesh->Dimension();
-
-      IsoparametricTransformation Tr;
-      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
-      mesh->GetElementTransformation(0, &Tr);
-      IntegrationRule ir;
-      MFIRs.GetVolumeIntegrationRule(Tr, ir);
-      if (dim > 1)
-      {
-         Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
-      }
-      else
-      {
-         Weights.SetSize(2 * ir.GetNPoints(), mesh->GetNE());
-      }
-
-      SetSize(ir.GetNPoints());
-      for (int ip = 0; ip < GetNPoints(); ip++)
-      {
-         IntPoint(ip).index = ip;
-         IntegrationPoint &intp = IntPoint(ip);
-         intp.x = ir.IntPoint(ip).x;
-         intp.y = ir.IntPoint(ip).y;
-         intp.z = ir.IntPoint(ip).z;
-         if (dim > 1)
-         {
-            Weights(ip, 0) = ir.IntPoint(ip).weight;
-         }
-         else
-         {
-            Weights(2 * ip, 0) = ir.IntPoint(ip).x;
-            Weights(2 * ip + 1, 0) = ir.IntPoint(ip).weight;
-         }
-      }
-
-      for (int elem = 1; elem < mesh->GetNE(); elem++)
-      {
-         mesh->GetElementTransformation(elem, &Tr);
-         MFIRs.GetVolumeIntegrationRule(Tr, ir);
-
-         for (int ip = 0; ip < GetNPoints(); ip++)
-         {
-            if (dim > 1)
-            {
-               Weights(ip, elem) = ir.IntPoint(ip).weight;
-            }
-            else
-            {
-               Weights(2 * ip, elem) = ir.IntPoint(ip).x;
-               Weights(2 * ip + 1, elem) = ir.IntPoint(ip).weight;
-            }
-         }
-      }
-   }
-
-   /// @brief Set the weights for the given element
-   void SetElement(int Element)
-   {
-      if (dim == 1)
-         for (int ip = 0; ip < GetNPoints(); ip++)
-         {
-            IntegrationPoint &intp = IntPoint(ip);
-            intp.x = Weights(2 * ip, Element);
-            intp.weight = Weights(2 * ip + 1, Element);
-         }
-      else
-         for (int ip = 0; ip < GetNPoints(); ip++)
-         {
-            IntegrationPoint &intp = IntPoint(ip);
-            intp.weight = Weights(ip, Element);
-         }
-   }
-
-   /// @brief Destructor of CIntegrationRule
-   ~CIntegrationRule() {}
-};
-/**
- @brief Class for surface linearform integrator
-
- Integrator to demonstrate the use of the surface integration rule on an
- implicit surface defined by a level-set.
- */
-class SurfaceLFIntegrator : public LinearFormIntegrator
-{
-protected:
-   /// @brief vector to evaluate the basis functions
-   Vector shape;
-
-   /// @brief surface integration rule
-   SIntegrationRule* SIntRule;
-
-   /// @brief coefficient representing the level-set defining the interface
-   Coefficient &LevelSet;
-
-   /// @brief coefficient representing the integrand
-   Coefficient &Q;
-
-public:
-   /**
-    @brief Constructor for the surface linear form integrator
-
-    Constructor for the surface linear form integrator to demonstrate the use
-    of the surface integration rule by means of moment-fitting.
-
-    @param [in] q coefficient representing the inegrand
-    @param [in] levelset level-set defining the implicit interfac
-    @param [in] ir surface integrtion rule to be used
-    */
-   SurfaceLFIntegrator(Coefficient &q, Coefficient &levelset,
-                       SIntegrationRule* ir)
-      : LinearFormIntegrator(), SIntRule(ir), LevelSet(levelset), Q(q) { }
-
-   /**
-    @brief Assembly of the element vector
-
-    Assemble the element vector of for the right hand side on the element given
-    by the FiniteElement and ElementTransformation.
-
-    @param [in] el finite Element the vector belongs to
-    @param [in] Tr transformation of finite element
-    @param [out] elvect vector containing the
-   */
-   virtual void AssembleRHSElementVect(const FiniteElement &el,
-                                       ElementTransformation &Tr,
-                                       Vector &elvect) override
-   {
-      int dof = el.GetDof();
-      shape.SetSize(dof);
-      elvect.SetSize(dof);
-      elvect = 0.;
-
-      // Update the surface integration rule for the current element
-      SIntRule->SetElementinclSurfaceWeight(Tr.ElementNo);
-
-      for (int ip = 0; ip < SIntRule->GetNPoints(); ip++)
-      {
-         Tr.SetIntPoint((&(SIntRule->IntPoint(ip))));
-         double val = Tr.Weight() * Q.Eval(Tr, SIntRule->IntPoint(ip));
-         el.CalcShape(SIntRule->IntPoint(ip), shape);
-         add(elvect, SIntRule->IntPoint(ip).weight * val, shape, elvect);
-      }
-   }
-};
-
-/**
- @brief Class for subdomain linearform integrator
-
- Integrator to demonstrate the use of the subdomain integration rule within
- an area defined by an implicit surface defined by a level-set.
- */
-class SubdomainLFIntegrator : public LinearFormIntegrator
-{
-protected:
-   /// @brief vector to evaluate the basis functions
-   Vector shape;
-
-   /// @brief surface integration rule
-   CIntegrationRule* CIntRule;
-
-   /// @brief coefficient representing the level-set defining the interface
-   Coefficient &LevelSet;
-
-   /// @brief coefficient representing the integrand
-   Coefficient &Q;
-
-public:
-   /**
-    @brief Constructor for the volumetric subdomain linear form integrator
-
-    Constructor for the subdomain linear form integrator to demonstrate the use
-    of the volumetric subdomain integration rule by means of moment-fitting.
-
-    @param [in] q coefficient representing the inegrand
-    @param [in] levelset level-set defining the implicit interfac
-    @param [in] ir subdomain integrtion rule to be used
-    */
-   SubdomainLFIntegrator(Coefficient &q, Coefficient &levelset,
-                         CIntegrationRule* ir)
-      : LinearFormIntegrator(), CIntRule(ir), LevelSet(levelset), Q(q) { }
-
-   /**
-    @brief Assembly of the element vector
-
-    Assemble the element vector of for the right hand side on the element given
-    by the FiniteElement and ElementTransformation.
-
-    @param [in] el finite Element the vector belongs to
-    @param [in] Tr transformation of finite element
-    @param [out] elvect vector containing the
-   */
-   virtual void AssembleRHSElementVect(const FiniteElement &el,
-                                       ElementTransformation &Tr,
-                                       Vector &elvect) override
-   {
-      int dof = el.GetDof();
-      shape.SetSize(dof);
-      elvect.SetSize(dof);
-      elvect = 0.;
-
-      // Update the subdomain integration rule
-      CIntRule->SetElement(Tr.ElementNo);
-
-      for (int ip = 0; ip < CIntRule->GetNPoints(); ip++)
-      {
-         Tr.SetIntPoint((&(CIntRule->IntPoint(ip))));
-         double val = Tr.Weight()
-                      * Q.Eval(Tr, CIntRule->IntPoint(ip));
-         el.CalcPhysShape(Tr, shape);
-         add(elvect, CIntRule->IntPoint(ip).weight * val, shape, elvect);
-      }
-   }
-};
-#endif // MFEM_USE_LAPACK
-
-int main(int argc, char *argv[])
-{
-#ifndef MFEM_USE_LAPACK
-   cout << "MFEM must be build with LAPACK for this example." << endl;
-   return EXIT_FAILURE;
-#else
-   // 1. Parse he command-line options.
-   int ref_levels = 3;
-   int order = 2;
-   const char *inttype = "surface2d";
-   bool visualization = true;
-   itype = IntegrationType::Surface2D;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&order, "-o", "--order", "Order of quadrature rule");
-   args.AddOption(&ref_levels, "-r", "--refine", "Number of meh refinements");
-   args.AddOption(&inttype, "-i", "--integrationtype",
-                  "IntegrationType to demonstrate");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.ParseCheck();
-
-   if (strcmp(inttype, "volumetric1d") == 0
-       || strcmp(inttype, "Volumetric1D") == 0)
-   {
-      itype = IntegrationType::Volumetric1D;
-   }
-   else if (strcmp(inttype, "surface2d") == 0
-            || strcmp(inttype, "Surface2D") == 0)
-   {
-      itype = IntegrationType::Surface2D;
-   }
-   else if (strcmp(inttype, "volumetric2d") == 0
-            || strcmp(inttype, "Volumetric2D") == 0)
-   {
-      itype = IntegrationType::Volumetric2D;
-   }
-   else if (strcmp(inttype, "surface3d") == 0
-            || strcmp(inttype, "Surface3d") == 0)
-   {
-      itype = IntegrationType::Surface3D;
-   }
-   else if (strcmp(inttype, "volumetric3d") == 0
-            || strcmp(inttype, "Volumetric3d") == 0)
-   {
-      itype = IntegrationType::Volumetric3D;
-   }
-
-   // 2. Construct and refine the mesh.
-   Mesh *mesh;
-   if (itype == IntegrationType::Volumetric1D)
-   {
-      mesh = new Mesh("../data/inline-segment.mesh");
-   }
-   if (itype == IntegrationType::Surface2D
-       || itype == IntegrationType::Volumetric2D)
-   {
-      mesh = new Mesh(2, 4, 1, 0, 2);
-      mesh->AddVertex(-1.6,-1.6);
-      mesh->AddVertex(1.6,-1.6);
-      mesh->AddVertex(1.6,1.6);
-      mesh->AddVertex(-1.6,1.6);
-      mesh->AddQuad(0,1,2,3);
-      mesh->FinalizeQuadMesh(1, 0, 1);
-   }
-   else if (itype == IntegrationType::Surface3D
-            || itype == IntegrationType::Volumetric3D)
-   {
-      mesh = new Mesh(3, 8, 1, 0, 3);
-      mesh->AddVertex(-1.6,-1.6,-1.6);
-      mesh->AddVertex(1.6,-1.6,-1.6);
-      mesh->AddVertex(1.6,1.6,-1.6);
-      mesh->AddVertex(-1.6,1.6,-1.6);
-      mesh->AddVertex(-1.6,-1.6,1.6);
-      mesh->AddVertex(1.6,-1.6,1.6);
-      mesh->AddVertex(1.6,1.6,1.6);
-      mesh->AddVertex(-1.6,1.6,1.6);
-      mesh->AddHex(0,1,2,3,4,5,6,7);
-      mesh->FinalizeHexMesh(1, 0, 1);
-   }
-
-   for (int lev = 0; lev < ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 3. Define the necessary finite element space on the mesh.
-   H1_FECollection fe_coll(1, mesh->Dimension());
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, &fe_coll);
-
-   // 4. Construction Coefficients for the level set and the integrand.
-   FunctionCoefficient levelset(lvlset);
-   FunctionCoefficient u(integrand);
-
-   // 5. Define the necessary Integration rules on element 0.
-   IsoparametricTransformation Tr;
-   mesh->GetElementTransformation(0, &Tr);
-   SIntegrationRule* sir = new SIntegrationRule(order, levelset, 2, mesh);
-   CIntegrationRule* cir = NULL;
-   if (itype == IntegrationType::Volumetric1D
-       || itype == IntegrationType::Volumetric2D
-       || itype == IntegrationType::Volumetric3D)
-   {
-      cir = new CIntegrationRule(order, levelset, 2, mesh);
-   }
-
-   // 6. Define and assemble the linear forms on the finite element space.
-   LinearForm surface(fespace);
-   LinearForm volume(fespace);
-
-   surface.AddDomainIntegrator(new SurfaceLFIntegrator(u, levelset, sir));
-   surface.Assemble();
-
-   if (itype == IntegrationType::Volumetric1D
-       || itype == IntegrationType::Volumetric2D
-       || itype == IntegrationType::Volumetric3D)
-   {
-      volume.AddDomainIntegrator(new SubdomainLFIntegrator(u, levelset, cir));
-      volume.Assemble();
-   }
-
-   // 7. Print information, computed values and errors to the console.
-   int qorder = 0;
-   int nbasis = 2 * (order + 1) + (int)(order * (order + 1) / 2);
-   IntegrationRules irs(0, Quadrature1D::GaussLegendre);
-   IntegrationRule ir = irs.Get(Geometry::SQUARE, qorder);
-   for (; ir.GetNPoints() <= nbasis; qorder++)
-   {
-      ir = irs.Get(Geometry::SQUARE, qorder);
-   }
-   cout << "============================================" << endl;
-   cout << "Mesh size dx:                       ";
-   if (itype != IntegrationType::Volumetric1D)
-   {
-      cout << 3.2 / pow(2., (double)ref_levels) << endl;
-   }
-   else
-   {
-      cout << .25 / pow(2., (double)ref_levels) << endl;
-   }
-   if (itype == IntegrationType::Surface2D
-       || itype == IntegrationType::Volumetric2D)
-   {
-      cout << "Number of div free basis functions: " << nbasis << endl;
-      cout << "Number of quadrature points:        " << ir.GetNPoints() << endl;
-   }
-   cout << scientific << setprecision(2);
-   cout << "============================================" << endl;
-   cout << "Computed value of surface integral: " << surface.Sum() << endl;
-   cout << "True value of surface integral:     " << Surface() << endl;
-   cout << "Absolut Error (Surface):            ";
-   cout << abs(surface.Sum() - Surface()) << endl;
-   cout << "Relative Error (Surface):           ";
-   cout << abs(surface.Sum() - Surface()) / Surface() << endl;
-   if (itype == IntegrationType::Volumetric1D
-       || itype == IntegrationType::Volumetric2D
-       || itype == IntegrationType::Volumetric3D)
-   {
-      cout << "--------------------------------------------" << endl;
-      cout << "Computed value of volume integral:  " << volume.Sum() << endl;
-      cout << "True value of volume integral:      " << Volume() << endl;
-      cout << "Absolut Error (Volume):             ";
-      cout << abs(volume.Sum() - Volume()) << endl;
-      cout << "Relative Error (Volume):            ";
-      cout << abs(volume.Sum() - Volume()) / Volume() << endl;
-   }
-   cout << "============================================" << endl;
-
-   // 8. Plot the level-set function on a high order finite element space.
-   if (visualization)
-   {
-      H1_FECollection fe_coll2(5, mesh->Dimension());
-      FiniteElementSpace fespace2(mesh, &fe_coll2);
-      FunctionCoefficient levelset_coeff(levelset);
-      GridFunction lgf(&fespace2);
-      lgf.ProjectCoefficient(levelset_coeff);
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << lgf << flush;
-      sol_sock << "keys pppppppppppppppppppppppppppcmmlRj\n";
-      sol_sock << "levellines " << 0. << " " << 0. << " " << 1 << "\n" << flush;
-   }
-
-   delete sir;
-   delete cir;
-   delete fespace;
-   delete mesh;
-   return EXIT_SUCCESS;
-#endif //MFEM_USE_LAPACK
-}
@@ -5,7 +5,6 @@
 // Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
 //               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh -nc -o 2
 //               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
 //               mpirun -np 4 ex3p -m ../data/beam-hex.mesh -o 2 -pa
 //               mpirun -np 4 ex3p -m ../data/escher.mesh
@@ -71,7 +70,6 @@ int main(int argc, char *argv[])
   int order = 1;
   bool static_cond = false;
   bool pa = false;
-   bool nc = false;
   const char *device_config = "cpu";
   bool visualization = true;
 #ifdef MFEM_USE_AMGX
@@ -89,9 +87,6 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&nc, "-nc", "--non-conforming", "-c",
-                  "--conforming",
-                  "Mark the mesh as nonconforming before partitioning.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -129,11 +124,6 @@ int main(int argc, char *argv[])
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   dim = mesh->Dimension();
   int sdim = mesh->SpaceDimension();
-   if (nc)
-   {
-      // Can set to false to use conformal refinement for simplices.
-      mesh->EnsureNCMesh(true);
-   }

   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
@@ -450,8 +450,7 @@ int main(int argc, char *argv[])

 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_)
-   : TimeDependentOperator(M_.FESpace()->GetTrueVSize()),
-     M(M_), K(K_), b(b_), z(height)
+   : TimeDependentOperator(M_.Height()), M(M_), K(K_), b(b_), z(M_.Height())
 {
   Array<int> ess_tdof_list;
   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACY)
@@ -659,9 +659,9 @@ int main(int argc, char *argv[])
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
                           const Vector &b_, PrecType prec_type)
-   : TimeDependentOperator(M_.ParFESpace()->GetTrueVSize()), b(b_),
+   : TimeDependentOperator(M_.Height()), b(b_),
     M_solver(M_.ParFESpace()->GetComm()),
-     z(height)
+     z(M_.Height())
 {
   if (M_.GetAssemblyLevel()==AssemblyLevel::LEGACY)
   {
@@ -3,28 +3,28 @@
 //
 // Compile with: make ex1
 //
-// Sample runs:  ex1 -m ../../data/square-disc.mesh
-//               ex1 -m ../../data/star.mesh
-//               ex1 -m ../../data/star-mixed.mesh
-//               ex1 -m ../../data/escher.mesh
-//               ex1 -m ../../data/fichera.mesh
-//               ex1 -m ../../data/fichera-mixed.mesh
-//               ex1 -m ../../data/toroid-wedge.mesh
-//               ex1 -m ../../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../../data/star-surf.mesh
-//               ex1 -m ../../data/square-disc-surf.mesh
-//               ex1 -m ../../data/inline-segment.mesh
-//               ex1 -m ../../data/amr-quad.mesh
-//               ex1 -m ../../data/amr-hex.mesh
-//               ex1 -m ../../data/fichera-amr.mesh
-//               ex1 -m ../../data/mobius-strip.mesh
-//               ex1 -m ../../data/mobius-strip.mesh -o -1 -sc
+// Sample runs:  ex1 -m ../data/square-disc.mesh
+//               ex1 -m ../data/star.mesh
+//               ex1 -m ../data/star-mixed.mesh
+//               ex1 -m ../data/escher.mesh
+//               ex1 -m ../data/fichera.mesh
+//               ex1 -m ../data/fichera-mixed.mesh
+//               ex1 -m ../data/toroid-wedge.mesh
+//               ex1 -m ../data/square-disc-p2.vtk -o 2
+//               ex1 -m ../data/square-disc-p3.mesh -o 3
+//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
+//               ex1 -m ../data/star-mixed-p2.mesh -o 2
+//               ex1 -m ../data/disc-nurbs.mesh -o -1
+//               ex1 -m ../data/pipe-nurbs.mesh -o -1
+//               ex1 -m ../data/fichera-mixed-p2.mesh -o 2
+//               ex1 -m ../data/star-surf.mesh
+//               ex1 -m ../data/square-disc-surf.mesh
+//               ex1 -m ../data/inline-segment.mesh
+//               ex1 -m ../data/amr-quad.mesh
+//               ex1 -m ../data/amr-hex.mesh
+//               ex1 -m ../data/fichera-amr.mesh
+//               ex1 -m ../data/mobius-strip.mesh
+//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
 //
 // Device sample runs:
 //               ex1 -pa -d cuda
@@ -32,7 +32,7 @@
 //               ex1 -pa -d occa-cuda
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
-//               ex1 -m ../../data/beam-hex.mesh -pa -d cuda
+//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -22,19 +22,15 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
-   ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-   ex31 ex33 ex34 ex36 ex37
+  ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
+  ex31 ex33 ex34 ex36 ex37
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
-   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
-   ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-   ex37p
+  ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
+  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
+	ex37p
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-   ex24p ex25p ex26p ex34p ex35p
-
-ifeq ($(MFEM_USE_LAPACK),YES)
-   SEQ_EXAMPLES += ex38
-endif
+  ex24p ex25p ex26p ex34p ex35p

 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
@@ -273,13 +273,12 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
         if (sp_solver)
         {
-            STRUMPACKSolver * strumpack = new STRUMPACKSolver(MPI_COMM_WORLD, argc, argv);
+            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
            strumpack->SetPrintFactorStatistics(true);
            strumpack->SetPrintSolveStatistics(false);
            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            strumpack->SetMatching(strumpack::MatchingJob::NONE);
-            strumpack->SetCompression(strumpack::CompressionType::NONE);
+            strumpack->DisableMatching();
            strumpack->SetOperator(*Arow);
            strumpack->SetFromCommandLine();
            precond = strumpack;
@@ -17,7 +17,7 @@
 //               finite elements (velocity u) and piecewise discontinuous
 //               polynomials (pressure p).
 //
-//               The example demonstrates the use of the BlockOperator class, as
+//               The example demonstrates the use of the BlockMatrix class, as
 //               well as the collective saving of several grid functions in a
 //               VisIt (visit.llnl.gov) visualization format.
 //
@@ -520,10 +520,10 @@ int main(int argc, char *argv[])
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
                           const Vector &b_,bool M_in_lhs)
-   : TimeDependentOperator(M_.ParFESpace()->GetTrueVSize(), 0.0,
+   : TimeDependentOperator(M_.Height(), 0.0,
                           M_in_lhs ? TimeDependentOperator::IMPLICIT
                           : TimeDependentOperator::EXPLICIT),
-     b(b_), comm(M_.ParFESpace()->GetComm()), M_solver(comm), z(height),
+     b(b_), comm(M_.ParFESpace()->GetComm()), M_solver(comm), z(M_.Height()),
     iJacobian(NULL), rJacobian(NULL)
 {
   MAlev = M_.GetAssemblyLevel();
@@ -476,8 +476,7 @@ int main(int argc, char *argv[])

 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_)
-   : TimeDependentOperator(M_.FESpace()->GetTrueVSize()),
-     M(M_), K(K_), b(b_), z(height)
+   : TimeDependentOperator(M_.Height()), M(M_), K(K_), b(b_), z(M_.Height())
 {
   Array<int> ess_tdof_list;
   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACY)
@@ -679,10 +679,10 @@ int main(int argc, char *argv[])
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
                           const Vector &b_, PrecType prec_type)
-   : TimeDependentOperator(M_.ParFESpace()->GetTrueVSize()),
+   : TimeDependentOperator(M_.Height()),
     b(b_),
     M_solver(M_.ParFESpace()->GetComm()),
-     z(height)
+     z(M_.Height())
 {
   if (M_.GetAssemblyLevel()==AssemblyLevel::LEGACY)
   {
@@ -77,7 +77,6 @@ set(SRCS
  gridfunc.cpp
  hybridization.cpp
  intrules.cpp
-  intrules_cut.cpp
  ceed/interface/basis.cpp
  ceed/interface/restriction.cpp
  ceed/interface/operator.cpp
@@ -97,6 +96,9 @@ set(SRCS
  lor/lor_ads.cpp
  lor/lor_ams.cpp
  lor/lor_batched.cpp
+  lor/lor_h1.cpp
+  lor/lor_nd.cpp
+  lor/lor_rt.cpp
  multigrid.cpp
  nonlinearform.cpp
  nonlinearform_ext.cpp
@@ -184,7 +186,6 @@ set(HDRS
  gridfunc.hpp
  hybridization.hpp
  intrules.hpp
-  intrules_cut.hpp
  ceed/interface/basis.hpp
  ceed/interface/integrator.hpp
  ceed/interface/interface.hpp
@@ -101,7 +101,6 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)

   // Copy the pointers to the integrators
   domain_integs = bf->domain_integs;
-   domain_integs_marker = bf->domain_integs_marker;

   boundary_integs = bf->boundary_integs;
   boundary_integs_marker = bf->boundary_integs_marker;
@@ -394,8 +393,8 @@ void BilinearForm::Assemble(int skip_zeros)
   }

   ElementTransformation *eltrans;
-   DofTransformation *doftrans;
-   Mesh *mesh = fes->GetMesh();
+   DofTransformation * doftrans;
+   Mesh *mesh = fes -> GetMesh();
   DenseMatrix elmat, *elmat_p;

   if (mat == NULL)
@@ -432,7 +431,7 @@ void BilinearForm::Assemble(int skip_zeros)
      }

      // Element-wise integration
-      for (int i = 0; i < fes->GetNE(); i++)
+      for (int i = 0; i < fes -> GetNE(); i++)
      {
         doftrans = fes->GetElementVDofs(i, vdofs);
         if (element_matrices)
@@ -442,7 +441,6 @@ void BilinearForm::Assemble(int skip_zeros)
         else
         {
            const int elem_attr = fes->GetMesh()->GetAttribute(i);
-            eltrans = fes->GetElementTransformation(i);
            elmat.SetSize(0);
            for (int k = 0; k < domain_integs.Size(); k++)
            {
@@ -450,8 +448,9 @@ void BilinearForm::Assemble(int skip_zeros)
                    (*(domain_integs_marker[k]))[elem_attr-1] == 1)
                   && !domain_integs[k]->Patchwise())
               {
-                  domain_integs[k]->AssembleElementMatrix(*fes->GetFE(i),
-                                                          *eltrans, elemmat);
+                  const FiniteElement &fe = *fes->GetFE(i);
+                  eltrans = fes->GetElementTransformation(i);
+                  domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
                  if (elmat.Size() == 0)
                  {
                     elmat = elemmat;
@@ -493,7 +492,7 @@ void BilinearForm::Assemble(int skip_zeros)
      // Patch-wise integration
      if (fes->GetNURBSext())
      {
-         for (int p = 0; p < mesh->NURBSext->GetNP(); ++p)
+         for (int p=0; p<mesh->NURBSext->GetNP(); ++p)
         {
            bool vdofsSet = false;
            for (int k = 0; k < domain_integs.Size(); k++)
@@ -506,15 +505,15 @@ void BilinearForm::Assemble(int skip_zeros)
                     vdofsSet = true;
                  }

-                  SparseMatrix *spmat = nullptr;
+                  SparseMatrix* spmat = nullptr;
                  domain_integs[k]->AssemblePatchMatrix(p, *fes, spmat);
                  Array<int> cols;
                  Vector srow;

-                  for (int r = 0; r < spmat->Height(); ++r)
+                  for (int r=0; r<spmat->Height(); ++r)
                  {
                     spmat->GetRow(r, cols, srow);
-                     for (int i = 0; i < cols.Size(); ++i)
+                     for (int i=0; i<cols.Size(); ++i)
                     {
                        cols[i] = vdofs[cols[i]];
                     }
@@ -551,14 +550,14 @@ void BilinearForm::Assemble(int skip_zeros)
         }
      }

-      for (int i = 0; i < fes->GetNBE(); i++)
+      for (int i = 0; i < fes -> GetNBE(); i++)
      {
         const int bdr_attr = mesh->GetBdrAttribute(i);
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }

         const FiniteElement &be = *fes->GetBE(i);
-         doftrans = fes->GetBdrElementVDofs(i, vdofs);
-         eltrans = fes->GetBdrElementTransformation(i);
+         doftrans = fes -> GetBdrElementVDofs (i, vdofs);
+         eltrans = fes -> GetBdrElementTransformation (i);
         int k = 0;
         for (; k < boundary_integs.Size(); k++)
         {
@@ -605,11 +604,11 @@ void BilinearForm::Assemble(int skip_zeros)
      int nfaces = mesh->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
      {
-         tr = mesh->GetInteriorFaceTransformations(i);
+         tr = mesh -> GetInteriorFaceTransformations (i);
         if (tr != NULL)
         {
-            fes->GetElementVDofs(tr->Elem1No, vdofs);
-            fes->GetElementVDofs(tr->Elem2No, vdofs2);
+            fes -> GetElementVDofs (tr -> Elem1No, vdofs);
+            fes -> GetElementVDofs (tr -> Elem2No, vdofs2);
            vdofs.Append (vdofs2);
            for (int k = 0; k < interior_face_integs.Size(); k++)
            {
@@ -617,7 +616,7 @@ void BilinearForm::Assemble(int skip_zeros)
               AssembleFaceMatrix(*fes->GetFE(tr->Elem1No),
                                  *fes->GetFE(tr->Elem2No),
                                  *tr, elemmat);
-               mat->AddSubMatrix(vdofs, vdofs, elemmat, skip_zeros);
+               mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
            }
         }
      }
@@ -649,16 +648,16 @@ void BilinearForm::Assemble(int skip_zeros)
         }
      }

-      for (int i = 0; i < fes->GetNBE(); i++)
+      for (int i = 0; i < fes -> GetNBE(); i++)
      {
         const int bdr_attr = mesh->GetBdrAttribute(i);
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }

-         tr = mesh->GetBdrFaceTransformations(i);
+         tr = mesh -> GetBdrFaceTransformations (i);
         if (tr != NULL)
         {
-            fes->GetElementVDofs(tr->Elem1No, vdofs);
-            fe1 = fes->GetFE(tr->Elem1No);
+            fes -> GetElementVDofs (tr -> Elem1No, vdofs);
+            fe1 = fes -> GetFE (tr -> Elem1No);
            // The fe2 object is really a dummy and not used on the boundaries,
            // but we can't dereference a NULL pointer, and we don't want to
            // actually make a fake element.
@@ -669,9 +668,9 @@ void BilinearForm::Assemble(int skip_zeros)
                   (*boundary_face_integs_marker[k])[bdr_attr-1] == 0)
               { continue; }

-               boundary_face_integs[k]->AssembleFaceMatrix(*fe1, *fe2, *tr,
-                                                           elemmat);
-               mat->AddSubMatrix(vdofs, vdofs, elemmat, skip_zeros);
+               boundary_face_integs[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr,
+                                                              elemmat);
+               mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
            }
         }
      }
@@ -1205,7 +1204,6 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   mat = NULL;
   mat_e = NULL;
   extern_bfs = 0;
-   element_matrices = NULL;
   assembly = AssemblyLevel::LEGACY;
   ext = NULL;
 }
@@ -1220,19 +1218,15 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   mat = NULL;
   mat_e = NULL;
   extern_bfs = 1;
-   element_matrices = NULL;
   ext = NULL;

   // Copy the pointers to the integrators
   domain_integs = mbf->domain_integs;
-   domain_integs_marker = mbf->domain_integs_marker;
-
   boundary_integs = mbf->boundary_integs;
-   boundary_integs_marker = mbf->boundary_integs_marker;
-
   trace_face_integs = mbf->trace_face_integs;
-
   boundary_trace_face_integs = mbf->boundary_trace_face_integs;
+
+   boundary_integs_marker = mbf->boundary_integs_marker;
   boundary_trace_face_integs_marker = mbf->boundary_trace_face_integs_marker;

   assembly = AssemblyLevel::LEGACY;
@@ -1355,14 +1349,6 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const
 void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
   domain_integs.Append (bfi);
-   domain_integs_marker.Append(NULL); // NULL marker means apply everywhere
-}
-
-void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi,
-                                             Array<int> &elem_marker)
-{
-   domain_integs.Append (bfi);
-   domain_integs_marker.Append(&elem_marker);
 }

 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
@@ -1397,7 +1383,7 @@ void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
   boundary_trace_face_integs_marker.Append(&bdr_marker);
 }

-void MixedBilinearForm::Assemble(int skip_zeros)
+void MixedBilinearForm::Assemble (int skip_zeros)
 {
   if (ext)
   {
@@ -1406,71 +1392,39 @@ void MixedBilinearForm::Assemble(int skip_zeros)
   }

   ElementTransformation *eltrans;
-   DofTransformation *dom_dof_trans;
-   DofTransformation *ran_dof_trans;
-   DenseMatrix elmat, *elmat_p;
+   DofTransformation * dom_dof_trans;
+   DofTransformation * ran_dof_trans;
+   DenseMatrix elmat;

-   Mesh *mesh = test_fes->GetMesh();
+   Mesh *mesh = test_fes -> GetMesh();

   if (mat == NULL)
   {
      mat = new SparseMatrix(height, width);
   }

-#ifdef MFEM_USE_LEGACY_OPENMP
-   int free_element_matrices = 0;
-   if (!element_matrices)
-   {
-      ComputeElementMatrices();
-      free_element_matrices = 1;
-   }
-#endif
-
   if (domain_integs.Size())
   {
-      for (int k = 0; k < domain_integs.Size(); k++)
+      for (int i = 0; i < test_fes -> GetNE(); i++)
      {
-         if (domain_integs_marker[k] != NULL)
-         {
-            MFEM_VERIFY(domain_integs_marker[k]->Size() ==
-                        (mesh->attributes.Size() ? mesh->attributes.Max() : 0),
-                        "invalid element marker for domain integrator #"
-                        << k << ", counting from zero");
-         }
-      }
+         dom_dof_trans = trial_fes -> GetElementVDofs (i, trial_vdofs);
+         ran_dof_trans = test_fes  -> GetElementVDofs (i, test_vdofs);
+         eltrans = test_fes -> GetElementTransformation (i);

-      for (int i = 0; i < test_fes->GetNE(); i++)
-      {
-         const int elem_attr = mesh->GetAttribute(i);
-         dom_dof_trans = trial_fes->GetElementVDofs(i, trial_vdofs);
-         ran_dof_trans = test_fes->GetElementVDofs(i, test_vdofs);
-         if (element_matrices)
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
+         for (int k = 0; k < domain_integs.Size(); k++)
         {
-            elmat_p = &(*element_matrices)(i);
+            domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
+                                                        *test_fes  -> GetFE(i),
+                                                        *eltrans, elemmat);
+            elmat += elemmat;
         }
-         else
+         if (ran_dof_trans || dom_dof_trans)
         {
-            eltrans = test_fes->GetElementTransformation(i);
-            elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-            elmat = 0.0;
-            for (int k = 0; k < domain_integs.Size(); k++)
-            {
-               if (domain_integs_marker[k] == NULL ||
-                   (*(domain_integs_marker[k]))[elem_attr-1] == 1)
-               {
-                  domain_integs[k]->AssembleElementMatrix2(*trial_fes->GetFE(i),
-                                                           *test_fes->GetFE(i),
-                                                           *eltrans, elemmat);
-                  elmat += elemmat;
-               }
-            }
-            if (ran_dof_trans || dom_dof_trans)
-            {
-               TransformDual(ran_dof_trans, dom_dof_trans, elmat);
-            }
-            elmat_p = &elmat;
+            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
         }
-         mat->AddSubMatrix(test_vdofs, trial_vdofs, *elmat_p, skip_zeros);
+         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }

@@ -1497,14 +1451,14 @@ void MixedBilinearForm::Assemble(int skip_zeros)
         }
      }

-      for (int i = 0; i < test_fes->GetNBE(); i++)
+      for (int i = 0; i < test_fes -> GetNBE(); i++)
      {
         const int bdr_attr = mesh->GetBdrAttribute(i);
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }

-         dom_dof_trans = trial_fes->GetBdrElementVDofs(i, trial_vdofs);
-         ran_dof_trans = test_fes->GetBdrElementVDofs(i, test_vdofs);
-         eltrans = test_fes->GetBdrElementTransformation(i);
+         dom_dof_trans = trial_fes -> GetBdrElementVDofs (i, trial_vdofs);
+         ran_dof_trans = test_fes  -> GetBdrElementVDofs (i, test_vdofs);
+         eltrans = test_fes -> GetBdrElementTransformation (i);

         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
         elmat = 0.0;
@@ -1513,17 +1467,16 @@ void MixedBilinearForm::Assemble(int skip_zeros)
            if (boundary_integs_marker[k] &&
                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }

-            boundary_integs[k]->AssembleElementMatrix2(*trial_fes->GetBE(i),
-                                                       *test_fes->GetBE(i),
-                                                       *eltrans, elemmat);
+            boundary_integs[k]->AssembleElementMatrix2 (*trial_fes -> GetBE(i),
+                                                        *test_fes  -> GetBE(i),
+                                                        *eltrans, elemmat);
            elmat += elemmat;
         }
         if (ran_dof_trans || dom_dof_trans)
         {
            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
         }
-         elmat_p = &elmat;
-         mat->AddSubMatrix(test_vdofs, trial_vdofs, *elmat_p, skip_zeros);
+         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }

@@ -1621,13 +1574,6 @@ void MixedBilinearForm::Assemble(int skip_zeros)
         }
      }
   }
-
-#ifdef MFEM_USE_LEGACY_OPENMP
-   if (free_element_matrices)
-   {
-      FreeElementMatrices();
-   }
-#endif
 }

 void MixedBilinearForm::AssembleDiagonal_ADAt(const Vector &D,
@@ -1713,13 +1659,6 @@ void MixedBilinearForm::ConformingAssemble()

 void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (element_matrices)
-   {
-      elmat.SetSize(element_matrices->SizeI(), element_matrices->SizeJ());
-      elmat = element_matrices->GetData(i);
-      return;
-   }
-
   if (domain_integs.Size())
   {
      const FiniteElement &trial_fe = *trial_fes->GetFE(i);
@@ -1806,50 +1745,6 @@ void MixedBilinearForm::AssembleBdrElementMatrix(
   mat->AddSubMatrix(test_vdofs_, trial_vdofs_, elmat, skip_zeros);
 }

-void MixedBilinearForm::ComputeElementMatrices()
-{
-   if (element_matrices || domain_integs.Size() == 0 || trial_fes->GetNE() == 0)
-   {
-      return;
-   }
-
-   int num_elements = trial_fes->GetNE();
-   int trial_dofs_per_el = trial_fes->GetFE(0)->GetDof() * trial_fes->GetVDim();
-   int test_dofs_per_el = test_fes->GetFE(0)->GetDof() * test_fes->GetVDim();
-
-   element_matrices = new DenseTensor(test_dofs_per_el, trial_dofs_per_el,
-                                      num_elements);
-   DenseMatrix tmp;
-   IsoparametricTransformation eltrans;
-#ifdef MFEM_USE_LEGACY_OPENMP
-   #pragma omp parallel for private(tmp,eltrans)
-#endif
-   for (int i = 0; i < num_elements; i++)
-   {
-      DenseMatrix elmat(element_matrices->GetData(i),
-                        test_dofs_per_el, trial_dofs_per_el);
-      const FiniteElement &trial_fe = *trial_fes->GetFE(i);
-      const FiniteElement &test_fe = *test_fes->GetFE(i);
-#ifdef MFEM_DEBUG
-      if (trial_dofs_per_el != trial_fe.GetDof() * trial_fes->GetVDim())
-         mfem_error("MixedBilinearForm::ComputeElementMatrices:"
-                    " all elements must have same number of dofs");
-#endif
-      test_fes->GetElementTransformation(i, &eltrans);
-
-      domain_integs[0]->AssembleElementMatrix2(trial_fe, test_fe, eltrans,
-                                               elmat);
-      for (int k = 1; k < domain_integs.Size(); k++)
-      {
-         // note: some integrators may not be thread-safe
-         domain_integs[k]->AssembleElementMatrix2(trial_fe, test_fe, eltrans,
-                                                  tmp);
-         elmat += tmp;
-      }
-      elmat.ClearExternalData();
-   }
-}
-
 void MixedBilinearForm::EliminateTrialDofs (
   const Array<int> &bdr_attr_is_ess, const Vector &sol, Vector &rhs )
 {
@@ -2046,56 +1941,41 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      return;
   }

-   ElementTransformation *eltrans;
+   Array<int> dom_vdofs, ran_vdofs;
+   ElementTransformation *T;
   DofTransformation * dom_dof_trans;
   DofTransformation * ran_dof_trans;
-   DenseMatrix elmat;
-
-   Mesh *mesh = test_fes->GetMesh();
+   const FiniteElement *dom_fe, *ran_fe;
+   DenseMatrix totelmat, elmat;

   if (mat == NULL)
   {
      mat = new SparseMatrix(height, width);
   }

-   if (domain_integs.Size())
+   if (domain_integs.Size() > 0)
   {
-      for (int k = 0; k < domain_integs.Size(); k++)
-      {
-         if (domain_integs_marker[k] != NULL)
-         {
-            MFEM_VERIFY(domain_integs_marker[k]->Size() ==
-                        (mesh->attributes.Size() ? mesh->attributes.Max() : 0),
-                        "invalid element marker for domain integrator #"
-                        << k << ", counting from zero");
-         }
-      }
-
      for (int i = 0; i < test_fes->GetNE(); i++)
      {
-         const int elem_attr = mesh->GetAttribute(i);
-         dom_dof_trans = trial_fes->GetElementVDofs(i, trial_vdofs);
-         ran_dof_trans = test_fes->GetElementVDofs(i, test_vdofs);
-         eltrans = test_fes->GetElementTransformation(i);
+         dom_dof_trans = trial_fes->GetElementVDofs(i, dom_vdofs);
+         ran_dof_trans = test_fes->GetElementVDofs(i, ran_vdofs);
+         T = test_fes->GetElementTransformation(i);
+         dom_fe = trial_fes->GetFE(i);
+         ran_fe = test_fes->GetFE(i);

-         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-         elmat = 0.0;
-         for (int k = 0; k < domain_integs.Size(); k++)
+         domain_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                  totelmat);
+         for (int j = 1; j < domain_integs.Size(); j++)
         {
-            if (domain_integs_marker[k] == NULL ||
-                (*(domain_integs_marker[k]))[elem_attr-1] == 1)
-            {
-               domain_integs[k]->AssembleElementMatrix2(*trial_fes->GetFE(i),
-                                                        *test_fes->GetFE(i),
-                                                        *eltrans, elemmat);
-               elmat += elemmat;
-            }
+            domain_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                     elmat);
+            totelmat += elmat;
         }
         if (ran_dof_trans || dom_dof_trans)
         {
-            TransformPrimal(ran_dof_trans, dom_dof_trans, elemmat);
+            TransformPrimal(ran_dof_trans, dom_dof_trans, totelmat);
         }
-         mat->SetSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
+         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
      }
   }

@@ -2104,20 +1984,21 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      const int nfaces = test_fes->GetMesh()->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
      {
-         trial_fes->GetFaceVDofs(i, trial_vdofs);
-         test_fes->GetFaceVDofs(i, test_vdofs);
-         eltrans = test_fes->GetMesh()->GetFaceTransformation(i);
+         trial_fes->GetFaceVDofs(i, dom_vdofs);
+         test_fes->GetFaceVDofs(i, ran_vdofs);
+         T = test_fes->GetMesh()->GetFaceTransformation(i);
+         dom_fe = trial_fes->GetFaceElement(i);
+         ran_fe = test_fes->GetFaceElement(i);

-         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-         elmat = 0.0;
-         for (int k = 0; k < trace_face_integs.Size(); k++)
+         trace_face_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                      totelmat);
+         for (int j = 1; j < trace_face_integs.Size(); j++)
         {
-            trace_face_integs[k]->AssembleElementMatrix2(*trial_fes->GetFaceElement(i),
-                                                         *test_fes->GetFaceElement(i),
-                                                         *eltrans, elemmat);
-            elmat += elemmat;
+            trace_face_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                         elmat);
+            totelmat += elmat;
         }
-         mat->SetSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
+         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
      }
   }
 }
@@ -100,7 +100,7 @@ protected:
   /// Includes all by default.
   /// 0 - ignore attribute
   /// 1 - include attribute
-   Array<Array<int>*> domain_integs_marker; ///< Entries are not owned.
+   Array<Array<int>*>             domain_integs_marker;

   /// Set of Boundary Integrators to be applied.
   Array<BilinearFormIntegrator*> boundary_integs;
@@ -722,13 +722,10 @@ protected:

   /// Domain integrators.
   Array<BilinearFormIntegrator*> domain_integs;
-   /// Entries are not owned.
-   Array<Array<int>*> domain_integs_marker;

   /// Boundary integrators.
   Array<BilinearFormIntegrator*> boundary_integs;
-   /// Entries are not owned.
-   Array<Array<int>*> boundary_integs_marker;
+   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.

   /// Trace face (skeleton) integrators.
   Array<BilinearFormIntegrator*> trace_face_integs;
@@ -739,7 +736,6 @@ protected:
   Array<Array<int>*> boundary_trace_face_integs_marker;

   DenseMatrix elemmat;
-   DenseTensor *element_matrices; ///< Owned.
   Array<int>  trial_vdofs, test_vdofs;

 private:
@@ -809,16 +805,12 @@ public:
   /// Adds a domain integrator. Assumes ownership of @a bfi.
   void AddDomainIntegrator(BilinearFormIntegrator *bfi);

-   /// Adds a domain integrator. Assumes ownership of @a bfi.
-   void AddDomainIntegrator(BilinearFormIntegrator *bfi,
-                            Array<int> &elem_marker);
-
   /// Adds a boundary integrator. Assumes ownership of @a bfi.
   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi);

   /// Adds a boundary integrator. Assumes ownership of @a bfi.
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi,
-                              Array<int> &bdr_marker);
+   void AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
+                               Array<int> &bdr_marker);

   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.

@@ -828,18 +820,14 @@ public:
   void AddTraceFaceIntegrator(BilinearFormIntegrator *bfi);

   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi);
+   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi);

   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi,
-                                  Array<int> &bdr_marker);
+   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi,
+                                   Array<int> &bdr_marker);

   /// Access all integrators added with AddDomainIntegrator().
   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }
-   /** @brief Access all domain markers added with AddDomainIntegrator().
-       If no marker was specified when the integrator was added, the
-       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetDBFI_Marker() { return &domain_integs_marker; }

   /// Access all integrators added with AddBoundaryIntegrator().
   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
@@ -896,18 +884,7 @@ public:
       MixedBilinearForm becomes an operator on the conforming FE spaces. */
   void ConformingAssemble();

-   /// Compute and store internally all element matrices.
-   void ComputeElementMatrices();
-
-   /// Free the memory used by the element matrices.
-   void FreeElementMatrices()
-   { delete element_matrices; element_matrices = NULL; }
-
   /// Compute the element matrix of the given element
-   /** The element matrix is computed by calling the domain integrators
-       or the one stored internally by a prior call of ComputeElementMatrices()
-       is returned when available.
-   */
   void ComputeElementMatrix(int i, DenseMatrix &elmat);

   /// Compute the boundary element matrix of the given boundary element
@@ -1088,9 +1065,6 @@ public:
   /// Adds a domain interpolator. Assumes ownership of @a di.
   void AddDomainInterpolator(DiscreteInterpolator *di)
   { AddDomainIntegrator(di); }
-   void AddDomainInterpolator(DiscreteInterpolator *di,
-                              Array<int> &elem_marker)
-   { AddDomainIntegrator(di, elem_marker); }

   /// Adds a trace face interpolator. Assumes ownership of @a di.
   void AddTraceFaceInterpolator(DiscreteInterpolator *di)
@@ -1098,7 +1072,6 @@ public:

   /// Access all interpolators added with AddDomainInterpolator().
   Array<BilinearFormIntegrator*> *GetDI() { return &domain_integs; }
-   Array<Array<int>*> *GetDI_Marker() { return &domain_integs_marker; }

   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
   /** This method must be called before assembly. */
@@ -303,7 +303,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
      std::unordered_map<int,int> f_to_be;
      for (int i = 0; i < mesh.GetNBE(); ++i)
      {
-         const int f = mesh.GetBdrElementFaceIndex(i);
+         const int f = mesh.GetBdrElementEdgeIndex(i);
         f_to_be[f] = i;
      }
      const int nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary);
@@ -1340,11 +1340,10 @@ void MassIntegrator::AssembleElementMatrix2(
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
+      trial_fe.CalcShape(ip, shape);
+      test_fe.CalcShape(ip, te_shape);
+
      Trans.SetIntPoint (&ip);
-
-      trial_fe.CalcPhysShape(Trans, shape);
-      test_fe.CalcPhysShape(Trans, te_shape);
-
      w = Trans.Weight() * ip.weight;
      if (Q)
      {
@@ -220,12 +220,12 @@ double TransformedCoefficient::Eval(ElementTransformation &T,
 {
   if (Q2)
   {
-      return Transform2(Q1->Eval(T, ip, GetTime()),
-                        Q2->Eval(T, ip, GetTime()));
+      return (*Transform2)(Q1->Eval(T, ip, GetTime()),
+                           Q2->Eval(T, ip, GetTime()));
   }
   else
   {
-      return Transform1(Q1->Eval(T, ip, GetTime()));
+      return (*Transform1)(Q1->Eval(T, ip, GetTime()));
   }
 }

@@ -422,15 +422,15 @@ class TransformedCoefficient : public Coefficient
 private:
   Coefficient * Q1;
   Coefficient * Q2;
-   std::function<double(double)> Transform1;
-   std::function<double(double, double)> Transform2;
+   double (*Transform1)(double);
+   double (*Transform2)(double,double);

 public:
-   TransformedCoefficient (Coefficient * q, std::function<double(double)> F)
-      : Q1(q), Transform1(std::move(F)) { Q2 = 0; Transform2 = 0; }
+   TransformedCoefficient (Coefficient * q,double (*F)(double))
+      : Q1(q), Transform1(F) { Q2 = 0; Transform2 = 0; }
   TransformedCoefficient (Coefficient * q1,Coefficient * q2,
-                           std::function<double(double, double)> F)
-      : Q1(q1), Q2(q2), Transform2(std::move(F)) { Transform1 = 0; }
+                           double (*F)(double,double))
+      : Q1(q1), Q2(q2), Transform2(F) { Transform1 = 0; }

   /// Set the time for internally stored coefficients
   void SetTime(double t);
@@ -1243,25 +1243,28 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
         HypreParMatrix * Ah;
         A_i.Get(Ah);
         hypre_ParCSRMatrix *Aih = *Ah;
-#if !defined(HYPRE_USING_GPU)
-         ess_tdof_list.HostRead();
-         for (int k = 0; k < n; k++)
+         if (!HypreUsingGPU())
         {
-            const int j = ess_tdof_list[k];
-            Aih->diag->data[Aih->diag->i[j]] = 0.0;
+            ess_tdof_list.HostRead();
+            for (int k = 0; k < n; k++)
+            {
+               const int j = ess_tdof_list[k];
+               Aih->diag->data[Aih->diag->i[j]] = 0.0;
+            }
         }
-#else
-         Ah->HypreReadWrite();
-         const int *d_ess_tdof_list =
-            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
-         const int *d_diag_i = Aih->diag->i;
-         double *d_diag_data = Aih->diag->data;
-         MFEM_GPU_FORALL(k, n,
+         else
         {
-            const int j = d_ess_tdof_list[k];
-            d_diag_data[d_diag_i[j]] = 0.0;
-         });
-#endif
+            Ah->HypreReadWrite();
+            const int *d_ess_tdof_list =
+               ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
+            const int *d_diag_i = Aih->diag->i;
+            double *d_diag_data = Aih->diag->data;
+            MFEM_GPU_FORALL(k, n,
+            {
+               const int j = d_ess_tdof_list[k];
+               d_diag_data[d_diag_i[j]] = 0.0;
+            });
+         }
      }
      else
      {
@@ -14,166 +14,175 @@
 namespace mfem
 {

-void DofTransformation::TransformPrimal(double *v) const
-{
-   MFEM_ASSERT(dof_trans_,
-               "DofTransformation has no local transformation, call "
-               "SetDofTransformation first!");
-   int size = dof_trans_->Size();
-
-   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         dof_trans_->TransformPrimal(Fo_, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         dof_trans_->TransformPrimal(Fo_, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void DofTransformation::InvTransformPrimal(double *v) const
-{
-   MFEM_ASSERT(dof_trans_,
-               "DofTransformation has no local transformation, call "
-               "SetDofTransformation first!");
-   int size = dof_trans_->Height();
-
-   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         dof_trans_->InvTransformPrimal(Fo_, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         dof_trans_->InvTransformPrimal(Fo_, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void DofTransformation::TransformDual(double *v) const
-{
-   MFEM_ASSERT(dof_trans_,
-               "DofTransformation has no local transformation, call "
-               "SetDofTransformation first!");
-   int size = dof_trans_->Size();
-
-   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         dof_trans_->TransformDual(Fo_, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         dof_trans_->TransformDual(Fo_, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void DofTransformation::InvTransformDual(double *v) const
-{
-   MFEM_ASSERT(dof_trans_,
-               "DofTransformation has no local transformation, call "
-               "SetDofTransformation first!");
-   int size = dof_trans_->Size();
-
-   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         dof_trans_->InvTransformDual(Fo_, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         dof_trans_->InvTransformDual(Fo_, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
 void TransformPrimal(const DofTransformation *ran_dof_trans,
                     const DofTransformation *dom_dof_trans,
                     DenseMatrix &elmat)
 {
-   // No action if both transformations are NULL
-   if (ran_dof_trans)
+   if (ran_dof_trans && dom_dof_trans)
+   {
+      ran_dof_trans->TransformPrimalCols(elmat);
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else if (ran_dof_trans)
   {
      ran_dof_trans->TransformPrimalCols(elmat);
   }
-   if (dom_dof_trans)
+   else if (dom_dof_trans)
   {
      dom_dof_trans->TransformDualRows(elmat);
   }
+   else
+   {
+      // If both transformations are NULL this function should not be called
+   }
 }

 void TransformDual(const DofTransformation *ran_dof_trans,
                   const DofTransformation *dom_dof_trans,
                   DenseMatrix &elmat)
 {
-   // No action if both transformations are NULL
-   if (ran_dof_trans)
+   if (ran_dof_trans && dom_dof_trans)
+   {
+      ran_dof_trans->TransformDualCols(elmat);
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else if (ran_dof_trans)
   {
      ran_dof_trans->TransformDualCols(elmat);
   }
-   if (dom_dof_trans)
+   else if (dom_dof_trans)
   {
      dom_dof_trans->TransformDualRows(elmat);
   }
+   else
+   {
+      // If both transformations are NULL this function should not be called
+   }
+}
+
+void StatelessVDofTransformation::TransformPrimal(const Array<int> & face_ori,
+                                                  double *v) const
+{
+   int size = sdoftrans_->Size();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES || vdim_ == 1)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         sdoftrans_->TransformPrimal(face_ori, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         sdoftrans_->TransformPrimal(face_ori, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void StatelessVDofTransformation::InvTransformPrimal(
+   const Array<int> & face_ori,
+   double *v) const
+{
+   int size = sdoftrans_->Height();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         sdoftrans_->InvTransformPrimal(face_ori, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         sdoftrans_->InvTransformPrimal(face_ori, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void StatelessVDofTransformation::TransformDual(const Array<int> & face_ori,
+                                                double *v) const
+{
+   int size = sdoftrans_->Size();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         sdoftrans_->TransformDual(face_ori, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         sdoftrans_->TransformDual(face_ori, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void StatelessVDofTransformation::InvTransformDual(const Array<int> & face_ori,
+                                                   double *v) const
+{
+   int size = sdoftrans_->Size();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         sdoftrans_->InvTransformDual(face_ori, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         sdoftrans_->InvTransformDual(face_ori, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
 }

 // ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
-const double ND_DofTransformation::T_data[24] =
+const double ND_StatelessDofTransformation::T_data[24] =
 {
   1.0,  0.0,  0.0,  1.0,
   -1.0, -1.0,  0.0,  1.0,
@@ -183,11 +192,11 @@ const double ND_DofTransformation::T_data[24] =
   0.0,  1.0,  1.0,  0.0
 };

-const DenseTensor ND_DofTransformation
-::T(const_cast<double *>(ND_DofTransformation::T_data), 2, 2, 6);
+const DenseTensor ND_StatelessDofTransformation
+::T(const_cast<double*>(ND_StatelessDofTransformation::T_data), 2, 2, 6);

 // ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
-const double ND_DofTransformation::TInv_data[24] =
+const double ND_StatelessDofTransformation::TInv_data[24] =
 {
   1.0,  0.0,  0.0,  1.0,
   -1.0, -1.0,  0.0,  1.0,
@@ -197,11 +206,12 @@ const double ND_DofTransformation::TInv_data[24] =
   0.0,  1.0,  1.0,  0.0
 };

-const DenseTensor ND_DofTransformation
-::TInv(const_cast<double *>(TInv_data), 2, 2, 6);
+const DenseTensor ND_StatelessDofTransformation
+::TInv(const_cast<double*>(TInv_data), 2, 2, 6);

-ND_DofTransformation::ND_DofTransformation(int size, int p, int num_edges,
-                                           int num_tri_faces)
+ND_StatelessDofTransformation::ND_StatelessDofTransformation(int size, int p,
+                                                             int num_edges,
+                                                             int num_tri_faces)
   : StatelessDofTransformation(size)
   , order(p)
   , nedofs(p)
@@ -211,19 +221,18 @@ ND_DofTransformation::ND_DofTransformation(int size, int p, int num_edges,
 {
 }

-void ND_DofTransformation::TransformPrimal(const Array<int> & Fo,
-                                           double *v) const
+void ND_StatelessDofTransformation::TransformPrimal(const Array<int> & Fo,
+                                                    double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (IsIdentity()) { return; }
+   if (nfdofs < 2) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_DofTransformation");
+               "ND_StatelessDofTransformation");

   double data[2];
   Vector v2(data, 2);
-   DenseMatrix T2;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -231,25 +240,23 @@ void ND_DofTransformation::TransformPrimal(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         T2.UseExternalData(const_cast<double *>(T.GetData(Fo[f])), 2, 2);
-         T2.Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_DofTransformation::InvTransformPrimal(const Array<int> & Fo,
-                                              double *v) const
+void ND_StatelessDofTransformation::InvTransformPrimal(const Array<int> & Fo,
+                                                       double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (IsIdentity()) { return; }
+   if (nfdofs < 2) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_DofTransformation");
+               "ND_StatelessDofTransformation");

   double data[2];
   Vector v2(data, 2);
-   DenseMatrix T2Inv;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -257,24 +264,23 @@ void ND_DofTransformation::InvTransformPrimal(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         T2Inv.UseExternalData(const_cast<double *>(TInv.GetData(Fo[f])), 2, 2);
-         T2Inv.Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         TInv(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_DofTransformation::TransformDual(const Array<int> & Fo, double *v) const
+void ND_StatelessDofTransformation::TransformDual(const Array<int> & Fo,
+                                                  double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (IsIdentity()) { return; }
+   if (nfdofs < 2) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_DofTransformation");
+               "ND_StatelessDofTransformation");

   double data[2];
   Vector v2(data, 2);
-   DenseMatrix T2Inv;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -282,25 +288,23 @@ void ND_DofTransformation::TransformDual(const Array<int> & Fo, double *v) const
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         T2Inv.UseExternalData(const_cast<double *>(TInv.GetData(Fo[f])), 2, 2);
-         T2Inv.MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         TInv(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_DofTransformation::InvTransformDual(const Array<int> & Fo,
-                                            double *v) const
+void ND_StatelessDofTransformation::InvTransformDual(const Array<int> & Fo,
+                                                     double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (IsIdentity()) { return; }
+   if (nfdofs < 2) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_DofTransformation");
+               "ND_StatelessDofTransformation");

   double data[2];
   Vector v2(data, 2);
-   DenseMatrix T2;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -308,8 +312,7 @@ void ND_DofTransformation::InvTransformDual(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         T2.UseExternalData(const_cast<double *>(T.GetData(Fo[f])), 2, 2);
-         T2.MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }
@@ -80,9 +80,6 @@ public:
   inline int Width() const { return size_; }
   inline int NumCols() const { return size_; }

-   /// If the DofTransformation performs no transformation
-   virtual bool IsIdentity() const = 0;
-
   /** Transform local DoFs to align with the global DoFs. For example, this
       transformation can be used to map the local vector computed by
       FiniteElement::Project() to the transformed vector stored within a
@@ -118,8 +115,6 @@ public:
   inline void InvTransformDual(const Array<int> & face_orientation,
                                Vector &v) const
   { InvTransformDual(face_orientation, v.GetData()); }
-
-   virtual ~StatelessDofTransformation() = default;
 };

 /** The DofTransformation class is an extension of the
@@ -138,76 +133,35 @@ public:
    transferring finite element degrees of freedom between different meshes.
    For examples of its use see the TransferMap used by the SubMesh class.
   */
-class DofTransformation
+class DofTransformation : virtual public StatelessDofTransformation
 {
 protected:
-   Array<int> Fo_;
-   const StatelessDofTransformation * dof_trans_;
-   int vdim_;
-   int ordering_;
+   Array<int> Fo;
+
+   DofTransformation(int size)
+      : StatelessDofTransformation(size) {}

 public:
-   /** @brief Default constructor which requires that SetDofTransformation be
-       called before use. */
-   DofTransformation(int vdim = 1, int ordering = 0)
-      : dof_trans_(NULL)
-      , vdim_(vdim)
-      , ordering_(ordering)
-   {}
-
-   /// Constructor with a known StatelessDofTransformation
-   DofTransformation(const StatelessDofTransformation & dof_trans,
-                     int vdim = 1, int ordering = 0)
-      : dof_trans_(&dof_trans)
-      , vdim_(vdim)
-      , ordering_(ordering)
-   {}

   /** @brief Configure the transformation using face orientations for the
       current element. */
   /// The face_orientation array can be obtained from Mesh::GetElementFaces.
-   inline void SetFaceOrientations(const Array<int> & Fo)
-   { Fo_ = Fo; }
+   inline void SetFaceOrientations(const Array<int> & face_orientation)
+   { Fo = face_orientation; }

-   /// Return the face orientations for the current element
-   inline const Array<int> & GetFaceOrientations() const { return Fo_; }
+   inline const Array<int> & GetFaceOrientations() const { return Fo; }

-   /// Set or change the nested StatelessDofTransformation object
-   inline void SetDofTransformation(const StatelessDofTransformation & dof_trans)
-   {
-      dof_trans_ = &dof_trans;
-   }
-   inline void SetDofTransformation(const StatelessDofTransformation * dof_trans)
-   {
-      dof_trans_ = dof_trans;
-   }
-
-   /// Return the nested StatelessDofTransformation object
-   inline const StatelessDofTransformation * GetDofTransformation() const
-   { return dof_trans_; }
-
-   /// Set or change the vdim and ordering parameter
-   inline void SetVDim(int vdim = 1, int ordering = 0)
-   {
-      vdim_ = vdim;
-      ordering_ = ordering;
-   }
-
-   /// Return the current vdim value
-   inline int GetVDim() const { return vdim_; }
-
-   inline int Size() const { return dof_trans_->Size(); }
-   inline int Height() const { return dof_trans_->Height(); }
-   inline int NumRows() const { return dof_trans_->NumRows(); }
-   inline int Width() const { return dof_trans_->Width(); }
-   inline int NumCols() const { return dof_trans_->NumCols(); }
-   inline bool IsIdentity() const { return dof_trans_->IsIdentity(); }
+   using StatelessDofTransformation::TransformPrimal;
+   using StatelessDofTransformation::InvTransformPrimal;
+   using StatelessDofTransformation::TransformDual;
+   using StatelessDofTransformation::InvTransformDual;

   /** Transform local DoFs to align with the global DoFs. For example, this
       transformation can be used to map the local vector computed by
       FiniteElement::Project() to the transformed vector stored within a
       GridFunction object. */
-   void TransformPrimal(double *v) const;
+   inline void TransformPrimal(double *v) const
+   { TransformPrimal(Fo, v); }
   inline void TransformPrimal(Vector &v) const
   { TransformPrimal(v.GetData()); }

@@ -225,18 +179,21 @@ public:
       transform the vector obtained using GridFunction::GetSubVector before it
       can be used to compute a local interpolation.
   */
-   void InvTransformPrimal(double *v) const;
+   inline void InvTransformPrimal(double *v) const
+   { InvTransformPrimal(Fo, v); }
   inline void InvTransformPrimal(Vector &v) const
   { InvTransformPrimal(v.GetData()); }

   /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
       into a LinearForm object. */
-   void TransformDual(double *v) const;
+   inline void TransformDual(double *v) const
+   { TransformDual(Fo, v); }
   inline void TransformDual(Vector &v) const
   { TransformDual(v.GetData()); }

   /** Inverse Transform dual DoFs */
-   void InvTransformDual(double *v) const;
+   inline void InvTransformDual(double *v) const
+   { InvTransformDual(Fo, v); }
   inline void InvTransformDual(Vector &v) const
   { InvTransformDual(v.GetData()); }

@@ -268,6 +225,8 @@ public:
         TransformDual(V.GetColumn(c));
      }
   }
+
+   virtual ~DofTransformation() = default;
 };

 /** Transform a matrix of DoFs entries from different finite element spaces as
@@ -286,6 +245,145 @@ void TransformDual(const DofTransformation *ran_dof_trans,
                   const DofTransformation *dom_dof_trans,
                   DenseMatrix &elmat);

+/** The StatelessVDofTransformation class implements a nested transformation
+    where an arbitrary StatelessDofTransformation is replicated with a
+    vdim >= 1.
+*/
+class StatelessVDofTransformation : virtual public StatelessDofTransformation
+{
+protected:
+   int vdim_;
+   int ordering_;
+   StatelessDofTransformation * sdoftrans_;
+
+public:
+   /** @brief Default constructor which requires that SetDofTransformation be
+       called before use. */
+   StatelessVDofTransformation(int vdim = 1, int ordering = 0)
+      : StatelessDofTransformation(0)
+      , vdim_(vdim)
+      , ordering_(ordering)
+      , sdoftrans_(NULL)
+   {}
+
+   /// Constructor with a known StatelessDofTransformation
+   StatelessVDofTransformation(StatelessDofTransformation & doftrans,
+                               int vdim = 1,
+                               int ordering = 0)
+      : StatelessDofTransformation(vdim * doftrans.Size())
+      , vdim_(vdim)
+      , ordering_(ordering)
+      , sdoftrans_(&doftrans)
+   {}
+
+   /// Set or change the vdim parameter
+   inline void SetVDim(int vdim)
+   {
+      vdim_ = vdim;
+      if (sdoftrans_)
+      {
+         size_ = vdim_ * sdoftrans_->Size();
+      }
+   }
+
+   /// Return the current vdim value
+   inline int GetVDim() const { return vdim_; }
+
+   /// Set or change the nested StatelessDofTransformation object
+   inline void SetDofTransformation(StatelessDofTransformation & doftrans)
+   {
+      size_ = vdim_ * doftrans.Size();
+      sdoftrans_ = &doftrans;
+   }
+
+   /// Return the nested StatelessDofTransformation object
+   inline StatelessDofTransformation * GetDofTransformation() const
+   { return sdoftrans_; }
+
+   using StatelessDofTransformation::TransformPrimal;
+   using StatelessDofTransformation::InvTransformPrimal;
+   using StatelessDofTransformation::TransformDual;
+   using StatelessDofTransformation::InvTransformDual;
+
+   /** Specializations of these base class methods which account for the vdim
+       and ordering of the full set of DoFs.
+   */
+   void TransformPrimal(const Array<int> & face_ori, double *v) const;
+   void InvTransformPrimal(const Array<int> & face_ori, double *v) const;
+   void TransformDual(const Array<int> & face_ori, double *v) const;
+   void InvTransformDual(const Array<int> & face_ori, double *v) const;
+};
+
+/** The VDofTransformation class implements a nested transformation where an
+    arbitrary DofTransformation is replicated with a vdim >= 1.
+*/
+class VDofTransformation : public StatelessVDofTransformation,
+   public DofTransformation
+{
+protected:
+   DofTransformation * doftrans_;
+
+public:
+   /** @brief Default constructor which requires that SetDofTransformation be
+       called before use. */
+   VDofTransformation(int vdim = 1, int ordering = 0)
+      : StatelessDofTransformation(0)
+      , StatelessVDofTransformation(vdim, ordering)
+      , DofTransformation(0)
+      , doftrans_(NULL)
+   {}
+
+   /// Constructor with a known DofTransformation
+   /// @note The face orientations in @a doftrans will be copied into the
+   /// new VDofTransformation object.
+   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
+                      int ordering = 0)
+      : StatelessDofTransformation(vdim * doftrans.Size())
+      , StatelessVDofTransformation(doftrans, vdim, ordering)
+      , DofTransformation(vdim * doftrans.Size())
+      , doftrans_(&doftrans)
+   {
+      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
+   }
+
+   using StatelessVDofTransformation::SetDofTransformation;
+
+   /// Set or change the nested DofTransformation object
+   /// @note The face orientations in @a doftrans will be copied into the
+   /// VDofTransformation object.
+   void SetDofTransformation(DofTransformation & doftrans)
+   {
+      doftrans_ = &doftrans;
+      StatelessVDofTransformation::SetDofTransformation(doftrans);
+      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
+   }
+
+   /// Return the nested DofTransformation object
+   inline DofTransformation * GetDofTransformation() const { return doftrans_; }
+
+   /// Set new face orientations in both the VDofTransformation and the
+   /// DofTransformation contained within (if there is one).
+   inline void SetFaceOrientations(const Array<int> & face_orientation)
+   {
+      DofTransformation::SetFaceOrientations(face_orientation);
+      if (doftrans_) { doftrans_->SetFaceOrientations(face_orientation); }
+   }
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+   using DofTransformation::InvTransformDual;
+
+   inline void TransformPrimal(double *v) const
+   { TransformPrimal(Fo, v); }
+   inline void InvTransformPrimal(double *v) const
+   { InvTransformPrimal(Fo, v); }
+   inline void TransformDual(double *v) const
+   { TransformDual(Fo, v); }
+   inline void InvTransformDual(double *v) const
+   { InvTransformDual(Fo, v); }
+};
+
 /** Abstract base class for high-order Nedelec spaces on elements with
    triangular faces.

@@ -298,7 +396,7 @@ void TransformDual(const DofTransformation *ran_dof_trans,
    be accessed as DenseMatrices using the GetFaceTransform() and
    GetFaceInverseTransform() methods.
 */
-class ND_DofTransformation : public StatelessDofTransformation
+class ND_StatelessDofTransformation : virtual public StatelessDofTransformation
 {
 private:
   static const double T_data[24];
@@ -312,7 +410,8 @@ protected:
   const int nedges; // number of edges per element
   const int nfaces; // number of triangular faces per element

-   ND_DofTransformation(int size, int order, int num_edges, int num_tri_faces);
+   ND_StatelessDofTransformation(int size, int order,
+                                 int num_edges, int num_tri_faces);

 public:
   // Return the 2x2 transformation operator for the given face orientation
@@ -322,41 +421,116 @@ public:
   static const DenseMatrix & GetFaceInverseTransform(int ori)
   { return TInv(ori); }

-   bool IsIdentity() const override { return nfdofs < 2; }
+   void TransformPrimal(const Array<int> & face_orientation,
+                        double *v) const;

-   void TransformPrimal(const Array<int> & Fo, double *v) const override;
-   void InvTransformPrimal(const Array<int> & Fo, double *v) const override;
-   void TransformDual(const Array<int> & Fo, double *v) const override;
-   void InvTransformDual(const Array<int> & Fo, double *v) const override;
+   void InvTransformPrimal(const Array<int> & face_orientation,
+                           double *v) const;
+
+   void TransformDual(const Array<int> & face_orientation,
+                      double *v) const;
+
+   void InvTransformDual(const Array<int> & face_orientation,
+                         double *v) const;
 };

 /// Stateless DoF transformation implementation for the Nedelec basis on
 /// triangles
-class ND_TriDofTransformation : public ND_DofTransformation
+class ND_TriStatelessDofTransformation : public ND_StatelessDofTransformation
+{
+public:
+   ND_TriStatelessDofTransformation(int order)
+      : StatelessDofTransformation(order*(order + 2))
+      , ND_StatelessDofTransformation(order*(order + 2), order, 3, 1)
+   {}
+};
+
+/// DoF transformation implementation for the Nedelec basis on triangles
+class ND_TriDofTransformation : public DofTransformation,
+   public ND_TriStatelessDofTransformation
 {
 public:
   ND_TriDofTransformation(int order)
-      : ND_DofTransformation(order*(order + 2), order, 3, 1)
+      : StatelessDofTransformation(order*(order + 2))
+      , DofTransformation(order*(order + 2))
+      , ND_TriStatelessDofTransformation(order)
+   {}
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+   using DofTransformation::InvTransformDual;
+
+   using ND_TriStatelessDofTransformation::TransformPrimal;
+   using ND_TriStatelessDofTransformation::InvTransformPrimal;
+   using ND_TriStatelessDofTransformation::TransformDual;
+   using ND_TriStatelessDofTransformation::InvTransformDual;
+};
+
+/// DoF transformation implementation for the Nedelec basis on tetrahedra
+class ND_TetStatelessDofTransformation : public ND_StatelessDofTransformation
+{
+public:
+   ND_TetStatelessDofTransformation(int order)
+      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
+      , ND_StatelessDofTransformation(order*(order + 2)*(order + 3)/2, order,
+                                      6, 4)
   {}
 };

 /// DoF transformation implementation for the Nedelec basis on tetrahedra
-class ND_TetDofTransformation : public ND_DofTransformation
+class ND_TetDofTransformation : public DofTransformation,
+   public ND_TetStatelessDofTransformation
 {
 public:
   ND_TetDofTransformation(int order)
-      : ND_DofTransformation(order*(order + 2)*(order + 3)/2, order, 6, 4)
+      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
+      , DofTransformation(order*(order + 2)*(order + 3)/2)
+      , ND_TetStatelessDofTransformation(order)
+   {}
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+   using DofTransformation::InvTransformDual;
+
+   using ND_TetStatelessDofTransformation::TransformPrimal;
+   using ND_TetStatelessDofTransformation::InvTransformPrimal;
+   using ND_TetStatelessDofTransformation::TransformDual;
+   using ND_TetStatelessDofTransformation::InvTransformDual;
+};
+
+/// DoF transformation implementation for the Nedelec basis on wedge elements
+class ND_WedgeStatelessDofTransformation : public ND_StatelessDofTransformation
+{
+public:
+   ND_WedgeStatelessDofTransformation(int order)
+      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
+      , ND_StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2,
+                                      order, 9, 2)
   {}
 };

 /// DoF transformation implementation for the Nedelec basis on wedge elements
-class ND_WedgeDofTransformation : public ND_DofTransformation
+class ND_WedgeDofTransformation : public DofTransformation,
+   public ND_WedgeStatelessDofTransformation
 {
 public:
   ND_WedgeDofTransformation(int order)
-      : ND_DofTransformation(3 * order * ((order + 1) * (order + 2))/2,
-                             order, 9, 2)
+      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
+      , DofTransformation(3 * order * ((order + 1) * (order + 2))/2)
+      , ND_WedgeStatelessDofTransformation(order)
   {}
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+   using DofTransformation::InvTransformDual;
+
+   using ND_WedgeStatelessDofTransformation::TransformPrimal;
+   using ND_WedgeStatelessDofTransformation::InvTransformPrimal;
+   using ND_WedgeStatelessDofTransformation::TransformDual;
+   using ND_WedgeStatelessDofTransformation::InvTransformDual;
 };

 } // namespace mfem
@@ -355,11 +355,15 @@ int InverseElementTransformation::Transform(const Vector &pt,
         }
         else
         {
-            RefinedGeometry &RefG = *refiner.Refine(T->GetGeometryType(), order);
+            const int old_type = GlobGeometryRefiner.GetType();
+            GlobGeometryRefiner.SetType(qpts_type);
+            RefinedGeometry &RefG =
+               *GlobGeometryRefiner.Refine(T->GetGeometryType(), order);
            int closest_idx = (init_guess_type == ClosestPhysNode) ?
                              FindClosestPhysPoint(pt, RefG.RefPts) :
                              FindClosestRefPoint(pt, RefG.RefPts);
            ip0 = &RefG.RefPts.IntPoint(closest_idx);
+            GlobGeometryRefiner.SetType(old_type);
         }
         break;
      }
@@ -488,7 +492,6 @@ int IsoparametricTransformation::OrderGrad(const FiniteElement *fe) const
 void IsoparametricTransformation::Transform (const IntegrationPoint &ip,
                                             Vector &trans)
 {
-   MFEM_ASSERT(FElem != nullptr, "Must provide a valid FiniteElement object!");
   shape.SetSize(FElem->GetDof());
   trans.SetSize(PointMat.Height());

@@ -233,7 +233,7 @@ protected:
   // Parameters of the inversion algorithms:
   const IntegrationPoint *ip0;
   int init_guess_type; // algorithm to use
-   GeometryRefiner refiner; // geometry refiner for initial guess
+   int qpts_type; // Quadrature1D type for the initial guess type
   int rel_qpts_order; // num_1D_qpts = max(trans_order+rel_qpts_order,0)+1
   int solver_type; // solution strategy to use
   int max_iter; // max. number of Newton iterations
@@ -276,7 +276,7 @@ public:
      : T(Trans),
        ip0(NULL),
        init_guess_type(Center),
-        refiner(Quadrature1D::OpenHalfUniform),
+        qpts_type(Quadrature1D::OpenHalfUniform),
        rel_qpts_order(-1),
        solver_type(NewtonElementProject),
        max_iter(16),
@@ -301,7 +301,7 @@ public:
   { ip0 = &init_ip; SetInitialGuessType(GivenPoint); }

   /// Set the Quadrature1D type used for the `Closest*` initial guess types.
-   void SetInitGuessPointsType(int q_type) { refiner.SetType(q_type); }
+   void SetInitGuessPointsType(int q_type) { qpts_type = q_type; }

   /// Set the relative order used for the `Closest*` initial guess types.
   /** The number of points in each spatial direction is given by the formula
@@ -361,7 +361,7 @@ public:
 class IsoparametricTransformation : public ElementTransformation
 {
 private:
-   DenseMatrix dshape, d2shape;
+   DenseMatrix dshape,d2shape;
   Vector shape;

   const FiniteElement *FElem;
@@ -359,148 +359,135 @@ void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,

   // Hessian in physical coords
   lhm.Invert();
-   Mult(hess, lhm, Hessian);
+   Mult( hess, lhm, Hessian);
 }

 const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
                                             DofToQuad::Mode mode) const
 {
-   DofToQuad *d2q = nullptr;
   MFEM_VERIFY(mode == DofToQuad::FULL, "invalid mode requested");

-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-   #pragma omp critical (DofToQuad)
-#endif
+   for (int i = 0; i < dof2quad_array.Size(); i++)
   {
-      for (int i = 0; i < dof2quad_array.Size(); i++)
-      {
-         d2q = dof2quad_array[i];
-         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
-      }
-      if (!d2q)
-      {
+      const DofToQuad &d2q = *dof2quad_array[i];
+      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
+   }
+
 #ifdef MFEM_THREAD_SAFE
-         DenseMatrix vshape(dof, dim);
+   DenseMatrix vshape(dof, dim);
 #endif
-         d2q = new DofToQuad;
-         const int nqpt = ir.GetNPoints();
-         d2q->FE = this;
-         d2q->IntRule = &ir;
-         d2q->mode = mode;
-         d2q->ndof = dof;
-         d2q->nqpt = nqpt;
-         switch (range_type)
+
+   DofToQuad *d2q = new DofToQuad;
+   const int nqpt = ir.GetNPoints();
+   d2q->FE = this;
+   d2q->IntRule = &ir;
+   d2q->mode = mode;
+   d2q->ndof = dof;
+   d2q->nqpt = nqpt;
+   if (range_type == SCALAR)
+   {
+      d2q->B.SetSize(nqpt*dof);
+      d2q->Bt.SetSize(dof*nqpt);
+
+      Vector shape;
+      vshape.GetColumnReference(0, shape);
+      for (int i = 0; i < nqpt; i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         CalcShape(ip, shape);
+         for (int j = 0; j < dof; j++)
         {
-            case SCALAR:
-            {
-               d2q->B.SetSize(nqpt*dof);
-               d2q->Bt.SetSize(dof*nqpt);
-
-               Vector shape;
-               vshape.GetColumnReference(0, shape);
-               for (int i = 0; i < nqpt; i++)
-               {
-                  const IntegrationPoint &ip = ir.IntPoint(i);
-                  CalcShape(ip, shape);
-                  for (int j = 0; j < dof; j++)
-                  {
-                     d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = shape(j);
-                  }
-               }
-               break;
-            }
-            case VECTOR:
-            {
-               d2q->B.SetSize(nqpt*dim*dof);
-               d2q->Bt.SetSize(dof*nqpt*dim);
-
-               for (int i = 0; i < nqpt; i++)
-               {
-                  const IntegrationPoint &ip = ir.IntPoint(i);
-                  CalcVShape(ip, vshape);
-                  for (int d = 0; d < dim; d++)
-                  {
-                     for (int j = 0; j < dof; j++)
-                     {
-                        d2q->B[i+nqpt*(d+dim*j)] =
-                           d2q->Bt[j+dof*(i+nqpt*d)] = vshape(j, d);
-                     }
-                  }
-               }
-               break;
-            }
-            case UNKNOWN_RANGE_TYPE:
-               // Skip B and Bt for unknown range type
-               break;
+            d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = shape(j);
         }
-         switch (deriv_type)
-         {
-            case GRAD:
-            {
-               d2q->G.SetSize(nqpt*dim*dof);
-               d2q->Gt.SetSize(dof*nqpt*dim);
-
-               for (int i = 0; i < nqpt; i++)
-               {
-                  const IntegrationPoint &ip = ir.IntPoint(i);
-                  CalcDShape(ip, vshape);
-                  for (int d = 0; d < dim; d++)
-                  {
-                     for (int j = 0; j < dof; j++)
-                     {
-                        d2q->G[i+nqpt*(d+dim*j)] =
-                           d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j, d);
-                     }
-                  }
-               }
-               break;
-            }
-            case DIV:
-            {
-               d2q->G.SetSize(nqpt*dof);
-               d2q->Gt.SetSize(dof*nqpt);
-
-               Vector divshape;
-               vshape.GetColumnReference(0, divshape);
-               for (int i = 0; i < nqpt; i++)
-               {
-                  const IntegrationPoint &ip = ir.IntPoint(i);
-                  CalcDivShape(ip, divshape);
-                  for (int j = 0; j < dof; j++)
-                  {
-                     d2q->G[i+nqpt*j] = d2q->Gt[j+dof*i] = divshape(j);
-                  }
-               }
-               break;
-            }
-            case CURL:
-            {
-               d2q->G.SetSize(nqpt*cdim*dof);
-               d2q->Gt.SetSize(dof*nqpt*cdim);
-
-               DenseMatrix curlshape(vshape.GetData(), dof, cdim);  // cdim <= dim
-               for (int i = 0; i < nqpt; i++)
-               {
-                  const IntegrationPoint &ip = ir.IntPoint(i);
-                  CalcCurlShape(ip, curlshape);
-                  for (int d = 0; d < cdim; d++)
-                  {
-                     for (int j = 0; j < dof; j++)
-                     {
-                        d2q->G[i+nqpt*(d+cdim*j)] =
-                           d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
-                     }
-                  }
-               }
-               break;
-            }
-            case NONE:
-               // Skip G and Gt for unknown derivative type
-               break;
-         }
-         dof2quad_array.Append(d2q);
      }
   }
+   else if (range_type == VECTOR)
+   {
+      d2q->B.SetSize(nqpt*dim*dof);
+      d2q->Bt.SetSize(dof*nqpt*dim);
+
+      for (int i = 0; i < nqpt; i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         CalcVShape(ip, vshape);
+         for (int d = 0; d < dim; d++)
+         {
+            for (int j = 0; j < dof; j++)
+            {
+               d2q->B[i+nqpt*(d+dim*j)] = d2q->Bt[j+dof*(i+nqpt*d)] = vshape(j, d);
+            }
+         }
+      }
+   }
+   else
+   {
+      // Skip B and Bt for unknown range type
+   }
+   switch (deriv_type)
+   {
+      case GRAD:
+      {
+         d2q->G.SetSize(nqpt*dim*dof);
+         d2q->Gt.SetSize(dof*nqpt*dim);
+
+         for (int i = 0; i < nqpt; i++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(i);
+            CalcDShape(ip, vshape);
+            for (int d = 0; d < dim; d++)
+            {
+               for (int j = 0; j < dof; j++)
+               {
+                  d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j, d);
+               }
+            }
+         }
+         break;
+      }
+      case DIV:
+      {
+         d2q->G.SetSize(nqpt*dof);
+         d2q->Gt.SetSize(dof*nqpt);
+
+         Vector divshape;
+         vshape.GetColumnReference(0, divshape);
+         for (int i = 0; i < nqpt; i++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(i);
+            CalcDivShape(ip, divshape);
+            for (int j = 0; j < dof; j++)
+            {
+               d2q->G[i+nqpt*j] = d2q->Gt[j+dof*i] = divshape(j);
+            }
+         }
+         break;
+      }
+      case CURL:
+      {
+         d2q->G.SetSize(nqpt*cdim*dof);
+         d2q->Gt.SetSize(dof*nqpt*cdim);
+
+         DenseMatrix curlshape(vshape.GetData(), dof, cdim);  // cdim <= dim
+         for (int i = 0; i < nqpt; i++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(i);
+            CalcCurlShape(ip, curlshape);
+            for (int d = 0; d < cdim; d++)
+            {
+               for (int j = 0; j < dof; j++)
+               {
+                  d2q->G[i+nqpt*(d+cdim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
+               }
+            }
+         }
+         break;
+      }
+      case NONE:
+      default:
+         // Skip G and Gt for unknown derivative type
+         break;
+   }
+   dof2quad_array.Append(d2q);
   return *d2q;
 }

@@ -917,14 +904,14 @@ VectorFiniteElement::VectorFiniteElement(int D, Geometry::Type G,
 }

 void VectorFiniteElement::CalcShape(
-   const IntegrationPoint &ip, Vector &shape) const
+   const IntegrationPoint &ip, Vector &shape ) const
 {
   mfem_error("Error: Cannot use scalar CalcShape(...) function with\n"
              "   VectorFiniteElements!");
 }

 void VectorFiniteElement::CalcDShape(
-   const IntegrationPoint &ip, DenseMatrix &dshape) const
+   const IntegrationPoint &ip, DenseMatrix &dshape ) const
 {
   mfem_error("Error: Cannot use scalar CalcDShape(...) function with\n"
              "   VectorFiniteElements!");
@@ -2196,72 +2183,51 @@ void Poly_1D::CalcChebyshev(const int p, const double x, double *u, double *d,

 const double *Poly_1D::GetPoints(const int p, const int btype)
 {
-   Array<double*> *pts;
   BasisType::Check(btype);
   const int qtype = BasisType::GetQuadrature1D(btype);
+
   if (qtype == Quadrature1D::Invalid) { return NULL; }

-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-   #pragma omp critical (Poly1DGetPoints)
-#endif
+   if (points_container.find(btype) == points_container.end())
   {
-      auto it = points_container.find(btype);
-      if (it != points_container.end())
-      {
-         pts = it->second;
-      }
-      else
-      {
-         pts = new Array<double*>(h_mt);
-         points_container[btype] = pts;
-      }
-      if (pts->Size() <= p)
-      {
-         pts->SetSize(p + 1, NULL);
-      }
-      if ((*pts)[p] == NULL)
-      {
-         (*pts)[p] = new double[p + 1];
-         quad_func.GivePolyPoints(p + 1, (*pts)[p], qtype);
-      }
+      points_container[btype] = new Array<double*>(h_mt);
   }
-   return (*pts)[p];
+   Array<double*> &pts = *points_container[btype];
+   if (pts.Size() <= p)
+   {
+      pts.SetSize(p + 1, NULL);
+   }
+   if (pts[p] == NULL)
+   {
+      pts[p] = new double[p + 1];
+      quad_func.GivePolyPoints(p+1, pts[p], qtype);
+   }
+   return pts[p];
 }

 Poly_1D::Basis &Poly_1D::GetBasis(const int p, const int btype)
 {
-   Array<Basis*> *bases;
   BasisType::Check(btype);

-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-   #pragma omp critical (Poly1DGetBasis)
-#endif
+   if ( bases_container.find(btype) == bases_container.end() )
   {
-      auto it = bases_container.find(btype);
-      if (it != bases_container.end())
-      {
-         bases = it->second;
-      }
-      else
-      {
-         // we haven't been asked for basis or points of this type yet
-         bases = new Array<Basis*>(h_mt);
-         bases_container[btype] = bases;
-      }
-      if (bases->Size() <= p)
-      {
-         bases->SetSize(p + 1, NULL);
-      }
-      if ((*bases)[p] == NULL)
-      {
-         EvalType etype;
-         if (btype == BasisType::Positive) { etype = Positive; }
-         else if (btype == BasisType::IntegratedGLL) { etype = Integrated; }
-         else { etype = Barycentric; }
-         (*bases)[p] = new Basis(p, GetPoints(p, btype), etype);
-      }
+      // we haven't been asked for basis or points of this type yet
+      bases_container[btype] = new Array<Basis*>(h_mt);
   }
-   return *(*bases)[p];
+   Array<Basis*> &bases = *bases_container[btype];
+   if (bases.Size() <= p)
+   {
+      bases.SetSize(p + 1, NULL);
+   }
+   if (bases[p] == NULL)
+   {
+      EvalType etype;
+      if (btype == BasisType::Positive) { etype = Positive; }
+      else if (btype == BasisType::IntegratedGLL) { etype = Integrated; }
+      else { etype = Barycentric; }
+      bases[p] = new Basis(p, GetPoints(p, btype), etype);
+   }
+   return *bases[p];
 }

 Poly_1D::~Poly_1D()
@@ -2270,7 +2236,7 @@ Poly_1D::~Poly_1D()
        it != points_container.end() ; ++it)
   {
      Array<double*>& pts = *it->second;
-      for (int i = 0; i < pts.Size(); ++i)
+      for ( int i = 0 ; i < pts.Size() ; ++i )
      {
         delete [] pts[i];
      }
@@ -2281,7 +2247,7 @@ Poly_1D::~Poly_1D()
        it != bases_container.end() ; ++it)
   {
      Array<Basis*>& bases = *it->second;
-      for (int i = 0; i < bases.Size(); ++i)
+      for ( int i = 0 ; i < bases.Size() ; ++i )
      {
         delete bases[i];
      }
@@ -2495,47 +2461,39 @@ const DofToQuad &TensorBasisElement::GetTensorDofToQuad(
   DofToQuad::Mode mode, const Poly_1D::Basis &basis, bool closed,
   Array<DofToQuad*> &dof2quad_array)
 {
-   DofToQuad *d2q = nullptr;
   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");

-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-   #pragma omp critical (DofToQuad)
-#endif
+   for (int i = 0; i < dof2quad_array.Size(); i++)
   {
-      for (int i = 0; i < dof2quad_array.Size(); i++)
+      const DofToQuad &d2q = *dof2quad_array[i];
+      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
+   }
+
+   DofToQuad *d2q = new DofToQuad;
+   const int ndof = closed ? fe.GetOrder() + 1 : fe.GetOrder();
+   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
+   d2q->FE = &fe;
+   d2q->IntRule = &ir;
+   d2q->mode = mode;
+   d2q->ndof = ndof;
+   d2q->nqpt = nqpt;
+   d2q->B.SetSize(nqpt*ndof);
+   d2q->Bt.SetSize(ndof*nqpt);
+   d2q->G.SetSize(nqpt*ndof);
+   d2q->Gt.SetSize(ndof*nqpt);
+   Vector val(ndof), grad(ndof);
+   for (int i = 0; i < nqpt; i++)
+   {
+      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
+      // of the 1D rule.
+      basis.Eval(ir.IntPoint(i).x, val, grad);
+      for (int j = 0; j < ndof; j++)
      {
-         d2q = dof2quad_array[i];
-         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
-      }
-      if (!d2q)
-      {
-         d2q = new DofToQuad;
-         const int ndof = closed ? fe.GetOrder() + 1 : fe.GetOrder();
-         const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
-         d2q->FE = &fe;
-         d2q->IntRule = &ir;
-         d2q->mode = mode;
-         d2q->ndof = ndof;
-         d2q->nqpt = nqpt;
-         d2q->B.SetSize(nqpt*ndof);
-         d2q->Bt.SetSize(ndof*nqpt);
-         d2q->G.SetSize(nqpt*ndof);
-         d2q->Gt.SetSize(ndof*nqpt);
-         Vector val(ndof), grad(ndof);
-         for (int i = 0; i < nqpt; i++)
-         {
-            // The first 'nqpt' points in 'ir' have the same x-coordinates as those
-            // of the 1D rule.
-            basis.Eval(ir.IntPoint(i).x, val, grad);
-            for (int j = 0; j < ndof; j++)
-            {
-               d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
-               d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
-            }
-         }
-         dof2quad_array.Append(d2q);
+         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
+         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
      }
   }
+   dof2quad_array.Append(d2q);
   return *d2q;
 }

@@ -250,7 +250,7 @@ protected:
   /// Container for all DofToQuad objects created by the FiniteElement.
   /** Multiple DofToQuad objects may be needed when different quadrature rules
       or different DofToQuad::Mode are used. */
-   mutable Array<DofToQuad *> dof2quad_array;
+   mutable Array<DofToQuad*> dof2quad_array;

 public:
   /// Enumeration for range_type and deriv_range_type
@@ -596,7 +596,7 @@ public:
   /** @brief Return a DoF transformation object for this particular type of
       basis.
   */
-   virtual const StatelessDofTransformation *GetDofTransformation() const
+   virtual StatelessDofTransformation * GetDofTransformation() const
   { return NULL; }

   /// Deconstruct the FiniteElement
@@ -1026,8 +1026,8 @@ public:
   };

 private:
-   typedef std::map<int, Array<double*>*> PointsMap;
-   typedef std::map<int, Array<Basis*>*> BasisMap;
+   typedef std::map< int, Array<double*>* > PointsMap;
+   typedef std::map< int, Array<Basis*>* > BasisMap;

   MemoryType h_mt;
   PointsMap points_container;
@@ -6031,7 +6031,7 @@ void RT0PyrFiniteElement::CalcVShape(const IntegrationPoint &ip,
   shape(1,2) = z;

   shape(2,0) = x * (2.0 - z) * ozi;
-   shape(2,1) = - y * z * ozi;
+   shape(2,1) = - y * z * ozi;;
   shape(2,2) = z;

   shape(3,0) = - x * z * ozi;
@@ -179,7 +179,7 @@ class ND_TetrahedronElement : public VectorFiniteElement
   Array<int> dof2tk;
   DenseMatrixInverse Ti;

-   ND_TetDofTransformation doftrans;
+   mutable ND_TetStatelessDofTransformation doftrans;

 public:
   /// Construct the ND_TetrahedronElement of order @a p
@@ -201,7 +201,7 @@ public:
                                  ElementTransformation &Trans,
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
-   virtual const StatelessDofTransformation *GetDofTransformation() const
+   virtual StatelessDofTransformation * GetDofTransformation() const
   { return &doftrans; }
   using FiniteElement::Project;
   virtual void Project(VectorCoefficient &vc,
@@ -242,7 +242,7 @@ class ND_TriangleElement : public VectorFiniteElement
   Array<int> dof2tk;
   DenseMatrixInverse Ti;

-   ND_TriDofTransformation doftrans;
+   mutable ND_TriStatelessDofTransformation doftrans;

 public:
   /// Construct the ND_TriangleElement of order @a p
@@ -264,7 +264,7 @@ public:
                                  ElementTransformation &Trans,
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
-   virtual const StatelessDofTransformation *GetDofTransformation() const
+   virtual StatelessDofTransformation * GetDofTransformation() const
   { return &doftrans; }
   using FiniteElement::Project;
   virtual void Project(VectorCoefficient &vc,
@@ -346,7 +346,7 @@ private:
 #endif
   Array<int> dof2tk, t_dof, s_dof;

-   ND_WedgeDofTransformation doftrans;
+   mutable ND_WedgeStatelessDofTransformation doftrans;

   H1_TriangleElement H1TriangleFE;
   ND_TriangleElement NDTriangleFE;
@@ -379,7 +379,7 @@ public:
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }

-   virtual const StatelessDofTransformation *GetDofTransformation() const
+   virtual StatelessDofTransformation * GetDofTransformation() const
   { return &doftrans; }

   using FiniteElement::Project;
@@ -59,7 +59,7 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
   int p = (this)->GetOrder();
   double x = ip.x, y = ip.y;

-   Poly_1D::Basis &edgeNodalBasis = poly1d.GetBasis(p, BasisType::GaussLobatto);
+   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
   Vector nodalX(p+1);
   Vector nodalY(p+1);

@@ -113,9 +113,10 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
   {
      double *legX = new double[p-1];
      double *legY = new double[p-1];
+      Poly_1D *storeLegendre = new Poly_1D();

-      Poly_1D::CalcLegendre(p-2, x, legX);
-      Poly_1D::CalcLegendre(p-2, y, legY);
+      storeLegendre->CalcLegendre(p-2, x, legX);
+      storeLegendre->CalcLegendre(p-2, y, legY);

      int interior_total = 0;
      for (int j = 4; j < p + 1; j++)
@@ -130,6 +131,7 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,

      delete[] legX;
      delete[] legY;
+      delete storeLegendre;
   }
 }

@@ -139,7 +141,7 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
   int p = (this)->GetOrder();
   double x = ip.x, y = ip.y;

-   Poly_1D::Basis &edgeNodalBasis = poly1d.GetBasis(p, BasisType::GaussLobatto);
+   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
   Vector nodalX(p+1);
   Vector DnodalX(p+1);
   Vector nodalY(p+1);
@@ -201,9 +203,10 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
      double *legY = new double[p-1];
      double *DlegX = new double[p-1];
      double *DlegY = new double[p-1];
+      Poly_1D *storeLegendre = new Poly_1D();

-      Poly_1D::CalcLegendre(p-2, x, legX, DlegX);
-      Poly_1D::CalcLegendre(p-2, y, legY, DlegY);
+      storeLegendre->CalcLegendre(p-2, x, legX, DlegX);
+      storeLegendre->CalcLegendre(p-2, y, legY, DlegY);

      int interior_total = 0;
      for (int j = 4; j < p + 1; j++)
@@ -221,6 +224,7 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
      delete[] legY;
      delete[] DlegX;
      delete[] DlegY;
+      delete storeLegendre;
   }
 }

@@ -2896,7 +2896,7 @@ ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
 }

-const StatelessDofTransformation *
+StatelessDofTransformation *
 ND_FECollection::DofTransformationForGeometry(Geometry::Type GeomType) const
 {
   if (!Geometry::IsTensorProduct(GeomType) && this->GetOrder() > 1)
@@ -63,7 +63,7 @@ public:
   /** @brief Returns a DoF transformation object compatible with this basis
       and geometry type.
   */
-   virtual const StatelessDofTransformation *
+   virtual StatelessDofTransformation *
   DofTransformationForGeometry(Geometry::Type GeomType) const
   { return NULL; }

@@ -483,7 +483,7 @@ public:
   int DofForGeometry(Geometry::Type GeomType) const override
   { return ND_dof[GeomType]; }

-   const StatelessDofTransformation *
+   StatelessDofTransformation *
   DofTransformationForGeometry(Geometry::Type GeomType) const override;

   const int *DofOrderForOrientation(Geometry::Type GeomType,
@@ -13,7 +13,6 @@
 #define MFEM_FEM_HPP

 #include "intrules.hpp"
-#include "intrules_cut.hpp"
 #include "geom.hpp"
 #include "fe.hpp"
 #include "fe_coll.hpp"
@@ -63,6 +63,7 @@ FiniteElementSpace::FiniteElementSpace()
     elem_dof(NULL), elem_fos(NULL), bdr_elem_dof(NULL), bdr_elem_fos(NULL),
     face_dof(NULL),
     NURBSext(NULL), own_ext(false),
+     DoFTrans(0), VDoFTrans(vdim, ordering),
     cP_is_set(false),
     Th(Operator::ANY_TYPE),
     sequence(0), mesh_sequence(0), orders_changed(false), relaxed_hp(false)
@@ -71,6 +72,7 @@ FiniteElementSpace::FiniteElementSpace()
 FiniteElementSpace::FiniteElementSpace(const FiniteElementSpace &orig,
                                       Mesh *mesh_,
                                       const FiniteElementCollection *fec_)
+   : VDoFTrans(orig.vdim, orig.ordering)
 {
   mesh_ = mesh_ ? mesh_ : orig.mesh;
   fec_ = fec_ ? fec_ : orig.fec;
@@ -210,7 +212,7 @@ void FiniteElementSpace::GetVDofs(int vd, Array<int>& dofs, int ndofs_) const
   }
 }

-void FiniteElementSpace::DofsToVDofs(Array<int> &dofs, int ndofs_) const
+void FiniteElementSpace::DofsToVDofs (Array<int> &dofs, int ndofs_) const
 {
   if (vdim == 1) { return; }
   if (ndofs_ < 0) { ndofs_ = this->ndofs; }
@@ -262,7 +264,7 @@ int FiniteElementSpace::DofToVDof(int dof, int vd, int ndofs_) const
 }

 // static function
-void FiniteElementSpace::AdjustVDofs(Array<int> &vdofs)
+void FiniteElementSpace::AdjustVDofs (Array<int> &vdofs)
 {
   int n = vdofs.Size(), *vdof = vdofs;
   for (int i = 0; i < n; i++)
@@ -275,36 +277,36 @@ void FiniteElementSpace::AdjustVDofs(Array<int> &vdofs)
   }
 }

-void FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs,
-                                         DofTransformation &doftrans) const
-{
-   GetElementDofs(i, vdofs, doftrans);
-   DofsToVDofs(vdofs);
-   doftrans.SetVDim(vdim, ordering);
-}
-
 DofTransformation *
 FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
 {
-   DoFTrans.SetDofTransformation(NULL);
-   GetElementVDofs(i, vdofs, DoFTrans);
-   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
-}
-
-void FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs,
-                                            DofTransformation &doftrans) const
-{
-   GetBdrElementDofs(i, vdofs, doftrans);
+   DofTransformation * doftrans = GetElementDofs(i, vdofs);
   DofsToVDofs(vdofs);
-   doftrans.SetVDim(vdim, ordering);
+   if (vdim == 1 || doftrans == NULL)
+   {
+      return doftrans;
+   }
+   else
+   {
+      VDoFTrans.SetDofTransformation(*doftrans);
+      return &VDoFTrans;
+   }
 }

 DofTransformation *
 FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
 {
-   DoFTrans.SetDofTransformation(NULL);
-   GetBdrElementVDofs(i, vdofs, DoFTrans);
-   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
+   DofTransformation * doftrans = GetBdrElementDofs(i, vdofs);
+   DofsToVDofs(vdofs);
+   if (vdim == 1 || doftrans == NULL)
+   {
+      return doftrans;
+   }
+   else
+   {
+      VDoFTrans.SetDofTransformation(*doftrans);
+      return &VDoFTrans;
+   }
 }

 void FiniteElementSpace::GetPatchVDofs(int i, Array<int> &vdofs) const
@@ -775,9 +777,9 @@ FiniteElementSpace::H2L_GlobalRestrictionMatrix (FiniteElementSpace *lfes)
   return R;
 }

-void FiniteElementSpace::AddDependencies(
-   SparseMatrix& deps, Array<int>& master_dofs, Array<int>& slave_dofs,
-   DenseMatrix& I, int skipfirst)
+void FiniteElementSpace
+::AddDependencies(SparseMatrix& deps, Array<int>& master_dofs,
+                  Array<int>& slave_dofs, DenseMatrix& I, int skipfirst)
 {
   for (int i = skipfirst; i < slave_dofs.Size(); i++)
   {
@@ -800,9 +802,11 @@ void FiniteElementSpace::AddDependencies(
   }
 }

-void FiniteElementSpace::AddEdgeFaceDependencies(
-   SparseMatrix &deps, Array<int> &master_dofs, const FiniteElement *master_fe,
-   Array<int> &slave_dofs, int slave_face, const DenseMatrix *pm) const
+void FiniteElementSpace
+::AddEdgeFaceDependencies(SparseMatrix &deps, Array<int> &master_dofs,
+                          const FiniteElement *master_fe,
+                          Array<int> &slave_dofs, int slave_face,
+                          const DenseMatrix *pm) const
 {
   // In variable-order spaces in 3D, we need to only constrain interior face
   // DOFs (this is done one level up), since edge dependencies can be more
@@ -1529,12 +1533,12 @@ SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
                                localP);
 }

-FiniteElementSpace::RefinementOperator::RefinementOperator(
-   const FiniteElementSpace* fespace, Table* old_elem_dof, Table* old_elem_fos,
-   int old_ndofs)
-   : fespace(fespace),
-     old_elem_dof(old_elem_dof),
-     old_elem_fos(old_elem_fos)
+FiniteElementSpace::RefinementOperator::RefinementOperator
+(const FiniteElementSpace* fespace, Table* old_elem_dof, Table* old_elem_fos,
+ int old_ndofs)
+   : fespace(fespace)
+   , old_elem_dof(old_elem_dof)
+   , old_elem_fos(old_elem_fos)
 {
   MFEM_VERIFY(fespace->GetNE() >= old_elem_dof->Size(),
               "Previous mesh is not coarser.");
@@ -1549,7 +1553,7 @@ FiniteElementSpace::RefinementOperator::RefinementOperator(
      fespace->GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
   }

-   ConstructDoFTransArray();
+   ConstructDoFTrans();
 }

 FiniteElementSpace::RefinementOperator::RefinementOperator(
@@ -1574,58 +1578,59 @@ FiniteElementSpace::RefinementOperator::RefinementOperator(
      old_elem_fos = new Table(*coarse_fes->GetElementToFaceOrientationTable());
   }

-   ConstructDoFTransArray();
+   ConstructDoFTrans();
 }

 FiniteElementSpace::RefinementOperator::~RefinementOperator()
 {
   delete old_elem_dof;
   delete old_elem_fos;
-   for (int i=0; i<old_DoFTransArray.Size(); i++)
+   for (int i=0; i<old_DoFTrans.Size(); i++)
   {
-      delete old_DoFTransArray[i];
+      delete old_DoFTrans[i];
   }
 }

-void FiniteElementSpace::RefinementOperator::ConstructDoFTransArray()
+void FiniteElementSpace::RefinementOperator
+::ConstructDoFTrans()
 {
-   old_DoFTransArray.SetSize(Geometry::NUM_GEOMETRIES);
-   for (int i=0; i<old_DoFTransArray.Size(); i++)
+   old_DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<old_DoFTrans.Size(); i++)
   {
-      old_DoFTransArray[i] = NULL;
+      old_DoFTrans[i] = NULL;
   }

   const FiniteElementCollection *fec_ref = fespace->FEColl();
   if (dynamic_cast<const ND_FECollection*>(fec_ref))
   {
-      const FiniteElement *nd_tri =
+      const FiniteElement * nd_tri =
         fec_ref->FiniteElementForGeometry(Geometry::TRIANGLE);
      if (nd_tri)
      {
-         old_DoFTransArray[Geometry::TRIANGLE] =
+         old_DoFTrans[Geometry::TRIANGLE] =
            new ND_TriDofTransformation(nd_tri->GetOrder());
      }

-      const FiniteElement *nd_tet =
+      const FiniteElement * nd_tet =
         fec_ref->FiniteElementForGeometry(Geometry::TETRAHEDRON);
      if (nd_tet)
      {
-         old_DoFTransArray[Geometry::TETRAHEDRON] =
+         old_DoFTrans[Geometry::TETRAHEDRON] =
            new ND_TetDofTransformation(nd_tet->GetOrder());
      }

-      const FiniteElement *nd_pri =
+      const FiniteElement * nd_pri =
         fec_ref->FiniteElementForGeometry(Geometry::PRISM);
      if (nd_pri)
      {
-         old_DoFTransArray[Geometry::PRISM] =
+         old_DoFTrans[Geometry::PRISM] =
            new ND_WedgeDofTransformation(nd_pri->GetOrder());
      }
   }
 }

-void FiniteElementSpace::RefinementOperator::Mult(const Vector &x,
-                                                  Vector &y) const
+void FiniteElementSpace::RefinementOperator
+::Mult(const Vector &x, Vector &y) const
 {
   Mesh* mesh_ref = fespace->GetMesh();
   const CoarseFineTransformations &trans_ref =
@@ -1657,7 +1662,6 @@ void FiniteElementSpace::RefinementOperator::Mult(const Vector &x,
            fespace->DofsToVDofs(vd, vdofs);
            old_dofs.Copy(old_vdofs);
            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
-
            x.GetSubVector(old_vdofs, subX);
            lP.Mult(subX, subY);
            y.SetSubVector(vdofs, subY);
@@ -1666,30 +1670,40 @@ void FiniteElementSpace::RefinementOperator::Mult(const Vector &x,
      else
      {
         old_elem_fos->GetRow(emb.parent, old_Fo);
-         old_DoFTrans.SetDofTransformation(*old_DoFTransArray[geom]);
-         old_DoFTrans.SetFaceOrientations(old_Fo);
+         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
+
+         DofTransformation *new_doftrans = NULL;
+         VDofTransformation *vdoftrans =
+            dynamic_cast<VDofTransformation*>(doftrans);
+         if (vdoftrans)
+         {
+            new_doftrans = doftrans;
+            doftrans = vdoftrans->GetDofTransformation();
+         }

-         doftrans->SetVDim();
         for (int vd = 0; vd < rvdim; vd++)
         {
            dofs.Copy(vdofs);
            fespace->DofsToVDofs(vd, vdofs);
            old_dofs.Copy(old_vdofs);
            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
-
            x.GetSubVector(old_vdofs, subX);
-            old_DoFTrans.InvTransformPrimal(subX);
+            old_DoFTrans[geom]->InvTransformPrimal(subX);
            lP.Mult(subX, subY);
            doftrans->TransformPrimal(subY);
            y.SetSubVector(vdofs, subY);
         }
-         doftrans->SetVDim(rvdim, fespace->GetOrdering());
+
+         if (vdoftrans)
+         {
+            doftrans = new_doftrans;
+         }
      }
   }
 }

-void FiniteElementSpace::RefinementOperator::MultTranspose(const Vector &x,
-                                                           Vector &y) const
+void FiniteElementSpace::RefinementOperator
+::MultTranspose(const Vector &x, Vector &y) const
 {
   y = 0.0;

@@ -1713,7 +1727,7 @@ void FiniteElementSpace::RefinementOperator::MultTranspose(const Vector &x,
      const Geometry::Type geom = mesh_ref->GetElementBaseGeometry(k);
      const DenseMatrix &lP = localP[geom](emb.matrix);

-      DofTransformation *doftrans = fespace->GetElementDofs(k, f_dofs);
+      DofTransformation * doftrans = fespace->GetElementDofs(k, f_dofs);
      old_elem_dof->GetRow(emb.parent, c_dofs);

      if (!doftrans)
@@ -1728,6 +1742,7 @@ void FiniteElementSpace::RefinementOperator::MultTranspose(const Vector &x,
            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);

            x.GetSubVector(f_vdofs, subX);
+
            for (int p = 0; p < f_dofs.Size(); ++p)
            {
               if (processed[DecodeDof(f_dofs[p])])
@@ -1735,6 +1750,7 @@ void FiniteElementSpace::RefinementOperator::MultTranspose(const Vector &x,
                  subX[p] = 0.0;
               }
            }
+
            lP.MultTranspose(subX, subY);
            y.AddElementVector(c_vdofs, subY);
         }
@@ -1744,10 +1760,17 @@ void FiniteElementSpace::RefinementOperator::MultTranspose(const Vector &x,
         subYt.SetSize(lP.Width());

         old_elem_fos->GetRow(emb.parent, old_Fo);
-         old_DoFTrans.SetDofTransformation(*old_DoFTransArray[geom]);
-         old_DoFTrans.SetFaceOrientations(old_Fo);
+         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
+
+         DofTransformation *new_doftrans = NULL;
+         VDofTransformation *vdoftrans =
+            dynamic_cast<VDofTransformation*>(doftrans);
+         if (vdoftrans)
+         {
+            new_doftrans = doftrans;
+            doftrans = vdoftrans->GetDofTransformation();
+         }

-         doftrans->SetVDim();
         for (int vd = 0; vd < rvdim; vd++)
         {
            f_dofs.Copy(f_vdofs);
@@ -1764,11 +1787,16 @@ void FiniteElementSpace::RefinementOperator::MultTranspose(const Vector &x,
                  subX[p] = 0.0;
               }
            }
+
            lP.MultTranspose(subX, subYt);
-            old_DoFTrans.TransformDual(subYt);
+            old_DoFTrans[geom]->TransformDual(subYt);
            y.AddElementVector(c_vdofs, subYt);
         }
-         doftrans->SetVDim(rvdim, fespace->GetOrdering());
+
+         if (vdoftrans)
+         {
+            doftrans = new_doftrans;
+         }
      }

      for (int p = 0; p < f_dofs.Size(); ++p)
@@ -1996,8 +2024,8 @@ FiniteElementSpace::DerefinementOperator::~DerefinementOperator()
   delete coarse_elem_dof;
 }

-void FiniteElementSpace::DerefinementOperator::Mult(const Vector &x,
-                                                    Vector &y) const
+void FiniteElementSpace::DerefinementOperator
+::Mult(const Vector &x, Vector &y) const
 {
   Array<int> c_vdofs, f_vdofs;
   Vector loc_x, loc_y;
@@ -2199,7 +2227,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh_, NURBSExtension *NURBSext_,
      R_transpose.reset();
      cP_is_set = false;

-      ConstructDoFTransArray();
+      ConstructDoFTrans();
   }
   else
   {
@@ -2211,39 +2239,40 @@ void FiniteElementSpace::Constructor(Mesh *mesh_, NURBSExtension *NURBSext_,
   BuildElementToDofTable();
 }

-void FiniteElementSpace::ConstructDoFTransArray()
+void FiniteElementSpace::ConstructDoFTrans()
 {
-   DestroyDoFTransArray();
+   DestroyDoFTrans();

-   DoFTransArray.SetSize(Geometry::NUM_GEOMETRIES);
-   for (int i=0; i<DoFTransArray.Size(); i++)
+   VDoFTrans.SetVDim(vdim);
+   DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<DoFTrans.Size(); i++)
   {
-      DoFTransArray[i] = NULL;
+      DoFTrans[i] = NULL;
   }
   if (mesh->Dimension() < 3) { return; }
   if (dynamic_cast<const ND_FECollection*>(fec))
   {
-      const FiniteElement *nd_tri =
+      const FiniteElement * nd_tri =
         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
      if (nd_tri)
      {
-         DoFTransArray[Geometry::TRIANGLE] =
+         DoFTrans[Geometry::TRIANGLE] =
            new ND_TriDofTransformation(nd_tri->GetOrder());
      }

-      const FiniteElement *nd_tet =
+      const FiniteElement * nd_tet =
         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
      if (nd_tet)
      {
-         DoFTransArray[Geometry::TETRAHEDRON] =
+         DoFTrans[Geometry::TETRAHEDRON] =
            new ND_TetDofTransformation(nd_tet->GetOrder());
      }

-      const FiniteElement *nd_pri =
+      const FiniteElement * nd_pri =
         fec->FiniteElementForGeometry(Geometry::PRISM);
      if (nd_pri)
      {
-         DoFTransArray[Geometry::PRISM] =
+         DoFTrans[Geometry::PRISM] =
            new ND_WedgeDofTransformation(nd_pri->GetOrder());
      }
   }
@@ -2295,7 +2324,7 @@ void FiniteElementSpace::BuildNURBSFaceToDofTable() const
   face_to_be = -1;
   for (int b = 0; b < GetNBE(); b++)
   {
-      int f = mesh->GetBdrElementFaceIndex(b);
+      int f = mesh->GetBdrElementEdgeIndex(b);
      face_to_be[f] = b;
   }

@@ -2399,7 +2428,6 @@ void FiniteElementSpace::Construct()
      {
         // the simple case: all edges are of the same order
         nedofs = mesh->GetNEdges() * fec->GetNumDof(Geometry::SEGMENT, order);
-         var_edge_dofs.Clear(); // ensure any old var_edge_dof table is dumped.
      }
   }

@@ -2418,7 +2446,6 @@ void FiniteElementSpace::Construct()
         // the simple case: all faces are of the same geometry and order
         uni_fdof = fec->GetNumDof(mesh->GetFaceGeometry(0), order);
         nfdofs = mesh->GetNFaces() * uni_fdof;
-         var_face_dofs.Clear(); // ensure any old var_face_dof table is dumped.
      }
   }

@@ -2447,7 +2474,7 @@ void FiniteElementSpace::Construct()

   ndofs = nvdofs + nedofs + nfdofs + nbdofs;

-   ConstructDoFTransArray();
+   ConstructDoFTrans();

   // record the current mesh sequence number to detect refinement etc.
   mesh_sequence = mesh->GetSequence();
@@ -2472,8 +2499,9 @@ int FiniteElementSpace::MinOrder(VarOrderBits bits)
   return 0;
 }

-void FiniteElementSpace::CalcEdgeFaceVarOrders(
-   Array<VarOrderBits> &edge_orders, Array<VarOrderBits> &face_orders) const
+void FiniteElementSpace
+::CalcEdgeFaceVarOrders(Array<VarOrderBits> &edge_orders,
+                        Array<VarOrderBits> &face_orders) const
 {
   MFEM_ASSERT(IsVariableOrder(), "");
   MFEM_ASSERT(Nonconforming(), "");
@@ -2628,6 +2656,7 @@ int FiniteElementSpace::MakeDofTable(int ent_dim,
            int dofs = fec->GetNumDof(geom, order);
            list.Append(Connection(i, total_dofs));
            total_dofs += dofs;
+
            if (var_ent_order) { var_ent_order->Append(order); }
         }
      }
@@ -2638,6 +2667,7 @@ int FiniteElementSpace::MakeDofTable(int ent_dim,

   // build the table
   entity_dofs.MakeFromList(num_ent+1, list);
+
   return total_dofs;
 }

@@ -2697,8 +2727,8 @@ int FiniteElementSpace::GetNVariants(int entity, int index) const
 static const char* msg_orders_changed =
   "Element orders changed, you need to Update() the space first.";

-void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs,
-                                        DofTransformation &doftrans) const
+DofTransformation *
+FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

@@ -2706,16 +2736,13 @@ void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs,
   {
      elem_dof->GetRow(elem, dofs);

-      if (DoFTransArray[mesh->GetElementBaseGeometry(elem)])
+      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
      {
         Array<int> Fo;
         elem_fos -> GetRow (elem, Fo);
-         doftrans.SetDofTransformation(
-            *DoFTransArray[mesh->GetElementBaseGeometry(elem)]);
-         doftrans.SetFaceOrientations(Fo);
-         doftrans.SetVDim();
+         DoFTrans[mesh->GetElementBaseGeometry(elem)]->SetFaceOrientations(Fo);
      }
-      return;
+      return DoFTrans[mesh->GetElementBaseGeometry(elem)];
   }

   Array<int> V, E, Eo, F, Fo; // TODO: LocalArray
@@ -2739,12 +2766,10 @@ void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs,
      {
         nfd += fec->GetNumDof(mesh->GetFaceGeometry(F[i]), order);
      }
-      if (DoFTransArray[mesh->GetElementBaseGeometry(elem)])
+      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
      {
-         doftrans.SetDofTransformation(
-            *DoFTransArray[mesh->GetElementBaseGeometry(elem)]);
-         doftrans.SetFaceOrientations(Fo);
-         doftrans.SetVDim();
+         DoFTrans[mesh->GetElementBaseGeometry(elem)]
+         -> SetFaceOrientations(Fo);
      }
   }

@@ -2803,18 +2828,54 @@ void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs,
         dofs.Append(bbase + j);
      }
   }
+   return DoFTrans[mesh->GetElementBaseGeometry(elem)];
 }

-DofTransformation *FiniteElementSpace::GetElementDofs(int elem,
-                                                      Array<int> &dofs) const
+void FiniteElementSpace::GetPatchDofs(int patch, Array<int> &dofs) const
 {
-   DoFTrans.SetDofTransformation(NULL);
-   GetElementDofs(elem, dofs, DoFTrans);
-   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
+   MFEM_ASSERT(NURBSext,
+               "FiniteElementSpace::GetPatchDofs needs a NURBSExtension");
+   NURBSext->GetPatchDofs(patch, dofs);
 }

-void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs,
-                                           DofTransformation &doftrans) const
+const FiniteElement *FiniteElementSpace::GetFE(int i) const
+{
+   if (i < 0 || i >= mesh->GetNE())
+   {
+      if (mesh->GetNE() == 0)
+      {
+         MFEM_ABORT("Empty MPI partitions are not permitted!");
+      }
+      MFEM_ABORT("Invalid element id:" << i << "; minimum allowed:" << 0 <<
+                 ", maximum allowed:" << mesh->GetNE()-1);
+   }
+
+   const FiniteElement *FE =
+      fec->GetFE(mesh->GetElementGeometry(i), GetElementOrderImpl(i));
+
+   if (NURBSext)
+   {
+      NURBSext->LoadFE(i, FE);
+   }
+   else
+   {
+#ifdef MFEM_DEBUG
+      // consistency check: fec->GetOrder() and FE->GetOrder() should return
+      // the same value (for standard, constant-order spaces)
+      if (!IsVariableOrder() && FE->GetDim() > 0)
+      {
+         MFEM_ASSERT(FE->GetOrder() == fec->GetOrder(),
+                     "internal error: " <<
+                     FE->GetOrder() << " != " << fec->GetOrder());
+      }
+#endif
+   }
+
+   return FE;
+}
+
+DofTransformation *
+FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

@@ -2822,19 +2883,17 @@ void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs,
   {
      bdr_elem_dof->GetRow(bel, dofs);

-      if (DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)])
+      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
      {
         Array<int> Fo;
         bdr_elem_fos -> GetRow (bel, Fo);
-         doftrans.SetDofTransformation(
-            *DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)]);
-         doftrans.SetFaceOrientations(Fo);
-         doftrans.SetVDim();
+         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
+         SetFaceOrientations(Fo);
      }
-      return;
+      return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
   }

-   Array<int> V, E, Eo; // TODO: LocalArray
+   Array<int> V, E, Eo, Fo; // TODO: LocalArray
   int F, oF;

   int dim = mesh->Dimension();
@@ -2858,14 +2917,11 @@ void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs,
   {
      mesh->GetBdrElementFace(bel, &F, &oF);

-      if (DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)])
+      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
      {
-         mfem::Array<int> Fo(1);
-         Fo[0] = oF;
-         doftrans.SetDofTransformation(
-            *DoFTransArray[mesh->GetBdrElementBaseGeometry(bel)]);
-         doftrans.SetFaceOrientations(Fo);
-         doftrans.SetVDim();
+         Fo.Append(oF);
+         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
+         SetFaceOrientations(Fo);
      }
   }

@@ -2907,14 +2963,8 @@ void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs,
         dofs.Append(EncodeDof(nvdofs + nedofs + fbase, ind[j]));
      }
   }
-}

-DofTransformation *FiniteElementSpace::GetBdrElementDofs(int bel,
-                                                         Array<int> &dofs) const
-{
-   DoFTrans.SetDofTransformation(NULL);
-   GetBdrElementDofs(bel, dofs, DoFTrans);
-   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
+   return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
 }

 int FiniteElementSpace::GetFaceDofs(int face, Array<int> &dofs,
@@ -2946,14 +2996,7 @@ int FiniteElementSpace::GetFaceDofs(int face, Array<int> &dofs,

      order = !IsVariableOrder() ? fec->GetOrder() :
              var_face_orders[var_face_dofs.GetI()[face] + variant];
-      MFEM_ASSERT(fec->GetNumDof(fgeom, order) == nf, [&]()
-      {
-         std::stringstream msg;
-         msg << "fec->GetNumDof(" << (fgeom == Geometry::SQUARE ? "square" : "triangle")
-             << ", " << order << ") = " << fec->GetNumDof(fgeom, order) << " nf " << nf;
-         msg << " face " << face << " variant " << variant << std::endl;
-         return msg.str();
-      }());
+      MFEM_ASSERT(fec->GetNumDof(fgeom, order) == nf, "");
   }
   else
   {
@@ -3084,6 +3127,18 @@ int FiniteElementSpace::GetNumElementInteriorDofs(int i) const
                         GetElementOrderImpl(i));
 }

+void FiniteElementSpace::GetEdgeInteriorDofs(int i, Array<int> &dofs) const
+{
+   MFEM_VERIFY(!IsVariableOrder(), "not implemented");
+
+   int ne = fec->DofForGeometry(Geometry::SEGMENT);
+   dofs.SetSize (ne);
+   for (int j = 0, k = nvdofs+i*ne; j < ne; j++, k++)
+   {
+      dofs[j] = k;
+   }
+}
+
 void FiniteElementSpace::GetFaceInteriorDofs(int i, Array<int> &dofs) const
 {
   MFEM_VERIFY(!IsVariableOrder(), "not implemented");
@@ -3108,61 +3163,6 @@ void FiniteElementSpace::GetFaceInteriorDofs(int i, Array<int> &dofs) const
   }
 }

-void FiniteElementSpace::GetEdgeInteriorDofs(int i, Array<int> &dofs) const
-{
-   MFEM_VERIFY(!IsVariableOrder(), "not implemented");
-
-   int ne = fec->DofForGeometry(Geometry::SEGMENT);
-   dofs.SetSize (ne);
-   for (int j = 0, k = nvdofs+i*ne; j < ne; j++, k++)
-   {
-      dofs[j] = k;
-   }
-}
-
-void FiniteElementSpace::GetPatchDofs(int patch, Array<int> &dofs) const
-{
-   MFEM_ASSERT(NURBSext,
-               "FiniteElementSpace::GetPatchDofs needs a NURBSExtension");
-   NURBSext->GetPatchDofs(patch, dofs);
-}
-
-const FiniteElement *FiniteElementSpace::GetFE(int i) const
-{
-   if (i < 0 || i >= mesh->GetNE())
-   {
-      if (mesh->GetNE() == 0)
-      {
-         MFEM_ABORT("Empty MPI partitions are not permitted!");
-      }
-      MFEM_ABORT("Invalid element id:" << i << "; minimum allowed:" << 0 <<
-                 ", maximum allowed:" << mesh->GetNE()-1);
-   }
-
-   const FiniteElement *FE =
-      fec->GetFE(mesh->GetElementGeometry(i), GetElementOrderImpl(i));
-
-   if (NURBSext)
-   {
-      NURBSext->LoadFE(i, FE);
-   }
-   else
-   {
-#ifdef MFEM_DEBUG
-      // consistency check: fec->GetOrder() and FE->GetOrder() should return
-      // the same value (for standard, constant-order spaces)
-      if (!IsVariableOrder() && FE->GetDim() > 0)
-      {
-         MFEM_ASSERT(FE->GetOrder() == fec->GetOrder(),
-                     "internal error: " <<
-                     FE->GetOrder() << " != " << fec->GetOrder());
-      }
-#endif
-   }
-
-   return FE;
-}
-
 const FiniteElement *FiniteElementSpace::GetBE(int i) const
 {
   int order = fec->GetOrder();
@@ -3235,8 +3235,8 @@ const FiniteElement *FiniteElementSpace::GetEdgeElement(int i,
   return fec->GetFE(Geometry::SEGMENT, eo);
 }

-const FiniteElement *FiniteElementSpace::GetTraceElement(
-   int i, Geometry::Type geom_type) const
+const FiniteElement *FiniteElementSpace
+::GetTraceElement(int i, Geometry::Type geom_type) const
 {
   return fec->TraceFiniteElementForGeometry(geom_type);
 }
@@ -3276,7 +3276,7 @@ void FiniteElementSpace::Destroy()
   }
   E2BFQ_array.SetSize(0);

-   DestroyDoFTransArray();
+   DestroyDoFTrans();

   dof_elem_array.DeleteAll();
   dof_ldof_array.DeleteAll();
@@ -3294,18 +3294,19 @@ void FiniteElementSpace::Destroy()
      delete bdr_elem_dof;
      delete bdr_elem_fos;
      delete face_dof;
+
      delete [] bdofs;
   }
   ceed::RemoveBasisAndRestriction(this);
 }

-void FiniteElementSpace::DestroyDoFTransArray()
+void FiniteElementSpace::DestroyDoFTrans()
 {
-   for (int i = 0; i < DoFTransArray.Size(); i++)
+   for (int i = 0; i < DoFTrans.Size(); i++)
   {
-      delete DoFTransArray[i];
+      delete DoFTrans[i];
   }
-   DoFTransArray.SetSize(0);
+   DoFTrans.SetSize(0);
 }

 void FiniteElementSpace::GetTransferOperator(
@@ -271,8 +271,8 @@ protected:
   int own_ext;
   mutable Array<int> face_to_be; // NURBS FE space only

-   Array<StatelessDofTransformation *> DoFTransArray;
-   mutable DofTransformation DoFTrans;
+   Array<DofTransformation*> DoFTrans;
+   mutable VDofTransformation VDoFTrans;

   /** Matrix representing the prolongation from the global conforming dofs to
       a set of intermediate partially conforming dofs, e.g. the dofs associated
@@ -328,8 +328,8 @@ protected:
   void Construct();
   void Destroy();

-   void ConstructDoFTransArray();
-   void DestroyDoFTransArray();
+   void ConstructDoFTrans();
+   void DestroyDoFTrans();

   void BuildElementToDofTable() const;
   void BuildBdrElementToDofTable() const;
@@ -416,10 +416,10 @@ protected:
      Table* old_elem_dof; // Owned.
      Table* old_elem_fos; // Owned.

-      Array<StatelessDofTransformation*> old_DoFTransArray;
-      mutable DofTransformation old_DoFTrans;
+      Array<DofTransformation*> old_DoFTrans;
+      mutable VDofTransformation old_VDoFTrans;

-      void ConstructDoFTransArray();
+      void ConstructDoFTrans();

   public:
      /** Construct the operator based on the elem_dof table of the original
@@ -803,16 +803,7 @@ public:
   /// with triangular faces.
   ///
   /// @note The returned object should NOT be deleted by the caller.
-   DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;
-
-   /// @brief The same as GetElementDofs(), but with a user-allocated
-   /// DofTransformation object. @a doftrans must be allocated in advance and
-   /// will be owned by the caller. The user can use the
-   /// DofTransformation::GetDofTransformation method on the returned
-   /// @a doftrans object to detect if the DofTransformation should actually be
-   /// used.
-   virtual void GetElementDofs(int elem, Array<int> &dofs,
-                               DofTransformation &doftrans) const;
+   virtual DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;

   /// @brief Returns indices of degrees of freedom for boundary element 'bel'.
   /// The returned indices are offsets into an @ref ldof vector. See also
@@ -826,16 +817,13 @@ public:
   /// with triangular faces.
   ///
   /// @note The returned object should NOT be deleted by the caller.
-   DofTransformation *GetBdrElementDofs(int bel, Array<int> &dofs) const;
+   virtual DofTransformation *GetBdrElementDofs(int bel,
+                                                Array<int> &dofs) const;

-   /// @brief The same as GetBdrElementDofs(), but with a user-allocated
-   /// DofTransformation object. @a doftrans must be allocated in advance and
-   /// will be owned by the caller. The user can use the
-   /// DofTransformation::GetDofTransformation method on the returned
-   /// @a doftrans object to detect if the DofTransformation should actually be
-   /// used.
-   virtual void GetBdrElementDofs(int bel, Array<int> &dofs,
-                                  DofTransformation &doftrans) const;
+   /** @brief Returns indices of degrees of freedom for NURBS patch index
+    @a patch. Cartesian ordering is used, for the tensor-product degrees of
+    freedom. */
+   void GetPatchDofs(int patch, Array<int> &dofs) const;

   /// @brief Returns the indices of the degrees of freedom for the specified
   /// face, including the DOFs for the edges and the vertices of the face.
@@ -882,13 +870,6 @@ public:
   /// GetElementInteriorVDofs().
   void GetElementInteriorDofs(int i, Array<int> &dofs) const;

-   /// @brief Returns the number of degrees of freedom associated with the
-   /// interior of the specified element.
-   ///
-   /// See GetElementInteriorDofs() for more information or to obtain the
-   /// relevant indices.
-   int GetNumElementInteriorDofs(int i) const;
-
   /// @brief Returns the indices of the degrees of freedom for the interior
   /// of the specified face.
   ///
@@ -901,6 +882,13 @@ public:
   /// GetFaceInteriorVDofs().
   void GetFaceInteriorDofs(int i, Array<int> &dofs) const;

+   /// @brief Returns the number of degrees of freedom associated with the
+   /// interior of the specified element.
+   ///
+   /// See GetElementInteriorDofs() for more information or to obtain the
+   /// relevant indices.
+   int GetNumElementInteriorDofs(int i) const;
+
   /// @brief Returns the indices of the degrees of freedom for the interior
   /// of the specified edge.
   ///
@@ -909,11 +897,6 @@ public:
   void GetEdgeInteriorDofs(int i, Array<int> &dofs) const;
   ///@}

-   /** @brief Returns indices of degrees of freedom for NURBS patch index
-    @a patch. Cartesian ordering is used, for the tensor-product degrees of
-    freedom. */
-   void GetPatchDofs(int patch, Array<int> &dofs) const;
-
   /// @anchor dof2vdof @name DoF To VDoF Conversion methods
   /// These methods convert between local dof and local vector dof using the
   /// appropriate relationship based on the Ordering::Type defined in this
@@ -1040,15 +1023,6 @@ public:
   /// @note The returned object should NOT be deleted by the caller.
   DofTransformation *GetElementVDofs(int i, Array<int> &vdofs) const;

-   /// @brief The same as GetElementVDofs(), but with a user-allocated
-   /// DofTransformation object. @a doftrans must be allocated in advance and
-   /// will be owned by the caller. The user can use the
-   /// DofTransformation::GetDofTransformation method on the returned
-   /// @a doftrans object to detect if the DofTransformation should actually be
-   /// used.
-   void GetElementVDofs(int i, Array<int> &vdofs,
-                        DofTransformation &doftrans) const;
-
   /// @brief Returns indices of degrees of freedom for @a i'th boundary
   /// element.
   /// The returned indices are offsets into an @ref ldof vector with @b vdim
@@ -1064,15 +1038,6 @@ public:
   /// @note The returned object should NOT be deleted by the caller.
   DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;

-   /// @brief The same as GetBdrElementVDofs(), but with a user-allocated
-   /// DofTransformation object. @a doftrans must be allocated in advance and
-   /// will be owned by the caller. The user can use the
-   /// DofTransformation::GetDofTransformation method on the returned
-   /// @a doftrans object to detect if the DofTransformation should actually be
-   /// used.
-   void GetBdrElementVDofs(int i, Array<int> &vdofs,
-                           DofTransformation &doftrans) const;
-
   /// Returns indices of degrees of freedom in @a vdofs for NURBS patch @a i.
   void GetPatchVDofs(int i, Array<int> &vdofs) const;

@@ -31,13 +31,13 @@ FmsBasisTypeToMfemBasis(FmsBasisType b)
   switch (b)
   {
      case FMS_NODAL_GAUSS_OPEN:
-         retval = mfem::BasisType::GaussLegendre;
+         retval = mfem::BasisType::GaussLegendre;;
         break;
      case FMS_NODAL_GAUSS_CLOSED:
-         retval = mfem::BasisType::GaussLobatto;
+         retval = mfem::BasisType::GaussLobatto;;
         break;
      case FMS_POSITIVE:
-         retval = mfem::BasisType::Positive;
+         retval = mfem::BasisType::Positive;;
         break;
      case FMS_NODAL_UNIFORM_OPEN:
         retval = mfem::BasisType::OpenUniform;
@@ -1812,22 +1812,22 @@ MeshToFmsMesh(const Mesh *mmesh, FmsMesh *fmesh, FmsComponent *volume)
      switch (betype)
      {
         case Element::POINT:
-            bdr_eles[FMS_VERTEX].push_back(mmesh->GetBdrElementFaceIndex(i));
+            bdr_eles[FMS_VERTEX].push_back(mmesh->GetBdrElementEdgeIndex(i));
            break;
         case Element::SEGMENT:
-            bdr_eles[FMS_EDGE].push_back(mmesh->GetBdrElementFaceIndex(i));
+            bdr_eles[FMS_EDGE].push_back(mmesh->GetBdrElementEdgeIndex(i));
            break;
         case Element::TRIANGLE:
-            bdr_eles[FMS_TRIANGLE].push_back(mmesh->GetBdrElementFaceIndex(i));
+            bdr_eles[FMS_TRIANGLE].push_back(mmesh->GetBdrElementEdgeIndex(i));
            break;
         case Element::QUADRILATERAL:
-            bdr_eles[FMS_QUADRILATERAL].push_back(mmesh->GetBdrElementFaceIndex(i));
+            bdr_eles[FMS_QUADRILATERAL].push_back(mmesh->GetBdrElementEdgeIndex(i));
            break;
         case Element::TETRAHEDRON:
-            bdr_eles[FMS_TETRAHEDRON].push_back(mmesh->GetBdrElementFaceIndex(i));
+            bdr_eles[FMS_TETRAHEDRON].push_back(mmesh->GetBdrElementEdgeIndex(i));
            break;
         case Element::HEXAHEDRON:
-            bdr_eles[FMS_HEXAHEDRON].push_back(mmesh->GetBdrElementFaceIndex(i));
+            bdr_eles[FMS_HEXAHEDRON].push_back(mmesh->GetBdrElementEdgeIndex(i));
            break;
         default:
            MFEM_WARNING("Unsupported boundary element " << betype << " at boundary index "
@@ -65,10 +65,10 @@ public:

   /** @brief Return an IntegrationRule consisting of all vertices of the given
       Geometry::Type, @a GeomType. */
-   const IntegrationRule *GetVertices(int GeomType) const;
+   const IntegrationRule *GetVertices(int GeomType);

   /// Return the center of the given Geometry::Type, @a GeomType.
-   const IntegrationPoint &GetCenter(int GeomType) const
+   const IntegrationPoint &GetCenter(int GeomType)
   { return GeomCenter[GeomType]; }

   /// Get a random point in the reference element specified by @a GeomType.
@@ -97,9 +97,9 @@ public:

   const DenseMatrix &GetGeomToPerfGeomJac(int GeomType) const
   { return *GeomToPerfGeomJac[GeomType]; }
-   const DenseMatrix *GetPerfGeomToGeomJac(int GeomType) const
+   DenseMatrix *GetPerfGeomToGeomJac(int GeomType)
   { return PerfGeomToGeomJac[GeomType]; }
-   void GetPerfPointMat(int GeomType, DenseMatrix &pm) const;
+   void GetPerfPointMat(int GeomType, DenseMatrix &pm);
   void JacToPerfJac(int GeomType, const DenseMatrix &J,
                     DenseMatrix &PJ) const;

@@ -123,7 +123,7 @@ public:
   }

   /// Return the number of boundary "faces" of a given Geometry::Type.
-   int NumBdr(int GeomType) const { return NumBdrArray[GeomType]; }
+   int NumBdr(int GeomType) { return NumBdrArray[GeomType]; }
 };

 template <> struct
@@ -317,27 +317,27 @@ public:
   int Type;

   RefinedGeometry(int NPts, int NRefG, int NRefE, int NBdrE = 0) :
-      RefPts(NPts), RefGeoms(NRefG), RefEdges(NRefE), NumBdrEdges(NBdrE) {}
+      RefPts(NPts), RefGeoms(NRefG), RefEdges(NRefE), NumBdrEdges(NBdrE) { }
 };

 class GeometryRefiner
 {
 private:
-   int Type; // Quadrature1D type (ClosedUniform is default)
+   int type; // Quadrature1D type (ClosedUniform is default)
   Array<RefinedGeometry *> RGeom[Geometry::NumGeom];
   Array<IntegrationRule *> IntPts[Geometry::NumGeom];

-   RefinedGeometry *FindInRGeom(Geometry::Type Geom, int Times,
-                                int ETimes) const;
-   IntegrationRule *FindInIntPts(Geometry::Type Geom, int NPts) const;
+   RefinedGeometry *FindInRGeom(Geometry::Type Geom, int Times, int ETimes,
+                                int Type);
+   IntegrationRule *FindInIntPts(Geometry::Type Geom, int NPts);

 public:
-   GeometryRefiner(int t = Quadrature1D::ClosedUniform) : Type(t) {}
+   GeometryRefiner();

   /// Set the Quadrature1D type of points to use for subdivision.
-   void SetType(int t) { Type = t; }
+   void SetType(const int t) { type = t; }
   /// Get the Quadrature1D type of points used for subdivision.
-   int GetType() const { return Type; }
+   int GetType() const { return type; }

   RefinedGeometry *Refine(Geometry::Type Geom, int Times, int ETimes = 1);

@@ -345,10 +345,10 @@ public:
   const IntegrationRule *RefineInterior(Geometry::Type Geom, int Times);

   /// Get the Refinement level based on number of points
-   static int GetRefinementLevelFromPoints(Geometry::Type Geom, int Npts);
+   virtual int GetRefinementLevelFromPoints(Geometry::Type Geom, int Npts);

   /// Get the Refinement level based on number of elements
-   static int GetRefinementLevelFromElems(Geometry::Type geom, int Npts);
+   virtual int GetRefinementLevelFromElems(Geometry::Type geom, int Npts);

   ~GeometryRefiner();
 };
@@ -720,6 +720,56 @@ void GridFunction::GetVectorValues(int i, const IntegrationRule &ir,
   GetVectorValues(*Tr, ir, vals);
 }

+void be_to_bfe(Geometry::Type geom, int o, const IntegrationPoint &ip,
+               IntegrationPoint &fip)
+{
+   if (geom == Geometry::TRIANGLE)
+   {
+      if (o == 2)
+      {
+         fip.x = 1.0 - ip.x - ip.y;
+         fip.y = ip.x;
+      }
+      else if (o == 4)
+      {
+         fip.x = ip.y;
+         fip.y = 1.0 - ip.x - ip.y;
+      }
+      else
+      {
+         fip.x = ip.x;
+         fip.y = ip.y;
+      }
+      fip.z = ip.z;
+   }
+   else
+   {
+      if (o == 2)
+      {
+         fip.x = ip.y;
+         fip.y = 1.0 - ip.x;
+      }
+      else if (o == 4)
+      {
+         fip.x = 1.0 - ip.x;
+         fip.y = 1.0 - ip.y;
+      }
+      else if (o == 6)
+      {
+         fip.x = 1.0 - ip.y;
+         fip.y = ip.x;
+      }
+      else
+      {
+         fip.x = ip.x;
+         fip.y = ip.y;
+      }
+      fip.z = ip.z;
+   }
+   fip.weight = ip.weight;
+   fip.index  = ip.index;
+}
+
 double GridFunction::GetValue(ElementTransformation &T,
                              const IntegrationPoint &ip,
                              int comp, Vector *tr) const
@@ -784,15 +834,18 @@ double GridFunction::GetValue(ElementTransformation &T,
            // boundary so we'll evaluate it in the neighboring element.
            FaceElementTransformations * FET =
               fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
-            MFEM_ASSERT(FET != nullptr,
-                        "FaceElementTransformation must be valid for a boundary element");

-            // Boundary elements and boundary faces may have different
+            // Boundary elements and Boundary Faces may have different
            // orientations so adjust the integration point if necessary.
-            int f, o;
-            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
-            IntegrationPoint fip =
-               Mesh::TransformBdrElementToFace(FET->GetGeometryType(), o, ip);
+            int o = 0;
+            if (fes->GetMesh()->Dimension() == 3)
+            {
+               int f;
+               fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+            }
+
+            IntegrationPoint fip;
+            be_to_bfe(FET->GetGeometryType(), o, ip, fip);

            // Compute and set the point in element 1 from fip
            FET->SetAllIntPoints(&fip);
@@ -920,15 +973,18 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
            // the boundary so we'll evaluate it in the neighboring element.
            FaceElementTransformations * FET =
               fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
-            MFEM_ASSERT(FET != nullptr,
-                        "FaceElementTransformation must be valid for a boundary element");

-            // Boundary elements and boundary faces may have different
+            // Boundary elements and Boundary Faces may have different
            // orientations so adjust the integration point if necessary.
-            int f, o;
-            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
-            IntegrationPoint fip =
-               Mesh::TransformBdrElementToFace(FET->GetGeometryType(), o, ip);
+            int o = 0;
+            if (fes->GetMesh()->Dimension() == 3)
+            {
+               int f;
+               fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+            }
+
+            IntegrationPoint fip;
+            be_to_bfe(FET->GetGeometryType(), o, ip, fip);

            // Compute and set the point in element 1 from fip
            FET->SetAllIntPoints(&fip);
@@ -941,8 +997,6 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
      {
         FaceElementTransformations * FET =
            dynamic_cast<FaceElementTransformations *>(&T);
-         MFEM_ASSERT(FET != nullptr,
-                     "FaceElementTransformation must be valid for a boundary element");

         // Evaluate in neighboring element for both continuous and
         // discontinuous fields (the integration point in T1 should have
@@ -1061,10 +1115,11 @@ int GridFunction::GetFaceVectorValues(
   int i, int side, const IntegrationRule &ir,
   DenseMatrix &vals, DenseMatrix &tr) const
 {
-   int di;
+   int n, di;
   FaceElementTransformations *Transf;

-   IntegrationRule eir(ir.GetNPoints());  // ---
+   n = ir.GetNPoints();
+   IntegrationRule eir(n);  // ---
   Transf = fes->GetMesh()->GetFaceElementTransformations(i, 0);
   if (side == 2)
   {
@@ -1086,14 +1141,12 @@ int GridFunction::GetFaceVectorValues(
   if (di == 0)
   {
      Transf = fes->GetMesh()->GetFaceElementTransformations(i, 5);
-      MFEM_ASSERT(Transf != nullptr, "FaceElementTransformation cannot be null!");
      Transf->Loc1.Transform(ir, eir);
      GetVectorValues(*Transf->Elem1, eir, vals, &tr);
   }
   else
   {
      Transf = fes->GetMesh()->GetFaceElementTransformations(i, 10);
-      MFEM_ASSERT(Transf != nullptr, "FaceElementTransformation cannot be null!");
      Transf->Loc2.Transform(ir, eir);
      GetVectorValues(*Transf->Elem2, eir, vals, &tr);
   }
@@ -1451,13 +1504,17 @@ double GridFunction::GetDivergence(ElementTransformation &T) const
         FaceElementTransformations * FET =
            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);

-         // Boundary elements and boundary faces may have different
+         // Boundary elements and Boundary Faces may have different
         // orientations so adjust the integration point if necessary.
-         int f, o;
-         fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
-         IntegrationPoint fip =
-            Mesh::TransformBdrElementToFace(FET->GetGeometryType(), o,
-                                            T.GetIntPoint());
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);

         // Compute and set the point in element 1 from fip
         FET->SetAllIntPoints(&fip);
@@ -1544,13 +1601,17 @@ void GridFunction::GetCurl(ElementTransformation &T, Vector &curl) const
         FaceElementTransformations * FET =
            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);

-         // Boundary elements and boundary faces may have different
+         // Boundary elements and Boundary Faces may have different
         // orientations so adjust the integration point if necessary.
-         int f, o;
-         fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
-         IntegrationPoint fip =
-            Mesh::TransformBdrElementToFace(FET->GetGeometryType(), o,
-                                            T.GetIntPoint());
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);

         // Compute and set the point in element 1 from fip
         FET->SetAllIntPoints(&fip);
@@ -1609,13 +1670,17 @@ void GridFunction::GetGradient(ElementTransformation &T, Vector &grad) const
         FaceElementTransformations * FET =
            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);

-         // Boundary elements and boundary faces may have different
+         // Boundary elements and Boundary Faces may have different
         // orientations so adjust the integration point if necessary.
-         int f, o;
-         fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
-         IntegrationPoint fip =
-            Mesh::TransformBdrElementToFace(FET->GetGeometryType(), o,
-                                            T.GetIntPoint());
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);

         // Compute and set the point in element 1 from fip
         FET->SetAllIntPoints(&fip);
@@ -1691,13 +1756,17 @@ void GridFunction::GetVectorGradient(
         FaceElementTransformations * FET =
            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);

-         // Boundary elements and boundary faces may have different
+         // Boundary elements and Boundary Faces may have different
         // orientations so adjust the integration point if necessary.
-         int f, o;
-         fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
-         IntegrationPoint fip =
-            Mesh::TransformBdrElementToFace(FET->GetGeometryType(), o,
-                                            T.GetIntPoint());
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);

         // Compute and set the point in element 1 from fip
         FET->SetAllIntPoints(&fip);
@@ -10,7 +10,6 @@
 // CONTRIBUTING.md for details.

 #include "gslib.hpp"
-#include "geom.hpp"

 #ifdef MFEM_USE_GSLIB

@@ -239,8 +238,7 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
   }

   // Map element number for simplices, and ref_pos from [-1,1] to [0,1] for
-   // both simplices and quads. Also sets code to 1 for points found on element
-   // faces/edges.
+   // both simplices and quads.
   MapRefPosAndElemIndices();
 }

@@ -683,9 +681,6 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()
   int nptorig = points_cnt,
       npt = points_cnt;

-   // tolerance for point to be marked as on element edge/face
-   double btol = 1e-12;
-
   GridFunction *gf_rst_map_temp = NULL;
   int nptsend = 0;

@@ -699,7 +694,7 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()

   // Pack data to send via crystal router
   struct gslib::array *outpt = new gslib::array;
-   struct out_pt { double r[3]; uint index, el, proc, code; };
+   struct out_pt { double r[3]; uint index, el, proc; };
   struct out_pt *pt;
   array_init(struct out_pt, outpt, nptsend);
   outpt->n=nptsend;
@@ -717,12 +712,12 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()
      pt->index = index;
      pt->proc  = gsl_proc[index];
      pt->el    = gsl_elem[index];
-      pt->code  = gsl_code[index];
      ++pt;
   }

   // Transfer data to target MPI ranks
   sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+
   // Map received points
   npt = outpt->n;
   pt = (struct out_pt *)outpt->ptr;
@@ -736,13 +731,7 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()
      const Geometry::Type gt = fe->GetGeomType();
      pt->el = mesh_elem;

-      if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
-      {
-         // check if it is on element boundary
-         pt->code = Geometry::CheckPoint(gt, ip, -btol) ? 0 : 1;
-         ++pt;
-         continue;
-      }
+      if (gt == Geometry::SQUARE || gt == Geometry::CUBE) { ++pt; continue; }
      else if (gt == Geometry::TRIANGLE)
      {
         gf_rst_map_temp = gf_rst_map[0];
@@ -769,10 +758,6 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()
      {
         pt->r[d] = mfem_ref(d);
      }
-
-      // check if point is on element boundary
-      ip.Set3(&pt->r[0]);
-      pt->code = Geometry::CheckPoint(gt, ip, -btol) ? 0 : 1;
      ++pt;
   }

@@ -789,7 +774,6 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()
      {
         gsl_mfem_ref(d + pt->index*dim) = pt->r[d];
      }
-      gsl_code[pt->index] = pt->code;
      ++pt;
   }
   array_free(outpt);
@@ -800,22 +784,12 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()
   {
      if (gsl_code[index] != 2 && gsl_proc[index] == gsl_comm->id)
      {
-
-         IntegrationPoint ip;
-         Vector mfem_ref(gsl_mfem_ref.GetData()+index*dim, dim);
-         ip.Set2(mfem_ref.GetData());
-         if (dim == 3) { ip.z = mfem_ref(2); }
-
         const int elem = gsl_elem[index];
         const int mesh_elem = split_element_map[elem];
         const FiniteElement *fe = mesh->GetNodalFESpace()->GetFE(mesh_elem);
         const Geometry::Type gt = fe->GetGeomType();
         gsl_mfem_elem[index] = mesh_elem;
-         if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
-         {
-            gsl_code[index] = Geometry::CheckPoint(gt, ip, -btol) ? 0 : 1;
-            continue;
-         }
+         if (gt == Geometry::SQUARE || gt == Geometry::CUBE) { continue; }
         else if (gt == Geometry::TRIANGLE)
         {
            gf_rst_map_temp = gf_rst_map[0];
@@ -834,12 +808,11 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()
         }

         int local_elem = split_element_index[elem];
-         gf_rst_map_temp->GetVectorValue(local_elem, ip, mfem_ref);
-
-         // Check if the point is on element boundary
+         IntegrationPoint ip;
+         Vector mfem_ref(gsl_mfem_ref.GetData()+index*dim, dim);
         ip.Set2(mfem_ref.GetData());
         if (dim == 3) { ip.z = mfem_ref(2); }
-         gsl_code[index]  = Geometry::CheckPoint(gt, ip, -btol) ? 0 : 1;
+         gf_rst_map_temp->GetVectorValue(local_elem, ip, mfem_ref);
      }
   }
 }
@@ -120,10 +120,6 @@ public:
   /// Return the serial hybridized matrix.
   SparseMatrix &GetMatrix() { return *H; }

-   /// Return the transpose of the serial constraint matrix.
-   SparseMatrix &GetConstraintMatrixTranspose()
-   { if (!Ct) mfem_error("Ct has not been constructed!"); return *Ct; }
-
 #ifdef MFEM_USE_MPI
   /// Return the parallel hybridized matrix.
   HypreParMatrix &GetParallelMatrix() { return *pH.Is<HypreParMatrix>(); }
@@ -737,7 +737,7 @@ void QuadratureFunctions1D::GivePolyPoints(const int np, double *pts,
         ClosedGL(np, &ir);
         break;
      }
-      case Quadrature1D::Invalid:
+      default:
      {
         MFEM_ABORT("Asking for an unknown type of 1D Quadrature points, "
                    "type = " << type);
@@ -831,10 +831,7 @@ void QuadratureFunctions1D::CalculateUniformWeights(IntegrationRule *ir,
         hinv = p+1;
         ihoffset = 1;
         break;
-      case Quadrature1D::GaussLegendre:
-      case Quadrature1D::GaussLobatto:
-      case Quadrature1D::ClosedGL:
-      case Quadrature1D::Invalid:
+      default:
         MFEM_ABORT("invalid Quadrature1D type: " << type);
   }
   // set w0 = (-1)^p*(p!)/(hinv^p)
@@ -943,10 +940,10 @@ IntegrationRules IntRules(0, Quadrature1D::GaussLegendre);

 IntegrationRules RefinedIntRules(1, Quadrature1D::GaussLegendre);

-IntegrationRules::IntegrationRules(int ref, int type)
-   : quad_type(type)
+IntegrationRules::IntegrationRules(int Ref, int type_):
+   quad_type(type_)
 {
-   refined = ref;
+   refined = Ref;

   if (refined < 0) { own_rules = 0; return; }

@@ -978,19 +975,11 @@ IntegrationRules::IntegrationRules(int ref, int type)

   CubeIntRules.SetSize(32, h_mt);
   CubeIntRules = NULL;
-
-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-   IntRuleLocks.SetSize(Geometry::NUM_GEOMETRIES, h_mt);
-   for (int i = 0; i < Geometry::NUM_GEOMETRIES; i++)
-   {
-      omp_init_lock(&IntRuleLocks[i]);
-   }
-#endif
 }

 const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
 {
-   Array<IntegrationRule *> *ir_array = NULL;
+   Array<IntegrationRule *> *ir_array;

   switch (GeomType)
   {
@@ -1002,9 +991,9 @@ const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
      case Geometry::CUBE:        ir_array = &CubeIntRules; break;
      case Geometry::PRISM:       ir_array = &PrismIntRules; break;
      case Geometry::PYRAMID:     ir_array = &PyramidIntRules; break;
-      case Geometry::INVALID:
-      case Geometry::NUM_GEOMETRIES:
-         MFEM_ABORT("Unknown type of reference element!");
+      default:
+         mfem_error("IntegrationRules::Get(...) : Unknown geometry type!");
+         ir_array = NULL;
   }

   if (Order < 0)
@@ -1012,35 +1001,36 @@ const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
      Order = 0;
   }

-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-   omp_set_lock(&IntRuleLocks[GeomType]);
-#endif
-
   if (!HaveIntRule(*ir_array, Order))
   {
-      IntegrationRule *ir = GenerateIntegrationRule(GeomType, Order);
-#ifdef MFEM_DEBUG
-      int RealOrder = Order;
-      while (RealOrder+1 < ir_array->Size() && (*ir_array)[RealOrder+1] == ir)
+#ifdef MFEM_USE_LEGACY_OPENMP
+      #pragma omp critical
+#endif
      {
-         RealOrder++;
-      }
-      MFEM_VERIFY(RealOrder == ir->GetOrder(), "internal error");
+         if (!HaveIntRule(*ir_array, Order))
+         {
+            IntegrationRule *ir = GenerateIntegrationRule(GeomType, Order);
+#ifdef MFEM_DEBUG
+            int RealOrder = Order;
+            while (RealOrder+1 < ir_array->Size() &&
+                   (*ir_array)[RealOrder+1] == ir)
+            {
+               RealOrder++;
+            }
+            MFEM_VERIFY(RealOrder == ir->GetOrder(), "internal error");
 #else
-      MFEM_CONTRACT_VAR(ir);
+            MFEM_CONTRACT_VAR(ir);
 #endif
+         }
+      }
   }

-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-   omp_unset_lock(&IntRuleLocks[GeomType]);
-#endif
-
   return *(*ir_array)[Order];
 }

 void IntegrationRules::Set(int GeomType, int Order, IntegrationRule &IntRule)
 {
-   Array<IntegrationRule *> *ir_array = NULL;
+   Array<IntegrationRule *> *ir_array;

   switch (GeomType)
   {
@@ -1052,15 +1042,11 @@ void IntegrationRules::Set(int GeomType, int Order, IntegrationRule &IntRule)
      case Geometry::CUBE:        ir_array = &CubeIntRules; break;
      case Geometry::PRISM:       ir_array = &PrismIntRules; break;
      case Geometry::PYRAMID:     ir_array = &PyramidIntRules; break;
-      case Geometry::INVALID:
-      case Geometry::NUM_GEOMETRIES:
-         MFEM_ABORT("Unknown type of reference element!");
+      default:
+         mfem_error("IntegrationRules::Set(...) : Unknown geometry type!");
+         ir_array = NULL;
   }

-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-   omp_set_lock(&IntRuleLocks[GeomType]);
-#endif
-
   if (HaveIntRule(*ir_array, Order))
   {
      MFEM_ABORT("Overwriting set rules is not supported!");
@@ -1069,19 +1055,16 @@ void IntegrationRules::Set(int GeomType, int Order, IntegrationRule &IntRule)
   AllocIntRule(*ir_array, Order);

   (*ir_array)[Order] = &IntRule;
-
-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-   omp_unset_lock(&IntRuleLocks[GeomType]);
-#endif
 }

-void IntegrationRules::DeleteIntRuleArray(
-   Array<IntegrationRule *> &ir_array) const
+void IntegrationRules::DeleteIntRuleArray(Array<IntegrationRule *> &ir_array)
 {
+   int i;
+   IntegrationRule *ir = NULL;
+
   // Many of the intrules have multiple contiguous copies in the ir_array
   // so we have to be careful to not delete them twice.
-   IntegrationRule *ir = NULL;
-   for (int i = 0; i < ir_array.Size(); i++)
+   for (i = 0; i < ir_array.Size(); i++)
   {
      if (ir_array[i] != NULL && ir_array[i] != ir)
      {
@@ -1093,13 +1076,6 @@ void IntegrationRules::DeleteIntRuleArray(

 IntegrationRules::~IntegrationRules()
 {
-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-   for (int i = 0; i < Geometry::NUM_GEOMETRIES; i++)
-   {
-      omp_destroy_lock(&IntRuleLocks[i]);
-   }
-#endif
-
   if (!own_rules) { return; }

   DeleteIntRuleArray(PointIntRules);
@@ -1134,11 +1110,10 @@ IntegrationRule *IntegrationRules::GenerateIntegrationRule(int GeomType,
         return PrismIntegrationRule(Order);
      case Geometry::PYRAMID:
         return PyramidIntegrationRule(Order);
-      case Geometry::INVALID:
-      case Geometry::NUM_GEOMETRIES:
-         MFEM_ABORT("Unknown type of reference element!");
+      default:
+         mfem_error("IntegrationRules::Set(...) : Unknown geometry type!");
+         return NULL;
   }
-   return NULL;
 }


@@ -1147,7 +1122,7 @@ IntegrationRule *IntegrationRules::PointIntegrationRule(int Order)
 {
   if (Order > 1)
   {
-      MFEM_ABORT("Point Integration Rule of Order > 1 not defined");
+      mfem_error("Point Integration Rule of Order > 1 not defined");
      return NULL;
   }

@@ -1210,7 +1185,7 @@ IntegrationRule *IntegrationRules::SegmentIntegrationRule(int Order)
         QuadratureFunctions1D::OpenHalfUniform(n, ir);
         break;
      }
-      case Quadrature1D::Invalid:
+      default:
      {
         MFEM_ABORT("unknown Quadrature1D type: " << quad_type);
      }
@@ -1787,8 +1762,8 @@ IntegrationRule *IntegrationRules::PyramidIntegrationRule(int Order)

   for (int k=0; k<npts; k++)
   {
-      const IntegrationPoint &ipc = irc.IntPoint(k);
-      IntegrationPoint &ipp = PyramidIntRules[Order]->IntPoint(k);
+      const IntegrationPoint & ipc = irc.IntPoint(k);
+      IntegrationPoint & ipp = PyramidIntRules[Order]->IntPoint(k);
      ipp.x = ipc.x * (1.0 - ipc.z);
      ipp.y = ipc.y * (1.0 - ipc.z);
      ipp.z = ipc.z;
@@ -1800,8 +1775,8 @@ IntegrationRule *IntegrationRules::PyramidIntegrationRule(int Order)
 // Integration rules for reference prism
 IntegrationRule *IntegrationRules::PrismIntegrationRule(int Order)
 {
-   const IntegrationRule &irt = Get(Geometry::TRIANGLE, Order);
-   const IntegrationRule &irs = Get(Geometry::SEGMENT, Order);
+   const IntegrationRule & irt = Get(Geometry::TRIANGLE, Order);
+   const IntegrationRule & irs = Get(Geometry::SEGMENT, Order);
   int nt = irt.GetNPoints();
   int ns = irs.GetNPoints();
   AllocIntRule(PrismIntRules, Order);
@@ -1815,12 +1790,12 @@ IntegrationRule *IntegrationRules::PrismIntegrationRule(int Order)

   for (int ks=0; ks<ns; ks++)
   {
-      const IntegrationPoint &ips = irs.IntPoint(ks);
+      const IntegrationPoint & ips = irs.IntPoint(ks);
      for (int kt=0; kt<nt; kt++)
      {
         int kp = ks * nt + kt;
-         const IntegrationPoint &ipt = irt.IntPoint(kt);
-         IntegrationPoint &ipp = PrismIntRules[Order]->IntPoint(kp);
+         const IntegrationPoint & ipt = irt.IntPoint(kt);
+         IntegrationPoint & ipp = PrismIntRules[Order]->IntPoint(kp);
         ipp.x = ipt.x;
         ipp.y = ipt.y;
         ipp.z = ips.x;
@@ -14,9 +14,6 @@

 #include "../config/config.hpp"
 #include "../general/array.hpp"
-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-#include <omp.h>
-#endif

 #include <vector>
 #include <map>
@@ -431,18 +428,14 @@ private:
   Array<IntegrationRule *> PrismIntRules;
   Array<IntegrationRule *> CubeIntRules;

-#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
-   Array<omp_lock_t> IntRuleLocks;
-#endif
-
-   void AllocIntRule(Array<IntegrationRule *> &ir_array, int Order) const
+   void AllocIntRule(Array<IntegrationRule *> &ir_array, int Order)
   {
      if (ir_array.Size() <= Order)
      {
         ir_array.SetSize(Order + 1, NULL);
      }
   }
-   bool HaveIntRule(Array<IntegrationRule *> &ir_array, int Order) const
+   bool HaveIntRule(Array<IntegrationRule *> &ir_array, int Order)
   {
      return (ir_array.Size() > Order && ir_array[Order] != NULL);
   }
@@ -450,7 +443,6 @@ private:
   {
      return Order | 1; // valid for all quad_type's
   }
-   void DeleteIntRuleArray(Array<IntegrationRule *> &ir_array) const;

   /// The following methods allocate new IntegrationRule objects without
   /// checking if they already exist.  To avoid memory leaks use
@@ -465,10 +457,12 @@ private:
   IntegrationRule *PrismIntegrationRule(int Order);
   IntegrationRule *CubeIntegrationRule(int Order);

+   void DeleteIntRuleArray(Array<IntegrationRule *> &ir_array);
+
 public:
   /// Sets initial sizes for the integration rule arrays, but rules
   /// are defined the first time they are requested with the Get method.
-   explicit IntegrationRules(int ref = 0,
+   explicit IntegrationRules(int Ref = 0,
                             int type = Quadrature1D::GaussLegendre);

   /// Returns an integration rule for given GeomType and Order.
@@ -137,7 +137,7 @@ bool LinearForm::SupportsDevice() const
      // Make sure every boundary element corresponds to a boundary face
      for (int be = 0; be < fes->GetNBE(); ++be)
      {
-         const int f = mesh.GetBdrElementFaceIndex(be);
+         const int f = mesh.GetBdrElementEdgeIndex(be);
         const auto face_info = mesh.GetFaceInformation(f);
         if (!face_info.IsBoundary())
         {
@@ -148,7 +148,7 @@ void LinearFormExtension::Update()
      std::unordered_map<int,int> f_to_be;
      for (int i = 0; i < mesh.GetNBE(); ++i)
      {
-         const int f = mesh.GetBdrElementFaceIndex(i);
+         const int f = mesh.GetBdrElementEdgeIndex(i);
         f_to_be[f] = i;
      }
      MFEM_VERIFY(size_t(nf_bdr) == f_to_be.size(), "Incompatible sizes");
@@ -395,8 +395,8 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
   {
      const IntegrationPoint &ip = ir->IntPoint(i);

-      Tr.SetIntPoint (&ip);
      Q.Eval(vec, Tr, ip);
+      Tr.SetIntPoint (&ip);
      vec *= Tr.Weight() * ip.weight;
      el.CalcShape(ip, shape);
      for (int k = 0; k < vdim; k++)
@@ -257,13 +257,13 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   // vertices of the LOR mesh. The vertex coordinates are already computed in
   // E-vector format and passed in in X_vert.
   //
-   // In this function, we need to convert X_vert (which has the shape (sdim,
+   // In this function, we need to convert X_vert (which has the shape (dim,
   // ndof_per_el, nel_ho)) to T-DOF format.
   //
-   // We place the results in the vector xyz_tvec, which has shape (ntdofs, sdim)
+   // We place the results in the vector xyz_tvec, which has shape (ntdofs, dim)
   // and then make the hypre vectors x, y, and z point to subvectors.
   //
-   // When the space dimension is 2, z is NULL.
+   // In 2D, z is NULL.

   // Create the H1 vertex space and get the element restriction
   ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
@@ -275,23 +275,24 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   const int nel_ho = vert_fes.GetNE();
   const int ndp1 = order + 1;
   const int ndof_per_el = static_cast<int>(pow(ndp1, dim));
-   const int sdim = vert_fes.GetMesh()->SpaceDimension();
+   const int sdim = dim;
   const int ntdofs = R->Height();

   const MemoryClass mc = GetHypreMemoryClass();
   bool dev = (mc == MemoryClass::DEVICE);

-   xyz_tvec = new Vector(ntdofs*sdim);
+   xyz_tvec = new Vector(ntdofs*dim);

-   auto xyz_tv = Reshape(HypreWrite(xyz_tvec->GetMemory()), ntdofs, sdim);
+   auto xyz_tv = Reshape(HypreWrite(xyz_tvec->GetMemory()), ntdofs, dim);
   const auto xyz_e =
-      Reshape(HypreRead(X_vert.GetMemory()), sdim, ndof_per_el, nel_ho);
+      Reshape(HypreRead(X_vert.GetMemory()), dim, ndof_per_el, nel_ho);
   const auto d_offsets = HypreRead(el_restr->Offsets().GetMemory());
   const auto d_indices = HypreRead(el_restr->Indices().GetMemory());
   const auto ltdof_ldof = HypreRead(R->GetMemoryJ());

   // Go from E-vector format directly to T-vector format
-   MFEM_HYPRE_FORALL(i, ntdofs,
+   //MFEM_HYPRE_FORALL(i, ntdofs,
+   mfem::forall_switch(HypreUsingGPU(), ntdofs, [=] MFEM_HOST_DEVICE (int i)
   {
      const int j = d_offsets[ltdof_ldof[i]];
      for (int c = 0; c < sdim; ++c)
@@ -309,7 +310,7 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
   x = new HypreParVector(vert_fes.GetComm(), glob_size, d_x_ptr, cols, dev);
   double *d_y_ptr = xyz_tv + 1*ntdofs;
   y = new HypreParVector(vert_fes.GetComm(), glob_size, d_y_ptr, cols, dev);
-   if (sdim == 3)
+   if (dim == 3)
   {
      double *d_z_ptr = xyz_tv + 2*ntdofs;
      z = new HypreParVector(vert_fes.GetComm(), glob_size, d_z_ptr, cols, dev);
@@ -77,7 +77,6 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,

   // Get nodal points at the LOR vertices
   const int dim = mesh_ho.Dimension();
-   const int sdim = mesh_ho.SpaceDimension();
   const int nel_ho = mesh_ho.GetNE();
   const int order = fes_ho.GetMaxElementOrder();
   const int nd1d = order + 1;
@@ -95,7 +94,7 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
   IntegrationRule ir = GetCollocatedIntRule(fes_ho);

   // Map from nodal E-vector to Q-vector at the LOR vertex points
-   X_vert.SetSize(sdim*ndof_per_el*nel_ho);
+   X_vert.SetSize(dim*ndof_per_el*nel_ho);
   const QuadratureInterpolator *quad_interp =
      nodal_fes->GetQuadratureInterpolator(ir);
   quad_interp->SetOutputLayout(QVectorLayout::byVDIM);
@@ -381,49 +380,44 @@ void BatchedLORAssembly::SparseIJToCSR(OperatorHandle &A) const
   FillJAndData(*A_mat);
 }

-template <int ORDER, int SDIM, typename LOR_KERNEL>
-static void Assemble_(LOR_KERNEL &kernel, int dim)
-{
-   if (dim == 2) { kernel.template Assemble2D<ORDER,SDIM>(); }
-   else if (dim == 3) { kernel.template Assemble3D<ORDER>(); }
-   else { MFEM_ABORT("Unsupported dimension"); }
-}
-
-template <int ORDER, typename LOR_KERNEL>
-static void Assemble_(LOR_KERNEL &kernel, int dim, int sdim)
-{
-   if (sdim == 2) { Assemble_<ORDER,2>(kernel, dim); }
-   else if (sdim == 3) { Assemble_<ORDER,3>(kernel, dim); }
-   else { MFEM_ABORT("Unsupported space dimension."); }
-}
-
-template <typename LOR_KERNEL>
-static void Assemble_(LOR_KERNEL &kernel, int dim, int sdim, int order)
-{
-   switch (order)
-   {
-      case 1: Assemble_<1>(kernel, dim, sdim); break;
-      case 2: Assemble_<2>(kernel, dim, sdim); break;
-      case 3: Assemble_<3>(kernel, dim, sdim); break;
-      case 4: Assemble_<4>(kernel, dim, sdim); break;
-      case 5: Assemble_<5>(kernel, dim, sdim); break;
-      case 6: Assemble_<6>(kernel, dim, sdim); break;
-      case 7: Assemble_<7>(kernel, dim, sdim); break;
-      case 8: Assemble_<8>(kernel, dim, sdim); break;
-      default: MFEM_ABORT("No kernel order " << order << "!");
-   }
-}
-
 template <typename LOR_KERNEL>
 void BatchedLORAssembly::AssemblyKernel(BilinearForm &a)
 {
   LOR_KERNEL kernel(a, fes_ho, X_vert, sparse_ij, sparse_mapping);

   const int dim = fes_ho.GetMesh()->Dimension();
-   const int sdim = fes_ho.GetMesh()->SpaceDimension();
   const int order = fes_ho.GetMaxElementOrder();

-   Assemble_(kernel, dim, sdim, order);
+   if (dim == 2)
+   {
+      switch (order)
+      {
+         case 1: kernel.template Assemble2D<1>(); break;
+         case 2: kernel.template Assemble2D<2>(); break;
+         case 3: kernel.template Assemble2D<3>(); break;
+         case 4: kernel.template Assemble2D<4>(); break;
+         case 5: kernel.template Assemble2D<5>(); break;
+         case 6: kernel.template Assemble2D<6>(); break;
+         case 7: kernel.template Assemble2D<7>(); break;
+         case 8: kernel.template Assemble2D<8>(); break;
+         default: MFEM_ABORT("No kernel order " << order << "!");
+      }
+   }
+   else if (dim == 3)
+   {
+      switch (order)
+      {
+         case 1: kernel.template Assemble3D<1>(); break;
+         case 2: kernel.template Assemble3D<2>(); break;
+         case 3: kernel.template Assemble3D<3>(); break;
+         case 4: kernel.template Assemble3D<4>(); break;
+         case 5: kernel.template Assemble3D<5>(); break;
+         case 6: kernel.template Assemble3D<6>(); break;
+         case 7: kernel.template Assemble3D<7>(); break;
+         case 8: kernel.template Assemble3D<8>(); break;
+         default: MFEM_ABORT("No kernel order " << order << "!");
+      }
+   }
 }

 void BatchedLORAssembly::AssembleWithoutBC(BilinearForm &a, OperatorHandle &A)
@@ -9,6 +9,7 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

+#include "lor_h1.hpp"
 #include "lor_util.hpp"
 #include "../../linalg/dtensor.hpp"
 #include "../../general/forall.hpp"
@@ -16,7 +17,7 @@
 namespace mfem
 {

-template <int ORDER, int SDIM>
+template <int ORDER>
 void BatchedLOR_H1::Assemble2D()
 {
   const int nel_ho = fes_ho.GetNE();
@@ -73,8 +74,31 @@ void BatchedLOR_H1::Assemble2D()

            for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }

-            SetupLORQuadData2D<ORDER,SDIM,false,false>(X, iel_ho, kx, ky, Q, false);
+            double vx[4], vy[4];
+            LORVertexCoordinates2D<ORDER>(X, iel_ho, kx, ky, vx, vy);

+            for (int iqy=0; iqy<2; ++iqy)
+            {
+               for (int iqx=0; iqx<2; ++iqx)
+               {
+                  const double x = iqx;
+                  const double y = iqy;
+                  const double w = 1.0/4.0;
+
+                  double J_[2*2];
+                  DeviceTensor<2> J(J_, 2, 2);
+
+                  Jacobian2D(x, y, vx, vy, J);
+
+                  const double detJ = Det2D(J);
+                  const double w_detJ = w/detJ;
+
+                  Q(0,iqy,iqx) = w_detJ * (J(0,1)*J(0,1) + J(1,1)*J(1,1)); // 1,1
+                  Q(1,iqy,iqx) = -w_detJ * (J(0,1)*J(0,0) + J(1,1)*J(1,0)); // 1,2
+                  Q(2,iqy,iqx) = w_detJ * (J(0,0)*J(0,0) + J(1,0)*J(1,0)); // 2,2
+                  Q(3,iqy,iqx) = w*detJ;
+               }
+            }
            for (int iqx=0; iqx<2; ++iqx)
            {
               for (int iqy=0; iqy<2; ++iqy)
@@ -495,4 +519,34 @@ void BatchedLOR_H1::Assemble3D()
   }
 }

+// Explicit template instantiations
+template void BatchedLOR_H1::Assemble2D<1>();
+template void BatchedLOR_H1::Assemble2D<2>();
+template void BatchedLOR_H1::Assemble2D<3>();
+template void BatchedLOR_H1::Assemble2D<4>();
+template void BatchedLOR_H1::Assemble2D<5>();
+template void BatchedLOR_H1::Assemble2D<6>();
+template void BatchedLOR_H1::Assemble2D<7>();
+template void BatchedLOR_H1::Assemble2D<8>();
+
+template void BatchedLOR_H1::Assemble3D<1>();
+template void BatchedLOR_H1::Assemble3D<2>();
+template void BatchedLOR_H1::Assemble3D<3>();
+template void BatchedLOR_H1::Assemble3D<4>();
+template void BatchedLOR_H1::Assemble3D<5>();
+template void BatchedLOR_H1::Assemble3D<6>();
+template void BatchedLOR_H1::Assemble3D<7>();
+template void BatchedLOR_H1::Assemble3D<8>();
+
+BatchedLOR_H1::BatchedLOR_H1(BilinearForm &a,
+                             FiniteElementSpace &fes_ho_,
+                             Vector &X_vert_,
+                             Vector &sparse_ij_,
+                             Array<int> &sparse_mapping_)
+   : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
+{
+   ProjectLORCoefficient<MassIntegrator>(a, c1);
+   ProjectLORCoefficient<DiffusionIntegrator>(a, c2);
+}
+
 } // namespace mfem
@@ -22,22 +22,15 @@ namespace mfem
 class BatchedLOR_H1 : BatchedLORKernel
 {
 public:
-   template <int ORDER, int SDIM> void Assemble2D();
+   template <int ORDER> void Assemble2D();
   template <int ORDER> void Assemble3D();
   BatchedLOR_H1(BilinearForm &a,
                 FiniteElementSpace &fes_ho_,
                 Vector &X_vert_,
                 Vector &sparse_ij_,
-                 Array<int> &sparse_mapping_)
-      : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
-   {
-      ProjectLORCoefficient<MassIntegrator>(a, c1);
-      ProjectLORCoefficient<DiffusionIntegrator>(a, c2);
-   }
+                 Array<int> &sparse_mapping_);
 };

 }

-#include "lor_h1_impl.hpp"
-
 #endif
@@ -9,6 +9,7 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

+#include "lor_nd.hpp"
 #include "lor_util.hpp"
 #include "../../linalg/dtensor.hpp"
 #include "../../general/forall.hpp"
@@ -16,7 +17,7 @@
 namespace mfem
 {

-template <int ORDER, int SDIM>
+template <int ORDER>
 void BatchedLOR_ND::Assemble2D()
 {
   const int nel_ho = fes_ho.GetNE();
@@ -82,8 +83,31 @@ void BatchedLOR_ND::Assemble2D()
            // local_mat is the local (dense) stiffness matrix
            for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }

-            SetupLORQuadData2D<ORDER,SDIM,false,true>(X, iel_ho, kx, ky, Q, true);
+            double vx[4], vy[4];
+            LORVertexCoordinates2D<ORDER>(X, iel_ho, kx, ky, vx, vy);

+            for (int iqx=0; iqx<2; ++iqx)
+            {
+               for (int iqy=0; iqy<2; ++iqy)
+               {
+                  const double x = iqx;
+                  const double y = iqy;
+                  const double w = 1.0/4.0;
+
+                  double J_[2*2];
+                  DeviceTensor<2> J(J_, 2, 2);
+
+                  Jacobian2D(x, y, vx, vy, J);
+
+                  const double detJ = Det2D(J);
+                  const double w_detJ = w/detJ;
+
+                  Q(0,iqy,iqx) = w_detJ * (J(0,1)*J(0,1) + J(1,1)*J(1,1)); // 1,1
+                  Q(1,iqy,iqx) = -w_detJ * (J(0,1)*J(0,0) + J(1,1)*J(1,0)); // 1,2
+                  Q(2,iqy,iqx) = w_detJ * (J(0,0)*J(0,0) + J(1,0)*J(1,0)); // 2,2
+                  Q(3,iqy,iqx) = w_detJ;
+               }
+            }
            for (int iqx=0; iqx<2; ++iqx)
            {
               for (int iqy=0; iqy<2; ++iqy)
@@ -539,4 +563,34 @@ void BatchedLOR_ND::Assemble3D()
   }
 }

+// Explicit template instantiations
+template void BatchedLOR_ND::Assemble2D<1>();
+template void BatchedLOR_ND::Assemble2D<2>();
+template void BatchedLOR_ND::Assemble2D<3>();
+template void BatchedLOR_ND::Assemble2D<4>();
+template void BatchedLOR_ND::Assemble2D<5>();
+template void BatchedLOR_ND::Assemble2D<6>();
+template void BatchedLOR_ND::Assemble2D<7>();
+template void BatchedLOR_ND::Assemble2D<8>();
+
+template void BatchedLOR_ND::Assemble3D<1>();
+template void BatchedLOR_ND::Assemble3D<2>();
+template void BatchedLOR_ND::Assemble3D<3>();
+template void BatchedLOR_ND::Assemble3D<4>();
+template void BatchedLOR_ND::Assemble3D<5>();
+template void BatchedLOR_ND::Assemble3D<6>();
+template void BatchedLOR_ND::Assemble3D<7>();
+template void BatchedLOR_ND::Assemble3D<8>();
+
+BatchedLOR_ND::BatchedLOR_ND(BilinearForm &a,
+                             FiniteElementSpace &fes_ho_,
+                             Vector &X_vert_,
+                             Vector &sparse_ij_,
+                             Array<int> &sparse_mapping_)
+   : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
+{
+   ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
+   ProjectLORCoefficient<CurlCurlIntegrator>(a, c2);
+}
+
 } // namespace mfem
@@ -22,22 +22,15 @@ namespace mfem
 class BatchedLOR_ND : BatchedLORKernel
 {
 public:
-   template <int ORDER, int SDIM> void Assemble2D();
+   template <int ORDER> void Assemble2D();
   template <int ORDER> void Assemble3D();
   BatchedLOR_ND(BilinearForm &a,
                 FiniteElementSpace &fes_ho_,
                 Vector &X_vert_,
                 Vector &sparse_ij_,
-                 Array<int> &sparse_mapping_)
-      : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
-   {
-      ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
-      ProjectLORCoefficient<CurlCurlIntegrator>(a, c2);
-   }
+                 Array<int> &sparse_mapping_);
 };

 }

-#include "lor_nd_impl.hpp"
-
 #endif
@@ -9,6 +9,7 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

+#include "lor_rt.hpp"
 #include "lor_util.hpp"
 #include "../../linalg/dtensor.hpp"
 #include "../../general/forall.hpp"
@@ -16,7 +17,7 @@
 namespace mfem
 {

-template <int ORDER, int SDIM>
+template <int ORDER>
 void BatchedLOR_RT::Assemble2D()
 {
   const int nel_ho = fes_ho.GetNE();
@@ -78,8 +79,31 @@ void BatchedLOR_RT::Assemble2D()
            // local_mat is the local (dense) stiffness matrix
            for (int i=0; i<sz_local_mat; ++i) { local_mat[i] = 0.0; }

-            SetupLORQuadData2D<ORDER,SDIM,true,false>(X, iel_ho, kx, ky, Q, true);
+            double vx[4], vy[4];
+            LORVertexCoordinates2D<ORDER>(X, iel_ho, kx, ky, vx, vy);

+            for (int iqx=0; iqx<2; ++iqx)
+            {
+               for (int iqy=0; iqy<2; ++iqy)
+               {
+                  const double x = iqx;
+                  const double y = iqy;
+                  const double w = 1.0/4.0;
+
+                  double J_[2*2];
+                  DeviceTensor<2> J(J_, 2, 2);
+
+                  Jacobian2D(x, y, vx, vy, J);
+
+                  const double detJ = Det2D(J);
+                  const double w_detJ = w/detJ;
+
+                  Q(0,iqy,iqx) = w_detJ * (J(0,0)*J(0,0) + J(1,0)*J(1,0)); // 1,1
+                  Q(1,iqy,iqx) = w_detJ * (J(0,0)*J(0,1) + J(1,0)*J(1,1)); // 1,2
+                  Q(2,iqy,iqx) = w_detJ * (J(0,1)*J(0,1) + J(1,1)*J(1,1)); // 2,2
+                  Q(3,iqy,iqx) = w_detJ;
+               }
+            }
            for (int iqx=0; iqx<2; ++iqx)
            {
               for (int iqy=0; iqy<2; ++iqy)
@@ -523,4 +547,34 @@ void BatchedLOR_RT::Assemble3D()
   }
 }

+// Explicit template instantiations
+template void BatchedLOR_RT::Assemble2D<1>();
+template void BatchedLOR_RT::Assemble2D<2>();
+template void BatchedLOR_RT::Assemble2D<3>();
+template void BatchedLOR_RT::Assemble2D<4>();
+template void BatchedLOR_RT::Assemble2D<5>();
+template void BatchedLOR_RT::Assemble2D<6>();
+template void BatchedLOR_RT::Assemble2D<7>();
+template void BatchedLOR_RT::Assemble2D<8>();
+
+template void BatchedLOR_RT::Assemble3D<1>();
+template void BatchedLOR_RT::Assemble3D<2>();
+template void BatchedLOR_RT::Assemble3D<3>();
+template void BatchedLOR_RT::Assemble3D<4>();
+template void BatchedLOR_RT::Assemble3D<5>();
+template void BatchedLOR_RT::Assemble3D<6>();
+template void BatchedLOR_RT::Assemble3D<7>();
+template void BatchedLOR_RT::Assemble3D<8>();
+
+BatchedLOR_RT::BatchedLOR_RT(BilinearForm &a,
+                             FiniteElementSpace &fes_ho_,
+                             Vector &X_vert_,
+                             Vector &sparse_ij_,
+                             Array<int> &sparse_mapping_)
+   : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
+{
+   ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
+   ProjectLORCoefficient<DivDivIntegrator>(a, c2);
+}
+
 } // namespace mfem
@@ -22,22 +22,15 @@ namespace mfem
 class BatchedLOR_RT : BatchedLORKernel
 {
 public:
-   template <int ORDER, int SDIM> void Assemble2D();
+   template <int ORDER> void Assemble2D();
   template <int ORDER> void Assemble3D();
   BatchedLOR_RT(BilinearForm &a,
                 FiniteElementSpace &fes_ho_,
                 Vector &X_vert_,
                 Vector &sparse_ij_,
-                 Array<int> &sparse_mapping_)
-      : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
-   {
-      ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
-      ProjectLORCoefficient<DivDivIntegrator>(a, c2);
-   }
+                 Array<int> &sparse_mapping_);
 };

 }

-#include "lor_rt_impl.hpp"
-
 #endif
@@ -20,22 +20,11 @@
 namespace mfem
 {

-MFEM_HOST_DEVICE inline double Det2D(DeviceMatrix &J)
-{
-   return J(0,0)*J(1,1) - J(1,0)*J(0,1);
-}
-
-MFEM_HOST_DEVICE inline double Det3D(DeviceMatrix &J)
-{
-   return J(0,0) * (J(1,1) * J(2,2) - J(2,1) * J(1,2)) -
-          J(1,0) * (J(0,1) * J(2,2) - J(2,1) * J(0,2)) +
-          J(2,0) * (J(0,1) * J(1,2) - J(1,1) * J(0,2));
-}
-
-template <int ORDER, int SDIM=2>
+template <int ORDER>
 MFEM_HOST_DEVICE inline void LORVertexCoordinates2D(
-   const double *X, int iel_ho, int kx, int ky, double **v)
+   const double *X, int iel_ho, int kx, int ky, double vx[4], double vy[4])
 {
+   const int dim = 2;
   const int nd1d = ORDER + 1;
   const int nvert_per_el = nd1d*nd1d;

@@ -44,31 +33,23 @@ MFEM_HOST_DEVICE inline void LORVertexCoordinates2D(
   const int v2 = kx + 1 + nd1d*(ky + 1);
   const int v3 = kx + nd1d*(ky + 1);

-   const int e0 = SDIM*(v0 + nvert_per_el*iel_ho);
-   const int e1 = SDIM*(v1 + nvert_per_el*iel_ho);
-   const int e2 = SDIM*(v2 + nvert_per_el*iel_ho);
-   const int e3 = SDIM*(v3 + nvert_per_el*iel_ho);
+   const int e0 = dim*(v0 + nvert_per_el*iel_ho);
+   const int e1 = dim*(v1 + nvert_per_el*iel_ho);
+   const int e2 = dim*(v2 + nvert_per_el*iel_ho);
+   const int e3 = dim*(v3 + nvert_per_el*iel_ho);

   // Vertex coordinates
-   v[0][0] = X[e0 + 0];
-   v[1][0] = X[e0 + 1];
+   vx[0] = X[e0 + 0];
+   vy[0] = X[e0 + 1];

-   v[0][1] = X[e1 + 0];
-   v[1][1] = X[e1 + 1];
+   vx[1] = X[e1 + 0];
+   vy[1] = X[e1 + 1];

-   v[0][2] = X[e2 + 0];
-   v[1][2] = X[e2 + 1];
+   vx[2] = X[e2 + 0];
+   vy[2] = X[e2 + 1];

-   v[0][3] = X[e3 + 0];
-   v[1][3] = X[e3 + 1];
-
-   if (SDIM == 3)
-   {
-      v[2][0] = X[e0 + 2];
-      v[2][1] = X[e1 + 2];
-      v[2][2] = X[e2 + 2];
-      v[2][3] = X[e3 + 2];
-   }
+   vx[3] = X[e3 + 0];
+   vy[3] = X[e3 + 1];
 }

 template <int ORDER>
@@ -131,80 +112,15 @@ MFEM_HOST_DEVICE inline void LORVertexCoordinates3D(
   vz[7] = X[e7 + 2];
 }

-template <int SDIM=2>
 MFEM_HOST_DEVICE inline void Jacobian2D(
-   const double x, const double y, double **v, DeviceMatrix &J);
-
-template <> MFEM_HOST_DEVICE inline void Jacobian2D<2>(
-   const double x, const double y, double **v, DeviceMatrix &J)
+   const double x, const double y, const double vx[4], const double vy[4],
+   DeviceMatrix &J)
 {
-   J(0,0) = -(1-y)*v[0][0] + (1-y)*v[0][1] + y*v[0][2] - y*v[0][3];
-   J(0,1) = -(1-x)*v[0][0] - x*v[0][1] + x*v[0][2] + (1-x)*v[0][3];
+   J(0,0) = -(1-y)*vx[0] + (1-y)*vx[1] + y*vx[2] - y*vx[3];
+   J(0,1) = -(1-x)*vx[0] - x*vx[1] + x*vx[2] + (1-x)*vx[3];

-   J(1,0) = -(1-y)*v[1][0] + (1-y)*v[1][1] + y*v[1][2] - y*v[1][3];
-   J(1,1) = -(1-x)*v[1][0] - x*v[1][1] + x*v[1][2] + (1-x)*v[1][3];
-}
-
-template <> MFEM_HOST_DEVICE inline void Jacobian2D<3>(
-   const double x, const double y, double **v, DeviceMatrix &J)
-{
-   J(0,0) = -(1-y)*v[0][0] + (1-y)*v[0][1] + y*v[0][2] - y*v[0][3];
-   J(0,1) = -(1-x)*v[0][0] - x*v[0][1] + x*v[0][2] + (1-x)*v[0][3];
-
-   J(1,0) = -(1-y)*v[1][0] + (1-y)*v[1][1] + y*v[1][2] - y*v[1][3];
-   J(1,1) = -(1-x)*v[1][0] - x*v[1][1] + x*v[1][2] + (1-x)*v[1][3];
-
-   J(2,0) = -(1-y)*v[2][0] + (1-y)*v[2][1] + y*v[2][2] - y*v[2][3];
-   J(2,1) = -(1-x)*v[2][0] - x*v[2][1] + x*v[2][2] + (1-x)*v[2][3];
-}
-
-template <int ORDER, int SDIM, bool RT, bool ND>
-MFEM_HOST_DEVICE inline void SetupLORQuadData2D(
-   const double *X, int iel_ho, int kx, int ky, DeviceTensor<3> &Q, bool piola)
-{
-   double vx[4], vy[4], vz[4];
-   double *v[] = {vx, vy, vz};
-   LORVertexCoordinates2D<ORDER,SDIM>(X, iel_ho, kx, ky, v);
-
-   for (int iqy=0; iqy<2; ++iqy)
-   {
-      for (int iqx=0; iqx<2; ++iqx)
-      {
-         const double x = iqx;
-         const double y = iqy;
-         const double w = 1.0/4.0;
-
-         double J_[SDIM*2];
-         DeviceTensor<2> J(J_, SDIM, 2);
-
-         Jacobian2D<SDIM>(x, y, v, J);
-
-         if (SDIM == 2)
-         {
-            const double detJ = Det2D(J);
-            const double w_detJ = w/detJ;
-            const double E = J(0,0)*J(0,0) + J(1,0)*J(1,0);
-            const double F = J(0,0)*J(0,1) + J(1,0)*J(1,1);
-            const double G = J(0,1)*J(0,1) + J(1,1)*J(1,1);
-            Q(0,iqy,iqx) = w_detJ * (RT ? E : G); // 1,1
-            Q(1,iqy,iqx) = w_detJ * (RT ? F : -F); // 1,2
-            Q(2,iqy,iqx) = w_detJ * (RT ? G : E); // 2,2
-            Q(3,iqy,iqx) = (ND || RT) ? w_detJ : w*detJ;
-         }
-         else
-         {
-            const double E = J(0,0)*J(0,0) + J(1,0)*J(1,0) + J(2,0)*J(2,0);
-            const double F = J(0,0)*J(0,1) + J(1,0)*J(1,1) + J(2,0)*J(2,1);
-            const double G = J(0,1)*J(0,1) + J(1,1)*J(1,1) + J(2,1)*J(2,1);
-            const double detJ = sqrt(E*G - F*F);
-            const double w_detJ = w/detJ;
-            Q(0,iqy,iqx) = w_detJ * (RT ? E : G); // 1,1
-            Q(1,iqy,iqx) = w_detJ * (RT ? F : -F); // 1,2
-            Q(2,iqy,iqx) = w_detJ * (RT ? G : E); // 2,2
-            Q(3,iqy,iqx) =  (ND || RT) ? w_detJ : w*detJ;
-         }
-      }
-   }
+   J(1,0) = -(1-y)*vy[0] + (1-y)*vy[1] + y*vy[2] - y*vy[3];
+   J(1,1) = -(1-x)*vy[0] - x*vy[1] + x*vy[2] + (1-x)*vy[3];
 }

 MFEM_HOST_DEVICE inline void Jacobian3D(
@@ -264,6 +180,18 @@ MFEM_HOST_DEVICE inline void Adjugate3D(const DeviceMatrix &J, DeviceMatrix &A)
   A(2,2) = (J(0,0) * J(1,1)) - (J(0,1) * J(1,0));
 }

+MFEM_HOST_DEVICE inline double Det2D(DeviceMatrix &J)
+{
+   return J(0,0)*J(1,1) - J(1,0)*J(0,1);
+}
+
+MFEM_HOST_DEVICE inline double Det3D(DeviceMatrix &J)
+{
+   return J(0,0) * (J(1,1) * J(2,2) - J(2,1) * J(1,2)) -
+          J(1,0) * (J(0,1) * J(2,2) - J(2,1) * J(0,2)) +
+          J(2,0) * (J(0,1) * J(1,2) - J(1,1) * J(0,2));
+}
+
 }

 #endif
@@ -97,37 +97,12 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
   const FiniteElement *fe;
   ElementTransformation *T;
   DofTransformation *doftrans;
-   Mesh *mesh = fes->GetMesh();
   double energy = 0.0;

   if (dnfi.Size())
   {
-      // Which attributes need to be processed?
-      Array<int> attr_marker(mesh->attributes.Size() ?
-                             mesh->attributes.Max() : 0);
-      attr_marker = 0;
-      for (int k = 0; k < dnfi.Size(); k++)
-      {
-         if (dnfi_marker[k] == NULL)
-         {
-            attr_marker = 1;
-            break;
-         }
-         Array<int> &marker = *dnfi_marker[k];
-         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
-                     "invalid marker for domain integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < attr_marker.Size(); i++)
-         {
-            attr_marker[i] |= marker[i];
-         }
-      }
-
      for (int i = 0; i < fes->GetNE(); i++)
      {
-         const int attr = mesh->GetAttribute(i);
-         if (attr_marker[attr-1] == 0) { continue; }
-
         fe = fes->GetFE(i);
         doftrans = fes->GetElementVDofs(i, vdofs);
         T = fes->GetElementTransformation(i);
@@ -135,9 +110,6 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
         if (doftrans) {doftrans->InvTransformPrimal(el_x); }
         for (int k = 0; k < dnfi.Size(); k++)
         {
-            if (dnfi_marker[k] &&
-                (*dnfi_marker[k])[attr-1] == 0) { continue; }
-
            energy += dnfi[k]->GetElementEnergy(*fe, *T, el_x);
         }
      }
@@ -203,32 +175,8 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const

   if (dnfi.Size())
   {
-      // Which attributes need to be processed?
-      Array<int> attr_marker(mesh->attributes.Size() ?
-                             mesh->attributes.Max() : 0);
-      attr_marker = 0;
-      for (int k = 0; k < dnfi.Size(); k++)
-      {
-         if (dnfi_marker[k] == NULL)
-         {
-            attr_marker = 1;
-            break;
-         }
-         Array<int> &marker = *dnfi_marker[k];
-         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
-                     "invalid marker for domain integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < attr_marker.Size(); i++)
-         {
-            attr_marker[i] |= marker[i];
-         }
-      }
-
      for (int i = 0; i < fes->GetNE(); i++)
      {
-         const int attr = mesh->GetAttribute(i);
-         if (attr_marker[attr-1] == 0) { continue; }
-
         fe = fes->GetFE(i);
         doftrans = fes->GetElementVDofs(i, vdofs);
         T = fes->GetElementTransformation(i);
@@ -236,9 +184,6 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const
         if (doftrans) {doftrans->InvTransformPrimal(el_x); }
         for (int k = 0; k < dnfi.Size(); k++)
         {
-            if (dnfi_marker[k] &&
-                (*dnfi_marker[k])[attr-1] == 0) { continue; }
-
            dnfi[k]->AssembleElementVector(*fe, *T, el_x, el_y);
            if (doftrans) {doftrans->TransformDual(el_y); }
            py.AddElementVector(vdofs, el_y);
@@ -377,32 +322,8 @@ Operator &NonlinearForm::GetGradient(const Vector &x) const

   if (dnfi.Size())
   {
-      // Which attributes need to be processed?
-      Array<int> attr_marker(mesh->attributes.Size() ?
-                             mesh->attributes.Max() : 0);
-      attr_marker = 0;
-      for (int k = 0; k < dnfi.Size(); k++)
-      {
-         if (dnfi_marker[k] == NULL)
-         {
-            attr_marker = 1;
-            break;
-         }
-         Array<int> &marker = *dnfi_marker[k];
-         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
-                     "invalid marker for domain integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < attr_marker.Size(); i++)
-         {
-            attr_marker[i] |= marker[i];
-         }
-      }
-
      for (int i = 0; i < fes->GetNE(); i++)
      {
-         const int attr = mesh->GetAttribute(i);
-         if (attr_marker[attr-1] == 0) { continue; }
-
         fe = fes->GetFE(i);
         doftrans = fes->GetElementVDofs(i, vdofs);
         T = fes->GetElementTransformation(i);
@@ -410,9 +331,6 @@ Operator &NonlinearForm::GetGradient(const Vector &x) const
         if (doftrans) {doftrans->InvTransformPrimal(el_x); }
         for (int k = 0; k < dnfi.Size(); k++)
         {
-            if (dnfi_marker[k] &&
-                (*dnfi_marker[k])[attr-1] == 0) { continue; }
-
            dnfi[k]->AssembleElementGrad(*fe, *T, el_x, elmat);
            if (doftrans) { doftrans->TransformDual(elmat); }
            Grad->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
@@ -643,6 +561,13 @@ BlockNonlinearForm::BlockNonlinearForm(Array<FiniteElementSpace *> &f) :
   SetSpaces(f);
 }

+void BlockNonlinearForm::AddBdrFaceIntegrator(BlockNonlinearFormIntegrator *nfi,
+                                              Array<int> &bdr_attr_marker)
+{
+   bfnfi.Append(nfi);
+   bfnfi_marker.Append(&bdr_attr_marker);
+}
+
 void BlockNonlinearForm::SetEssentialBC(
   const Array<Array<int> *> &bdr_attr_is_ess, Array<Vector *> &rhs)
 {
@@ -667,7 +592,6 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
   Array<const FiniteElement *> fe(fes.Size());
   ElementTransformation *T;
   DofTransformation *doftrans;
-   Mesh *mesh = fes[0]->GetMesh();
   double energy = 0.0;

   for (int i=0; i<fes.Size(); ++i)
@@ -677,33 +601,8 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
   }

   if (dnfi.Size())
-   {
-      // Which attributes need to be processed?
-      Array<int> attr_marker(mesh->attributes.Size() ?
-                             mesh->attributes.Max() : 0);
-      attr_marker = 0;
-      for (int k = 0; k < dnfi.Size(); k++)
-      {
-         if (dnfi_marker[k] == NULL)
-         {
-            attr_marker = 1;
-            break;
-         }
-         Array<int> &marker = *dnfi_marker[k];
-         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
-                     "invalid marker for domain integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < attr_marker.Size(); i++)
-         {
-            attr_marker[i] |= marker[i];
-         }
-      }
-
      for (int i = 0; i < fes[0]->GetNE(); ++i)
      {
-         const int attr = mesh->GetAttribute(i);
-         if (attr_marker[attr-1] == 0) { continue; }
-
         T = fes[0]->GetElementTransformation(i);
         for (int s=0; s<fes.Size(); ++s)
         {
@@ -715,13 +614,9 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const

         for (int k = 0; k < dnfi.Size(); ++k)
         {
-            if (dnfi_marker[k] &&
-                (*dnfi_marker[k])[attr-1] == 0) { continue; }
-
            energy += dnfi[k]->GetElementEnergy(fe, *T, el_x_const);
         }
      }
-   }

   // free the allocated memory
   for (int i = 0; i < fes.Size(); ++i)
@@ -761,7 +656,6 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
   Array<const FiniteElement *> fe2(fes.Size());
   ElementTransformation *T;
   Array<DofTransformation *> doftrans(fes.Size()); doftrans = nullptr;
-   Mesh *mesh = fes[0]->GetMesh();

   by.UseDevice(true);
   by = 0.0;
@@ -776,32 +670,8 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

   if (dnfi.Size())
   {
-      // Which attributes need to be processed?
-      Array<int> attr_marker(mesh->attributes.Size() ?
-                             mesh->attributes.Max() : 0);
-      attr_marker = 0;
-      for (int k = 0; k < dnfi.Size(); k++)
-      {
-         if (dnfi_marker[k] == NULL)
-         {
-            attr_marker = 1;
-            break;
-         }
-         Array<int> &marker = *dnfi_marker[k];
-         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
-                     "invalid marker for domain integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < attr_marker.Size(); i++)
-         {
-            attr_marker[i] |= marker[i];
-         }
-      }
-
      for (int i = 0; i < fes[0]->GetNE(); ++i)
      {
-         const int attr = mesh->GetAttribute(i);
-         if (attr_marker[attr-1] == 0) { continue; }
-
         T = fes[0]->GetElementTransformation(i);
         for (int s = 0; s < fes.Size(); ++s)
         {
@@ -813,9 +683,6 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

         for (int k = 0; k < dnfi.Size(); ++k)
         {
-            if (dnfi_marker[k] &&
-                (*dnfi_marker[k])[attr-1] == 0) { continue; }
-
            dnfi[k]->AssembleElementVector(fe, *T,
                                           el_x_const, el_y);

@@ -831,6 +698,7 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

   if (fnfi.Size())
   {
+      Mesh *mesh = fes[0]->GetMesh();
      FaceElementTransformations *tr;

      for (int i = 0; i < mesh->GetNumFaces(); ++i)
@@ -868,8 +736,8 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,

   if (bfnfi.Size())
   {
+      Mesh *mesh = fes[0]->GetMesh();
      FaceElementTransformations *tr;
-
      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
@@ -990,7 +858,6 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
   Array<const FiniteElement *>fe2(fes.Size());
   ElementTransformation * T;
   Array<DofTransformation *> doftrans(fes.Size()); doftrans = nullptr;
-   Mesh *mesh = fes[0]->GetMesh();

   for (int i=0; i<fes.Size(); ++i)
   {
@@ -1021,32 +888,8 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const

   if (dnfi.Size())
   {
-      // Which attributes need to be processed?
-      Array<int> attr_marker(mesh->attributes.Size() ?
-                             mesh->attributes.Max() : 0);
-      attr_marker = 0;
-      for (int k = 0; k < dnfi.Size(); k++)
-      {
-         if (dnfi_marker[k] == NULL)
-         {
-            attr_marker = 1;
-            break;
-         }
-         Array<int> &marker = *dnfi_marker[k];
-         MFEM_ASSERT(marker.Size() == attr_marker.Size(),
-                     "invalid marker for domain integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < attr_marker.Size(); i++)
-         {
-            attr_marker[i] |= marker[i];
-         }
-      }
-
      for (int i = 0; i < fes[0]->GetNE(); ++i)
      {
-         const int attr = mesh->GetAttribute(i);
-         if (attr_marker[attr-1] == 0) { continue; }
-
         T = fes[0]->GetElementTransformation(i);
         for (int s = 0; s < fes.Size(); ++s)
         {
@@ -1058,9 +901,6 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const

         for (int k = 0; k < dnfi.Size(); ++k)
         {
-            if (dnfi_marker[k] &&
-                (*dnfi_marker[k])[attr-1] == 0) { continue; }
-
            dnfi[k]->AssembleElementGrad(fe, *T, el_x_const, elmats);

            for (int j=0; j<fes.Size(); ++j)
@@ -1083,6 +923,7 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
   if (fnfi.Size())
   {
      FaceElementTransformations *tr;
+      Mesh *mesh = fes[0]->GetMesh();

      for (int i = 0; i < mesh->GetNumFaces(); ++i)
      {
@@ -1119,6 +960,7 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
   if (bfnfi.Size())
   {
      FaceElementTransformations *tr;
+      Mesh *mesh = fes[0]->GetMesh();

      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
@@ -37,7 +37,6 @@ protected:

   /// Set of Domain Integrators to be assembled (added).
   Array<NonlinearFormIntegrator*> dnfi; // owned
-   Array<Array<int>*>              dnfi_marker; // not owned

   /// Set of interior face Integrators to be assembled (added).
   Array<NonlinearFormIntegrator*> fnfi; // owned
@@ -109,12 +108,7 @@ public:

   /// Adds new Domain Integrator.
   void AddDomainIntegrator(NonlinearFormIntegrator *nlfi)
-   { dnfi.Append(nlfi); dnfi_marker.Append(NULL); }
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(NonlinearFormIntegrator *nlfi,
-                            Array<int> &elem_marker)
-   { dnfi.Append(nlfi); dnfi_marker.Append(&elem_marker); }
+   { dnfi.Append(nlfi); }

   /// Access all integrators added with AddDomainIntegrator().
   Array<NonlinearFormIntegrator*> *GetDNFI() { return &dnfi; }
@@ -233,14 +227,13 @@ protected:

   /// Set of Domain Integrators to be assembled (added).
   Array<BlockNonlinearFormIntegrator*> dnfi;
-   Array<Array<int>*>                   dnfi_marker;

   /// Set of interior face Integrators to be assembled (added).
   Array<BlockNonlinearFormIntegrator*> fnfi;

   /// Set of Boundary Face Integrators to be assembled (added).
   Array<BlockNonlinearFormIntegrator*> bfnfi;
-   Array<Array<int>*>                   bfnfi_marker;
+   Array<Array<int>*>           bfnfi_marker;

   /** Auxiliary block-vectors for wrapping input and output vectors or holding
       GridFunction-like block-vector data (e.g. in parallel). */
@@ -305,12 +298,7 @@ public:

   /// Adds new Domain Integrator.
   void AddDomainIntegrator(BlockNonlinearFormIntegrator *nlfi)
-   { dnfi.Append(nlfi); dnfi_marker.Append(NULL); }
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(BlockNonlinearFormIntegrator *nlfi,
-                            Array<int> &elem_marker)
-   { dnfi.Append(nlfi); dnfi_marker.Append(&elem_marker); }
+   { dnfi.Append(nlfi); }

   /// Adds new Interior Face Integrator.
   void AddInteriorFaceIntegrator(BlockNonlinearFormIntegrator *nlfi)
@@ -323,8 +311,7 @@ public:
   /** @brief Adds new Boundary Face Integrator, restricted to specific boundary
       attributes. */
   void AddBdrFaceIntegrator(BlockNonlinearFormIntegrator *nlfi,
-                             Array<int> &bdr_marker)
-   { bfnfi.Append(nlfi); bfnfi_marker.Append(&bdr_marker); }
+                             Array<int> &bdr_marker);

   virtual void SetEssentialBC(const Array<Array<int> *>&bdr_attr_is_ess,
                               Array<Vector *> &rhs);
@@ -266,9 +266,6 @@ public:
      test_pfes  = test_fes;
   }

-   /// Return the test parallel FE space associated with the ParMixedBilinearForm.
-   ParFiniteElementSpace *TestParFESpace() const { return test_pfes; }
-
   /// Returns the matrix assembled on the true dofs, i.e. P_test^t A P_trial.
   HypreParMatrix *ParallelAssemble();

@@ -466,54 +466,53 @@ void ParFiniteElementSpace::ApplyLDofSigns(Table &el_dof) const
   ApplyLDofSigns(all_dofs);
 }

-void ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs,
-                                           DofTransformation &doftrans) const
+DofTransformation *
+ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs) const
 {
   if (elem_dof)
   {
      elem_dof->GetRow(i, dofs);

-      if (DoFTransArray[mesh->GetElementBaseGeometry(i)])
+      if (DoFTrans[mesh->GetElementBaseGeometry(i)])
      {
         Array<int> Fo;
         elem_fos->GetRow(i, Fo);
-         doftrans.SetDofTransformation(
-            *DoFTransArray[mesh->GetElementBaseGeometry(i)]);
-         doftrans.SetFaceOrientations(Fo);
-         doftrans.SetVDim();
+         DoFTrans[mesh->GetElementBaseGeometry(i)]->SetFaceOrientations(Fo);
+         return DoFTrans[mesh->GetElementBaseGeometry(i)];
      }
-      return;
+      return NULL;
   }
-   FiniteElementSpace::GetElementDofs(i, dofs, doftrans);
+   DofTransformation * doftrans = FiniteElementSpace::GetElementDofs(i, dofs);
   if (Conforming())
   {
      ApplyLDofSigns(dofs);
   }
+   return doftrans;
 }

-void ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs,
-                                              DofTransformation &doftrans) const
+DofTransformation *
+ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs) const
 {
   if (bdr_elem_dof)
   {
      bdr_elem_dof->GetRow(i, dofs);

-      if (DoFTransArray[mesh->GetBdrElementBaseGeometry(i)])
+      if (DoFTrans[mesh->GetBdrElementBaseGeometry(i)])
      {
         Array<int> Fo;
-         bdr_elem_fos->GetRow(i, Fo);
-         doftrans.SetDofTransformation(
-            *DoFTransArray[mesh->GetBdrElementBaseGeometry(i)]);
-         doftrans.SetFaceOrientations(Fo);
-         doftrans.SetVDim();
+         bdr_elem_fos -> GetRow (i, Fo);
+         DoFTrans[mesh->GetBdrElementBaseGeometry(i)]->SetFaceOrientations(Fo);
+         return DoFTrans[mesh->GetBdrElementBaseGeometry(i)];
      }
-      return;
+      return NULL;
   }
-   FiniteElementSpace::GetBdrElementDofs(i, dofs, doftrans);
+   DofTransformation * doftrans =
+      FiniteElementSpace::GetBdrElementDofs(i, dofs);
   if (Conforming())
   {
      ApplyLDofSigns(dofs);
   }
+   return doftrans;
 }

 int ParFiniteElementSpace::GetFaceDofs(int i, Array<int> &dofs,
@@ -744,8 +743,6 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
      // Safe to assume 1-1 correspondence between shared dofs
      int ldof  = GetVSize();
      int ltdof = TrueVSize();
-      std::cout << MyRank << " (ldof, ltdof) = ("
-                << ldof << "," << ltdof << ")" << std::endl;

      HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
      HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
@@ -758,11 +755,7 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
      HYPRE_BigInt *cmap   = Memory<HYPRE_BigInt>(ldof-ltdof);

      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
-      std::cout << MyRank << " col_starts: [" << col_starts[0]
-                << "," << col_starts[1] << "]" << std::endl;
      HYPRE_BigInt *row_starts = GetDofOffsets();
-      std::cout << MyRank << " row_starts: [" << row_starts[0]
-                << "," << row_starts[1] << "]" << std::endl;

      Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(ldof-ltdof);

@@ -784,18 +777,6 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
         i_diag[i+1] = diag_counter;
         i_offd[i+1] = offd_counter;
      }
-      std::cout << MyRank << " i_diag: ";
-      for (int i = 0; i < ldof+1; ++i)
-      {
-         std::cout << i_diag[i] << " ";
-      }
-      std::cout << std::endl;
-      std::cout << MyRank << " j_diag: ";
-      for (int i = 0; i < ltdof; ++i)
-      {
-         std::cout << j_diag[i] << " ";
-      }
-      std::cout << std::endl;

      SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_counter);

@@ -804,24 +785,6 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
         cmap[i] = cmap_j_offd[i].one;
         j_offd[cmap_j_offd[i].two] = i;
      }
-      std::cout << MyRank << " i_offd: ";
-      for (int i = 0; i < ldof+1; ++i)
-      {
-         std::cout << i_offd[i] << " ";
-      }
-      std::cout << std::endl;
-      std::cout << MyRank << " j_offd: ";
-      for (int i = 0; i < ldof-ltdof; ++i)
-      {
-         std::cout << j_offd[i] << " ";
-      }
-      std::cout << std::endl;
-      std::cout << MyRank << " cmap: ";
-      for (int i = 0; i < ldof-ltdof; ++i)
-      {
-         std::cout << cmap[i] << " ";
-      }
-      std::cout << std::endl;

      P = new HypreParMatrix(MyComm, MyRank, NRanks, row_starts, col_starts,
                             i_diag, j_diag, i_offd, j_offd,
@@ -976,8 +939,8 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
         }
         else if (i_offd[i+1] == i_offd[i] + 2)
         {
-            const double *T =
-               ND_DofTransformation::GetFaceTransform(ltori[i]).GetData();
+            const double * T = ND_StatelessDofTransformation
+                               ::GetFaceTransform(ltori[i]).GetData();
            j_offd[i_offd[i] + 1] = j_offd[i_offd[i]] + 1;
            d_offd[i_offd[i]] = T[0]; d_offd[i_offd[i] + 1] = T[2];
            i++;
@@ -1491,28 +1454,29 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
   delete [] requests;
 }

-void ParFiniteElementSpace::GetFaceNbrElementVDofs(
-   int i, Array<int> &vdofs, DofTransformation &doftrans) const
-{
-   face_nbr_element_dof.GetRow(i, vdofs);
-
-   if (DoFTransArray[GetFaceNbrFE(i)->GetGeomType()])
-   {
-      Array<int> F, Fo;
-      pmesh->GetFaceNbrElementFaces(pmesh->GetNE() + i, F, Fo);
-      doftrans.SetDofTransformation(
-         *DoFTransArray[GetFaceNbrFE(i)->GetGeomType()]);
-      doftrans.SetFaceOrientations(Fo);
-      doftrans.SetVDim(vdim, ordering);
-   }
-}
-
 DofTransformation *ParFiniteElementSpace::GetFaceNbrElementVDofs(
   int i, Array<int> &vdofs) const
 {
-   DoFTrans.SetDofTransformation(NULL);
-   GetFaceNbrElementVDofs(i, vdofs, DoFTrans);
-   return DoFTrans.GetDofTransformation() ? &DoFTrans : NULL;
+   face_nbr_element_dof.GetRow(i, vdofs);
+
+   DofTransformation *doftrans = NULL;
+   Geometry::Type geom = GetFaceNbrFE(i)->GetGeomType();
+   if (DoFTrans[geom])
+   {
+      Array<int> F, Fo;
+      pmesh->GetFaceNbrElementFaces(pmesh->GetNE() + i, F, Fo);
+      doftrans = DoFTrans[geom];
+      doftrans->SetFaceOrientations(Fo);
+   }
+   if (vdim == 1 || doftrans == NULL)
+   {
+      return doftrans;
+   }
+   else
+   {
+      VDoFTrans.SetDofTransformation(*doftrans);
+      return &VDoFTrans;
+   }
 }

 void ParFiniteElementSpace::GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const
@@ -1982,8 +1946,9 @@ struct PMatrixRow
   void AddRow(const PMatrixRow &other, double coef)
   {
      elems.reserve(elems.size() + other.elems.size());
-      for (const PMatrixElement &oei : other.elems)
+      for (unsigned i = 0; i < other.elems.size(); i++)
      {
+         const PMatrixElement &oei = other.elems[i];
         elems.push_back(
            PMatrixElement(oei.column, oei.stride, coef * oei.value));
      }
@@ -2093,7 +2058,7 @@ void NeighborRowMessage::Encode(int rank)
   for (unsigned i = 0; i < rows.size(); i++)
   {
      const RowInfo &ri = rows[i];
-      const MeshId &id = *pncmesh->GetNCList(ri.entity).GetMeshIdAndType(ri.index).id;
+      const MeshId &id = pncmesh->GetNCList(ri.entity).LookUp(ri.index);
      ent_ids[ri.entity].Append(id);
      row_idx[ri.entity].Append(i);
      group_ids[ri.entity].Append(ri.group);
@@ -2167,33 +2132,25 @@ void NeighborRowMessage::Decode(int rank)
   rows.clear();
   rows.reserve(nrows);

-   // read rows ent = {0,1,2} means vertex, edge and face entity
+   // read rows
   for (int ent = 0, gi = 0; ent < 3; ent++)
   {
-      // extract the vertex list, edge list or face list.
      const Array<MeshId> &ids = ent_ids[ent];
      for (int i = 0; i < ids.Size(); i++)
      {
         const MeshId &id = ids[i];
-         // read the particular element dof value off the stream.
         int edof = bin_io::read<int>(stream);

-         // Handle orientation and sign change. This flips the sign on dofs
-         // where necessary, and for edges and faces also reorders if flipped,
-         // i.e. an edge with 1 -> 2 -> 3 -> 4 might become -4 -> -3 -> -2 -> -1
-         // This cannot treat all face dofs, as they can have rotations and
-         // reflections.
-         const int *ind = nullptr;
-         Geometry::Type geom = Geometry::Type::INVALID;
+         // handle orientation and sign change
+         const int *ind = NULL;
         if (ent == 1)
         {
-            // edge NC orientation is element defined.
            int eo = pncmesh->GetEdgeNCOrientation(id);
            ind = fec->DofOrderForOrientation(Geometry::SEGMENT, eo);
         }
         else if (ent == 2)
         {
-            geom = pncmesh->GetFaceGeometry(id.index);
+            Geometry::Type geom = pncmesh->GetFaceGeometry(id.index);
            int fo = pncmesh->GetFaceOrientation(id.index);
            ind = fec->DofOrderForOrientation(geom, fo);
         }
@@ -2208,14 +2165,13 @@ void NeighborRowMessage::Decode(int rank)
         // If edof arrived with a negative index, flip it, and the scaling.
         double s = (edof < 0) ? -1.0 : 1.0;
         edof = (edof < 0) ? -1 - edof : edof;
+
         if (ind && (edof = ind[edof]) < 0)
         {
            edof = -1 - edof;
            s *= -1.0;
         }

-         // Create a row for this entity, recording the index of the mesh
-         // element
         rows.push_back(RowInfo(ent, id.index, edof, group_ids[gi++]));
         rows.back().row.read(stream, s);

@@ -2225,63 +2181,6 @@ void NeighborRowMessage::Decode(int rank)
                   << rows.back().index << ", edof " << rows.back().edof
                   << std::endl;
 #endif
-
-         if (ent == 2 && fec->GetContType() == FiniteElementCollection::TANGENTIAL
-             && !Geometry::IsTensorProduct(geom))
-         {
-            // ND face dofs need to be processed together, as the transformation
-            // is given by a 2x2 matrix, so we manually apply an extra increment
-            // to the loop counter and add in a new row. Once these rows are
-            // placed, they represent the Identity transformation. To map across
-            // the processor boundary, we also need to apply a Primal
-            // Transformation (see doftrans.hpp) to a notional "global dof"
-            // orientation. For simplicity we perform the action of these 2x2
-            // matrices manually using the AddRow capability, followed by a
-            // Collapse.
-
-            // To perform the operations, we add and subtract initial versions
-            // of the rows, that represent [1 0; 0 1] in row major notation. The
-            // first row represents the 1 at (0,0) in [1 0; 0 1] The second row
-            // represents the 1 at (1,1) in [1 0; 0 1]
-
-            // We can safely bind this reference as rows was reserved above so
-            // there is no hidden copying that could result in a dangling
-            // reference.
-            auto &first_row = rows.back().row;
-            // This is the first "fundamental unit" used in the transformation.
-            const auto initial_first_row = first_row;
-            // Extract the next dof too, and apply any dof order transformation
-            // expected.
-            const MeshId &next_id = ids[++i];
-            const int fo = pncmesh->GetFaceOrientation(next_id.index);
-            ind = fec->DofOrderForOrientation(geom, fo);
-            edof = bin_io::read<int>(stream);
-
-            // If edof arrived with a negative index, flip it, and the scaling.
-            s = (edof < 0) ? -1.0 : 1.0;
-            edof = (edof < 0) ? -1 - edof : edof;
-            if (ind && (edof = ind[edof]) < 0)
-            {
-               edof = -1 - edof;
-               s *= -1.0;
-            }
-            rows.push_back(RowInfo(ent, next_id.index, edof, group_ids[gi++]));
-            rows.back().row.read(stream, s);
-            auto &second_row = rows.back().row;
-
-            // This is the second "fundamental unit" used in the transformation.
-            const auto initial_second_row = second_row;
-            const double *T =
-               ND_DofTransformation::GetFaceTransform(fo).GetData();
-
-            first_row.AddRow(initial_first_row, T[0] - 1.0);
-            first_row.AddRow(initial_second_row, T[2]);
-            second_row.AddRow(initial_first_row, T[1]);
-            second_row.AddRow(initial_second_row, T[3] - 1.0);
-
-            first_row.Collapse();
-            second_row.Collapse();
-         }
      }
   }
 }
@@ -2409,6 +2308,12 @@ int ParFiniteElementSpace
                                       Array<int> *dof_tdof,
                                       bool partial) const
 {
+   // TODO: general face DOF transformations in NeighborRowMessage::Decode()
+   MFEM_VERIFY(!(fec->GetOrder() >= 2
+                 && pmesh->HasGeometry(Geometry::TETRAHEDRON)
+                 && fec->GetContType() == FiniteElementCollection::TANGENTIAL),
+               "Nedelec NC tets of order >= 2 are not supported yet.");
+
   const bool dg = (nvdofs == 0 && nedofs == 0 && nfdofs == 0);

 #ifdef MFEM_PMATRIX_STATS
@@ -190,15 +190,15 @@ private:
   /// Updates the internal mesh pointer. @warning @a new_mesh must be
   /// <b>topologically identical</b> to the existing mesh. Used if the address
   /// of the Mesh object has changed, e.g. in @a Mesh::Swap.
-   void UpdateMeshPointer(Mesh *new_mesh) override;
+   virtual void UpdateMeshPointer(Mesh *new_mesh);

   /// Copies the prolongation and restriction matrices from @a fes.
   ///
   /// Used for low order preconditioning on non-conforming meshes. If the DOFs
   /// require a permutation, it will be supplied by non-NULL @a perm. NULL @a
   /// perm indicates that no permutation is required.
-   void CopyProlongationAndRestriction(const FiniteElementSpace &fes,
-                                       const Array<int> *perm) override;
+   virtual void CopyProlongationAndRestriction(const FiniteElementSpace &fes,
+                                               const Array<int> *perm);

 public:
   // Face-neighbor data
@@ -248,11 +248,7 @@ public:
       If the FiniteElementCollection, @a f, is NULL (default), the FE
       collection used by @a global_fes will be reused. If @a f is not NULL, it
       must be the same as, or a copy of, the FE collection used by
-       @a global_fes.
-
-       @note Currently the @a partitioning array is not used by this
-       constructor, it is required for general parallel variable-order support.
-   */
+       @a global_fes. */
   ParFiniteElementSpace(ParMesh *pm, const FiniteElementSpace *global_fes,
                         const int *partitioning,
                         const FiniteElementCollection *f = NULL);
@@ -286,38 +282,32 @@ public:
   { return Dof_TrueDof_Matrix()->GetGlobalNumCols(); }

   /// Return the number of local vector true dofs.
-   int GetTrueVSize() const override { return ltdof_size; }
+   virtual int GetTrueVSize() const { return ltdof_size; }

-   /// Returns indexes of degrees of freedom in array dofs for i'th element and
-   /// returns the DofTransformation data in a user-provided object.
-   using FiniteElementSpace::GetElementDofs;
-   void GetElementDofs(int i, Array<int> &dofs,
-                       DofTransformation &doftrans) const override;
+   /// Returns indexes of degrees of freedom in array dofs for i'th element.
+   virtual DofTransformation *GetElementDofs(int i, Array<int> &dofs) const;

-   /// Returns indexes of degrees of freedom for i'th boundary element and
-   /// returns the DofTransformation data in a user-provided object.
-   using FiniteElementSpace::GetBdrElementDofs;
-   void GetBdrElementDofs(int i, Array<int> &dofs,
-                          DofTransformation &doftrans) const override;
+   /// Returns indexes of degrees of freedom for i'th boundary element.
+   virtual DofTransformation *GetBdrElementDofs(int i, Array<int> &dofs) const;

   /** Returns the indexes of the degrees of freedom for i'th face
       including the dofs for the edges and the vertices of the face. */
-   int GetFaceDofs(int i, Array<int> &dofs, int variant = 0) const override;
+   virtual int GetFaceDofs(int i, Array<int> &dofs, int variant = 0) const;

   /** Returns pointer to the FiniteElement in the FiniteElementCollection
       associated with i'th element in the mesh object. If @a i is greater than
       or equal to the number of local mesh elements, @a i will be interpreted
       as a shifted index of a face neighbor element. */
-   const FiniteElement *GetFE(int i) const override;
+   virtual const FiniteElement *GetFE(int i) const;

   /** Returns an Operator that converts L-vectors to E-vectors on each face.
       The parallel version is different from the serial one because of the
       presence of shared faces. Shared faces are treated as interior faces,
       the returned operator handles the communication needed to get the
       shared face values from other MPI ranks */
-   const FaceRestriction *GetFaceRestriction(
+   virtual const FaceRestriction *GetFaceRestriction(
      ElementDofOrdering f_ordering, FaceType type,
-      L2FaceValues mul = L2FaceValues::DoubleValued) const override;
+      L2FaceValues mul = L2FaceValues::DoubleValued) const;

   void GetSharedEdgeDofs(int group, int ei, Array<int> &dofs) const;
   void GetSharedTriangleDofs(int group, int fi, Array<int> &dofs) const;
@@ -357,15 +347,15 @@ public:
   void Synchronize(Array<int> &ldof_marker) const;

   /// Determine the boundary degrees of freedom
-   void GetEssentialVDofs(const Array<int> &bdr_attr_is_ess,
-                          Array<int> &ess_dofs,
-                          int component = -1) const override;
+   virtual void GetEssentialVDofs(const Array<int> &bdr_attr_is_ess,
+                                  Array<int> &ess_dofs,
+                                  int component = -1) const;

   /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
       boundary attributes marked in the array bdr_attr_is_ess. */
-   void GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
-                             Array<int> &ess_tdof_list,
-                             int component = -1) override;
+   virtual void GetEssentialTrueDofs(const Array<int> &bdr_attr_is_ess,
+                                     Array<int> &ess_tdof_list,
+                                     int component = -1);

   /** If the given ldof is owned by the current processor, return its local
       tdof number, otherwise return -1 */
@@ -380,20 +370,18 @@ public:
   HYPRE_BigInt GetMyDofOffset() const;
   HYPRE_BigInt GetMyTDofOffset() const;

-   const Operator *GetProlongationMatrix() const override;
+   virtual const Operator *GetProlongationMatrix() const;
   /** Get an Operator that performs the action of GetRestrictionMatrix(),
       but potentially with a non-assembled optimized matrix-free
       implementation. */
-   const Operator *GetRestrictionOperator() const override;
+   virtual const Operator *GetRestrictionOperator() const;
   /// Get the R matrix which restricts a local dof vector to true dof vector.
-   const SparseMatrix *GetRestrictionMatrix() const override
+   virtual const SparseMatrix *GetRestrictionMatrix() const
   { Dof_TrueDof_Matrix(); return R; }

   // Face-neighbor functions
   void ExchangeFaceNbrData();
   int GetFaceNbrVSize() const { return num_face_nbr_dofs; }
-   void GetFaceNbrElementVDofs(int i, Array<int> &vdofs,
-                               DofTransformation &doftrans) const;
   DofTransformation *GetFaceNbrElementVDofs(int i, Array<int> &vdofs) const;
   void GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const;
   const FiniteElement *GetFaceNbrFE(int i) const;
@@ -414,15 +402,15 @@ public:
   // Transfer parallel true-dof data from coarse_fes, defined on a coarse mesh,
   // to this FE space, defined on a refined mesh. See full documentation in the
   // base class, FiniteElementSpace::GetTrueTransferOperator.
-   void GetTrueTransferOperator(const FiniteElementSpace &coarse_fes,
-                                OperatorHandle &T) const override;
+   virtual void GetTrueTransferOperator(const FiniteElementSpace &coarse_fes,
+                                        OperatorHandle &T) const;

   /** Reflect changes in the mesh. Calculate one of the refinement/derefinement
       /rebalance matrices, unless want_transform is false. */
-   void Update(bool want_transform = true) override;
+   virtual void Update(bool want_transform = true);

   /// Free ParGridFunction transformation matrix (if any), to save memory.
-   void UpdatesFinished() override
+   virtual void UpdatesFinished()
   {
      FiniteElementSpace::UpdatesFinished();
      old_dof_offsets.DeleteAll();
@@ -454,9 +442,9 @@ public:

   const GroupCommunicator &GetGroupCommunicator() const;

-   void Mult(const Vector &x, Vector &y) const override;
+   virtual void Mult(const Vector &x, Vector &y) const;

-   void MultTranspose(const Vector &x, Vector &y) const override;
+   virtual void MultTranspose(const Vector &x, Vector &y) const;
 };

 /// Auxiliary device class used by ParFiniteElementSpace.
@@ -505,9 +493,9 @@ public:

   virtual ~DeviceConformingProlongationOperator();

-   void Mult(const Vector &x, Vector &y) const override;
+   virtual void Mult(const Vector &x, Vector &y) const;

-   void MultTranspose(const Vector &x, Vector &y) const override;
+   virtual void MultTranspose(const Vector &x, Vector &y) const;
 };

 }
@@ -469,16 +469,17 @@ void ParGridFunction::GetVectorValue(ElementTransformation &T,
   }

   Array<int> vdofs;
-   DofTransformation * doftrans = pfes->GetFaceNbrElementVDofs(nbr_el_no, vdofs);
+   DofTransformation * doftrans = pfes->GetFaceNbrElementVDofs(nbr_el_no,
+                                                               vdofs);
+   const FiniteElement *fe = pfes->GetFaceNbrFE(nbr_el_no);
+
+   int dof = fe->GetDof();
   Vector loc_data;
   face_nbr_data.GetSubVector(vdofs, loc_data);
   if (doftrans)
   {
      doftrans->InvTransformPrimal(loc_data);
   }
-
-   const FiniteElement *fe = pfes->GetFaceNbrFE(nbr_el_no);
-   const int dof = fe->GetDof();
   if (fe->GetRangeType() == FiniteElement::SCALAR)
   {
      Vector shape(dof);
@@ -693,23 +694,7 @@ void ParGridFunction::ProjectBdrCoefficient(

 #ifdef MFEM_DEBUG
   Array<int> ess_vdofs_marker;
-   if (vcoeff) { pfes->GetEssentialVDofs(attr, ess_vdofs_marker); }
-   else
-   {
-      ess_vdofs_marker.SetSize(Size());
-      ess_vdofs_marker = 0;
-      for (int i = 0; i < fes->GetVDim(); i++)
-      {
-         if (!coeff[i]) { continue; }
-         Array<int> component_dof_marker;
-         pfes->GetEssentialVDofs(attr, component_dof_marker,i);
-         for (int j = 0; j<Size(); j++)
-         {
-            ess_vdofs_marker[j] = bool(ess_vdofs_marker[j]) ||
-                                  bool(component_dof_marker[j]);
-         }
-      }
-   }
+   pfes->GetEssentialVDofs(attr, ess_vdofs_marker);
   for (int i = 0; i < values_counter.Size(); i++)
   {
      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
@@ -212,7 +212,7 @@ public:
   virtual double GetValue(int i, const IntegrationPoint &ip,
                           int vdim = 1) const;
   double GetValue(ElementTransformation &T)
-   { return GetValue(T, T.GetIntPoint()); }
+   { return GetValue(T.ElementNo, T.GetIntPoint()); }

   // Redefine to handle the case when T describes a face-neighbor element
   virtual double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
@@ -177,7 +177,7 @@ int FaceQuadratureSpace::GetEntityIndex(const ElementTransformation &T) const
         return get_face_index(T.ElementNo);
      case ElementTransformation::BDR_ELEMENT:
      case ElementTransformation::BDR_FACE:
-         return get_face_index(mesh.GetBdrElementFaceIndex(T.ElementNo));
+         return get_face_index(mesh.GetBdrElementEdgeIndex(T.ElementNo));
      default:
         MFEM_ABORT("Invalid element type.");
         return -1;
@@ -462,52 +462,21 @@ void TMOP_Metric_009::AssembleH(const DenseMatrix &Jpt,
   ie.Assemble_ddI1b(weight, A.GetData());
 }

-double TMOP_Metric_014::EvalWMatrixForm(const DenseMatrix &Jpt) const
-{
-   // mu_14 = |J - I|^2.
-   DenseMatrix Mat(Jpt);
-   Mat(0,0) -= 1.0;
-   Mat(1,1) -= 1.0;
-   return Mat.FNorm2();
-}
-
+// mu_14 = |T-I|^2
 double TMOP_Metric_014::EvalW(const DenseMatrix &Jpt) const
 {
-   // mu_14 = |J - I|^2 = I1[J-I].
-   DenseMatrix Mat(Jpt);
-   Mat(0,0) -= 1.0;
-   Mat(1,1) -= 1.0;
+   MFEM_VERIFY(Jtr != NULL,
+               "Requires a target Jacobian, use SetTargetJacobian().");

-   ie.SetJacobian(Mat.GetData());
-   return ie.Get_I1();
-}
+   DenseMatrix Id(2,2);

-void TMOP_Metric_014::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
-{
-   // P = dI1[J-I] d/dJ[J-I] = dI1[J-I].
-   DenseMatrix JptMinusId = Jpt;
-   for (int i = 0; i < Jpt.Size(); i++)
-   {
-      JptMinusId(i, i) -= 1.0;
-   }
-   ie.SetJacobian(JptMinusId.GetData());
-   P = ie.Get_dI1();
-}
+   Id(0,0) = 1; Id(0,1) = 0;
+   Id(1,0) = 0; Id(1,1) = 1;

-void TMOP_Metric_014::AssembleH(const DenseMatrix &Jpt,
-                                const DenseMatrix &DS,
-                                const double weight,
-                                DenseMatrix &A) const
-{
-   // dP = ddI1[J-I].
-   DenseMatrix JptMinusId = Jpt;
-   for (int i = 0; i < Jpt.Size(); i++)
-   {
-      JptMinusId(i, i) -= 1.0;
-   }
-   ie.SetJacobian(JptMinusId.GetData());
-   ie.SetDerivativeMatrix(DS.Height(), DS.GetData());
-   ie.Assemble_ddI1(weight, A.GetData());
+   DenseMatrix Mat(2,2);
+   Mat = Jpt;
+   Mat.Add(-1,Id);
+   return Mat.FNorm2();
 }

 double TMOP_Metric_022::EvalW(const DenseMatrix &Jpt) const
@@ -4378,8 +4347,6 @@ UpdateAfterMeshPositionChange(const Vector &x_new,
 {
   if (discr_tc) { PA.Jtr_needs_update = true; }

-   if (PA.enabled) { UpdateCoefficientsPA(x_new); }
-
   Ordering::Type ordering = x_fes.GetOrdering();

   // Update the finite difference delta if FD are used.
@@ -373,20 +373,16 @@ public:
 /// 2D non-barrier Shape+Size+Orientation (VOS) metric (polyconvex).
 class TMOP_Metric_014 : public TMOP_QualityMetric
 {
-protected:
-   mutable InvariantsEvaluator2D<double> ie;
-
 public:
-   // W = |J - I|^2.
-   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
-
-   // W = I1[J-I].
+   // W = |T-I|^2.
   virtual double EvalW(const DenseMatrix &Jpt) const;

-   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+   { MFEM_ABORT("Not implemented"); }

   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
-                          const double weight, DenseMatrix &A) const;
+                          const double weight, DenseMatrix &A) const
+   { MFEM_ABORT("Not implemented"); }
 };

 /// 2D Shifted barrier form of shape metric (mu_2).
@@ -1821,27 +1817,17 @@ protected:

   // PA extension
   // ------------
-   // Jtr: all ref->target Jacobians, (dim x dim) Q-Vector as DenseTensor.
-   //      updated when needed, based on Jtr_needs_update.
-   //
   //  E: Q-vector for TMOP-energy
-   //     Used as temporary storage when the total energy is computed.
   //  O: Q-Vector of 1.0, used to compute sums using the dot product kernel.
   // X0: E-vector for initial nodal coordinates used for limiting.
-   //     Does not change during the TMOP iteration.
   //  H: Q-Vector for Hessian associated with the metric term.
-   //     Updated by every call to PANonlinearFormExtension::GetGradient().
   // C0: Q-Vector for spatial weight used for the limiting term.
-   //     Updated when the mesh nodes change.
   // LD: E-Vector constructed using limiting distance grid function (delta).
-   //     Does not change during the TMOP iteration.
   // H0: Q-Vector for Hessian associated with the limiting term.
-   //     Updated by every call to PANonlinearFormExtension::GetGradient().
-   // MC: Q-Vector for the metric Coefficient.
-   //     Updated when the mesh nodes change.
   //
-   // maps:     Dof2Quad map for fes associated with the nodal coordinates.
-   // maps_lim: Dof2Quad map for fes associated with the limiting dist GridFunc.
+   // maps:     Dof2Quad map for fespace associate with nodal coordinates.
+   // maps_lim: Dof2Quad map for fespace associated with the limiting distance
+   //            grid function.
   //
   // Jtr_debug_grad
   //     We keep track if Jtr was set by AssembleGradPA() in Jtr_debug_grad: it
@@ -1860,7 +1846,7 @@ protected:
      mutable DenseTensor Jtr;
      mutable bool Jtr_needs_update;
      mutable bool Jtr_debug_grad;
-      mutable Vector E, O, X0, H, C0, LD, H0, MC;
+      mutable Vector E, O, X0, H, C0, LD, H0;
      const DofToQuad *maps;
      const DofToQuad *maps_lim = nullptr;
      const GeometricFactors *geom;
@@ -1974,9 +1960,6 @@ protected:

   void AssemblePA_Limiting();
   void ComputeAllElementTargets(const Vector &xe = Vector()) const;
-   // Updates the Q-vectors for the metric_coeff and lim_coeff, based on the
-   // new physical positions of the quadrature points.
-   void UpdateCoefficientsPA(const Vector &x_loc);

   // Compute Min(Det(Jpt)) in the mesh, does not reduce over MPI.
   double ComputeMinDetT(const Vector &x, const FiniteElementSpace &fes);
@@ -176,42 +176,6 @@ void TMOP_Integrator::ComputeAllElementTargets(const Vector &xe) const
   targetC->ComputeAllElementTargets(*fes, ir, xe, PA.Jtr);
 }

-void TMOP_Integrator::UpdateCoefficientsPA(const Vector &x_loc)
-{
-   // Both are constant or not specified.
-   if (PA.MC.Size() == 1 && PA.C0.Size() == 1) { return; }
-
-   // Coefficients are always evaluated on the CPU for now.
-   PA.MC.HostWrite();
-   PA.C0.HostWrite();
-
-   const IntegrationRule &ir = *PA.ir;
-   auto T = new IsoparametricTransformation;
-   for (int e = 0; e < PA.ne; ++e)
-   {
-      // Uses the node positions in x_loc.
-      PA.fes->GetMesh()->GetElementTransformation(e, x_loc, T);
-
-      if (PA.MC.Size() > 1)
-      {
-         for (int q = 0; q < PA.nq; ++q)
-         {
-            PA.MC(q + e * PA.nq) = metric_coeff->Eval(*T, ir.IntPoint(q));
-         }
-      }
-
-      if (PA.C0.Size() > 1)
-      {
-         for (int q = 0; q < PA.nq; ++q)
-         {
-            PA.C0(q + e * PA.nq) = lim_coeff->Eval(*T, ir.IntPoint(q));
-         }
-      }
-   }
-
-   delete T;
-}
-
 void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
 {
   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
@@ -249,35 +213,6 @@ void TMOP_Integrator::AssemblePA(const FiniteElementSpace &fes)
   PA.O.SetSize(ne*nq, Device::GetDeviceMemoryType());
   PA.O = 1.0;

-   if (metric_coeff)
-   {
-      if (auto cc = dynamic_cast<ConstantCoefficient *>(metric_coeff))
-      {
-         PA.MC.SetSize(1, Device::GetMemoryType());
-         PA.MC.HostWrite();
-         PA.MC(0) = cc->constant;
-      }
-      else
-      {
-         PA.MC.SetSize(PA.nq * PA.ne, Device::GetMemoryType());
-         auto M0 = Reshape(PA.MC.HostWrite(), PA.nq, PA.ne);
-         for (int e = 0; e < PA.ne; ++e)
-         {
-            ElementTransformation& T = *PA.fes->GetElementTransformation(e);
-            for (int q = 0; q < ir.GetNPoints(); ++q)
-            {
-               M0(q,e) = metric_coeff->Eval(T, ir.IntPoint(q));
-            }
-         }
-      }
-   }
-   else
-   {
-      PA.MC.SetSize(1, Device::GetMemoryType());
-      PA.MC.HostWrite();
-      PA.MC(0) = 1.0;
-   }
-
   // Setup ref->target Jacobians, PA.Jtr, (dim x dim) Q-vector, DenseTensor
   PA.Jtr.SetSize(dim, dim, PA.ne*PA.nq, mt);
   PA.Jtr_needs_update = true;
@@ -258,7 +258,6 @@ void EvalH_094(const int e, const int qx, const int qy,
 MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
                           const Vector &x_,
                           const double metric_normal,
-                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -274,16 +273,11 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
               || mid == 80 || mid == 94,
               "2D metric not yet implemented!");

-   const bool const_m0 = mc_.Size() == 1;
-
   constexpr int DIM = 2;
   constexpr int NBZ = 1;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

-   const auto MC = const_m0 ?
-                   Reshape(mc_.Read(), 1, 1, 1) :
-                   Reshape(mc_.Read(), Q1D, Q1D, NE);
   const auto W = Reshape(w_.Read(), Q1D, Q1D);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
@@ -318,8 +312,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
         {
            const double *Jtr = &J(0,0,qx,qy,e);
            const double detJtr = kernels::Det<2>(Jtr);
-            const double m_coef = const_m0 ? MC(0,0,0) : MC(qx,qy,e);
-            const double weight = metric_normal * m_coef * W(qx,qy) * detJtr;
+            const double weight = metric_normal * W(qx,qy) * detJtr;

            // Jrt = Jtr^{-1}
            double Jrt[4];
@@ -354,7 +347,6 @@ void TMOP_Integrator::AssembleGradPA_2D(const Vector &X) const
   const int Q1D = PA.maps->nqpt;
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
-   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -367,7 +359,7 @@ void TMOP_Integrator::AssembleGradPA_2D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,MC,mp,M,N,W,B,G,J,H);
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,mp,M,N,W,B,G,J,H);
 }

 } // namespace mfem
@@ -312,7 +312,6 @@ void EvalH_338(const int e, const int qx, const int qy, const int qz,

 MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
                           const double metric_normal,
-                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const Vector &x_,
@@ -329,15 +328,10 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
               mid == 321 || mid == 332 || mid == 338,
               "3D metric not yet implemented!");

-   const bool const_m0 = mc_.Size() == 1;
-
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

-   const auto MC = const_m0 ?
-                   Reshape(mc_.Read(), 1, 1, 1, 1) :
-                   Reshape(mc_.Read(), Q1D, Q1D, Q1D, NE);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
@@ -375,9 +369,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
            {
               const double *Jtr = &J(0,0,qx,qy,qz,e);
               const double detJtr = kernels::Det<3>(Jtr);
-               const double m_coef = const_m0 ? MC(0,0,0,0) : MC(qx,qy,qz,e);
-               const double weight = metric_normal * m_coef *
-                                     W(qx,qy,qz) * detJtr;
+               const double weight = metric_normal * W(qx,qy,qz) * detJtr;

               // Jrt = Jtr^{-1}
               double Jrt[9];
@@ -446,7 +438,6 @@ void TMOP_Integrator::AssembleGradPA_3D(const Vector &X) const
   const int M = metric->Id();
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
-   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -459,7 +450,7 @@ void TMOP_Integrator::AssembleGradPA_3D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,MC,mp,M,X,N,W,B,G,J,H);
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,mp,M,X,N,W,B,G,J,H);
 }

 } // namespace mfem
@@ -98,7 +98,6 @@ void EvalP_094(const double *Jpt, const double *w, double *P)

 MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
                           const double metric_normal,
-                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -115,17 +114,12 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
               || mid == 80 || mid == 94,
               "2D metric not yet implemented!");

-   const bool const_m0 = mc_.Size() == 1;
-
   constexpr int DIM = 2;
   constexpr int NBZ = 1;

   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

-   const auto MC = const_m0 ?
-                   Reshape(mc_.Read(), 1, 1, 1) :
-                   Reshape(mc_.Read(), Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
   const auto W = Reshape(w_.Read(), Q1D, Q1D);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
@@ -160,9 +154,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
         {
            const double *Jtr = &J(0,0,qx,qy,e);
            const double detJtr = kernels::Det<2>(Jtr);
-            const double m_coef = const_m0 ? MC(0,0,0) : MC(qx,qy,e);
-            const double weight = metric_normal * m_coef *
-                                  W(qx,qy) * detJtr;
+            const double weight = metric_normal * W(qx,qy) * detJtr;

            // Jrt = Jtr^{-1}
            double Jrt[4];
@@ -212,7 +204,6 @@ void TMOP_Integrator::AddMultPA_2D(const Vector &X, Vector &Y) const
   const Array<double> &B = PA.maps->B;
   const Array<double> &G = PA.maps->G;
   const double mn = metric_normal;
-   const Vector &MC = PA.MC;

   Array<double> mp;
   if (auto m = dynamic_cast<TMOP_Combo_QualityMetric *>(metric))
@@ -220,7 +211,7 @@ void TMOP_Integrator::AddMultPA_2D(const Vector &X, Vector &Y) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,MC,mp,M,N,J,W,B,G,X,Y);
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,mp,M,N,J,W,B,G,X,Y);
 }

 } // namespace mfem
@@ -131,7 +131,6 @@ void EvalP_338(const double *J, const double *w, double *P)

 MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
                           const double metric_normal,
-                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -148,15 +147,10 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
               mid == 321 || mid == 332 || mid == 338,
               "3D metric not yet implemented!");

-   const bool const_m0 = mc_.Size() == 1;
-
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

-   const auto MC = const_m0 ?
-                   Reshape(mc_.Read(), 1, 1, 1, 1) :
-                   Reshape(mc_.Read(), Q1D, Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
   const auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
@@ -194,9 +188,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
            {
               const double *Jtr = &J(0,0,qx,qy,qz,e);
               const double detJtr = kernels::Det<3>(Jtr);
-               const double m_coef = const_m0 ? MC(0,0,0,0) : MC(qx,qy,qz,e);
-               const double weight = metric_normal * m_coef *
-                                     W(qx,qy,qz) * detJtr;
+               const double weight = metric_normal * W(qx,qy,qz) * detJtr;

               // Jrt = Jtr^{-1}
               double Jrt[9];
@@ -248,7 +240,6 @@ void TMOP_Integrator::AddMultPA_3D(const Vector &X, Vector &Y) const
   const Array<double> &B = PA.maps->B;
   const Array<double> &G = PA.maps->G;
   const double mn = metric_normal;
-   const Vector &MC = PA.MC;

   Array<double> mp;
   if (auto m = dynamic_cast<TMOP_Combo_QualityMetric *>(metric))
@@ -256,7 +247,7 @@ void TMOP_Integrator::AddMultPA_3D(const Vector &X, Vector &Y) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,MC,mp,M,N,J,W,B,G,X,Y);
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,mp,M,N,J,W,B,G,X,Y);
 }

 } // namespace mfem
@@ -73,7 +73,6 @@ double EvalW_094(const double *Jpt, const double *w)

 MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
                           const double metric_normal,
-                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -91,17 +90,12 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
               || mid == 80 || mid == 94,
               "2D metric not yet implemented!");

-   const bool const_m0 = mc_.Size() == 1;
-
   constexpr int DIM = 2;
   constexpr int NBZ = 1;

   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

-   const auto MC = const_m0 ?
-                   Reshape(mc_.Read(), 1, 1, 1) :
-                   Reshape(mc_.Read(), Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, NE);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
@@ -137,8 +131,7 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
         {
            const double *Jtr = &J(0,0,qx,qy,e);
            const double detJtr = kernels::Det<2>(Jtr);
-            const double m_coef = const_m0 ? MC(0,0,0) : MC(qx,qy,e);
-            const double weight = metric_normal * m_coef * W(qx,qy) * detJtr;
+            const double weight = metric_normal * W(qx,qy) * detJtr;

            // Jrt = Jtr^{-1}
            double Jrt[4];
@@ -176,7 +169,6 @@ double TMOP_Integrator::GetLocalStateEnergyPA_2D(const Vector &X) const
   const int Q1D = PA.maps->nqpt;
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
-   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -190,7 +182,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_2D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,mn,MC,mp,M,N,J,W,B,G,X,O,E);
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,mn,mp,M,N,J,W,B,G,X,O,E);
 }

 } // namespace mfem
@@ -82,7 +82,6 @@ double EvalW_338(const double *J, const double *w)

 MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
                           const double metric_normal,
-                           const Vector &mc_,
                           const Array<double> &metric_param,
                           const int mid,
                           const int NE,
@@ -100,15 +99,10 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
               mid == 321 || mid == 332 || mid == 338,
               "3D metric not yet implemented!");

-   const bool const_m0 = mc_.Size() == 1;
-
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;

-   const auto MC = const_m0 ?
-                   Reshape(mc_.Read(), 1, 1, 1, 1) :
-                   Reshape(mc_.Read(), Q1D, Q1D, Q1D, NE);
   const auto J = Reshape(j_.Read(), DIM, DIM, Q1D, Q1D, Q1D, NE);
   const auto b = Reshape(b_.Read(), Q1D, D1D);
   const auto g = Reshape(g_.Read(), Q1D, D1D);
@@ -147,9 +141,7 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
            {
               const double *Jtr = &J(0,0,qx,qy,qz,e);
               const double detJtr = kernels::Det<3>(Jtr);
-               const double m_coef = const_m0 ? MC(0,0,0,0) : MC(qx,qy,qz,e);
-               const double weight = metric_normal * m_coef *
-                                     W(qx,qy,qz) * detJtr;
+               const double weight = metric_normal * W(qx,qy,qz) * detJtr;

               // Jrt = Jtr^{-1}
               double Jrt[9];
@@ -189,7 +181,6 @@ double TMOP_Integrator::GetLocalStateEnergyPA_3D(const Vector &X) const
   const int Q1D = PA.maps->nqpt;
   const int id = (D1D << 4 ) | Q1D;
   const double mn = metric_normal;
-   const Vector &MC = PA.MC;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -203,7 +194,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_3D(const Vector &X) const
      m->GetWeights(mp);
   }

-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,MC,mp,M,N,J,W,B,G,O,X,E);
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,mp,M,N,J,W,B,G,O,X,E);
 }

 } // namespace mfem
@@ -781,13 +781,7 @@ std::unique_ptr<SparseMatrix>>
   int ndof_lor = fes_lor.GetNDofs();

   // If the local mesh is empty, skip all computations
-   if (nel_ho == 0)
-   {
-      return std::make_pair(
-                std::unique_ptr<SparseMatrix>(new SparseMatrix),
-                std::unique_ptr<SparseMatrix>(new SparseMatrix)
-             );
-   }
+   if (nel_ho == 0) { return {nullptr, nullptr}; }

   const CoarseFineTransformations& cf_tr = mesh_lor->GetRefinementTransforms();

@@ -26,10 +26,6 @@
 #include "sort_pairs.hpp"
 #include "globals.hpp"

-#ifdef MFEM_USE_STRUMPACK
-#include <StrumpackConfig.hpp> // STRUMPACK_USE_PTSCOTCH, etc.
-#endif
-
 #include <iostream>
 #include <map>

@@ -38,14 +34,6 @@ using namespace std;
 namespace mfem
 {

-#if defined(MFEM_USE_STRUMPACK) && \
-    (defined(STRUMPACK_USE_PTSCOTCH) || defined(STRUMPACK_USE_SLATE_SCALAPACK))
-int Mpi::default_thread_required = MPI_THREAD_MULTIPLE;
-#else
-int Mpi::default_thread_required = MPI_THREAD_SINGLE;
-#endif
-
-
 GroupTopology::GroupTopology(const GroupTopology &gt)
   : MyComm(gt.MyComm),
     group_lproc(gt.group_lproc)
@@ -22,6 +22,7 @@
 #include "globals.hpp"
 #include <mpi.h>

+
 namespace mfem
 {

@@ -31,34 +32,10 @@ namespace mfem
 class Mpi
 {
 public:
-   /// Singleton creation with Mpi::Init(argc, argv).
-   static void Init(int &argc, char **&argv,
-                    int required = default_thread_required,
-                    int *provided = nullptr)
-   { Init(&argc, &argv, required, provided); }
-   /// Singleton creation with Mpi::Init().
-   static void Init(int *argc = nullptr, char ***argv = nullptr,
-                    int required = default_thread_required,
-                    int *provided = nullptr)
-   {
-      MFEM_VERIFY(!IsInitialized(), "MPI already initialized!");
-      if (required == MPI_THREAD_SINGLE)
-      {
-         int mpi_err = MPI_Init(argc, argv);
-         MFEM_VERIFY(!mpi_err, "error in MPI_Init()!");
-         if (provided) { *provided = MPI_THREAD_SINGLE; }
-      }
-      else
-      {
-         int mpi_provided;
-         int mpi_err = MPI_Init_thread(argc, argv, required, &mpi_provided);
-         MFEM_VERIFY(!mpi_err, "error in MPI_Init()!");
-         if (provided) { *provided = mpi_provided; }
-      }
-      // The Mpi singleton object below needs to be created after MPI_Init() for
-      // some MPI implementations.
-      Singleton();
-   }
+   /// Singleton creation with Mpi::Init();
+   static void Init() { Init_(NULL, NULL); }
+   /// Singleton creation with Mpi::Init(argc,argv);
+   static void Init(int &argc, char **&argv) { Init_(&argc, &argv); }
   /// Finalize MPI (if it has been initialized and not yet already finalized).
   static void Finalize()
   {
@@ -94,19 +71,20 @@ public:
   }
   /// Return true if the rank in MPI_COMM_WORLD is zero.
   static bool Root() { return WorldRank() == 0; }
-   /// Default level of thread support for MPI_Init_thread.
-   static MFEM_EXPORT int default_thread_required;
 private:
-   /// Initialize the Mpi singleton.
-   static Mpi &Singleton()
+   /// Initialize MPI
+   static void Init_(int *argc, char ***argv)
   {
+      MFEM_VERIFY(!IsInitialized(), "MPI already initialized!")
+      MPI_Init(argc, argv);
+      // The "mpi" object below needs to be created after MPI_Init() for some
+      // MPI implementations
      static Mpi mpi;
-      return mpi;
   }
-   /// Finalize MPI.
+   /// Finalize MPI
   ~Mpi() { Finalize(); }
-   /// Prevent direct construction of objects of this class.
-   Mpi() {}
+   /// Prevent direct construction of objects of this class
+   Mpi() { }
 };

 /** @brief A simple convenience class based on the Mpi singleton class above.
@@ -56,7 +56,7 @@ void mfem_backtrace(int mode = 0, int depth = -1);

 /** @brief Function called when an error is encountered. Used by the macros
    MFEM_ABORT, MFEM_ASSERT, MFEM_VERIFY. */
-[[noreturn]] void mfem_error(const char *msg = NULL);
+void mfem_error(const char *msg = NULL);

 /// Function called by the macro MFEM_WARNING.
 void mfem_warning(const char *msg = NULL);
@@ -16,13 +16,13 @@
 #include <cstdlib>
 #include <errno.h>
 #ifndef _WIN32
+#include <netinet/in.h>
 #include <netdb.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <unistd.h>
 #else
-#include <winsock2.h>
-#include <ws2tcpip.h>
+#include <winsock.h>
 #ifdef _MSC_VER
 typedef int ssize_t;
 // Link with ws2_32.lib
@@ -51,66 +51,47 @@ int isockstream::establish()
 {
   // char myname[129];
   char   myname[] = "localhost";
-   int    sfd;
-   struct addrinfo hints, *res, *rp;
+   int    port;
+   struct sockaddr_in sa;
+   struct hostent *hp;

-   memset(&hints, 0, sizeof(hints));
-   hints.ai_family = AF_UNSPEC;
-   hints.ai_socktype = SOCK_STREAM;
-   hints.ai_protocol = 0;
+   memset(&sa, 0, sizeof(struct sockaddr_in));
+   // gethostname(myname, 128);
+   hp= gethostbyname(myname);

-   int s = getaddrinfo(myname, NULL, &hints, &res);
-   if (s != 0)
+   if (hp == NULL)
   {
-      mfem::err << "isockstream::establish(): getaddrinfo() failed!\n"
-                << "isockstream::establish(): getaddrinfo() returned: '"
+      mfem::err << "isockstream::establish(): gethostbyname() failed!\n"
+                << "isockstream::establish(): gethostname() returned: '"
                << myname << "'" << endl;
      error = 1;
      return (-1);
   }

-   // loop the list of address structures returned by getaddrinfo()
-   for (rp = res; rp != NULL; rp = rp->ai_next)
+   sa.sin_family= hp->h_addrtype;
+   sa.sin_port= htons(portnum);
+
+   if ((port = socket(AF_INET, SOCK_STREAM, 0)) < 0)
   {
-      if ((sfd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol)) < 0)
-      {
-         mfem::err << "isockstream::establish(): socket() failed!" << endl;
-         error = 2;
-         return (-1);
-      }
-
-      int on = 1;
-      if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on)) < 0)
-      {
-         mfem::err << "isockstream::establish(): setsockopt() failed!" << endl;
-         return (-1);
-      }
-
-#if defined(__APPLE__)
-      if (bind(sfd, (const struct sockaddr *)rp->ai_addr, rp->ai_addrlen) < 0)
-#else
-      if (bind(sfd, rp->ai_addr, rp->ai_addrlen) < 0)
-#endif
-      {
-         mfem::err << "isockstream::establish(): bind() failed!" << endl;
-         close(sfd);
-         error = 3;
-         continue;
-      }
-
-      break;
-   }
-
-   // No address succeeded
-   if (rp == NULL)
-   {
-      mfem::err << "Could not bind\n";
+      mfem::err << "isockstream::establish(): socket() failed!" << endl;
+      error = 2;
      return (-1);
   }

-   freeaddrinfo(res);
-   listen(sfd, 4);
-   return (sfd);
+   int on=1;
+   setsockopt(port, SOL_SOCKET, SO_REUSEADDR, (char *)(&on), sizeof(on));
+
+   if (bind(port,(const sockaddr*)&sa,(socklen_t)sizeof(struct sockaddr_in)) < 0)
+   {
+      mfem::err << "isockstream::establish(): bind() failed!" << endl;
+      close(port);
+      error = 3;
+      return (-1);
+   }
+
+   listen(port, 4);
+   error = 0;
+   return (port);
 }

 int isockstream::read_data(int s, char *buf, int n)
@@ -17,9 +17,6 @@
 #include <cstring> // std::memcpy
 #include <type_traits> // std::is_const
 #include <cstddef> // std::max_align_t
-#ifdef MFEM_USE_MPI
-#include <HYPRE_config.h> // HYPRE_USING_GPU
-#endif

 namespace mfem
 {
@@ -994,17 +991,7 @@ inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
   h_ptr = base.h_ptr + offset;
   if (!(base.flags & Registered))
   {
-      if (
-#if !defined(HYPRE_USING_GPU)
-         // If the following condition is true then MemoryManager::Exists()
-         // should also be true:
-         IsDeviceMemory(MemoryManager::GetDeviceMemoryType())
-#else
-         // When HYPRE_USING_GPU is defined we always register the 'base' if
-         // the MemoryManager::Exists():
-         MemoryManager::Exists()
-#endif
-      )
+      if (IsDeviceMemory(MemoryManager::GetDeviceMemoryType()))
      {
         // Register 'base':
         MemoryManager::Register_(base.h_ptr, nullptr, base.capacity*sizeof(T),
@@ -19,15 +19,15 @@
 #include <cstring>      // memset, memcpy, strerror
 #include <cerrno>       // errno
 #ifndef _WIN32
-#include <netdb.h>      // getaddrinfo
+#include <netdb.h>      // gethostbyname
 #include <arpa/inet.h>  // htons
 #include <sys/types.h>  // socket, setsockopt, connect, recv, send
 #include <sys/socket.h> // socket, setsockopt, connect, recv, send
 #include <unistd.h>     // close
+#include <netinet/in.h> // sockaddr_in
 #define closesocket (::close)
 #else
-#include <winsock2.h>
-#include <ws2tcpip.h>
+#include <winsock.h>
 #ifdef _MSC_VER
 typedef int ssize_t;
 // Link with ws2_32.lib
@@ -93,7 +93,8 @@ int socketbuf::attach(int sd)

 int socketbuf::open(const char hostname[], int port)
 {
-   struct addrinfo     hints, *res, *rp;
+   struct sockaddr_in  sa;
+   struct hostent     *hp;

   if (!wsInit_.Initialized())
   {
@@ -104,50 +105,42 @@ int socketbuf::open(const char hostname[], int port)
   setg(NULL, NULL, NULL);
   setp(obuf, obuf + buflen);

-   hints.ai_family = AF_UNSPEC;
-   hints.ai_socktype = SOCK_STREAM;
-   hints.ai_flags = 0;
-   hints.ai_protocol = 0;
-
-   std::string portStr = std::to_string(port);
-   int s = getaddrinfo(hostname, portStr.c_str(), &hints, &res);
-   if (s != 0)
+   hp = gethostbyname(hostname);
+   if (hp == NULL)
   {
      socket_descriptor = -3;
      return -1;
   }
-
-   for (rp = res; rp != NULL; rp = rp->ai_next)
+   memset(&sa, 0, sizeof(sa));
+   memcpy((char *)&sa.sin_addr, hp->h_addr, hp->h_length);
+   sa.sin_family = hp->h_addrtype;
+   sa.sin_port = htons(port);
+   socket_descriptor = socket(hp->h_addrtype, SOCK_STREAM, 0);
+   if (socket_descriptor < 0)
   {
-      socket_descriptor = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
-      if (socket_descriptor < 0)
-      {
-         continue;
-      }
-
-#if defined __APPLE__
-      // OS X does not support the MSG_NOSIGNAL option of send().
-      // Instead we can use the SO_NOSIGPIPE socket option.
-      int on = 1;
-      if (setsockopt(socket_descriptor, SOL_SOCKET, SO_NOSIGPIPE,
-                     &on, sizeof(on)) < 0)
-      {
-         closesocket(socket_descriptor);
-         socket_descriptor = -2;
-         return -1;
-      }
-#endif
-
-      if (connect(socket_descriptor, rp->ai_addr, rp->ai_addrlen) < 0)
-      {
-         closesocket(socket_descriptor);
-         socket_descriptor = -2;
-         continue;
-      }
-      break;
+      return -1;
   }

-   freeaddrinfo(res);
+#if defined __APPLE__
+   // OS X does not support the MSG_NOSIGNAL option of send().
+   // Instead we can use the SO_NOSIGPIPE socket option.
+   int on = 1;
+   if (setsockopt(socket_descriptor, SOL_SOCKET, SO_NOSIGPIPE,
+                  (char *)(&on), sizeof(on)) < 0)
+   {
+      closesocket(socket_descriptor);
+      socket_descriptor = -2;
+      return -1;
+   }
+#endif
+
+   if (connect(socket_descriptor,
+               (const struct sockaddr *)&sa, sizeof(sa)) < 0)
+   {
+      closesocket(socket_descriptor);
+      socket_descriptor = -2;
+      return -1;
+   }
   return 0;
 }

@@ -218,8 +218,7 @@ void Table::SetIJ(int *newI, int *newJ, int newsize)

 int Table::Push(int i, int j)
 {
-   MFEM_ASSERT(i >=0 &&
-               i<size, "Index out of bounds.  i = " << i << " size " << size);
+   MFEM_ASSERT( i >=0 && i<size, "Index out of bounds.  i = "<<i);

   for (int k = I[i], end = I[i+1]; k < end; k++)
   {
@@ -21,7 +21,6 @@
 #include <iomanip>
 #include <algorithm>
 #include <cmath>
-#include <cstring>

 namespace mfem
 {
@@ -29,14 +28,8 @@ namespace mfem
 namespace Ginkgo
 {

-// Create a GinkgoExecutor of type exec_type.
 GinkgoExecutor::GinkgoExecutor(ExecType exec_type)
 {
-#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
-   gko::version_info gko_version = gko::version_info::get();
-   bool gko_with_omp_support = (strcmp(gko_version.omp_version.tag,
-                                       "not compiled") != 0);
-#endif
   switch (exec_type)
   {
      case GinkgoExecutor::REFERENCE:
@@ -56,23 +49,13 @@ GinkgoExecutor::GinkgoExecutor(ExecType exec_type)
 #ifdef MFEM_USE_CUDA
            int current_device = 0;
            MFEM_GPU_CHECK(cudaGetDevice(&current_device));
-            if (gko_with_omp_support)
-            {
-               executor = gko::CudaExecutor::create(current_device,
-                                                    gko::OmpExecutor::create());
-            }
-            else
-            {
-               executor = gko::CudaExecutor::create(current_device,
-                                                    gko::ReferenceExecutor::create());
-            }
+            executor = gko::CudaExecutor::create(current_device,
+                                                 gko::OmpExecutor::create());
 #endif
         }
         else
-         {
            MFEM_ABORT("gko::CudaExecutor::get_num_devices() did not report "
                       "any valid devices.");
-         }
         break;
      }
      case GinkgoExecutor::HIP:
@@ -82,111 +65,24 @@ GinkgoExecutor::GinkgoExecutor(ExecType exec_type)
 #ifdef MFEM_USE_HIP
            int current_device = 0;
            MFEM_GPU_CHECK(hipGetDevice(&current_device));
-            if (gko_with_omp_support)
-            {
-               executor = gko::HipExecutor::create(current_device,
-                                                   gko::OmpExecutor::create());
-            }
-            else
-            {
-               executor = gko::HipExecutor::create(current_device,
-                                                   gko::ReferenceExecutor::create());
-            }
+            executor = gko::HipExecutor::create(current_device,
+                                                gko::OmpExecutor::create());
 #endif
         }
         else
-         {
-            MFEM_ABORT("gko::HipExecutor::get_num_devices() did not report "
-                       "any valid devices.");
-         }
+            mfem::err << "gko::HipExecutor::get_num_devices() did not report "
+                      << "any valid devices" << std::endl;
         break;
      }
      default:
-         MFEM_ABORT("Invalid ExecType specified");
+         mfem::err << "Invalid ExecType specified" << std::endl;
   }
 }

-// Create a GinkgoExecutor of type exec_type, with host_exec_type for the
-// related CPU Executor (only applicable to GPU backends).
-GinkgoExecutor::GinkgoExecutor(ExecType exec_type, ExecType host_exec_type)
-{
-   switch (exec_type)
-   {
-      case GinkgoExecutor::REFERENCE:
-      {
-         MFEM_WARNING("Parameter host_exec_type ignored for CPU GinkgoExecutor.");
-         executor = gko::ReferenceExecutor::create();
-         break;
-      }
-      case GinkgoExecutor::OMP:
-      {
-         MFEM_WARNING("Parameter host_exec_type ignored for CPU GinkgoExecutor.");
-         executor = gko::OmpExecutor::create();
-         break;
-      }
-      case GinkgoExecutor::CUDA:
-      {
-         if (gko::CudaExecutor::get_num_devices() > 0)
-         {
-#ifdef MFEM_USE_CUDA
-            int current_device = 0;
-            MFEM_GPU_CHECK(cudaGetDevice(&current_device));
-            if (host_exec_type == GinkgoExecutor::OMP)
-            {
-               executor = gko::CudaExecutor::create(current_device,
-                                                    gko::OmpExecutor::create());
-            }
-            else
-            {
-               executor = gko::CudaExecutor::create(current_device,
-                                                    gko::ReferenceExecutor::create());
-            }
-#endif
-         }
-         else
-         {
-            MFEM_ABORT("gko::CudaExecutor::get_num_devices() did not report "
-                       "any valid devices.");
-         }
-         break;
-      }
-      case GinkgoExecutor::HIP:
-      {
-         if (gko::HipExecutor::get_num_devices() > 0)
-         {
-#ifdef MFEM_USE_HIP
-            int current_device = 0;
-            MFEM_GPU_CHECK(hipGetDevice(&current_device));
-            if (host_exec_type == GinkgoExecutor::OMP)
-            {
-               executor = gko::HipExecutor::create(current_device,
-                                                   gko::OmpExecutor::create());
-            }
-            else
-            {
-               executor = gko::HipExecutor::create(current_device,
-                                                   gko::ReferenceExecutor::create());
-            }
-#endif
-         }
-         else
-         {
-            MFEM_ABORT("gko::HipExecutor::get_num_devices() did not report "
-                       "any valid devices.");
-         }
-         break;
-      }
-      default:
-         MFEM_ABORT("Invalid ExecType specified");
-   }
-}
-
-// Create a GinkgoExecutor to match MFEM's device configuration.
 GinkgoExecutor::GinkgoExecutor(Device &mfem_device)
 {
-   gko::version_info gko_version = gko::version_info::get();
-   bool gko_with_omp_support = (strcmp(gko_version.omp_version.tag,
-                                       "not compiled") != 0);
+
+   // Pick "best match" Executor based on MFEM device configuration.
   if (mfem_device.Allows(Backend::CUDA_MASK))
   {
      if (gko::CudaExecutor::get_num_devices() > 0)
@@ -194,23 +90,13 @@ GinkgoExecutor::GinkgoExecutor(Device &mfem_device)
 #ifdef MFEM_USE_CUDA
         int current_device = 0;
         MFEM_GPU_CHECK(cudaGetDevice(&current_device));
-         if (gko_with_omp_support)
-         {
-            executor = gko::CudaExecutor::create(current_device,
-                                                 gko::OmpExecutor::create());
-         }
-         else
-         {
-            executor = gko::CudaExecutor::create(current_device,
-                                                 gko::ReferenceExecutor::create());
-         }
+         executor = gko::CudaExecutor::create(current_device,
+                                              gko::OmpExecutor::create());
 #endif
      }
      else
-      {
         MFEM_ABORT("gko::CudaExecutor::get_num_devices() did not report "
                    "any valid devices.");
-      }
   }
   else if (mfem_device.Allows(Backend::HIP_MASK))
   {
@@ -219,123 +105,16 @@ GinkgoExecutor::GinkgoExecutor(Device &mfem_device)
 #ifdef MFEM_USE_HIP
         int current_device = 0;
         MFEM_GPU_CHECK(hipGetDevice(&current_device));
-         if (gko_with_omp_support)
-         {
-            executor = gko::HipExecutor::create(current_device,
-                                                gko::OmpExecutor::create());
-         }
-         else
-         {
-            executor = gko::HipExecutor::create(current_device,
-                                                gko::ReferenceExecutor::create());
-         }
+         executor = gko::HipExecutor::create(current_device, gko::OmpExecutor::create());
 #endif
      }
      else
-      {
         MFEM_ABORT("gko::HipExecutor::get_num_devices() did not report "
                    "any valid devices.");
-      }
   }
   else
   {
-      if (mfem_device.Allows(Backend::OMP_MASK))
-      {
-         // Also use OpenMP for Ginkgo, if Ginkgo supports it
-         if (gko_with_omp_support)
-         {
-            executor = gko::OmpExecutor::create();
-         }
-         else
-         {
-            executor = gko::ReferenceExecutor::create();
-         }
-      }
-      else
-      {
-         executor = gko::ReferenceExecutor::create();
-      }
-   }
-}
-
-// Create a GinkgoExecutor to match MFEM's device configuration, with
-// a specific host_exec_type for the associated CPU Executor (only
-// applicable to GPU backends).
-GinkgoExecutor::GinkgoExecutor(Device &mfem_device, ExecType host_exec_type)
-{
-
-   if (mfem_device.Allows(Backend::CUDA_MASK))
-   {
-      if (gko::CudaExecutor::get_num_devices() > 0)
-      {
-#ifdef MFEM_USE_CUDA
-         int current_device = 0;
-         MFEM_GPU_CHECK(cudaGetDevice(&current_device));
-         if (host_exec_type == GinkgoExecutor::OMP)
-         {
-            executor = gko::CudaExecutor::create(current_device,
-                                                 gko::OmpExecutor::create());
-         }
-         else
-         {
-            executor = gko::CudaExecutor::create(current_device,
-                                                 gko::ReferenceExecutor::create());
-         }
-#endif
-      }
-      else
-      {
-         MFEM_ABORT("gko::CudaExecutor::get_num_devices() did not report "
-                    "any valid devices.");
-      }
-   }
-   else if (mfem_device.Allows(Backend::HIP_MASK))
-   {
-      if (gko::HipExecutor::get_num_devices() > 0)
-      {
-#ifdef MFEM_USE_HIP
-         int current_device = 0;
-         MFEM_GPU_CHECK(hipGetDevice(&current_device));
-         if (host_exec_type == GinkgoExecutor::OMP)
-         {
-            executor = gko::HipExecutor::create(current_device,
-                                                gko::OmpExecutor::create());
-         }
-         else
-         {
-            executor = gko::HipExecutor::create(current_device,
-                                                gko::ReferenceExecutor::create());
-         }
-#endif
-      }
-      else
-      {
-         MFEM_ABORT("gko::HipExecutor::get_num_devices() did not report "
-                    "any valid devices.");
-      }
-   }
-   else
-   {
-      MFEM_WARNING("Parameter host_exec_type ignored for CPU GinkgoExecutor.");
-      if (mfem_device.Allows(Backend::OMP_MASK))
-      {
-         // Also use OpenMP for Ginkgo, if Ginkgo supports it
-         gko::version_info gko_version = gko::version_info::get();
-         bool gko_with_omp_support = (strcmp(gko_version.omp_version.tag,
-                                             "not compiled") != 0);
-         if (gko_with_omp_support)
-         {
-            executor = gko::OmpExecutor::create();
-         }
-         else
-         {
-            executor = gko::ReferenceExecutor::create();
-         }
-      }
-      else
-      {
-         executor = gko::ReferenceExecutor::create();
-      }
+      executor = gko::OmpExecutor::create();
   }
 }

@@ -416,7 +195,7 @@ const
                           gko::log::Logger::criterion_check_completed_mask);
 #endif
   residual_logger = std::make_shared<ResidualLogger<>>(executor,
-                                                        system_oper.get(),b);
+                                                        gko::lend(system_oper),b);

 }

@@ -555,7 +334,7 @@ GinkgoIterativeSolver::Mult(const Vector &x, Vector &y) const

   // Create the logger object to log some data from the solvers to confirm
   // convergence.
-   initialize_ginkgo_log(gko_x.get());
+   initialize_ginkgo_log(gko::lend(gko_x));

   MFEM_VERIFY(convergence_logger, "convergence logger not initialized" );
   if (print_level==1)
@@ -571,11 +350,7 @@ GinkgoIterativeSolver::Mult(const Vector &x, Vector &y) const
   combined_factory->add_logger(convergence_logger);

   // Finally, apply the solver to x and get the solution in y.
-#if MFEM_GINKGO_VERSION < 10600
   solver->apply(gko::lend(gko_x), gko::lend(gko_y));
-#else
-   solver->apply(gko_x, gko_y);
-#endif

   // Get the number of iterations taken to converge to the solution.
   final_iter = convergence_logger->get_num_iterations();
@@ -692,10 +467,6 @@ void GinkgoIterativeSolver::SetOperator(const Operator &op)
                       new OperatorWrapper(executor, op.Height(), &op));
   }

-   // Set MFEM Solver size values
-   height = op.Height();
-   width = op.Width();
-
   // Generate the solver from the solver using the system matrix or operator.
   solver = solver_gen->generate(system_oper);
 }
@@ -937,14 +708,11 @@ GMRESSolver::GMRESSolver(GinkgoExecutor &exec,
 void GMRESSolver::SetKDim(int dim)
 {
   m = dim;
-   using gmres = gko::solver::Gmres<double>;
-   // Create new solver factory with other parameters the same, but new value for krylov_dim
-   auto current_params = gko::as<gmres::Factory>(solver_gen)->get_parameters();
-   this->solver_gen = current_params.with_krylov_dim(static_cast<unsigned long>(m))
-                      .on(this->executor);
+   using gmres_type = gko::solver::Gmres<double>;
+   gko::as<gmres_type::Factory>(solver_gen)->get_parameters().krylov_dim = m;
   if (solver)
   {
-      gko::as<gmres>(solver)->set_krylov_dim(static_cast<unsigned long>(m));
+      gko::as<gmres_type>(solver)->set_krylov_dim(static_cast<unsigned long>(m));
   }
 }

@@ -1039,14 +807,11 @@ CBGMRESSolver::CBGMRESSolver(GinkgoExecutor &exec,
 void CBGMRESSolver::SetKDim(int dim)
 {
   m = dim;
-   using gmres = gko::solver::CbGmres<double>;
-   // Create new solver factory with other parameters the same, but new value for krylov_dim
-   auto current_params = gko::as<gmres::Factory>(solver_gen)->get_parameters();
-   this->solver_gen = current_params.with_krylov_dim(static_cast<unsigned long>(m))
-                      .on(this->executor);
+   using gmres_type = gko::solver::CbGmres<double>;
+   gko::as<gmres_type::Factory>(solver_gen)->get_parameters().krylov_dim = m;
   if (solver)
   {
-      gko::as<gmres>(solver)->set_krylov_dim(static_cast<unsigned long>(m));
+      gko::as<gmres_type>(solver)->set_krylov_dim(static_cast<unsigned long>(m));
   }
 }

@@ -1113,11 +878,7 @@ GinkgoPreconditioner::Mult(const Vector &x, Vector &y) const
                            gko_array<double>::view(executor,
                                                    y.Size(),
                                                    y.ReadWrite(on_device)), 1);
-#if MFEM_GINKGO_VERSION < 10600
   generated_precond.get()->apply(gko::lend(gko_x), gko::lend(gko_y));
-#else
-   generated_precond.get()->apply(gko_x, gko_y);
-#endif
 }

 void GinkgoPreconditioner::SetOperator(const Operator &op)
@@ -1156,10 +917,6 @@ void GinkgoPreconditioner::SetOperator(const Operator &op)

   generated_precond = precond_gen->generate(gko::give(gko_matrix));
   has_generated_precond = true;
-
-   // Set MFEM Solver size values
-   height = op.Height();
-   width = op.Width();
 }


@@ -1211,11 +968,7 @@ IluPreconditioner::IluPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ilu<>::build()
-#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
-#else
-                    .with_factorization(fact_factory)
-#endif
                    .on(executor);
   }
   else
@@ -1227,11 +980,7 @@ IluPreconditioner::IluPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ilu<>::build()
-#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
-#else
-                    .with_factorization(fact_factory)
-#endif
                    .on(executor);
   }

@@ -1269,15 +1018,9 @@ IluIsaiPreconditioner::IluIsaiPreconditioner(
         .on(executor);
      precond_gen = gko::preconditioner::Ilu<l_solver_type,
      u_solver_type>::build()
-#if MFEM_GINKGO_VERSION < 10700
      .with_factorization_factory(fact_factory)
      .with_l_solver_factory(l_solver_factory)
      .with_u_solver_factory(u_solver_factory)
-#else
-      .with_factorization(fact_factory)
-      .with_l_solver(l_solver_factory)
-      .with_u_solver(u_solver_factory)
-#endif
      .on(executor);

   }
@@ -1291,15 +1034,9 @@ IluIsaiPreconditioner::IluIsaiPreconditioner(
         .on(executor);
      precond_gen = gko::preconditioner::Ilu<l_solver_type,
      u_solver_type>::build()
-#if MFEM_GINKGO_VERSION < 10700
      .with_factorization_factory(fact_factory)
      .with_l_solver_factory(l_solver_factory)
      .with_u_solver_factory(u_solver_factory)
-#else
-      .with_factorization(fact_factory)
-      .with_l_solver(l_solver_factory)
-      .with_u_solver(u_solver_factory)
-#endif
      .on(executor);
   }
 }
@@ -1324,11 +1061,7 @@ IcPreconditioner::IcPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ic<>::build()
-#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
-#else
-                    .with_factorization(fact_factory)
-#endif
                    .on(executor);
   }
   else
@@ -1341,11 +1074,7 @@ IcPreconditioner::IcPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ic<>::build()
-#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
-#else
-                    .with_factorization(fact_factory)
-#endif
                    .on(executor);
   }
 }
@@ -1374,13 +1103,8 @@ IcIsaiPreconditioner::IcIsaiPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ic<l_solver_type>::build()
-#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
                    .with_l_solver_factory(l_solver_factory)
-#else
-                    .with_factorization(fact_factory)
-                    .with_l_solver(l_solver_factory)
-#endif
                    .on(executor);
   }
   else
@@ -1393,13 +1117,8 @@ IcIsaiPreconditioner::IcIsaiPreconditioner(
         .with_skip_sorting(skip_sort)
         .on(executor);
      precond_gen = gko::preconditioner::Ic<l_solver_type>::build()
-#if MFEM_GINKGO_VERSION < 10700
                    .with_factorization_factory(fact_factory)
                    .with_l_solver_factory(l_solver_factory)
-#else
-                    .with_factorization(fact_factory)
-                    .with_l_solver(l_solver_factory)
-#endif
                    .on(executor);
   }
 }
@@ -265,13 +265,9 @@ double compute_norm(const gko::matrix::Dense<ValueType> *b)
   // Initialize a result scalar containing the value 0.0.
   auto b_norm = gko::initialize<gko::matrix::Dense<ValueType>>({0.0}, exec);
   // Use the dense `compute_norm2` function to compute the norm.
-#if MFEM_GINKGO_VERSION < 10600
-   b->compute_norm2(gko::lend(b_norm));
-#else
-   b->compute_norm2(b_norm);
-#endif
+   b->compute_norm2(lend(b_norm));
   // Use the other utility function to return the norm contained in `b_norm``
-   return std::pow(get_norm(b_norm.get()),2);
+   return std::pow(get_norm(lend(b_norm)),2);
 }

 /**
@@ -323,43 +319,84 @@ struct ResidualLogger : gko::log::Logger

   using gko_dense = gko::matrix::Dense<ValueType>;

-   // Ginkgo 1.5 and older: version for solver that doesn't log implicit res norm
-   void on_iteration_complete(const gko::LinOp *op,
-                              const gko::size_type &iteration,
-                              const gko::LinOp *residual,
-                              const gko::LinOp *solution,
-                              const gko::LinOp *residual_norm) const override
-   {
-      iteration_complete_core(iteration, residual, solution, residual_norm,
-                              nullptr);
-   }
-   // Ginkgo 1.5 and older: version with implicit residual norm
-   void on_iteration_complete(const gko::LinOp *op,
+   // Customize the logging hook which is called every time an iteration is
+   // completed
+   void on_iteration_complete(const gko::LinOp *,
                              const gko::size_type &iteration,
                              const gko::LinOp *residual,
                              const gko::LinOp *solution,
                              const gko::LinOp *residual_norm,
                              const gko::LinOp *implicit_sq_residual_norm) const override
   {
-      iteration_complete_core(iteration, residual, solution, residual_norm,
-                              implicit_sq_residual_norm);
+      // If the solver shares the current solution vector and we want to
+      // compute the residual from that
+      if (solution && compute_real_residual)
+      {
+         // Store the matrix's executor
+         auto exec = matrix->get_executor();
+         // Compute the real residual vector by calling apply on the system
+         // First, compute res = A * x
+         matrix->apply(gko::lend(solution), gko::lend(res));
+         // Now do res = res - b, depending on which vector/oper type
+         // Check if b is a Ginkgo vector or wrapped MFEM Vector
+         if (dynamic_cast<const VectorWrapper*>(b))
+         {
+            const VectorWrapper *b_cast = gko::as<const VectorWrapper>(b);
+            // Copy the MFEM Vector stored in b
+            VectorWrapper *res_cast = gko::as<VectorWrapper>(res);
+            res_cast->get_mfem_vec_ref() -= b_cast->get_mfem_vec_const_ref();
+         }
+         else
+         {
+            // Create a scalar containing the value -1.0
+            auto neg_one = gko::initialize<gko_dense>({-1.0}, exec);
+            res->add_scaled(gko::lend(neg_one), gko::lend(b));
+         }
+
+         // Compute the norm of the residual vector and add it to the
+         // `residual_norms` vector
+         residual_norms.push_back(compute_norm(gko::lend(res)));
+      }
+      else
+      {
+         // If the solver shares an implicit or recurrent residual norm, log its value
+         if (implicit_sq_residual_norm)
+         {
+            auto dense_norm = gko::as<gko_dense>(implicit_sq_residual_norm);
+            // Add the norm to the `residual_norms` vector
+            residual_norms.push_back(get_norm(dense_norm));
+            // Otherwise, use the recurrent residual vector
+         }
+         else if (residual_norm)
+         {
+            auto dense_norm = gko::as<gko_dense>(residual_norm);
+            // Add the norm to the `residual_norms` vector
+            residual_norms.push_back(get_norm(dense_norm));
+            // Otherwise, use the recurrent residual vector
+         }
+         else
+         {
+            auto dense_residual = gko::as<gko_dense>(residual);
+            // Compute the residual vector's norm
+            auto norm = compute_norm(gko::lend(dense_residual));
+            // Add the computed norm to the `residual_norms` vector
+            residual_norms.push_back(norm);
+         }
+      }
+      // Add the current iteration number to the `iterations` vector
+      iterations.push_back(iteration);
   }
-#if MFEM_GINKGO_VERSION > 10500
-   // Ginkgo 1.6 and newer
+
+   // Version for solver that doesn't log implicit res norm
   void on_iteration_complete(const gko::LinOp *op,
-                              const gko::LinOp *rhs,
-                              const gko::LinOp *solution,
                              const gko::size_type &iteration,
                              const gko::LinOp *residual,
-                              const gko::LinOp *residual_norm,
-                              const gko::LinOp *implicit_sq_residual_norm,
-                              const gko::array<gko::stopping_status>* status,
-                              bool stopped) const override
+                              const gko::LinOp *solution,
+                              const gko::LinOp *residual_norm) const override
   {
-      iteration_complete_core(iteration, residual, solution, residual_norm,
-                              implicit_sq_residual_norm);
+      on_iteration_complete(op, iteration, residual, solution, residual_norm,
+                            nullptr);
   }
-#endif

   // Construct the logger and store the system matrix and b vectors
   ResidualLogger(std::shared_ptr<const gko::Executor> exec,
@@ -391,89 +428,6 @@ struct ResidualLogger : gko::log::Logger
   }

 private:
-   // Customize the logging hook which is called every time an iteration is
-   // completed.
-   void iteration_complete_core(const gko::size_type &iteration,
-                                const gko::LinOp *residual,
-                                const gko::LinOp *solution,
-                                const gko::LinOp *residual_norm,
-                                const gko::LinOp *implicit_sq_residual_norm) const
-   {
-      // If the solver shares the current solution vector and we want to
-      // compute the residual from that
-      if (solution && compute_real_residual)
-      {
-         // Store the matrix's executor
-         auto exec = matrix->get_executor();
-         // Compute the real residual vector by calling apply on the system
-         // First, compute res = A * x
-#if MFEM_GINKGO_VERSION < 10600
-         matrix->apply(gko::lend(solution), gko::lend(res));
-#else
-         matrix->apply(solution, res);
-#endif
-         // Now do res = res - b, depending on which vector/oper type
-         // Check if b is a Ginkgo vector or wrapped MFEM Vector
-         if (dynamic_cast<const VectorWrapper*>(b))
-         {
-            const VectorWrapper *b_cast = gko::as<const VectorWrapper>(b);
-            // Copy the MFEM Vector stored in b
-            VectorWrapper *res_cast = gko::as<VectorWrapper>(res);
-            res_cast->get_mfem_vec_ref() -= b_cast->get_mfem_vec_const_ref();
-         }
-         else
-         {
-            // Create a scalar containing the value -1.0
-            auto neg_one = gko::initialize<gko_dense>({-1.0}, exec);
-#if MFEM_GINKGO_VERSION < 10600
-            res->add_scaled(gko::lend(neg_one), gko::lend(b));
-#else
-            res->add_scaled(neg_one, b);
-#endif
-         }
-
-         // Compute the norm of the residual vector and add it to the
-         // `residual_norms` vector
-#if MFEM_GINKGO_VERSION < 10600
-         residual_norms.push_back(compute_norm(gko::lend(res)));
-#else
-         residual_norms.push_back(compute_norm(res));
-#endif
-      }
-      else
-      {
-         // If the solver shares an implicit or recurrent residual norm, log its value
-         if (implicit_sq_residual_norm)
-         {
-            auto dense_norm = gko::as<gko_dense>(implicit_sq_residual_norm);
-            // Add the norm to the `residual_norms` vector
-            residual_norms.push_back(get_norm(dense_norm));
-            // Otherwise, use the recurrent residual vector
-         }
-         else if (residual_norm)
-         {
-            auto dense_norm = gko::as<gko_dense>(residual_norm);
-            // Add the norm to the `residual_norms` vector
-            residual_norms.push_back(get_norm(dense_norm));
-            // Otherwise, use the recurrent residual vector
-         }
-         else
-         {
-            auto dense_residual = gko::as<gko_dense>(residual);
-            // Compute the residual vector's norm
-#if MFEM_GINKGO_VERSION < 10600
-            auto norm = compute_norm(gko::lend(dense_residual));
-#else
-            auto norm = compute_norm(dense_residual);
-#endif
-            // Add the computed norm to the `residual_norms` vector
-            residual_norms.push_back(norm);
-         }
-      }
-      // Add the current iteration number to the `iterations` vector
-      iterations.push_back(iteration);
-   }
-
   // Pointer to the system matrix
   const gko::LinOp *matrix;
   // Pointer to the right hand sides
@@ -515,40 +469,18 @@ public:
   /**
    * Constructor.
    * Takes an @p GinkgoExecType argument and creates an Executor.
-    * In Ginkgo, GPU Executors must have an associated host Executor.
-    * This routine will select a CPU Executor based on the OpenMP support
-    * for Ginkgo.
    */
   GinkgoExecutor(ExecType exec_type);

-   /**
-    * Constructor.
-    * Takes an @p GinkgoExecType argument and creates an Executor.
-    * In Ginkgo, GPU Executors must have an associated host Executor.
-    * This routine allows for explicite setting of the CPU Executor
-    * for GPU backends.
-    */
-   GinkgoExecutor(ExecType exec_type, ExecType host_exec_type);
-
   /**
    * Constructor.
    * Takes an MFEM @p Device object and creates an Executor
    * that "matches" (e.g., if MFEM is using the CPU, Ginkgo
-    * will choose the Reference or OmpExecutor based on MFEM's
-    * configuration and Ginkgo's capabilities; if MFEM is using
-    * CUDA, Ginkgo will choose the CudaExecutor with a default
-    * CPU Executor based on Ginkgo's OpenMP support).
+    * will choose the OmpExecutor; if MFEM is using CUDA,
+    * Ginkgo will choose the CudaExecutor).
    */
   GinkgoExecutor(Device &mfem_device);

-   /**
-    * Constructor.
-    * Takes an MFEM @p Device object and creates an Executor
-    * that "matches", but allows the user to specify the host
-    * Executor for GPU backends.
-    */
-   GinkgoExecutor(Device &mfem_device, ExecType host_exec_type);
-
   /**
    * Destructor.
    */
@@ -862,10 +794,8 @@ public:
   {
      rel_tol = rtol;
      this->update_stop_factory();
-      auto current_params = gko::as<typename SolverType::Factory>
-                            (solver_gen)->get_parameters();
-      this->solver_gen = current_params.with_criteria(this->combined_factory)
-                         .on(this->executor);
+      gko::as<typename SolverType::Factory>(solver_gen)->get_parameters().criteria =
+      { combined_factory };
      if (solver)
      {
         gko::as<SolverType>(solver)->set_stop_criterion_factory(combined_factory);
@@ -876,10 +806,8 @@ public:
   {
      abs_tol = atol;
      this->update_stop_factory();
-      auto current_params = gko::as<typename SolverType::Factory>
-                            (solver_gen)->get_parameters();
-      this->solver_gen = current_params.with_criteria(this->combined_factory)
-                         .on(this->executor);
+      gko::as<typename SolverType::Factory>(solver_gen)->get_parameters().criteria =
+      { combined_factory };
      if (solver)
      {
         gko::as<SolverType>(solver)->set_stop_criterion_factory(combined_factory);
@@ -890,10 +818,8 @@ public:
   {
      max_iter = max_it;
      this->update_stop_factory();
-      auto current_params = gko::as<typename SolverType::Factory>
-                            (solver_gen)->get_parameters();
-      this->solver_gen = current_params.with_criteria(this->combined_factory)
-                         .on(this->executor);
+      gko::as<typename SolverType::Factory>(solver_gen)->get_parameters().criteria =
+      { combined_factory };
      if (solver)
      {
         gko::as<SolverType>(solver)->set_stop_criterion_factory(combined_factory);
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Tom Stitt	cf1c58ee1c	Merge remote-tracking branch 'origin/hypre-runtime-compute-policy' into apps-dev	2023-10-27 09:22:16 -07:00
Tom Stitt	02579a1d23	Merge remote-tracking branch 'origin/feature/askinner/hdiv-saddle-point-solver' into apps-dev	2023-10-27 09:22:12 -07:00
Tom Stitt	66d9ead7b1	Merge remote-tracking branch 'origin/master' into hypre-runtime-compute-policy	2023-10-27 09:03:44 -07:00
Tom Stitt	4115a9ad5d	handle more HYPRE_USING_{GPU,HIP,CUDA}	2023-08-29 21:16:35 -07:00
Tom Stitt	a8cb5babce	add support for runtime hypre compute policy	2023-08-25 14:36:14 -07:00