WIP: updates to MG setup

WIP: add more GMG options
2021-02-01 13:50:01 -05:00 · 2021-01-26 18:57:58 -05:00 · 2021-01-21 19:59:58 -05:00 · 2021-01-20 18:22:59 -05:00 · 2021-01-20 18:21:47 -05:00 · 2021-01-19 12:46:34 -08:00
123 changed files with 15082 additions and 1247 deletions
@@ -215,6 +215,7 @@ miniapps/meshing/polar-nc.mesh

 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
+miniapps/navier/navier_kovasznay_vs
 miniapps/navier/navier_tgv
 miniapps/navier/navier_shear
 miniapps/navier/navier_3dfoc
@@ -11,6 +11,9 @@
 Version 4.2.1 (development)
 ===========================

+- Added matrix-free GPU-enabled implementations of GradientInterpolator and
+  IdentityInterpolator.
+
 - Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
  See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.

@@ -20,11 +23,46 @@ Version 4.2.1 (development)
 - Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
  help identify elements with poor mesh quality.

+- Added support for the "BR2" discontinuous Galerkin discretization for
+  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
+
+- Generalized the Multigrid class to support non-geometric multigrid. The
+  previous functionality, based on FiniteElementSpaceHierarchy, is now available
+  in the derived class GeometricMultigrid.
+
 - Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.

 - Implemented a filter method for the Navier miniapp to stabilize highly
  turbulent flows in direct numerical simulation.

+- Added partial assembly and device support to Example 25/25p, with diagonal
+  preconditioning.
+
+- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+
+- Added new mesh quality metrics and improved the untangling capabilities of the
+  TMOP-based mesh optimization algorithms.
+
+- Changed the interface for the error estimator.
+
+- Implemented the parallel Kelly error indicator for scalar-valued problems.
+
+- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
+  efficient evaluation of symmetric matrix coefficients. This replaces the now
+  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
+  for clarity, which is a typedef of VectorCoefficient.
+
+- Added support for AMG preconditioners for non-symmetric systems (e.g.
+  advection-dominated problems) using hypre's approximate ideal restriction
+  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
+  example 9/9p.
+
+- Implemented an adaptive linear solver tolerance option for NewtonSolver based
+  on the algorithm of Eisenstat and Walker.
+
+- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
+  linear interpolators and GridFunction "GetValue" methods.
+

 Version 4.2, released on October 30, 2020
 =========================================
@@ -9,12 +9,13 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-cmake_minimum_required(VERSION 2.8.11)
+# The variable CMAKE_CXX_STANDARD and related were introduced in CMake v3.1
+cmake_minimum_required(VERSION 3.1)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
  "Path to optional user configuration file.")

 # Require C++11 and disable compiler-specific extensions
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)

@@ -94,7 +95,7 @@ if (MFEM_USE_CUDA)
      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
   endif()
   enable_language(CUDA)
-   set(CMAKE_CUDA_STANDARD 11)
+   set(CMAKE_CUDA_STANDARD 14)
   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
   set(CMAKE_CUDA_EXTENSIONS OFF)
   set(CUDA_FLAGS "--expt-extended-lambda")
@@ -710,10 +710,10 @@ The specific libraries and their options are:
  Versions: libCEED >= 0.7.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
+  Beginning with MFEM v4.3, only RAJA v0.13.0+ is supported.
  URL: https://github.com/LLNL/RAJA
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 0.10.0.
+  Versions: RAJA >= 0.13.0.

 - Umpire, used when MFEM_USE_UMPIRE = YES.
  URL: https://github.com/LLNL/Umpire
@@ -25,16 +25,16 @@ mfem_find_package(METIS METIS METIS_DIR "include;Lib" "metis.h"

 int main()
 {
-    int n = 10;
-    int nparts = 5;
-    int edgecut;
-    int* partitioning = new int[10];
-    int* I = partitioning,
+    idx_t n = 10;
+    idx_t nparts = 5;
+    idx_t edgecut;
+    idx_t* partitioning = new idx_t[10];
+    idx_t* I = partitioning,
       * J = partitioning;

-    int ncon = 1;
+    idx_t ncon = 1;
    int err;
-    int options[40];
+    idx_t options[40];

    METIS_SetDefaultOptions(options);
    options[10] = 1; // set METIS_OPTION_CONTIG
@@ -754,7 +754,13 @@ function(mfem_export_mk_files)
  set(MFEM_CXX ${CMAKE_CXX_COMPILER})
  set(MFEM_HOST_CXX ${MFEM_CXX})
  set(MFEM_CPPFLAGS "")
-  string(STRIP "${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
+  get_target_property(cxx_std mfem CXX_STANDARD)
+  # For now, we ignore the setting of the CXX_EXTENSIONS property. If this
+  # property is set, then we need to use a variable like:
+  #    CMAKE_CXX11_EXTENSION_COMPILE_OPTION
+  set(cxx_std_flag ${CMAKE_CXX${cxx_std}_STANDARD_COMPILE_OPTION})
+  string(STRIP
+         "${cxx_std_flag} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
         MFEM_CXXFLAGS)
  set(MFEM_TPLFLAGS "")
  foreach(dir ${MFEM_TPL_INCLUDE_DIRS})
@@ -161,7 +161,7 @@ endif
 ZLIB_DIR =
 ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include)
 ZLIB_LIB = $(if $(ZLIB_DIR),$(ZLIB_RPATH) -L$(ZLIB_DIR)/lib ,)-lz
-ZLIB_RPATH = -Wl,-rpath,$(ZLIB_DIR)/lib
+ZLIB_RPATH = $(XLINKER)-rpath,$(ZLIB_DIR)/lib

 LIBUNWIND_OPT = -g
 LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
@@ -231,19 +231,21 @@ MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite
 LIB_RT = $(if $(NOTMAC),-lrt,)
 SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse
 SUITESPARSE_OPT = -I$(SUITESPARSE_DIR)/include
-SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
- -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd -lccolamd\
- -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)
+SUITESPARSE_LIB = $(XLINKER)-rpath,$(SUITESPARSE_DIR)/lib\
+ -L$(SUITESPARSE_DIR)/lib -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd\
+ -lccolamd -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)

 # SuperLU library configuration
 ifeq ($(MFEM_USE_SUPERLU5),YES)
   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0
+   SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\
+      -lsuperlu_dist_5.1.0
 else
   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64 -lsuperlu_dist -lblas
+   SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64\
+      -lsuperlu_dist -lblas
 endif

 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
@@ -269,7 +271,7 @@ MPI_FORTRAN_LIB = -lmpifort
 # MUMPS library configuration
 MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = -Wl,-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
+MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
 -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)

 # STRUMPACK library configuration
@@ -299,8 +301,8 @@ GNUTLS_LIB = -lgnutls
 NETCDF_DIR = $(HOME)/local
 HDF5_DIR   = $(HOME)/local
 NETCDF_OPT = -I$(NETCDF_DIR)/include -I$(HDF5_DIR)/include $(ZLIB_OPT)
-NETCDF_LIB = -Wl,-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
- -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
+NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
+ $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
 -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB)

 # PETSc library configuration (version greater or equal to 3.8 or the dev branch)
@@ -312,9 +314,10 @@ PETSC_INC_VAR = PETSC_CC_INCLUDES
 PETSC_LIB_VAR = PETSC_EXTERNAL_LIB_BASIC
 ifeq ($(PETSC_FOUND),YES)
   PETSC_OPT := $(shell sed -n "s/$(PETSC_INC_VAR) = *//p" $(PETSC_VARS))
-   PETSC_LIB := $(shell sed -n "s/$(PETSC_LIB_VAR) = *//p" $(PETSC_VARS))
-   PETSC_LIB := -Wl,-rpath,$(abspath $(PETSC_DIR))/lib\
-      -L$(abspath $(PETSC_DIR))/lib -lpetsc $(PETSC_LIB)
+   PETSC_DEP := $(shell sed -n "s/$(PETSC_LIB_VAR) = *//p" $(PETSC_VARS))
+   PETSC_LIB = $(XLINKER)-rpath,$(abspath $(PETSC_DIR))/lib\
+      -L$(abspath $(PETSC_DIR))/lib -lpetsc\
+      $(subst $(CXX_XLINKER),$(XLINKER),$(PETSC_DEP))
 endif

 SLEPC_DIR := $(MFEM_DIR)/../slepc
@@ -326,9 +329,10 @@ ifeq ($(SLEPC_FOUND),YES)
   SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS))
   # Some additional external libraries might be defined in this file
   -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables
-   SLEPC_LIB := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
-   SLEPC_LIB := -Wl,-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
-      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc $(SLEPC_LIB)
+   SLEPC_DEP := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
+   SLEPC_LIB = $(XLINKER)-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
+      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc\
+      $(subst $(CXX_XLINKER),$(XLINKER),$(SLEPC_DEP))
 endif

 # MPFR library configuration
@@ -339,7 +343,7 @@ MPFR_LIB = -lmpfr
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit
 CONDUIT_LIB = \
-   -Wl,-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
+   $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
   -lconduit -lconduit_relay -lconduit_blueprint  -ldl

 # Check if Conduit was built with hdf5 support, by looking
@@ -347,7 +351,7 @@ CONDUIT_LIB = \
 CONDUIT_HDF5_HEADER=$(CONDUIT_DIR)/include/conduit/conduit_relay_hdf5.hpp
 ifneq (,$(wildcard $(CONDUIT_HDF5_HEADER)))
   CONDUIT_OPT += -I$(HDF5_DIR)/include
-   CONDUIT_LIB += -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
+   CONDUIT_LIB += $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
                  -lhdf5 $(ZLIB_LIB)
 endif

@@ -357,9 +361,9 @@ SIDRE_DIR = @MFEM_DIR@/../axom
 SIDRE_OPT = -I$(SIDRE_DIR)/include -I$(CONDUIT_DIR)/include/conduit\
 -I$(HDF5_DIR)/include
 SIDRE_LIB = \
-   -Wl,-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
-   -Wl,-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
-   -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
+   $(XLINKER)-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
+   $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
+   $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
   -laxom -lconduit -lconduit_relay -lconduit_blueprint -lhdf5 $(ZLIB_LIB) -ldl

 # PUMI
@@ -415,9 +419,9 @@ MKL_CPARDISO_DIR ?=
 MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64
 MKL_LIBRARY_SUBDIR ?= lib
 MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include
-MKL_CPARDISO_LIB = -Wl,-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
-                   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
-                   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
+MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
+   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
+   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core

 # If YES, enable some informational messages
 VERBOSE = NO
@@ -31,14 +31,14 @@ POINTS 27 double
 7 0.5 1
 8 0.5 1
 CELLS 8 56
-6 0 9 18 1 10 19
-6 1 10 19 2 11 20
-6 2 11 20 3 12 21
-6 3 12 21 4 13 22
-6 4 13 22 5 14 23
-6 5 14 23 6 15 24
-6 6 15 24 7 16 25
-6 7 16 25 8 17 26
+6 0 18 9 1 19 10
+6 1 19 10 2 20 11
+6 2 20 11 3 21 12
+6 3 21 12 4 22 13
+6 4 22 13 5 23 14
+6 5 23 14 6 24 15
+6 6 24 15 7 25 16
+6 7 25 16 8 26 17
 CELL_TYPES 8
 13
 13
@@ -205,10 +205,14 @@ int main(int argc, char *argv[])

      if (amgx_solver)
      {
+         amgx.SetConvergenceCheck(true);
         amgx.Mult(B,X);
      }
      else
      {
+         // Omit convergence check at the AmgX level when using as a
+         // preconditioner.
+         amgx.SetConvergenceCheck(false);
         PCG(*A.As<SparseMatrix>(), amgx, B, X, 3, 40, 1e-12, 0.0);
      }
   }
@@ -264,6 +264,7 @@ int main(int argc, char *argv[])
      }

      amgx.SetOperator(*A.As<HypreParMatrix>());
+      amgx.SetConvergenceCheck(true);
      amgx.Mult(B, X);

      // Release MPI communicators and resources created by AmgX
@@ -30,6 +30,7 @@
 // Device sample runs:
 //               ex1 -pa -d cuda
 //               ex1 -pa -d raja-cuda
+//             * ex1 -pa -d raja-hip
 //               ex1 -pa -d occa-cuda
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
@@ -178,6 +178,7 @@ int main(int argc, char *argv[])
   double visc = 1e-2;
   double mu = 0.25;
   double K = 5.0;
+   bool adaptive_lin_rtol = true;
   bool visualization = true;
   int vis_steps = 1;

@@ -206,6 +207,9 @@ int main(int argc, char *argv[])
                  "Shear modulus in the Neo-Hookean hyperelastic model.");
   args.AddOption(&K, "-K", "--bulk-modulus",
                  "Bulk modulus in the Neo-Hookean hyperelastic model.");
+   args.AddOption(&adaptive_lin_rtol, "-alrtol", "--adaptive-lin-rtol",
+                  "-no-alrtol", "--no-adaptive-lin-rtol",
+                  "Enable or disable adaptive linear solver rtol.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -574,6 +578,7 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
   newton_solver.SetPrintLevel(1); // print Newton iterations
   newton_solver.SetRelTol(rel_tol);
   newton_solver.SetAbsTol(0.0);
+   newton_solver.SetAdaptiveLinRtol(2, 0.5, 0.9);
   newton_solver.SetMaxIter(10);
 }

@@ -5,6 +5,7 @@
 // Sample runs:  ex14 -m ../data/inline-quad.mesh -o 0
 //               ex14 -m ../data/star.mesh -r 4 -o 2
 //               ex14 -m ../data/star-mixed.mesh -r 4 -o 2
+//               ex14 -m ../data/star-mixed.mesh -r 2 -o 2 -k 0 -e 1
 //               ex14 -m ../data/escher.mesh -s 1
 //               ex14 -m ../data/fichera.mesh -s 1 -k 1
 //               ex14 -m ../data/fichera-mixed.mesh -s 1 -k 1
@@ -44,6 +45,7 @@ int main(int argc, char *argv[])
   int order = 1;
   double sigma = -1.0;
   double kappa = -1.0;
+   double eta = 0.0;
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -54,11 +56,12 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree) >= 0.");
   args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the two DG penalty parameters, typically +1/-1."
+                  "One of the three DG penalty parameters, typically +1/-1."
                  " See the documentation of class DGDiffusionIntegrator.");
   args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the two DG penalty parameters, should be positive."
+                  "One of the three DG penalty parameters, should be positive."
                  " Negative values are replaced with (order+1)^2.");
+   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -130,6 +133,11 @@ int main(int argc, char *argv[])
   a->AddDomainIntegrator(new DiffusionIntegrator(one));
   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   if (eta > 0)
+   {
+      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+   }
   a->Assemble();
   a->Finalize();
   const SparseMatrix &A = a->SpMat();
@@ -5,6 +5,7 @@
 // Sample runs:  mpirun -np 4 ex14p -m ../data/inline-quad.mesh -o 0
 //               mpirun -np 4 ex14p -m ../data/star.mesh -o 2
 //               mpirun -np 4 ex14p -m ../data/star-mixed.mesh -o 2
+//               mpirun -np 4 ex14p -m ../data/star-mixed.mesh -o 2 -k 0 -e 1
 //               mpirun -np 4 ex14p -m ../data/escher.mesh -s 1
 //               mpirun -np 4 ex14p -m ../data/fichera.mesh -s 1 -k 1
 //               mpirun -np 4 ex14p -m ../data/fichera-mixed.mesh -s 1 -k 1
@@ -82,6 +83,7 @@ int main(int argc, char *argv[])
   int order = 1;
   double sigma = -1.0;
   double kappa = -1.0;
+   double eta = 0.0;
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -95,11 +97,12 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree) >= 0.");
   args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the two DG penalty parameters, typically +1/-1."
+                  "One of the three DG penalty parameters, typically +1/-1."
                  " See the documentation of class DGDiffusionIntegrator.");
   args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the two DG penalty parameters, should be positive."
+                  "One of the three DG penalty parameters, should be positive."
                  " Negative values are replaced with (order+1)^2.");
+   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -194,6 +197,11 @@ int main(int argc, char *argv[])
   a->AddDomainIntegrator(new DiffusionIntegrator(one));
   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   if (eta > 0)
+   {
+      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+   }
   a->Assemble();
   a->Finalize();

@@ -19,6 +19,13 @@
 //               mpirun -np 4 ex15p -m ../data/square-disc.mesh
 //               mpirun -np 4 ex15p -m ../data/escher.mesh -r 2 -tf 0.3
 //
+//               Different estimators:
+//
+//               mpirun -np 4 ex15p -est 0 -e 1e-4
+//               mpirun -np 4 ex15p -est 1 -e 1e-6
+//               mpirun -np 4 ex15p -est 1 -o 3 -tf 0.3
+//               mpirun -np 4 ex15p -est 2 -o 2
+//
 // Description:  Building on Example 6, this example demonstrates dynamic AMR.
 //               The mesh is adapted to a time-dependent solution by refinement
 //               as well as by derefinement. For simplicity, the solution is
@@ -28,8 +35,11 @@
 //               At each outer iteration the right hand side function is changed
 //               to mimic a time dependent problem.  Within each inner iteration
 //               the problem is solved on a sequence of meshes which are locally
-//               refined according to a simple ZZ error estimator.  At the end
-//               of the inner iteration the error estimates are also used to
+//               refined according to a chosen error estimator. Currently there
+//               are three error estimators supported: A L2 formulation of the
+//               Zienkiewicz-Zhu error estimator (0), a Kelly error indicator (1)
+//               and a traditional Zienkiewicz-Zhu error estimator (2). At the
+//               end of the inner iteration the error estimates are also used to
 //               identify any elements which may be over-refined and a single
 //               derefinement step is performed.  After each refinement or
 //               derefinement step a rebalance operation is performed to keep
@@ -87,6 +97,7 @@ int main(int argc, char *argv[])
   int nc_limit = 3;         // maximum level of hanging nodes
   bool visualization = true;
   bool visit = false;
+   int which_estimator = 0;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -107,6 +118,9 @@ int main(int argc, char *argv[])
                  "Maximum level of hanging nodes.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
+   args.AddOption(&which_estimator, "-est", "--estimator",
+                  "Which estimator to use: "
+                  "0 = L2ZZ, 1 = Kelly, 2 = ZZ. Defaults to L2ZZ.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -214,17 +228,43 @@ int main(int argc, char *argv[])
   //     provide the method ComputeElementFlux. We supply an L2 space for the
   //     discontinuous flux and an H(div) space for the smoothed flux.
   L2_FECollection flux_fec(order, dim);
-   ParFiniteElementSpace flux_fes(&pmesh, &flux_fec, sdim);
   RT_FECollection smooth_flux_fec(order-1, dim);
-   ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec);
-   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, smooth_flux_fes);
+   ErrorEstimator* estimator;
+   switch (which_estimator)
+   {
+      case 1:
+      {
+         auto flux_fes = new ParFiniteElementSpace(&pmesh, &flux_fec, sdim);
+         estimator = new KellyErrorEstimator(*integ, x, flux_fes);
+         break;
+      }
+      case 2:
+      {
+         auto flux_fes = new ParFiniteElementSpace(&pmesh, &fec, sdim);
+         estimator = new ZienkiewiczZhuEstimator(*integ, x, flux_fes);
+         break;
+      }
+
+      default:
+         if (myid == 0)
+         {
+            std::cout << "Unkown estimator. Falling back to L2ZZ." << std::endl;
+         }
+      case 0:
+      {
+         auto flux_fes = new ParFiniteElementSpace(&pmesh, &flux_fec, sdim);
+         auto smooth_flux_fes = new ParFiniteElementSpace(&pmesh, &smooth_flux_fec);
+         estimator = new L2ZienkiewiczZhuEstimator(*integ, x, flux_fes, smooth_flux_fes);
+         break;
+      }
+   }

   // 11. As in Example 6p, we also need a refiner. This time the refinement
   //     strategy is based on a fixed threshold that is applied locally to each
   //     element. The global threshold is turned off by setting the total error
   //     fraction to zero. We also enforce a maximum refinement ratio between
   //     adjacent elements.
-   ThresholdRefiner refiner(estimator);
+   ThresholdRefiner refiner(*estimator);
   refiner.SetTotalErrorFraction(0.0); // use purely local threshold
   refiner.SetLocalErrorGoal(max_elem_error);
   refiner.PreferConformingRefinement();
@@ -233,7 +273,7 @@ int main(int argc, char *argv[])
   // 12. A derefiner selects groups of elements that can be coarsened to form
   //     a larger element. A conservative enough threshold needs to be set to
   //     prevent derefining elements that would immediately be refined again.
-   ThresholdDerefiner derefiner(estimator);
+   ThresholdDerefiner derefiner(*estimator);
   derefiner.SetThreshold(hysteresis * max_elem_error);
   derefiner.SetNCLimit(nc_limit);

@@ -316,7 +356,7 @@ int main(int argc, char *argv[])
         refiner.Apply(pmesh);
         if (myid == 0)
         {
-            cout << ", total error: " << estimator.GetTotalError() << endl;
+            cout << ", total error: " << estimator->GetTotalError() << endl;
         }

         // 21. Quit the AMR loop if the termination criterion has been met
@@ -346,6 +386,8 @@ int main(int argc, char *argv[])
      }
   }

+   delete estimator;
+
   // 25. Exit
   MPI_Finalize();
   return 0;
@@ -10,6 +10,10 @@
 //               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh
 //               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh
 //
+// Device sample runs:
+//               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh -pa -d cuda
+//               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh -pa -d cuda
+//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -157,7 +161,10 @@ int main(int argc, char *argv[])
   int iprob = 4;
   double freq = 5.0;
   bool herm_conv = true;
+   bool umf_solver = false;
   bool visualization = 1;
+   bool pa = false;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -176,15 +183,28 @@ int main(int argc, char *argv[])
                  "Frequency (in Hz).");
   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
                  "--no-hermitian", "Use convention for Hermitian operators.");
+#ifdef MFEM_USE_SUITESPARSE
+   args.AddOption(&umf_solver, "-umf", "--umfpack", "-no-umf",
+                  "--no-umfpack", "Use the UMFPack Solver.");
+#endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();

   if (iprob > 4) { iprob = 4; }
   prob = (prob_type)iprob;

-   // 2. Setup the mesh
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Setup the mesh
   if (!mesh_file)
   {
      exact_known = true;
@@ -225,7 +245,7 @@ int main(int argc, char *argv[])
   // Setup PML length
   Array2D<double> length(dim, 2); length = 0.0;

-   // 3. Setup the Cartesian PML region.
+   // 4. Setup the Cartesian PML region.
   switch (prob)
   {
      case disc:
@@ -251,19 +271,19 @@ int main(int argc, char *argv[])
   comp_domain_bdr = pml->GetCompDomainBdr();
   domain_bdr = pml->GetDomainBdr();

-   // 4. Refine the mesh to increase the resolution.
+   // 5. Refine the mesh to increase the resolution.
   for (int l = 0; l < ref_levels; l++)
   {
      mesh->UniformRefinement();
   }

-   // 5. Reorient mesh in case of a tet mesh
+   // 6. Reorient mesh in case of a tet mesh
   mesh->ReorientTetMesh();

   // Set element attributes in order to distinguish elements in the PML region
   pml->SetAttributes(mesh);

-   // 6. Define a finite element space on the mesh. Here we use the Nedelec
+   // 7. Define a finite element space on the mesh. Here we use the Nedelec
   //    finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
@@ -271,7 +291,7 @@ int main(int argc, char *argv[])

   cout << "Number of finite element unknowns: " << size << endl;

-   // 7. Determine the list of true essential boundary dofs. In this example,
+   // 8. Determine the list of true essential boundary dofs. In this example,
   //    the boundary conditions are defined based on the specific mesh and the
   //    problem type.
   Array<int> ess_tdof_list;
@@ -313,12 +333,12 @@ int main(int argc, char *argv[])
   }
   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-   // 8. Setup Complex Operator convention
+   // 9. Setup Complex Operator convention
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system.
+   // 10. Set up the linear form b(.) which corresponds to the right-hand side of
+   //     the FEM linear system.
   VectorFunctionCoefficient f(dim, source);
   ComplexLinearForm b(fespace, conv);
   if (prob == load_src)
@@ -328,7 +348,7 @@ int main(int argc, char *argv[])
   b.Vector::operator=(0.0);
   b.Assemble();

-   // 10. Define the solution vector x as a complex finite element grid function
+   // 11. Define the solution vector x as a complex finite element grid function
   //     corresponding to fespace.
   ComplexGridFunction x(fespace);
   x = 0.0;
@@ -336,7 +356,7 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
   x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);

-   // 11. Set up the sesquilinear form a(.,.)
+   // 12. Set up the sesquilinear form a(.,.)
   //
   //     In Comp
   //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -390,32 +410,35 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                         new VectorFEMassIntegrator(restr_c2_Im));

-   // 12. Assemble the bilinear form and the corresponding linear system,
+   // 13. Assemble the bilinear form and the corresponding linear system,
   //     applying any necessary transformations such as: assembly, eliminating
   //     boundary conditions, applying conforming constraints for
   //     non-conforming AMR, etc.
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a.Assemble(0);

   OperatorPtr A;
   Vector B, X;
   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);

-   // 13. Solve using a direct or an iterative solver
+   // 14. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUITESPARSE
+   if (!pa && umf_solver)
   {
      ComplexUMFPackSolver csolver(*A.As<ComplexSparseMatrix>());
      csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
      csolver.SetPrintLevel(1);
      csolver.Mult(B, X);
   }
-#else
-   // 13a. Set up the Bilinear form a(.,.) for the preconditioner
+#endif
+   // 14a. Set up the Bilinear form a(.,.) for the preconditioner
   //
   //    In Comp
   //    Domain:   1/mu (Curl E, Curl F) + omega^2 * epsilon (E,F)
   //
   //    In PML:   1/mu (abs(1/det(J) J^T J) Curl E, Curl F)
   //              + omega^2 * epsilon (abs(det(J) * (J^T J)^-1) * E, F)
+   if (pa || !umf_solver)
   {
      ConstantCoefficient absomeg(pow(omega, 2) * epsilon);
      RestrictedCoefficient restr_absomeg(absomeg,attr);
@@ -435,39 +458,57 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));

+      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
      prec.Assemble();

-      OperatorPtr PCOpAh;
-      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
-      // 13b. Define and apply a GMRES solver for AU=B with a block diagonal
-      //      preconditioner based on the Gauss-Seidel sparse smoother.
+      // 14b. Define and apply a GMRES solver for AU=B with a block diagonal
+      //      preconditioner based on the Gauss-Seidel or Jacobi sparse smoother.
      Array<int> offsets(3);
      offsets[0] = 0;
      offsets[1] = fespace->GetTrueVSize();
      offsets[2] = fespace->GetTrueVSize();
      offsets.PartialSum();

-      GSSmoother gs00(*PCOpAh.As<SparseMatrix>());
-      BlockDiagonalPreconditioner BlockGS(offsets);
-      ScaledOperator gs11(&gs00,
-                          (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
-      BlockGS.SetDiagonalBlock(0,&gs00);
-      BlockGS.SetDiagonalBlock(1,&gs11);
+      Operator *pc_r = nullptr;
+      Operator *pc_i = nullptr;
+      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
+      if (pa)
+      {
+         // Jacobi Smoother
+         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
+         ScaledOperator *d11 = new ScaledOperator(d00, s);
+         pc_r = d00;
+         pc_i = d11;
+      }
+      else
+      {
+         OperatorPtr PCOpAh;
+         prec.SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
+         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
+         // Gauss-Seidel Smoother
+         GSSmoother *gs00 = new GSSmoother(*PCOpAh.As<SparseMatrix>());
+         ScaledOperator *gs11 = new ScaledOperator(gs00, s);
+         pc_r = gs00;
+         pc_i = gs11;
+      }
+
+      BlockDiagonalPreconditioner BlockDP(offsets);
+      BlockDP.SetDiagonalBlock(0, pc_r);
+      BlockDP.SetDiagonalBlock(1, pc_i);

      GMRESSolver gmres;
      gmres.SetPrintLevel(1);
      gmres.SetKDim(200);
-      gmres.SetMaxIter(2000);
+      gmres.SetMaxIter(pa ? 5000 : 2000);
      gmres.SetRelTol(1e-5);
      gmres.SetAbsTol(0.0);
      gmres.SetOperator(*A);
-      gmres.SetPreconditioner(BlockGS);
+      gmres.SetPreconditioner(BlockDP);
      gmres.Mult(B, X);
   }
-#endif

-   // 14. Recover the solution as a finite element grid function and compute the
+   // 15. Recover the solution as a finite element grid function and compute the
   //     errors if the exact solution is known.
   a.RecoverFEMSolution(X, b, x);

@@ -504,7 +545,7 @@ int main(int argc, char *argv[])
           << sqrt(L2Error_Re*L2Error_Re + L2Error_Im*L2Error_Im) << "\n\n";
   }

-   // 15. Save the refined mesh and the solution. This output can be viewed
+   // 16. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m mesh -g sol".
   {
      ofstream mesh_ofs("ex25.mesh");
@@ -519,7 +560,7 @@ int main(int argc, char *argv[])
      x.imag().Save(sol_i_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 17. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      // Define visualization keys for GLVis (see GLVis documentation)
@@ -570,7 +611,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 17. Free the used memory.
+   // 18. Free the used memory.
   delete pml;
   delete fespace;
   delete fec;
@@ -916,7 +957,14 @@ void CartesianPML::SetBoundaries()

 void CartesianPML::SetAttributes(Mesh *mesh_)
 {
+   // Initialize bdr attributes
+   for (int i = 0; i < mesh_->GetNBE(); ++i)
+   {
+      mesh_->GetBdrElement(i)->SetAttribute(i+1);
+   }
+
   int nrelem = mesh_->GetNE();
+
   elems.SetSize(nrelem);

   // Loop through the elements and identify which of them are in the PML
@@ -10,6 +10,10 @@
 //               mpirun -np 4 ex25p -o 2 -f 8.0 -rs 2 -rp 2 -prob 4 -m ../data/inline-quad.mesh
 //               mpirun -np 4 ex25p -o 2 -f 2.0 -rs 1 -rp 1 -prob 4 -m ../data/inline-hex.mesh
 //
+// Device sample runs:
+//               mpirun -np 4 ex25p -o 1 -f 3.0 -rs 3 -rp 1 -prob 2 -pa -d cuda
+//               mpirun -np 4 ex25p -o 2 -f 1.0 -rs 1 -rp 1 -prob 3 -pa -d cuda
+//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -167,6 +171,8 @@ int main(int argc, char *argv[])
   bool slu_solver  = false;
   bool mumps_solver = false;
   bool visualization = 1;
+   bool pa = false;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -198,6 +204,10 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (slu_solver && mumps_solver)
   {
@@ -211,7 +221,12 @@ int main(int argc, char *argv[])
   if (iprob > 4) { iprob = 4; }
   prob = (prob_type)iprob;

-   // 3. Setup the (serial) mesh on all processors.
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Setup the (serial) mesh on all processors.
   if (!mesh_file)
   {
      exact_known = true;
@@ -259,7 +274,7 @@ int main(int argc, char *argv[])
   // Setup PML length
   Array2D<double> length(dim, 2); length = 0.0;

-   // 4. Setup the Cartesian PML region.
+   // 5. Setup the Cartesian PML region.
   switch (prob)
   {
      case disc:
@@ -285,13 +300,13 @@ int main(int argc, char *argv[])
   comp_domain_bdr = pml->GetCompDomainBdr();
   domain_bdr = pml->GetDomainBdr();

-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 6. Refine the serial mesh on all processors to increase the resolution.
   for (int l = 0; l < ref_levels; l++)
   {
      mesh->UniformRefinement();
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh.
+   // 7. Define a parallel mesh by a partitioning of the serial mesh.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -301,13 +316,13 @@ int main(int argc, char *argv[])
      }
   }

-   // 6a. Reorient mesh in case of a tet mesh
+   // 7a. Reorient mesh in case of a tet mesh
   pmesh->ReorientTetMesh();

-   // 7. Set element attributes in order to distinguish elements in the PML
+   // 8. Set element attributes in order to distinguish elements in the PML
   pml->SetAttributes(pmesh);

-   // 8. Define a parallel finite element space on the parallel mesh. Here we
+   // 9. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -317,9 +332,9 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 9. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    based on the specific mesh and the problem type.
+   // 10. Determine the list of true (i.e. parallel conforming) essential
+   //     boundary dofs. In this example, the boundary conditions are defined
+   //     based on the specific mesh and the problem type.
   Array<int> ess_tdof_list;
   Array<int> ess_bdr;
   if (pmesh->bdr_attributes.Size())
@@ -359,11 +374,11 @@ int main(int argc, char *argv[])
   }
   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-   // 10. Setup Complex Operator convention
+   // 11. Setup Complex Operator convention
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 11. Set up the parallel linear form b(.) which corresponds to the
+   // 12. Set up the parallel linear form b(.) which corresponds to the
   //     right-hand side of the FEM linear system.
   VectorFunctionCoefficient f(dim, source);
   ParComplexLinearForm b(fespace, conv);
@@ -374,7 +389,7 @@ int main(int argc, char *argv[])
   b.Vector::operator=(0.0);
   b.Assemble();

-   // 12. Define the solution vector x as a parallel complex finite element grid
+   // 13. Define the solution vector x as a parallel complex finite element grid
   //     function corresponding to fespace.
   ParComplexGridFunction x(fespace);
   x = 0.0;
@@ -382,7 +397,7 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
   x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);

-   // 13. Set up the parallel sesquilinear form a(.,.)
+   // 14. Set up the parallel sesquilinear form a(.,.)
   //
   //     In Comp
   //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -436,19 +451,20 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                         new VectorFEMassIntegrator(restr_c2_Im));

-   // 14. Assemble the parallel bilinear form and the corresponding linear
+   // 15. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, etc.
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a.Assemble();

   OperatorPtr Ah;
   Vector B, X;
   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);

-   // 15. Solve using a direct or an iterative solver
+   // 16. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUPERLU
-   if (slu_solver)
+   if (!pa && slu_solver)
   {
      // Transform to monolithic HypreParMatrix
      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
@@ -463,7 +479,7 @@ int main(int argc, char *argv[])
   }
 #endif
 #ifdef MFEM_USE_MUMPS
-   if (mumps_solver)
+   if (!pa && mumps_solver)
   {
      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
      MUMPSSolver mumps;
@@ -481,7 +497,7 @@ int main(int argc, char *argv[])
   //
   //    In PML:   1/mu (abs(1/det(J) J^T J) Curl E, Curl F)
   //              + omega^2 * epsilon (abs(det(J) * (J^T J)^-1) * E, F)
-   if (!slu_solver && !mumps_solver)
+   if (pa || (!slu_solver && !mumps_solver))
   {
      ConstantCoefficient absomeg(pow(omega, 2) * epsilon);
      RestrictedCoefficient restr_absomeg(absomeg,attr);
@@ -501,11 +517,9 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));

+      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
      prec.Assemble();

-      OperatorPtr PCOpAh;
-      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
      // 16b. Define and apply a parallel GMRES solver for AU=B with a block
      //      diagonal preconditioner based on hypre's AMS preconditioner.
      Array<int> offsets(3);
@@ -514,21 +528,41 @@ int main(int argc, char *argv[])
      offsets[2] = fespace->GetTrueVSize();
      offsets.PartialSum();

-      HypreAMS ams00(*PCOpAh.As<HypreParMatrix>(),fespace);
-      BlockDiagonalPreconditioner BlockAMS(offsets);
-      ScaledOperator ams11(&ams00,
-                           (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
-      BlockAMS.SetDiagonalBlock(0,&ams00);
-      BlockAMS.SetDiagonalBlock(1,&ams11);
+      Operator *pc_r = nullptr;
+      Operator *pc_i = nullptr;
+      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
+      if (pa)
+      {
+         // Jacobi Smoother
+         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
+         ScaledOperator *d11 = new ScaledOperator(d00, s);
+         pc_r = d00;
+         pc_i = d11;
+      }
+      else
+      {
+         OperatorPtr PCOpAh;
+         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
+         // Hypre AMS
+         HypreAMS *ams00 = new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace);
+         ScaledOperator *ams11 = new ScaledOperator(ams00, s);
+         pc_r = ams00;
+         pc_i = ams11;
+      }
+
+      BlockDiagonalPreconditioner BlockDP(offsets);
+      BlockDP.SetDiagonalBlock(0, pc_r);
+      BlockDP.SetDiagonalBlock(1, pc_i);

      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetPrintLevel(1);
      gmres.SetKDim(200);
-      gmres.SetMaxIter(2000);
+      gmres.SetMaxIter(pa ? 5000 : 2000);
      gmres.SetRelTol(1e-5);
      gmres.SetAbsTol(0.0);
      gmres.SetOperator(*Ah);
-      gmres.SetPreconditioner(BlockAMS);
+      gmres.SetPreconditioner(BlockDP);
      gmres.Mult(B, X);
   }

@@ -1003,8 +1037,12 @@ void CartesianPML::SetBoundaries()

 void CartesianPML::SetAttributes(ParMesh *pmesh)
 {
-   int myid;
-   MPI_Comm_rank(MPI_COMM_WORLD,&myid);
+   // Initialize bdr attributes
+   for (int i = 0; i < pmesh->GetNBE(); ++i)
+   {
+      pmesh->GetBdrElement(i)->SetAttribute(i+1);
+   }
+
   int nrelem = pmesh->GetNE();

   // Initialize list with 1
@@ -40,7 +40,7 @@ using namespace mfem;
 // in the FiniteElementSpaceHierarchy. The preconditioner uses a CG solver on
 // the coarsest level and second order Chebyshev accelerated smoothers on the
 // other levels.
-class DiffusionMultigrid : public Multigrid
+class DiffusionMultigrid : public GeometricMultigrid
 {
 private:
   ConstantCoefficient one;
@@ -49,7 +49,7 @@ public:
   // Constructs a diffusion multigrid for the given FiniteElementSpaceHierarchy
   // and the array of essential boundaries
   DiffusionMultigrid(FiniteElementSpaceHierarchy& fespaces, Array<int>& ess_bdr)
-      : Multigrid(fespaces), one(1.0)
+      : GeometricMultigrid(fespaces), one(1.0)
   {
      ConstructCoarseOperatorAndSolver(fespaces.GetFESpaceAtLevel(0), ess_bdr);

@@ -37,7 +37,7 @@ using namespace mfem;
 // all spaces except the coarsest one in the ParFiniteElementSpaceHierarchy.
 // The multigrid uses a PCG solver preconditioned with AMG on the coarsest level
 // and second order Chebyshev accelerated smoothers on the other levels.
-class DiffusionMultigrid : public Multigrid
+class DiffusionMultigrid : public GeometricMultigrid
 {
 private:
   ConstantCoefficient one;
@@ -48,7 +48,7 @@ public:
   // and the array of essential boundaries
   DiffusionMultigrid(ParFiniteElementSpaceHierarchy& fespaces,
                      Array<int>& ess_bdr)
-      : Multigrid(fespaces), one(1.0)
+      : GeometricMultigrid(fespaces), one(1.0)
   {
      ConstructCoarseOperatorAndSolver(fespaces.GetFESpaceAtLevel(0), ess_bdr);

@@ -22,6 +22,7 @@
 //
 // Device sample runs:
 //               mpirun -np 4 ex3p -m ../data/star.mesh -pa -d cuda
+//               mpirun -np 4 ex3p -m ../data/star.mesh -no-pa -d cuda
 //               mpirun -np 4 ex3p -m ../data/star.mesh -pa -d raja-cuda
 //               mpirun -np 4 ex3p -m ../data/star.mesh -pa -d raja-omp
 //               mpirun -np 4 ex3p -m ../data/beam-hex.mesh -pa -d cuda
@@ -290,15 +290,16 @@ int main(int argc, char *argv[])
      k.SetAssemblyLevel(AssemblyLevel::FULL);
   }
   m.AddDomainIntegrator(new MassIntegrator);
-   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+   constexpr double alpha = -1.0;
+   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
   k.AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
   k.AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));

   LinearForm b(&fes);
   b.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
+      new BoundaryFlowIntegrator(inflow, velocity, alpha));

   m.Assemble();
   int skip_zeros = 0;
@@ -64,6 +64,66 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;

+// Type of preconditioner for implicit time integrator
+enum class PrecType : int
+{
+   ILU = 0,
+   AIR = 1
+};
+
+#if MFEM_HYPRE_VERSION >= 21800
+// Algebraic multigrid preconditioner for advective problems based on
+// approximate ideal restriction (AIR). Most effective when matrix is
+// first scaled by DG block inverse, and AIR applied to scaled matrix.
+// See https://doi.org/10.1137/17M1144350.
+class AIR_prec : public Solver
+{
+private:
+   const HypreParMatrix *A;
+   // Copy of A scaled by block-diagonal inverse
+   HypreParMatrix A_s;
+
+   HypreBoomerAMG *AIR_solver;
+   int blocksize;
+
+public:
+   AIR_prec(int blocksize_) : AIR_solver(NULL), blocksize(blocksize_) { }
+
+   void SetOperator(const Operator &op)
+   {
+      width = op.Width();
+      height = op.Height();
+
+      A = dynamic_cast<const HypreParMatrix *>(&op);
+      MFEM_VERIFY(A != NULL, "AIR_prec requires a HypreParMatrix.")
+
+      // Scale A by block-diagonal inverse
+      BlockInverseScale(A, &A_s, NULL, NULL, blocksize,
+                        BlockInverseScaleJob::MATRIX_ONLY);
+      delete AIR_solver;
+      AIR_solver = new HypreBoomerAMG(A_s);
+      AIR_solver->SetAdvectiveOptions(1, "", "FA");
+      AIR_solver->SetPrintLevel(0);
+      AIR_solver->SetMaxLevels(50);
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      // Scale the rhs by block inverse and solve system
+      HypreParVector z_s;
+      BlockInverseScale(A, NULL, &x, &z_s, blocksize,
+                        BlockInverseScaleJob::RHS_ONLY);
+      AIR_solver->Mult(z_s, y);
+   }
+
+   ~AIR_prec()
+   {
+      delete AIR_solver;
+   }
+};
+#endif
+
+
 class DG_Solver : public Solver
 {
 private:
@@ -71,24 +131,37 @@ private:
   SparseMatrix M_diag;
   HypreParMatrix *A;
   GMRESSolver linear_solver;
-   BlockILU prec;
+   Solver *prec;
   double dt;
 public:
-   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
+   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes,
+             PrecType prec_type)
      : M(M_),
        K(K_),
        A(NULL),
        linear_solver(M.GetComm()),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
        dt(-1.0)
   {
+      int block_size = fes.GetFE(0)->GetDof();
+      if (prec_type == PrecType::ILU)
+      {
+         prec = new BlockILU(block_size,
+                             BlockILU::Reordering::MINIMUM_DISCARDED_FILL);
+      }
+      else if (prec_type == PrecType::AIR)
+      {
+#if MFEM_HYPRE_VERSION >= 21800
+         prec = new AIR_prec(block_size);
+#else
+         MFEM_ABORT("Must have MFEM_HYPRE_VERSION >= 21800 to use AIR.\n");
+#endif
+      }
      linear_solver.iterative_mode = false;
      linear_solver.SetRelTol(1e-9);
      linear_solver.SetAbsTol(0.0);
      linear_solver.SetMaxIter(100);
      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
+      linear_solver.SetPreconditioner(*prec);

      M.GetDiag(M_diag);
   }
@@ -121,10 +194,12 @@ public:

   ~DG_Solver()
   {
+      delete prec;
      delete A;
   }
 };

+
 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
    and advection matrices, and b describes the flow on the boundary. This can
@@ -142,7 +217,8 @@ private:
   mutable Vector z;

 public:
-   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b);
+   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b,
+                PrecType prec_type);

   virtual void Mult(const Vector &x, Vector &y) const;
   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
@@ -154,10 +230,9 @@ public:
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();

   // 2. Parse command-line options.
   problem = 0;
@@ -178,7 +253,11 @@ int main(int argc, char *argv[])
   bool adios2 = false;
   bool binary = false;
   int vis_steps = 5;
-
+#if MFEM_HYPRE_VERSION >= 21800
+   PrecType prec_type = PrecType::AIR;
+#else
+   PrecType prec_type = PrecType::ILU;
+#endif
   int precision = 8;
   cout.precision(precision);

@@ -212,6 +291,8 @@ int main(int argc, char *argv[])
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
                  "Time step.");
+   args.AddOption((int *)&prec_type, "-pt", "--prec-type", "Preconditioner for "
+                  "implicit solves. 0 for ILU, 1 for pAIR-AMG.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -232,20 +313,19 @@ int main(int argc, char *argv[])
   args.Parse();
   if (!args.Good())
   {
-      if (myid == 0)
+      if (mpi.Root())
      {
         args.PrintUsage(cout);
      }
-      MPI_Finalize();
      return 1;
   }
-   if (myid == 0)
+   if (mpi.Root())
   {
      args.PrintOptions(cout);
   }

   Device device(device_config);
-   if (myid == 0) { device.Print(); }
+   if (mpi.Root()) { device.Print(); }

   // 3. Read the serial mesh from the given mesh file on all processors. We can
   //    handle geometrically periodic meshes in this code.
@@ -272,12 +352,11 @@ int main(int argc, char *argv[])
      case 23: ode_solver = new SDIRK23Solver; break;
      case 24: ode_solver = new SDIRK34Solver; break;
      default:
-         if (myid == 0)
+         if (mpi.Root())
         {
            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
         }
         delete mesh;
-         MPI_Finalize();
         return 3;
   }

@@ -311,7 +390,7 @@ int main(int argc, char *argv[])
   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);

   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
-   if (myid == 0)
+   if (mpi.Root())
   {
      cout << "Number of unknowns: " << global_vSize << endl;
   }
@@ -342,15 +421,16 @@ int main(int argc, char *argv[])
   }

   m->AddDomainIntegrator(new MassIntegrator);
-   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+   constexpr double alpha = -1.0;
+   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
   k->AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
   k->AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));

   ParLinearForm *b = new ParLinearForm(fes);
   b->AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
+      new BoundaryFlowIntegrator(inflow, velocity, alpha));

   int skip_zeros = 0;
   m->Assemble();
@@ -451,11 +531,11 @@ int main(int argc, char *argv[])
      sout.open(vishost, visport);
      if (!sout)
      {
-         if (myid == 0)
+         if (mpi.Root())
            cout << "Unable to connect to GLVis server at "
                 << vishost << ':' << visport << endl;
         visualization = false;
-         if (myid == 0)
+         if (mpi.Root())
         {
            cout << "GLVis visualization disabled.\n";
         }
@@ -467,7 +547,7 @@ int main(int argc, char *argv[])
         sout << "solution\n" << *pmesh << *u;
         sout << "pause\n";
         sout << flush;
-         if (myid == 0)
+         if (mpi.Root())
            cout << "GLVis visualization paused."
                 << " Press space (in the GLVis window) to resume it.\n";
      }
@@ -476,7 +556,7 @@ int main(int argc, char *argv[])
   // 10. Define the time-dependent evolution operator describing the ODE
   //     right-hand side, and perform time-integration (looping over the time
   //     iterations, ti, with a time-step dt).
-   FE_Evolution adv(*m, *k, *B);
+   FE_Evolution adv(*m, *k, *B, prec_type);

   double t = 0.0;
   adv.SetTime(t);
@@ -493,7 +573,7 @@ int main(int argc, char *argv[])

      if (done || ti % vis_steps == 0)
      {
-         if (myid == 0)
+         if (mpi.Root())
         {
            cout << "time step: " << ti << ", time: " << t << endl;
         }
@@ -564,16 +644,14 @@ int main(int argc, char *argv[])
 #endif
   delete dc;

-   MPI_Finalize();
   return 0;
 }


 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
-                           const Vector &_b)
-   : TimeDependentOperator(_M.Height()),
-     b(_b),
+                           const Vector &_b, PrecType prec_type)
+   : TimeDependentOperator(_M.Height()), b(_b),
     M_solver(_M.ParFESpace()->GetComm()),
     z(_M.Height())
 {
@@ -598,7 +676,7 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
      HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
      M_prec = hypre_prec;

-      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
+      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace(), prec_type);
   }
   else
   {
@@ -614,6 +692,10 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
   M_solver.SetPrintLevel(0);
 }

+// Solve the equation:
+//    u_t = M^{-1}(Ku + b),
+// by solving associated linear system
+//    (M - dt*K) d = K*u + b
 void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
 {
   K->Mult(x, z);
@@ -12,6 +12,9 @@
 set(GINKGO_EXAMPLES_SRCS)
 list(APPEND GINKGO_EXAMPLES_SRCS
  ex1.cpp
+  ex1lor.cpp
+  ex1lorcoeff.cpp
+  ex1tetlorcoeff.cpp
 )

 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
@@ -33,6 +36,8 @@ add_mfem_examples(GINKGO_EXAMPLES_SRCS ${PFX} "" test_ginkgo)

 # Command line options for the tests.
 set(EX1_COMMON_OPTS ex1 -m ../data/star.mesh --use_gko_solver)
+set(EX1LOR_COMMON_OPTS ex1lor -m ../../data/beam-hex.mesh -pc)
+
 set(EX1_TEST_OPTS ${EX9_COMMON_OPTS})

 # Add the tests: one test per source file.
@@ -0,0 +1,17 @@
+{
+    "config_version": 2,
+    "solver": {
+        "solver": "AMG",
+        "smoother": {
+            "scope": "jacobi",
+            "solver": "JACOBI_L1"
+        },
+        "presweeps": 1,
+        "interpolator": "D2",
+        "max_row_sum" : 0.9,
+        "strength_threshold" : 0.25,
+        "postsweeps": 1,
+        "max_iters": 1,
+        "cycle": "V"
+    }
+}
@@ -0,0 +1,721 @@
+//                          MFEM Example 1, modified
+//
+// This code has been modified from `ex1.cpp` provided in the examples of
+// MFEM.  The sections not marked as related to Ginkgo are largely unchanged
+// from the version provided by MFEM.
+//
+// This code also contains portions from `miniapps/performance/ex1.cpp`
+// pertaining to the LOR preconditioner.  The preconditioner types
+// used in this example are applied to the SparseMatrix created from
+// the LOR mesh, so they are all LOR preconditioners with different
+// subtypes.
+//
+// The default mesh option is "beam-hex.mesh", provided by MFEM.
+// Important non-default options:
+//  -m [file] : Mesh file.
+//  -d "cuda" : Use the MFEM cuda backend and Ginkgo CudaExecutor.
+//  -pc-type "gko:ilu" : Use the Ginkgo ILU preconditioner (default is Block
+//                       Jacobi)
+//  -pc-type "none" : No LOR preconditioner
+//
+//  Options only for the Block Jacobi preconditioner (default:)
+//  -pc-so "none" : Don't let Ginkgo automatically pick options for precision
+//                  reduction in the storage of the Block Jacobi preconditioner
+//  -pc-acc [value] : Accuracy parameter.
+//
+// MFEM's provided information about `ex1.cpp`:
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void PermuteSparseMatrix(SparseMatrix &A, Array<int> &pinv, double shift=0.0)
+{
+   int n = pinv.Size();
+
+   SparseMatrix PA(A);
+
+   Array<int> p(n);
+   for (int i=0; i<n; ++i)
+   {
+      p[pinv[i]] = i;
+   }
+
+   // Set LU = A(P,P) using the permutation generated above
+   const int *IA = A.GetI();
+   const int *JA = A.GetJ();
+   const double *VA = A.GetData();
+
+   int *I = PA.GetI();
+   int *J = PA.GetJ();
+   double *V = PA.GetData();
+
+   I[0] = 0;
+   for (int i=0; i<n; ++i)
+   {
+      int pi = p[i];
+      int nnz_pi = IA[pi+1] - IA[pi];
+      I[i+1] = I[i] + nnz_pi;
+      for (int jj=0; jj<nnz_pi; ++jj)
+      {
+         int pj = JA[IA[pi] + jj];
+         int j = pinv[pj];
+
+         J[I[i] + jj] = j;
+         V[I[i] + jj] = VA[IA[pi] + jj];
+         if (i == j) { V[I[i] + jj] += shift; }
+      }
+   }
+
+   PA.SortColumnIndices();
+   PA.Swap(A);
+}
+
+// helper functions for cg/pcg solve with timer and iter count return
+int cg_solve(const Operator &A, const Vector &b, Vector &x,
+             int print_iter, int max_num_iter,
+             double RTOLERANCE, double ATOLERANCE, double &it_time)
+{
+
+   CGSolver cg;
+   cg.SetPrintLevel(print_iter);
+   cg.SetMaxIter(max_num_iter);
+   cg.SetRelTol(sqrt(RTOLERANCE));
+   cg.SetAbsTol(sqrt(ATOLERANCE));
+   cg.SetOperator(A);
+
+   tic_toc.Clear();
+   tic_toc.Start();
+
+   cg.Mult(b, x);
+
+   tic_toc.Stop();
+   it_time = tic_toc.RealTime();
+
+   return cg.GetNumIterations();
+}
+
+int pcg_solve(const Operator &A, Solver &B, const Vector &b, Vector &x,
+              int print_iter, int max_num_iter,
+              double RTOLERANCE, double ATOLERANCE, double &it_time)
+{
+
+   CGSolver pcg;
+   pcg.SetPrintLevel(print_iter);
+   pcg.SetMaxIter(max_num_iter);
+   pcg.SetRelTol(sqrt(RTOLERANCE));
+   pcg.SetAbsTol(sqrt(ATOLERANCE));
+   pcg.SetOperator(A);
+   pcg.SetPreconditioner(B);
+
+   tic_toc.Clear();
+   tic_toc.Start();
+
+   pcg.Mult(b, x);
+
+   tic_toc.Stop();
+   it_time = tic_toc.RealTime();
+
+   return pcg.GetNumIterations();
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/beam-hex.mesh";
+   int ref_levels = 3;
+   int order = 2;
+   const char *basis_type = "G"; // Gauss-Lobatto
+   bool static_cond = false;
+   bool pa = true;
+   const char *device_config = "cpu";
+   bool visualization = true;
+   const char *pc_type = "gko:bj";
+   const char *pc_storage_opt = "auto";
+   double pc_acc = 1.e-1;
+   int pc_max_bs = 32;
+   int permute = 0;
+   bool output_sol = false;
+   bool output_pc = false;
+   int isai_sparsity_power = 1;
+   int par_ilu_its = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&ref_levels, "-l", "--refinement-levels",
+                  "Number of uniform refinement levels for mesh.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&pc_type, "-pc-type", "--preconditioner-type",
+                  "Type of preconditioner used on LOR matrix.");
+   args.AddOption(&pc_storage_opt, "-pc-so",
+                  "--preconditioner-storage-optimization",
+                  "Type of precision storage optimization to use for Ginkgo BlockJacobi.");
+   args.AddOption(&pc_acc, "-pc-acc", "--preconditioner-accuracy",
+                  "Accuracy parameter for Ginkgo BlockJacobi.");
+   args.AddOption(&pc_max_bs, "-pc-mbs", "--preconditioner-max-block-size",
+                  "Maximum block size for Ginkgo BlockJacobi.");
+   args.AddOption(&permute, "-per", "--permutation",
+                  "Specify preconditioner permutation.");
+   args.AddOption(&output_sol, "-out", "--output-solution-and-mesh", "-no-out",
+                  "--no-solution-and-mesh-output",
+                  "Output mesh and solution for inspection.");
+   args.AddOption(&output_pc, "-out-pc", "--output-lor-matrix-and-mesh",
+                  "-no-out-pc",
+                  "--no-lor-matrix-and-mesh-output",
+                  "Output LOR mesh and sparse matrix for inspection.");
+   args.AddOption(&isai_sparsity_power, "-isai-sp", "--isai-sparsity-power",
+                  "Power to use for sparsity pattern of ISAI in Ginkgo ILU-ISAI.");
+   args.AddOption(&par_ilu_its, "-pilu-its", "--par-ilu-iterations",
+                  "Number of iterations for the Ginkgo ParILU algorithm.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   enum PCType { NONE, GKO_BLOCK_JACOBI, GKO_ILU, GKO_ILU_ISAI, GKO_CUILU, GKO_CUILU_ISAI, MFEM_GS, MFEM_UMFPACK };
+   PCType pc_choice;
+   bool pc = true;
+   const char *trisolve_type = "exact"; //only used for ILU
+   if (!strcmp(pc_type, "gko:bj")) { pc_choice = GKO_BLOCK_JACOBI; }
+   else if (!strcmp(pc_type, "gko:ilu")) { pc_choice = GKO_ILU; }
+   else if (!strcmp(pc_type, "gko:ilu-isai"))
+   {
+      pc_choice = GKO_ILU_ISAI;
+      trisolve_type = "isai";
+   }
+   else if (!strcmp(pc_type, "gko:cuilu")) { pc_choice = GKO_CUILU; }
+   else if (!strcmp(pc_type, "gko:cuilu-isai"))
+   {
+      pc_choice = GKO_CUILU_ISAI;
+      trisolve_type = "isai";
+   }
+   else if (!strcmp(pc_type, "mfem:gs")) { pc_choice = MFEM_GS; }
+   else if (!strcmp(pc_type, "mfem:umf"))
+   {
+#ifdef MFEM_USE_SUITESPARSE
+      pc_choice = MFEM_UMFPACK;
+#else
+      mfem_error("Preconditioner requires SuiteSparse");
+#endif
+   }
+   else if (!strcmp(pc_type, "none"))
+   {
+      pc_choice = NONE;
+      pc = false;
+   }
+   else
+   {
+      mfem_error("Invalid Preconditioner specified");
+      return 3;
+   }
+
+   // ---------------------------------------------------------------
+   // -------------------- Start Ginkgo set-up ----------------------
+
+   // Create Ginkgo executor.
+
+   // This will point to the selected executor default executor
+   std::shared_ptr<gko::Executor> executor;
+
+   // We will always need an OpenMP executor.
+   auto omp_executor = gko::OmpExecutor::create();
+
+   // If the user has requested to use CUDA, then build a
+   // CudaExecutor and set `executor` to it; otherwise,
+   // use the OmpExecutor
+   if (!strcmp(device_config, "cuda"))
+   {
+      auto cuda_executor =
+         gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+      executor = cuda_executor;
+   }
+   else
+   {
+      executor = omp_executor;
+   }
+
+   // --------------------- End Ginkgo set-up -----------------------
+   // ---------------------------------------------------------------
+
+   // See class BasisType in fem/fe_coll.hpp for available basis types
+   int basis = BasisType::GetType(basis_type[0]);
+   cout << "Using " << BasisType::Name(basis) << " basis ..." << endl;
+
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement.
+   {
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+   cout << "Total elements in refined mesh: " <<  mesh->GetNE() << std::endl;
+
+   // 5. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim, basis);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+      cout << "Using isoparametric FEs: " << fec->Name() << endl;
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim, basis);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
+        << endl;
+
+   // Create the LOR mesh and finite element space. In the settings of this
+   // example, we can transfer between HO and LOR with the identity operator.
+   Mesh *mesh_lor = NULL;
+   FiniteElementCollection *fec_lor = NULL;
+   FiniteElementSpace *fespace_lor = NULL;
+   Array<int> *inv_reordering = NULL;
+   if (pc)
+   {
+      int basis_lor = basis;
+      if (basis == BasisType::Positive) { basis_lor=BasisType::ClosedUniform; }
+      mesh_lor = new Mesh(mesh, order, basis_lor);
+      fec_lor = new H1_FECollection(1, dim);
+      fespace_lor = new FiniteElementSpace(mesh_lor, fec_lor);
+
+      if (permute == 1)
+      {
+
+         tic_toc.Clear();
+         tic_toc.Start();
+
+         const Table &pre_reorder_dofs = fespace_lor->GetElementToDofTable();
+         const Table pre_reorder_dofs_copy(pre_reorder_dofs);
+         fespace_lor->ReorderElementToDofTable();
+         const Table &post_reorder_dofs = fespace_lor->GetElementToDofTable();
+
+         inv_reordering = new Array<int>(fespace_lor->GetTrueVSize());
+         for (int i = 0; i < pre_reorder_dofs.Size(); i++)
+         {
+
+            Array<int> old_row;
+            Array<int> new_row;
+            pre_reorder_dofs_copy.GetRow(i, old_row);
+            post_reorder_dofs.GetRow(i, new_row);
+            for (int j = 0; j < pre_reorder_dofs_copy.RowSize(i); j++)
+            {
+               int new_dof = new_row[j];
+               int old_dof = old_row[j];
+               (*inv_reordering)[old_dof] = new_dof;
+            }
+         }
+
+         tic_toc.Stop();
+         cout << "Real time spent reordering: " <<
+              tic_toc.RealTime() << "\n";
+      }
+   }
+
+   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking all
+   //    the boundary attributes from the mesh as essential (Dirichlet) and
+   //    converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (mesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Array<int> ess_pc_tdof_list(ess_tdof_list.Size());
+
+   if (permute == 1)
+   {
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_pc_tdof_list.operator[](i) = inv_reordering->operator[](
+                                             ess_tdof_list.operator[](i));
+      }
+   }
+   else
+   {
+
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_pc_tdof_list);
+   }
+
+   // Array<int> ess_bdr(mesh->bdr_attributes.Max());
+   // ess_bdr = 1;
+   // fespace->GetEssentialTrueDofs(ess_bdr, ess_pc_tdof_list);
+
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side
+   // of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+   LinearForm *b = new LinearForm(fespace);
+
+   ConstantCoefficient one(1.0);
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();
+
+   // 8. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
+   GridFunction x(fespace);
+   x = 0.0;
+
+   // 9. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the
+   //    Diffusion domain integrator.
+   BilinearForm *a = new BilinearForm(fespace);
+   BilinearForm *a_pc = NULL;
+   if (pc) { a_pc = new BilinearForm(fespace_lor); }
+   if (pa)
+   {
+      a->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   }
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 10. Assemble the bilinear form and the corresponding linear system,
+   //     applying any necessary transformations such as: eliminating boundary
+   //     conditions, applying conforming constraints for non-conforming AMR,
+   //     static condensation, etc.
+   if (static_cond)
+   {
+      a->EnableStaticCondensation();
+   }
+   a->Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   // 11. Solve the linear system A X = B.
+   double it_time = 0.;
+   int total_its = 0;
+
+   SparseMatrix A_pc;
+   if (pc)
+   {
+
+      tic_toc.Clear();
+      tic_toc.Start();
+
+      a_pc->AddDomainIntegrator(new DiffusionIntegrator(one));
+      a_pc->UsePrecomputedSparsity();
+      a_pc->Assemble();
+
+      tic_toc.Stop();
+      cout << "Real time assembling A_pc SparseMatrix: " <<
+           tic_toc.RealTime() << "\n";
+
+      tic_toc.Clear();
+      tic_toc.Start();
+
+      a_pc->FormSystemMatrix(ess_pc_tdof_list, A_pc);
+
+      tic_toc.Stop();
+      cout << "Real time adjusting A_pc for essential BC: " <<
+           tic_toc.RealTime() << "\n";
+
+
+      if (permute == 2)
+      {
+         tic_toc.Clear();
+         tic_toc.Start();
+
+         Array<int> perm(fespace_lor->GetTrueVSize());
+         SparseMatrix A_pc_tmp(A_pc);
+         MinimumDiscardedFillOrdering(A_pc_tmp, perm);
+         inv_reordering = new Array<int>(fespace_lor->GetTrueVSize());
+         for (int i=0; i<perm.Size(); ++i)
+         {
+            (*inv_reordering)[perm[i]] = i;
+         }
+         PermuteSparseMatrix(A_pc, *inv_reordering);
+
+         tic_toc.Stop();
+         cout << "Real time spent reordering: " <<
+              tic_toc.RealTime() << "\n";
+      }
+
+
+      if (pc_choice == GKO_BLOCK_JACOBI)
+      {
+
+         // Create Ginkgo Jacobi preconditioner
+         if (permute)
+         {
+            tic_toc.Clear();
+            tic_toc.Start();
+            GinkgoWrappers::GinkgoJacobiPreconditioner M(executor, A_pc, *inv_reordering,
+                                                         pc_storage_opt,
+                                                         pc_acc, pc_max_bs);
+            tic_toc.Stop();
+            cout << "Real time creating Ginkgo BlockJacobi preconditioner: " <<
+                 tic_toc.RealTime() << "\n";
+
+            // Use preconditioned CG
+            total_its = pcg_solve(*A, M, B, X, 0, X.Size(), 1e-12, 0.0, it_time);
+
+            cout << "Real time in PCG: " << it_time << "\n";
+         }
+         else
+         {
+            tic_toc.Clear();
+            tic_toc.Start();
+            GinkgoWrappers::GinkgoJacobiPreconditioner M(executor, A_pc, pc_storage_opt,
+                                                         pc_acc, pc_max_bs);
+            tic_toc.Stop();
+            cout << "Real time creating Ginkgo BlockJacobi preconditioner: " <<
+                 tic_toc.RealTime() << "\n";
+
+            // Use preconditioned CG
+            total_its = pcg_solve(*A, M, B, X, 0, X.Size(), 1e-12, 0.0, it_time);
+
+            cout << "Real time in PCG: " << it_time << "\n";
+         }
+
+      }
+      else if (pc_choice == GKO_ILU || pc_choice == GKO_ILU_ISAI)
+      {
+
+         // Create Ginkgo ILU preconditioner
+
+         if (permute)
+         {
+
+            tic_toc.Clear();
+            tic_toc.Start();
+
+            GinkgoWrappers::GinkgoIluPreconditioner M(executor, A_pc, *inv_reordering,
+                                                      trisolve_type, isai_sparsity_power, par_ilu_its);
+
+            tic_toc.Stop();
+            cout << "Real time creating Ginkgo Ilu preconditioner: " <<
+                 tic_toc.RealTime() << "\n";
+
+            // Use preconditioned CG
+            total_its = pcg_solve(*A, M, B, X, 0, X.Size(), 1e-12, 0.0, it_time);
+
+            cout << "Real time in PCG: " << it_time << "\n";
+
+         }
+         else
+         {
+
+            tic_toc.Clear();
+            tic_toc.Start();
+
+            GinkgoWrappers::GinkgoIluPreconditioner M(executor, A_pc, trisolve_type,
+                                                      isai_sparsity_power, par_ilu_its);
+
+            tic_toc.Stop();
+            cout << "Real time creating Ginkgo Ilu preconditioner: " <<
+                 tic_toc.RealTime() << "\n";
+
+            // Use preconditioned CG
+            total_its = pcg_solve(*A, M, B, X, 0, X.Size(), 1e-12, 0.0, it_time);
+
+            cout << "Real time in PCG: " << it_time << "\n";
+
+         }
+      }
+      else if (pc_choice == GKO_CUILU || pc_choice == GKO_CUILU_ISAI)
+      {
+
+         // Create Ginkgo CuILU preconditioner (uses cuSPARSE for factorization)
+
+         if (permute)
+         {
+
+            tic_toc.Clear();
+            tic_toc.Start();
+
+            GinkgoWrappers::GinkgoCuIluPreconditioner M(executor, A_pc, *inv_reordering,
+                                                      trisolve_type, isai_sparsity_power);
+
+            tic_toc.Stop();
+            cout << "Real time creating Ginkgo CuIlu preconditioner: " <<
+                 tic_toc.RealTime() << "\n";
+
+            // Use preconditioned CG
+            total_its = pcg_solve(*A, M, B, X, 0, X.Size(), 1e-12, 0.0, it_time);
+
+            cout << "Real time in PCG: " << it_time << "\n";
+
+         }
+         else
+         {
+
+            tic_toc.Clear();
+            tic_toc.Start();
+
+            GinkgoWrappers::GinkgoCuIluPreconditioner M(executor, A_pc, trisolve_type,
+                                                      isai_sparsity_power);
+
+            tic_toc.Stop();
+            cout << "Real time creating Ginkgo CuIlu preconditioner: " <<
+                 tic_toc.RealTime() << "\n";
+
+            // Use preconditioned CG
+            total_its = pcg_solve(*A, M, B, X, 0, X.Size(), 1e-12, 0.0, it_time);
+
+            cout << "Real time in PCG: " << it_time << "\n";
+
+         }
+      }
+      else if (pc_choice == MFEM_GS)
+      {
+
+         // Create MFEM preconditioner
+         tic_toc.Clear();
+         tic_toc.Start();
+
+         GSSmoother M(A_pc);
+
+         tic_toc.Stop();
+         cout << "Real time creating MFEM GS preconditioner: " <<
+              tic_toc.RealTime() << "\n";
+
+         // Use preconditioned CG
+         total_its = pcg_solve(*A, M, B, X, 0, X.Size(), 1e-12, 0.0, it_time);
+
+         cout << "Real time in PCG: " << it_time << "\n";
+
+      }
+      else if (pc_choice == MFEM_UMFPACK)
+      {
+
+#ifdef MFEM_USE_SUITESPARSE
+         // Create MFEM preconditioner
+         tic_toc.Clear();
+         tic_toc.Start();
+
+         UMFPackSolver M;
+         M.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+         M.SetOperator(A_pc);
+
+         tic_toc.Stop();
+         cout << "Real time creating MFEM UMFPACK preconditioner: " <<
+              tic_toc.RealTime() << "\n";
+
+         // Use preconditioned CG
+         total_its = pcg_solve(*A, M, B, X, 0, X.Size(), 1e-12, 0.0, it_time);
+
+         cout << "Real time in PCG: " << it_time << "\n";
+#endif
+      }
+   }
+   else
+   {
+
+      total_its = cg_solve(*A, B, X, 0, X.Size(), 1e-12, 0.0, it_time);
+
+      cout << "Real time in CG: " << it_time << "\n";
+   }
+
+   cout << "Total iterations: " << total_its << "\n";
+   cout << "Avg time per iteration: " << it_time/double(total_its) << "\n";
+
+   // 12. Recover the solution as a finite element grid function.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 13. Save the refined mesh and the solution. This output can be viewed
+   // later
+   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
+
+   if (output_sol)
+   {
+      ofstream mesh_ofs("refined.mesh");
+      mesh_ofs.precision(8);
+      mesh->Print(mesh_ofs);
+      ofstream sol_ofs("sol.gf");
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   if (pc && output_pc)
+   {
+      ofstream mesh_lor_ofs("lor-refined.mesh");
+      mesh_lor_ofs.precision(8);
+      mesh_lor->Print(mesh_lor_ofs);
+
+      ofstream apc_lor_ofs("lor-mat.dat");
+      mesh_lor_ofs.precision(8);
+      A_pc.PrintCSR(apc_lor_ofs);
+   }
+
+   // 14. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int visport = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << x << flush;
+   }
+
+   // 15. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   delete fespace_lor;
+   delete fec_lor;
+   delete mesh_lor;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete mesh;
+}
+
@@ -0,0 +1,217 @@
+#ifndef __MULTIGRIDPC_HPP__
+#define __MULTIGRIDPC_HPP__
+
+#include "mfem.hpp"
+#include <memory>
+
+namespace mfem
+{
+
+struct SolverConfig
+{
+   enum SolverType
+   {
+     JACOBI = 0,
+     AMGX = 1,
+     CHEBYSHEV = 2,
+     GINKGO_CUIC = 3,
+     GINKGO_CUIC_ISAI = 4 
+   };
+   SolverType type;
+   SolverType smoother_type;
+   AssemblyLevel upper_level_asm;
+   const char *amgx_config;
+   std::shared_ptr<gko::Executor> gko_exec;
+#ifdef MFEM_SIMPLEX_LOR
+   bool simplex_lor = true;
+#endif
+   SolverConfig(SolverType type_, SolverType sm_type_, AssemblyLevel upper_asm_, 
+                const char *amgx_config_, std::shared_ptr<gko::Executor> gko_exec_) : type(type_),
+     smoother_type(sm_type_),
+     upper_level_asm(upper_asm_) 
+     {
+       amgx_config = amgx_config_;
+       gko_exec = gko_exec_; 
+     } 
+};
+
+struct MGRefinement
+{  
+   enum Type { P_MG, H_MG };
+   Type type;
+   int order;
+   MGRefinement(Type type_, int order_) : type(type_), order(order_) { }
+   static MGRefinement p(int order_) { return MGRefinement(P_MG, order_); }
+   static MGRefinement h() { return MGRefinement(H_MG, 0); }
+};
+
+bool NeedsLOR(SolverConfig config)
+{
+   switch (config.type)
+   {
+      case SolverConfig::GINKGO_CUIC:
+      case SolverConfig::GINKGO_CUIC_ISAI:
+         return true;
+      default:
+         return false;
+   }
+}
+
+struct DiffusionMultigrid : GeometricMultigrid
+{
+   Coefficient &coeff;
+   OperatorPtr A_coarse;
+
+   DiffusionMultigrid(
+      FiniteElementSpaceHierarchy& hierarchy,
+      Coefficient &coeff_,
+      Array<int>& ess_bdr,
+      SolverConfig solver_config)
+       : GeometricMultigrid(hierarchy), coeff(coeff_)
+    {
+       ConstructCoarseOperatorAndSolver(
+          solver_config, hierarchy.GetFESpaceAtLevel(0), ess_bdr);
+       int nlevels = hierarchy.GetNumLevels();
+       for (int i=1; i<nlevels; ++i)
+       {
+          ConstructOperatorAndSmoother(solver_config, hierarchy.GetFESpaceAtLevel(i), ess_bdr);
+       }
+    }
+
+   void ConstructBilinearForm(
+      FiniteElementSpace &fespace, Array<int> &ess_bdr, AssemblyLevel asm_lvl)
+    {
+       BilinearForm* form = new BilinearForm(&fespace);
+       form->SetAssemblyLevel(asm_lvl);
+       form->SetDiagonalPolicy(DIAG_ONE);
+       form->AddDomainIntegrator(new DiffusionIntegrator(coeff));
+       form->Assemble();
+       bfs.Append(form);
+    
+       essentialTrueDofs.Append(new Array<int>());
+       fespace.GetEssentialTrueDofs(ess_bdr, *essentialTrueDofs.Last());
+    }
+
+   void ConstructOperatorAndSmoother(SolverConfig solver_config,
+      FiniteElementSpace& fespace, Array<int>& ess_bdr)
+    {
+       ConstructBilinearForm(fespace, ess_bdr, solver_config.upper_level_asm);
+    
+       OperatorPtr opr;
+       bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), opr);
+       opr.SetOperatorOwner(false);
+   
+       switch (solver_config.smoother_type)
+       {
+         case SolverConfig::CHEBYSHEV: 
+         {
+           Vector diag(fespace.GetTrueVSize());
+           bfs.Last()->AssembleDiagonal(diag);
+    
+           Solver* smoother = new OperatorChebyshevSmoother(
+              opr.Ptr(), diag, *essentialTrueDofs.Last(), 2);
+    
+           if (solver_config.upper_level_asm == AssemblyLevel::PARTIAL) 
+           {
+             AddLevel(opr.Ptr(), smoother, true, true);
+           }
+           else
+           {
+             AddLevel(opr.Ptr(), smoother, false, true);
+           }
+           break;
+         }
+         case SolverConfig::GINKGO_CUIC:
+         {
+            SparseMatrix *A_lvl = dynamic_cast<SparseMatrix*>(opr.Ptr());
+            Solver *smoother = new GinkgoWrappers::GinkgoCuIcPreconditioner(
+                                              solver_config.gko_exec, *A_lvl,
+                                                        "exact", 1);
+           if (solver_config.upper_level_asm == AssemblyLevel::PARTIAL) 
+           {
+             AddLevel(opr.Ptr(), smoother, true, true);
+           }
+           else
+           {
+             AddLevel(opr.Ptr(), smoother, false, true);
+           }
+           break;
+         }
+         case SolverConfig::GINKGO_CUIC_ISAI:
+         {
+            SparseMatrix *A_lvl = dynamic_cast<SparseMatrix*>(opr.Ptr());
+            Solver *smoother = new GinkgoWrappers::GinkgoCuIcPreconditioner(
+                                              solver_config.gko_exec, *A_lvl,
+                                                        "isai", 1);
+           if (solver_config.upper_level_asm == AssemblyLevel::PARTIAL) 
+           {
+             AddLevel(opr.Ptr(), smoother, true, true);
+           }
+           else
+           {
+             AddLevel(opr.Ptr(), smoother, false, true);
+           }
+           break;
+         }
+       }
+      
+    }
+
+   void ConstructCoarseOperatorAndSolver(
+      SolverConfig config, FiniteElementSpace& fespace, Array<int>& ess_bdr)
+    {
+       ConstructBilinearForm(fespace, ess_bdr, AssemblyLevel::LEGACYFULL);
+       BilinearForm &a = *bfs.Last();
+       Array<int> &ess_dofs = *essentialTrueDofs.Last();
+    
+       bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), A_coarse);
+    
+       OperatorPtr A_prec;
+       {
+          A_prec = A_coarse;
+       }
+    
+       Solver *coarse_solver;
+       switch (config.type)
+       {
+          case SolverConfig::JACOBI:
+             coarse_solver = new OperatorJacobiSmoother(a, ess_dofs);
+             break;
+    #ifdef MFEM_USE_AMGX
+          case SolverConfig::AMGX:
+          {
+             AmgXSolver *amg = new AmgXSolver;
+             amg->ReadParameters(config.amgx_config, AmgXSolver::EXTERNAL);
+             amg->InitSerial();
+             amg->SetOperator(*A_prec.As<SparseMatrix>());
+             coarse_solver = amg;
+             break;
+          }
+    #endif
+          case SolverConfig::GINKGO_CUIC:
+          {
+            SparseMatrix *A_lvl = dynamic_cast<SparseMatrix*>(A_prec.Ptr());
+            Solver *gko_solver = new GinkgoWrappers::GinkgoCuIcPreconditioner(
+                                              config.gko_exec, *A_lvl,
+                                                        "exact", 1);
+            coarse_solver = gko_solver;
+            break;
+          }
+          case SolverConfig::GINKGO_CUIC_ISAI:
+          {
+            SparseMatrix *A_lvl = dynamic_cast<SparseMatrix*>(A_prec.Ptr());
+            Solver *gko_solver = new GinkgoWrappers::GinkgoCuIcPreconditioner(
+                                              config.gko_exec, *A_lvl,
+                                                        "isai", 1);
+            coarse_solver = gko_solver;
+            break;
+          }
+          default:
+             MFEM_ABORT("Not available.")
+       }
+    
+       AddLevel(A_coarse.Ptr(), coarse_solver, false, true);
+    }
+};
+}
+#endif                                                             
@@ -13,6 +13,7 @@ set(SRCS
  bilinearform.cpp
  bilinearform_ext.cpp
  bilininteg.cpp
+  bilininteg_br2.cpp
  bilininteg_convection_pa.cpp
  bilininteg_convection_ea.cpp
  bilininteg_dgtrace_pa.cpp
@@ -1437,9 +1437,9 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         ftr = mesh->GetBdrFaceTransformations(i);
         if (ftr)
         {
-            trial_fes->GetFaceVDofs(i, tr_vdofs);
+            trial_fes->GetFaceVDofs(ftr->ElementNo, tr_vdofs);
            test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
-            trial_face_fe = trial_fes->GetFaceElement(i);
+            trial_face_fe = trial_fes->GetFaceElement(ftr->ElementNo);
            test_fe1 = test_fes->GetFE(ftr->Elem1No);
            // The test_fe2 object is really a dummy and not used on the
            // boundaries, but we can't dereference a NULL pointer, and we don't
@@ -1770,9 +1770,41 @@ MixedBilinearForm::~MixedBilinearForm()
   delete ext;
 }

+void DiscreteLinearOperator::SetAssemblyLevel(AssemblyLevel assembly_level)
+{
+   if (ext)
+   {
+      MFEM_ABORT("the assembly level has already been set!");
+   }
+   assembly = assembly_level;
+   switch (assembly)
+   {
+      case AssemblyLevel::LEGACYFULL:
+      case AssemblyLevel::FULL:
+         // Use the original implementation for now
+         break;
+      case AssemblyLevel::ELEMENT:
+         mfem_error("Element assembly not supported yet... stay tuned!");
+         break;
+      case AssemblyLevel::PARTIAL:
+         ext = new PADiscreteLinearOperatorExtension(this);
+         break;
+      case AssemblyLevel::NONE:
+         mfem_error("Matrix-free action not supported yet... stay tuned!");
+         break;
+      default:
+         mfem_error("Unknown assembly level");
+   }
+}

 void DiscreteLinearOperator::Assemble(int skip_zeros)
 {
+   if (ext)
+   {
+      ext->Assemble();
+      return;
+   }
+
   Array<int> dom_vdofs, ran_vdofs;
   ElementTransformation *T;
   const FiniteElement *dom_fe, *ran_fe;
@@ -299,8 +299,17 @@ public:
   /// Returns a reference to the sparse matrix:  \f$ M \f$
   SparseMatrix &SpMat()
   {
-      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
-      return *mat;
+      FABilinearFormExtension *fa_ext = dynamic_cast<FABilinearFormExtension*>
+                                        (this->ext);
+      if (fa_ext)
+      {
+         return fa_ext->SpMat();
+      }
+      else
+      {
+         MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
+         return *mat;
+      }
   }

   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
@@ -376,6 +385,13 @@ public:
   /// Get the output finite element space prolongation matrix
   virtual const Operator *GetOutputProlongation() const
   { return GetProlongation(); }
+   /** @brief Returns the output fe space restriction matrix, transposed
+
+       Logically, this is the transpose of GetOutputRestriction, but in
+       practice it is convenient to have it in transposed form for
+       construction of RAP operators in matrix-free methods. */
+   virtual const Operator *GetOutputRestrictionTranspose() const
+   { return GetOutputProlongation(); }
   /// Get the output finite element space restriction matrix
   virtual const Operator *GetOutputRestriction() const
   { return GetRestriction(); }
@@ -977,9 +993,18 @@ public:
   /// Access all interpolators added with AddDomainInterpolator().
   Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }

+   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
+   /** This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level);
+
   /** @brief Construct the internal matrix representation of the discrete
       linear operator. */
   virtual void Assemble(int skip_zeros = 1);
+
+   /** @brief Get the output finite element space restriction matrix in
+       transposed form. */
+   virtual const Operator *GetOutputRestrictionTranspose() const
+   { return test_fes->GetRestrictionTransposeOperator(); }
 };

 }
@@ -1021,7 +1021,6 @@ void PAMixedBilinearFormExtension::Update()
      localTrial.UseDevice(true);
      localTrial.SetSize(elem_restrict_trial->Height(),
                         Device::GetMemoryType());
-
   }
   if (elem_restrict_test)
   {
@@ -1221,4 +1220,134 @@ void PAMixedBilinearFormExtension::AssembleDiagonal_ADAt(const Vector &D,
   }
 }

+PADiscreteLinearOperatorExtension::PADiscreteLinearOperatorExtension(
+   DiscreteLinearOperator *linop) :
+   PAMixedBilinearFormExtension(linop)
+{
+}
+
+const
+Operator *PADiscreteLinearOperatorExtension::GetOutputRestrictionTranspose()
+const
+{
+   return a->GetOutputRestrictionTranspose();
+}
+
+void PADiscreteLinearOperatorExtension::Assemble()
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int integratorCount = integrators.Size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->AssemblePA(*trialFes, *testFes);
+   }
+
+   test_multiplicity.UseDevice(true);
+   test_multiplicity.SetSize(elem_restrict_test->Width()); // l-vector
+   Vector ones(elem_restrict_test->Height()); // e-vector
+   ones = 1.0;
+
+   const ElementRestriction* elem_restrict =
+      dynamic_cast<const ElementRestriction*>(elem_restrict_test);
+   if (elem_restrict)
+   {
+      elem_restrict->MultTransposeUnsigned(ones, test_multiplicity);
+   }
+   else
+   {
+      mfem_error("A real ElementRestriction is required in this setting!");
+   }
+
+   auto tm = test_multiplicity.ReadWrite();
+   MFEM_FORALL(i, test_multiplicity.Size(),
+   {
+      tm[i] = 1.0 / tm[i];
+   });
+}
+
+void PADiscreteLinearOperatorExtension::AddMult(
+   const Vector &x, Vector &y, const double c) const
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int iSz = integrators.Size();
+
+   // * G operation
+   SetupMultInputs(elem_restrict_trial, x, localTrial,
+                   elem_restrict_test, y, localTest, c);
+
+   // * B^TDB operation
+   for (int i = 0; i < iSz; ++i)
+   {
+      integrators[i]->AddMultPA(localTrial, localTest);
+   }
+
+   // do a kind of "set" rather than "add" in the below
+   // operation as compared to the BilinearForm case
+   // * G^T operation (kind of...)
+   const ElementRestriction* elem_restrict =
+      dynamic_cast<const ElementRestriction*>(elem_restrict_test);
+   if (elem_restrict)
+   {
+      tempY.SetSize(y.Size());
+      elem_restrict->MultLeftInverse(localTest, tempY);
+      y += tempY;
+   }
+   else
+   {
+      mfem_error("In this setting you need a real ElementRestriction!");
+   }
+}
+
+void PADiscreteLinearOperatorExtension::AddMultTranspose(
+   const Vector &x, Vector &y, const double c) const
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int iSz = integrators.Size();
+
+   // do a kind of "set" rather than "add" in the below
+   // operation as compared to the BilinearForm case
+   // * G operation (kinda)
+   Vector xscaled(x);
+   MFEM_VERIFY(x.Size() == test_multiplicity.Size(), "Input vector of wrong size");
+   auto xs = xscaled.ReadWrite();
+   auto tm = test_multiplicity.Read();
+   MFEM_FORALL(i, x.Size(),
+   {
+      xs[i] *= tm[i];
+   });
+   SetupMultInputs(elem_restrict_test, xscaled, localTest,
+                   elem_restrict_trial, y, localTrial, c);
+
+   // * B^TD^TB operation
+   for (int i = 0; i < iSz; ++i)
+   {
+      integrators[i]->AddMultTransposePA(localTest, localTrial);
+   }
+
+   // * G^T operation
+   if (elem_restrict_trial)
+   {
+      tempY.SetSize(y.Size());
+      elem_restrict_trial->MultTranspose(localTrial, tempY);
+      y += tempY;
+   }
+   else
+   {
+      mfem_error("Trial ElementRestriction not defined");
+   }
+}
+
+void PADiscreteLinearOperatorExtension::FormRectangularSystemOperator(
+   const Array<int>& ess1, const Array<int>& ess2, OperatorHandle &A)
+{
+   const Operator *Pi = this->GetProlongation();
+   const Operator *RoT = this->GetOutputRestrictionTranspose();
+   Operator *rap = SetupRAP(Pi, RoT);
+
+   RectangularConstrainedOperator *Arco
+      = new RectangularConstrainedOperator(rap, ess1, ess2, rap != this);
+
+   A.Reset(Arco);
+}
+
 } // namespace mfem
@@ -21,6 +21,7 @@ namespace mfem

 class BilinearForm;
 class MixedBilinearForm;
+class DiscreteLinearOperator;

 /// Class extending the BilinearForm class to support different AssemblyLevels.
 /**  FA - Full Assembly
@@ -128,6 +129,10 @@ public:
   void Assemble();
   void Mult(const Vector &x, Vector &y) const;
   void MultTranspose(const Vector &x, Vector &y) const;
+   SparseMatrix &SpMat()
+   {
+      return mat;
+   }
 };

 /// Data and methods for matrix-free bilinear forms
@@ -212,7 +217,7 @@ protected:
   mutable Vector localTrial, localTest, tempY;
   const Operator *elem_restrict_trial; // Not owned
   const Operator *elem_restrict_test;  // Not owned
-private:
+
   /// Helper function to set up inputs/outputs for Mult or MultTranspose
   void SetupMultInputs(const Operator *elem_restrict_x,
                        const Vector &x, Vector &localX,
@@ -258,6 +263,35 @@ public:
   void Update();
 };

+
+/**
+   @brief Partial assembly extension for DiscreteLinearOperator
+
+   This acts very much like PAMixedBilinearFormExtension, but its
+   FormRectangularSystemOperator implementation emulates 'Set' rather than
+   'Add' in the assembly case.
+*/
+class PADiscreteLinearOperatorExtension : public PAMixedBilinearFormExtension
+{
+public:
+   PADiscreteLinearOperatorExtension(DiscreteLinearOperator *linop);
+
+   /// Partial assembly of all internal integrators
+   void Assemble();
+
+   void AddMult(const Vector &x, Vector &y, const double c) const;
+
+   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;
+
+   void FormRectangularSystemOperator(const Array<int>&, const Array<int>&,
+                                      OperatorHandle& A);
+
+   const Operator * GetOutputRestrictionTranspose() const;
+
+private:
+   Vector test_multiplicity;
+};
+
 }

 #endif
@@ -22,14 +22,14 @@ namespace mfem

 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
+   mfem_error ("BilinearFormIntegrator::AssemblePA(fes)\n"
               "   is not implemented for this class.");
 }

 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&,
                                        const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
+   mfem_error ("BilinearFormIntegrator::AssemblePA(fes, fes)\n"
               "   is not implemented for this class.");
 }

@@ -92,7 +92,7 @@ void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const

 void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::MultAssembledTranspose(...)\n"
+   mfem_error ("BilinearFormIntegrator::AddMultTransposePA(...)\n"
               "   is not implemented for this class.");
 }

@@ -229,6 +229,159 @@ void SumIntegrator::AssembleElementMatrix(
   }
 }

+void SumIntegrator::AssembleElementMatrix2(
+   const FiniteElement &el1, const FiniteElement &el2,
+   ElementTransformation &Trans, DenseMatrix &elmat)
+{
+   MFEM_ASSERT(integrators.Size() > 0, "empty SumIntegrator.");
+
+   integrators[0]->AssembleElementMatrix2(el1, el2, Trans, elmat);
+   for (int i = 1; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssembleElementMatrix2(el1, el2, Trans, elem_mat);
+      elmat += elem_mat;
+   }
+}
+
+void SumIntegrator::AssembleFaceMatrix(
+   const FiniteElement &el1, const FiniteElement &el2,
+   FaceElementTransformations &Trans, DenseMatrix &elmat)
+{
+   MFEM_ASSERT(integrators.Size() > 0, "empty SumIntegrator.");
+
+   integrators[0]->AssembleFaceMatrix(el1, el2, Trans, elmat);
+   for (int i = 1; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssembleFaceMatrix(el1, el2, Trans, elem_mat);
+      elmat += elem_mat;
+   }
+}
+
+void SumIntegrator::AssembleFaceMatrix(
+   const FiniteElement &tr_fe,
+   const FiniteElement &te_fe1, const FiniteElement &te_fe2,
+   FaceElementTransformations &Trans, DenseMatrix &elmat)
+{
+   MFEM_ASSERT(integrators.Size() > 0, "empty SumIntegrator.");
+
+   integrators[0]->AssembleFaceMatrix(tr_fe, te_fe1, te_fe2, Trans, elmat);
+   for (int i = 1; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssembleFaceMatrix(tr_fe, te_fe1, te_fe2, Trans, elem_mat);
+      elmat += elem_mat;
+   }
+}
+
+void SumIntegrator::AssemblePA(const FiniteElementSpace& fes)
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssemblePA(fes);
+   }
+}
+
+void SumIntegrator::AssembleDiagonalPA(Vector &diag)
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssembleDiagonalPA(diag);
+   }
+}
+
+void SumIntegrator::AssemblePAInteriorFaces(const FiniteElementSpace &fes)
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssemblePAInteriorFaces(fes);
+   }
+}
+
+void SumIntegrator::AssemblePABoundaryFaces(const FiniteElementSpace &fes)
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssemblePABoundaryFaces(fes);
+   }
+}
+
+void SumIntegrator::AddMultPA(const Vector& x, Vector& y) const
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AddMultPA(x, y);
+   }
+}
+
+void SumIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AddMultTransposePA(x, y);
+   }
+}
+
+void SumIntegrator::AssembleMF(const FiniteElementSpace &fes)
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssembleMF(fes);
+   }
+}
+
+void SumIntegrator::AddMultMF(const Vector& x, Vector& y) const
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AddMultTransposeMF(x, y);
+   }
+}
+
+void SumIntegrator::AddMultTransposeMF(const Vector &x, Vector &y) const
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AddMultMF(x, y);
+   }
+}
+
+void SumIntegrator::AssembleDiagonalMF(Vector &diag)
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssembleDiagonalMF(diag);
+   }
+}
+
+void SumIntegrator::AssembleEA(const FiniteElementSpace &fes, Vector &emat,
+                               const bool add)
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssembleEA(fes, emat, add);
+   }
+}
+
+void SumIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace &fes,
+                                            Vector &ea_data_int,
+                                            Vector &ea_data_ext,
+                                            const bool add)
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssembleEAInteriorFaces(fes,ea_data_int,ea_data_ext,add);
+   }
+}
+
+void SumIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace &fes,
+                                            Vector &ea_data_bdr,
+                                            const bool add)
+{
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->AssembleEABoundaryFaces(fes, ea_data_bdr, add);
+   }
+}
+
 SumIntegrator::~SumIntegrator()
 {
   if (own_integrators)
@@ -1913,12 +2066,12 @@ void VectorFEMassIntegrator::AssembleElementMatrix(
   double w;

 #ifdef MFEM_THREAD_SAFE
-   Vector D(VQ ? VQ->GetVDim() : 0);
+   Vector D(DQ ? DQ->GetVDim() : 0);
   DenseMatrix trial_vshape(dof, spaceDim);
   DenseMatrix K(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #else
   trial_vshape.SetSize(dof, spaceDim);
-   D.SetSize(VQ ? VQ->GetVDim() : 0);
+   D.SetSize(DQ ? DQ->GetVDim() : 0);
   K.SetSize(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #endif
   DenseMatrix tmp(trial_vshape.Height(), K.Width());
@@ -1950,9 +2103,9 @@ void VectorFEMassIntegrator::AssembleElementMatrix(
         Mult(trial_vshape,K,tmp);
         AddMultABt(tmp,trial_vshape,elmat);
      }
-      else if (VQ)
+      else if (DQ)
      {
-         VQ->Eval(D, Trans, ip);
+         DQ->Eval(D, Trans, ip);
         D *= w;
         AddMultADAt(trial_vshape, D, elmat);
      }
@@ -1984,12 +2137,12 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
 #ifdef MFEM_THREAD_SAFE
      DenseMatrix trial_vshape(trial_dof, spaceDim);
      Vector shape(test_dof);
-      Vector D(VQ ? VQ->GetVDim() : 0);
+      Vector D(DQ ? DQ->GetVDim() : 0);
      DenseMatrix K(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #else
      trial_vshape.SetSize(trial_dof, spaceDim);
      shape.SetSize(test_dof);
-      D.SetSize(VQ ? VQ->GetVDim() : 0);
+      D.SetSize(DQ ? DQ->GetVDim() : 0);
      K.SetSize(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #endif

@@ -2013,9 +2166,9 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
         test_fe.CalcShape(ip, shape);

         w = ip.weight * Trans.Weight();
-         if (VQ)
+         if (DQ)
         {
-            VQ->Eval(D, Trans, ip);
+            DQ->Eval(D, Trans, ip);
            D *= w;
            for (int d = 0; d < vdim; d++)
            {
@@ -2081,12 +2234,12 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
 #ifdef MFEM_THREAD_SAFE
      DenseMatrix trial_vshape(trial_dof,spaceDim);
      DenseMatrix test_vshape(test_dof,spaceDim);
-      Vector D(VQ ? VQ->GetVDim() : 0);
+      Vector D(DQ ? DQ->GetVDim() : 0);
      DenseMatrix K(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #else
      trial_vshape.SetSize(trial_dof,spaceDim);
      test_vshape.SetSize(test_dof,spaceDim);
-      D.SetSize(VQ ? VQ->GetVDim() : 0);
+      D.SetSize(DQ ? DQ->GetVDim() : 0);
      K.SetSize(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #endif
      DenseMatrix tmp(test_vshape.Height(), K.Width());
@@ -2118,9 +2271,9 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
            Mult(test_vshape,K,tmp);
            AddMultABt(tmp,trial_vshape,elmat);
         }
-         else if (VQ)
+         else if (DQ)
         {
-            VQ->Eval(D, Trans, ip);
+            DQ->Eval(D, Trans, ip);
            D *= w;
            AddMultADBt(test_vshape,D,trial_vshape,elmat);
         }
@@ -3515,6 +3668,47 @@ VectorScalarProductInterpolator::AssembleElementMatrix2(
 }


+void
+ScalarCrossProductInterpolator::AssembleElementMatrix2(
+   const FiniteElement &dom_fe,
+   const FiniteElement &ran_fe,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   // Vector coefficient product with vector shape functions
+   struct VCrossVShapeCoefficient : public VectorCoefficient
+   {
+      VectorCoefficient &VQ;
+      const FiniteElement &fe;
+      DenseMatrix vshape;
+      Vector vc;
+
+      VCrossVShapeCoefficient(VectorCoefficient &vq, const FiniteElement &fe_)
+         : VectorCoefficient(fe_.GetDof()), VQ(vq), fe(fe_),
+           vshape(vdim, vq.GetVDim()), vc(vq.GetVDim()) { }
+
+      virtual void Eval(Vector &V, ElementTransformation &T,
+                        const IntegrationPoint &ip)
+      {
+         V.SetSize(vdim);
+         VQ.Eval(vc, T, ip);
+         fe.CalcPhysVShape(T, vshape);
+         for (int k = 0; k < vdim; k++)
+         {
+            V(k) = vc(0) * vshape(k,1) - vc(1) * vshape(k,0);
+         }
+      }
+   };
+
+   VCrossVShapeCoefficient dom_shape_coeff(*VQ, dom_fe);
+
+   elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());
+
+   Vector elmat_as_vec(elmat.Data(), elmat.Height()*elmat.Width());
+
+   ran_fe.Project(dom_shape_coeff, Trans, elmat_as_vec);
+}
+
 void
 VectorCrossProductInterpolator::AssembleElementMatrix2(
   const FiniteElement &dom_fe,
@@ -355,7 +355,7 @@ class SumIntegrator : public BilinearFormIntegrator
 {
 private:
   int own_integrators;
-   DenseMatrix elem_mat;
+   mutable DenseMatrix elem_mat;
   Array<BilinearFormIntegrator*> integrators;

 public:
@@ -367,6 +367,55 @@ public:
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
+   using BilinearFormIntegrator::AssembleFaceMatrix;
+   virtual void AssembleFaceMatrix(const FiniteElement &el1,
+                                   const FiniteElement &el2,
+                                   FaceElementTransformations &Trans,
+                                   DenseMatrix &elmat);
+
+   virtual void AssembleFaceMatrix(const FiniteElement &trial_face_fe,
+                                   const FiniteElement &test_fe1,
+                                   const FiniteElement &test_fe2,
+                                   FaceElementTransformations &Trans,
+                                   DenseMatrix &elmat);
+
+   using BilinearFormIntegrator::AssemblePA;
+   virtual void AssemblePA(const FiniteElementSpace& fes);
+
+   virtual void AssembleDiagonalPA(Vector &diag);
+
+   virtual void AssemblePAInteriorFaces(const FiniteElementSpace &fes);
+
+   virtual void AssemblePABoundaryFaces(const FiniteElementSpace &fes);
+
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+
+   virtual void AddMultPA(const Vector& x, Vector& y) const;
+
+   virtual void AssembleMF(const FiniteElementSpace &fes);
+
+   virtual void AddMultMF(const Vector &x, Vector &y) const;
+
+   virtual void AddMultTransposeMF(const Vector &x, Vector &y) const;
+
+   virtual void AssembleDiagonalMF(Vector &diag);
+
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
+                           const bool add);
+
+   virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
+                                        Vector &ea_data_int,
+                                        Vector &ea_data_ext,
+                                        const bool add);
+
+   virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
+                                        Vector &ea_data_bdr,
+                                        const bool add);

   virtual ~SumIntegrator();
 };
@@ -465,8 +514,8 @@ protected:
      : same_calc_shape(false), Q(NULL), VQ(NULL), DQ(NULL), MQ(NULL) {}
   MixedVectorIntegrator(Coefficient &q)
      : same_calc_shape(false), Q(&q), VQ(NULL), DQ(NULL), MQ(NULL) {}
-   MixedVectorIntegrator(VectorCoefficient &dq, bool diag = true)
-      : same_calc_shape(false), Q(NULL), VQ(diag?NULL:&dq), DQ(diag?&dq:NULL),
+   MixedVectorIntegrator(VectorCoefficient &vq, bool diag = true)
+      : same_calc_shape(false), Q(NULL), VQ(diag?NULL:&vq), DQ(diag?&vq:NULL),
        MQ(NULL) {}
   MixedVectorIntegrator(MatrixCoefficient &mq)
      : same_calc_shape(false), Q(NULL), VQ(NULL), DQ(NULL), MQ(&mq) {}
@@ -503,7 +552,7 @@ protected:

   Coefficient *Q;
   VectorCoefficient *VQ;
-   VectorCoefficient *DQ;
+   DiagonalMatrixCoefficient *DQ;
   MatrixCoefficient *MQ;

 private:
@@ -901,7 +950,7 @@ public:
   MixedVectorMassIntegrator() { same_calc_shape = true; }
   MixedVectorMassIntegrator(Coefficient &q)
      : MixedVectorIntegrator(q) { same_calc_shape = true; }
-   MixedVectorMassIntegrator(VectorCoefficient &dq)
+   MixedVectorMassIntegrator(DiagonalMatrixCoefficient &dq)
      : MixedVectorIntegrator(dq, true) { same_calc_shape = true; }
   MixedVectorMassIntegrator(MatrixCoefficient &mq)
      : MixedVectorIntegrator(mq) { same_calc_shape = true; }
@@ -1019,7 +1068,7 @@ public:
   MixedGradGradIntegrator() { same_calc_shape = true; }
   MixedGradGradIntegrator(Coefficient &q)
      : MixedVectorIntegrator(q) { same_calc_shape = true; }
-   MixedGradGradIntegrator(VectorCoefficient &dq)
+   MixedGradGradIntegrator(DiagonalMatrixCoefficient &dq)
      : MixedVectorIntegrator(dq, true) { same_calc_shape = true; }
   MixedGradGradIntegrator(MatrixCoefficient &mq)
      : MixedVectorIntegrator(mq) { same_calc_shape = true; }
@@ -1107,7 +1156,7 @@ public:
   MixedCurlCurlIntegrator() { same_calc_shape = true; }
   MixedCurlCurlIntegrator(Coefficient &q)
      : MixedVectorIntegrator(q) { same_calc_shape = true; }
-   MixedCurlCurlIntegrator(VectorCoefficient &dq)
+   MixedCurlCurlIntegrator(DiagonalMatrixCoefficient &dq)
      : MixedVectorIntegrator(dq, true) { same_calc_shape = true; }
   MixedCurlCurlIntegrator(MatrixCoefficient &mq)
      : MixedVectorIntegrator(mq) { same_calc_shape = true; }
@@ -1651,7 +1700,7 @@ public:
   MixedVectorGradientIntegrator() {}
   MixedVectorGradientIntegrator(Coefficient &q)
      : MixedVectorIntegrator(q) {}
-   MixedVectorGradientIntegrator(VectorCoefficient &dq)
+   MixedVectorGradientIntegrator(DiagonalMatrixCoefficient &dq)
      : MixedVectorIntegrator(dq, true) {}
   MixedVectorGradientIntegrator(MatrixCoefficient &mq)
      : MixedVectorIntegrator(mq) {}
@@ -1705,7 +1754,7 @@ public:
   MixedVectorCurlIntegrator() {}
   MixedVectorCurlIntegrator(Coefficient &q)
      : MixedVectorIntegrator(q) {}
-   MixedVectorCurlIntegrator(VectorCoefficient &dq)
+   MixedVectorCurlIntegrator(DiagonalMatrixCoefficient &dq)
      : MixedVectorIntegrator(dq, true) {}
   MixedVectorCurlIntegrator(MatrixCoefficient &mq)
      : MixedVectorIntegrator(mq) {}
@@ -1760,7 +1809,7 @@ public:
   MixedVectorWeakCurlIntegrator() {}
   MixedVectorWeakCurlIntegrator(Coefficient &q)
      : MixedVectorIntegrator(q) {}
-   MixedVectorWeakCurlIntegrator(VectorCoefficient &dq)
+   MixedVectorWeakCurlIntegrator(DiagonalMatrixCoefficient &dq)
      : MixedVectorIntegrator(dq, true) {}
   MixedVectorWeakCurlIntegrator(MatrixCoefficient &mq)
      : MixedVectorIntegrator(mq) {}
@@ -1813,7 +1862,7 @@ public:
   MixedVectorWeakDivergenceIntegrator() {}
   MixedVectorWeakDivergenceIntegrator(Coefficient &q)
      : MixedVectorIntegrator(q) {}
-   MixedVectorWeakDivergenceIntegrator(VectorCoefficient &dq)
+   MixedVectorWeakDivergenceIntegrator(DiagonalMatrixCoefficient &dq)
      : MixedVectorIntegrator(dq, true) {}
   MixedVectorWeakDivergenceIntegrator(MatrixCoefficient &mq)
      : MixedVectorIntegrator(mq) {}
@@ -1844,8 +1893,10 @@ protected:
 };

 /** Class for integrating the bilinear form a(u,v) := (Q grad u, v) where Q is a
-    scalar coefficient, and v is a vector with components v_i in the same space
-    as u. */
+    scalar coefficient, and v is a vector with components v_i in the same (H1) space
+    as u.
+
+    See also MixedVectorGradientIntegrator when v is in H(curl). */
 class GradientIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -1900,6 +1951,7 @@ protected:
   Coefficient *Q;
   VectorCoefficient *VQ;
   MatrixCoefficient *MQ;
+   SymmetricMatrixCoefficient *SMQ;

 private:
   Vector vec, pointflux, shape;
@@ -1922,19 +1974,28 @@ private:
 public:
   /// Construct a diffusion integrator with coefficient Q = 1
   DiffusionIntegrator()
-      : Q(NULL), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
+      : Q(NULL), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL),
+        ceedDataPtr(NULL) { }

   /// Construct a diffusion integrator with a scalar coefficient q
   DiffusionIntegrator(Coefficient &q)
-      : Q(&q), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
+      : Q(&q), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL),
+        ceedDataPtr(NULL) { }

   /// Construct a diffusion integrator with a vector coefficient q
   DiffusionIntegrator(VectorCoefficient &q)
-      : Q(NULL), VQ(&q), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
+      : Q(NULL), VQ(&q), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL),
+        ceedDataPtr(NULL) { }

   /// Construct a diffusion integrator with a matrix coefficient q
   DiffusionIntegrator(MatrixCoefficient &q)
-      : Q(NULL), VQ(NULL), MQ(&q), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
+      : Q(NULL), VQ(NULL), MQ(&q), SMQ(NULL), maps(NULL), geom(NULL),
+        ceedDataPtr(NULL) { }
+
+   /// Construct a diffusion integrator with a symmetric matrix coefficient q
+   DiffusionIntegrator(SymmetricMatrixCoefficient &q)
+      : Q(NULL), VQ(NULL), MQ(NULL), SMQ(&q), maps(NULL), geom(NULL),
+        ceedDataPtr(NULL) { }

   virtual ~DiffusionIntegrator()
   {
@@ -1984,6 +2045,8 @@ public:

   virtual void AddMultPA(const Vector&, Vector&) const;

+   virtual void AddMultTransposePA(const Vector&, Vector&) const;
+
   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                         const FiniteElement &test_fe);
 };
@@ -2047,6 +2110,8 @@ public:

   virtual void AddMultPA(const Vector&, Vector&) const;

+   virtual void AddMultTransposePA(const Vector&, Vector&) const;
+
   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                         const FiniteElement &test_fe,
                                         ElementTransformation &Trans);
@@ -2108,6 +2173,17 @@ public:
                                         ElementTransformation &Trans);
 };

+// Alias for @ConvectionIntegrator.
+using NonconservativeConvectionIntegrator = ConvectionIntegrator;
+
+/// -alpha (u, q . grad v), negative transpose of ConvectionIntegrator
+class ConservativeConvectionIntegrator : public TransposeIntegrator
+{
+public:
+   ConservativeConvectionIntegrator(VectorCoefficient &q, double a = 1.0)
+      : TransposeIntegrator(new ConvectionIntegrator(q, -a)) { }
+};
+
 /// alpha (q . grad u, v) using the "group" FE discretization
 class GroupConvectionIntegrator : public BilinearFormIntegrator
 {
@@ -2333,8 +2409,9 @@ private:

 protected:
   Coefficient *Q;
-   VectorCoefficient *DQ;
+   DiagonalMatrixCoefficient *DQ;
   MatrixCoefficient *MQ;
+   SymmetricMatrixCoefficient *SMQ;

   // PA extension
   Vector pa_data;
@@ -2345,14 +2422,18 @@ protected:
   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
-   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
+   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; SMQ = NULL; }
   /// Construct a bilinear form integrator for Nedelec elements
   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(&q) { DQ = NULL; MQ = NULL; }
-   CurlCurlIntegrator(VectorCoefficient &dq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), DQ(&dq) { Q = NULL; MQ = NULL; }
+      BilinearFormIntegrator(ir), Q(&q), DQ(NULL), MQ(NULL), SMQ(NULL) { }
+   CurlCurlIntegrator(DiagonalMatrixCoefficient &dq,
+                      const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(NULL), DQ(&dq), MQ(NULL), SMQ(NULL) { }
   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), MQ(&mq) { Q = NULL; DQ = NULL; }
+      BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(&mq), SMQ(NULL) { }
+   CurlCurlIntegrator(SymmetricMatrixCoefficient &smq,
+                      const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(NULL), SMQ(&smq) { }

   /* Given a particular Finite Element, compute the
      element curl-curl matrix elmat */
@@ -2409,8 +2490,9 @@ public:
 class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
-   void Init(Coefficient *q, VectorCoefficient *vq, MatrixCoefficient *mq)
-   { Q = q; VQ = vq; MQ = mq; }
+   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq,
+             SymmetricMatrixCoefficient *smq)
+   { Q = q; DQ = dq; MQ = mq; SMQ = smq; }

 #ifndef MFEM_THREAD_SAFE
   Vector shape;
@@ -2423,8 +2505,9 @@ private:

 protected:
   Coefficient *Q;
-   VectorCoefficient *VQ;
+   DiagonalMatrixCoefficient *DQ;
   MatrixCoefficient *MQ;
+   SymmetricMatrixCoefficient *SMQ;

   // PA extension
   Vector pa_data;
@@ -2437,13 +2520,15 @@ protected:
   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
-   VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
-   VectorFEMassIntegrator(Coefficient *_q) { Init(_q, NULL, NULL); }
-   VectorFEMassIntegrator(Coefficient &q) { Init(&q, NULL, NULL); }
-   VectorFEMassIntegrator(VectorCoefficient *_vq) { Init(NULL, _vq, NULL); }
-   VectorFEMassIntegrator(VectorCoefficient &vq) { Init(NULL, &vq, NULL); }
-   VectorFEMassIntegrator(MatrixCoefficient *_mq) { Init(NULL, NULL, _mq); }
-   VectorFEMassIntegrator(MatrixCoefficient &mq) { Init(NULL, NULL, &mq); }
+   VectorFEMassIntegrator() { Init(NULL, NULL, NULL, NULL); }
+   VectorFEMassIntegrator(Coefficient *_q) { Init(_q, NULL, NULL, NULL); }
+   VectorFEMassIntegrator(Coefficient &q) { Init(&q, NULL, NULL, NULL); }
+   VectorFEMassIntegrator(DiagonalMatrixCoefficient *_dq) { Init(NULL, _dq, NULL, NULL); }
+   VectorFEMassIntegrator(DiagonalMatrixCoefficient &dq) { Init(NULL, &dq, NULL, NULL); }
+   VectorFEMassIntegrator(MatrixCoefficient *_mq) { Init(NULL, NULL, _mq, NULL); }
+   VectorFEMassIntegrator(MatrixCoefficient &mq) { Init(NULL, NULL, &mq, NULL); }
+   VectorFEMassIntegrator(SymmetricMatrixCoefficient &smq) { Init(NULL, NULL, NULL, &smq); }
+   VectorFEMassIntegrator(SymmetricMatrixCoefficient *smq) { Init(NULL, NULL, NULL, smq); }

   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
@@ -2652,11 +2737,28 @@ public:
 /** Integrator for the DG form:
    alpha < rho_u (u.n) {v},[w] > + beta < rho_u |u.n| [v],[w] >,
    where v and w are the trial and test variables, respectively, and rho/u are
-    given scalar/vector coefficients. The vector coefficient, u, is assumed to
-    be continuous across the faces and when given the scalar coefficient, rho,
-    is assumed to be discontinuous. The integrator uses the upwind value of rho,
-    rho_u, which is value from the side into which the vector coefficient, u,
-    points. */
+    given scalar/vector coefficients. {v} represents the average value of v on
+    the face and [v] is the jump such that {v}=(v1+v2)/2 and [v]=(v1-v2) for the
+    face between elements 1 and 2. For boundary elements, v2=0. The vector
+    coefficient, u, is assumed to be continuous across the faces and when given
+    the scalar coefficient, rho, is assumed to be discontinuous. The integrator
+    uses the upwind value of rho, rho_u, which is value from the side into which
+    the vector coefficient, u, points.
+
+    One use case for this integrator is to discretize the operator -u.grad(v)
+    with a DG formulation. The resulting formulation uses the
+    ConvectionIntegrator (with coefficient u, and parameter alpha = -1) and the
+    transpose of the DGTraceIntegrator (with coefficient u, and parameters alpha
+    = 1, beta = -1/2 to use the upwind face flux, see also
+    NonconservativeDGTraceIntegrator). This discretization and the handling of
+    the inflow and outflow boundaries is illustrated in Example 9/9p.
+
+    Another use case for this integrator is to discretize the operator -div(u v)
+    with a DG formulation. The resulting formulation is conservative and
+    consists of the ConservativeConvectionIntegrator (with coefficient u, and
+    parameter alpha = -1) plus the DGTraceIntegrator (with coefficient u, and
+    parameters alpha = -1, beta = -1/2 to use the upwind face flux).
+    */
 class DGTraceIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2673,13 +2775,17 @@ private:
   Vector shape1, shape2;

 public:
-   /// Construct integrator with rho = 1.
-   DGTraceIntegrator(VectorCoefficient &_u, double a, double b)
-   { rho = NULL; u = &_u; alpha = a; beta = b; }
+   /// Construct integrator with rho = 1, b = 0.5*a.
+   DGTraceIntegrator(VectorCoefficient &u_, double a)
+   { rho = NULL; u = &u_; alpha = a; beta = 0.5*a; }

-   DGTraceIntegrator(Coefficient &_rho, VectorCoefficient &_u,
+   /// Construct integrator with rho = 1.
+   DGTraceIntegrator(VectorCoefficient &u_, double a, double b)
+   { rho = NULL; u = &u_; alpha = a; beta = b; }
+
+   DGTraceIntegrator(Coefficient &_rho, VectorCoefficient &u_,
                     double a, double b)
-   { rho = &_rho; u = &_u; alpha = a; beta = b; }
+   { rho = &_rho; u = &u_; alpha = a; beta = b; }

   using BilinearFormIntegrator::AssembleFaceMatrix;
   virtual void AssembleFaceMatrix(const FiniteElement &el1,
@@ -2713,6 +2819,30 @@ private:
   void SetupPA(const FiniteElementSpace &fes, FaceType type);
 };

+// Alias for @a DGTraceIntegrator.
+using ConservativeDGTraceIntegrator = DGTraceIntegrator;
+
+/** Integrator that represents the face terms used for the non-conservative
+    DG discretization of the convection equation:
+    -alpha < rho_u (u.n) {v},[w] > + beta < rho_u |u.n| [v],[w] >.
+
+    This integrator can be used with together with ConvectionIntegrator to
+    implement an upwind DG discretization in non-conservative form, see ex9 and
+    ex9p. */
+class NonconservativeDGTraceIntegrator : public TransposeIntegrator
+{
+public:
+   NonconservativeDGTraceIntegrator(VectorCoefficient &u, double a)
+      : TransposeIntegrator(new DGTraceIntegrator(u, -a, 0.5*a)) { }
+
+   NonconservativeDGTraceIntegrator(VectorCoefficient &u, double a, double b)
+      : TransposeIntegrator(new DGTraceIntegrator(u, -a, b)) { }
+
+   NonconservativeDGTraceIntegrator(Coefficient &rho, VectorCoefficient &u,
+                                    double a, double b)
+      : TransposeIntegrator(new DGTraceIntegrator(rho, u, -a, b)) { }
+};
+
 /** Integrator for the DG form:

    - < {(Q grad(u)).n}, [v] > + sigma < [u], {(Q grad(v)).n} >
@@ -2750,6 +2880,51 @@ public:
                                   DenseMatrix &elmat);
 };

+/** Integrator for the "BR2" diffusion stabilization term
+
+    sum_e eta (r_e([u]), r_e([v]))
+
+    where r_e is the lifting operator defined on each edge e. The parameter eta
+    can be chosen to be one to obtain a stable discretization. The constructor
+    for this integrator requires the finite element space because the lifting
+    operator depends on the element-wise inverse mass matrix.
+
+    BR2 stands for the second method of Bassi and Rebay:
+
+    - F. Bassi and S. Rebay. A high order discontinuous Galerkin method for
+      compressible turbulent flows. In B. Cockburn, G. E. Karniadakis, and
+      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77–88. Springer
+      Berlin Heidelberg, 2000.
+    - D. N. Arnold, F. Brezzi, B. Cockburn, and L. D. Marini. Unified analysis
+      of discontinuous Galerkin methods for elliptic problems. SIAM Journal on
+      Numerical Analysis, 39(5):1749–1779, 2002.
+*/
+class DGDiffusionBR2Integrator : public BilinearFormIntegrator
+{
+protected:
+   double eta;
+
+   // Block factorizations of local mass matrices, with offsets for the case of
+   // not equally sized blocks (mixed meshes, p-refinement)
+   Array<double> Minv;
+   Array<int> ipiv;
+   Array<int> ipiv_offsets, Minv_offsets;
+
+   Vector shape1, shape2;
+
+   DenseMatrix R11, R12, R21, R22;
+   DenseMatrix MinvR11, MinvR12, MinvR21, MinvR22;
+   DenseMatrix Re, MinvRe;
+
+public:
+   DGDiffusionBR2Integrator(class FiniteElementSpace *fes, double e = 1.0);
+   using BilinearFormIntegrator::AssembleFaceMatrix;
+   virtual void AssembleFaceMatrix(const FiniteElement &el1,
+                                   const FiniteElement &el2,
+                                   FaceElementTransformations &Trans,
+                                   DenseMatrix &elmat);
+};
+
 /** Integrator for the DG elasticity form, for the formulations see:
    - PhD Thesis of Jonas De Basabe, High-Order Finite %Element Methods for
      Seismic Wave Propagation, UT Austin, 2009, p. 23, and references therein
@@ -2908,11 +3083,36 @@ class DiscreteInterpolator : public BilinearFormIntegrator { };
 class GradientInterpolator : public DiscreteInterpolator
 {
 public:
+   GradientInterpolator() : dofquad_fe(NULL) { }
+   virtual ~GradientInterpolator() { delete dofquad_fe; }
+
   virtual void AssembleElementMatrix2(const FiniteElement &h1_fe,
                                       const FiniteElement &nd_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat)
   { nd_fe.ProjectGrad(h1_fe, Trans, elmat); }
+
+   using BilinearFormIntegrator::AssemblePA;
+
+   /** @brief Setup method for PA data.
+
+       @param[in] trial_fes   H1 Lagrange space
+       @param[in] test_fes    H(curl) Nedelec space
+    */
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes);
+
+   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+
+private:
+   /// 1D finite element that generates and owns the 1D DofToQuad maps below
+   FiniteElement * dofquad_fe;
+
+   bool B_id; // is the B basis operator (maps_C_C) the identity?
+   const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
+   const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
+   int dim, ne, o_dofs1D, c_dofs1D;
 };


@@ -2927,6 +3127,24 @@ public:
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat)
   { ran_fe.Project(dom_fe, Trans, elmat); }
+
+   using BilinearFormIntegrator::AssemblePA;
+
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes);
+
+   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+
+private:
+   /// 1D finite element that generates and owns the 1D DofToQuad maps below
+   FiniteElement * dofquad_fe;
+
+   const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
+   const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
+   int dim, ne, o_dofs1D, c_dofs1D;
+
+   Vector pa_data;
 };


@@ -3026,6 +3244,22 @@ protected:
   VectorCoefficient *VQ;
 };

+/** Interpolator of the 2D cross product between a vector coefficient and an
+    H(curl)-conforming field onto an L2-conforming field. */
+class ScalarCrossProductInterpolator : public DiscreteInterpolator
+{
+public:
+   ScalarCrossProductInterpolator(VectorCoefficient & vc)
+      : VQ(&vc) { }
+
+   virtual void AssembleElementMatrix2(const FiniteElement &nd_fe,
+                                       const FiniteElement &l2_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+protected:
+   VectorCoefficient *VQ;
+};
+
 /** Interpolator of the cross product between a vector coefficient and an
    H(curl)-conforming field onto an H(div)-conforming field. The range space
    can also be vector L2. */
@@ -0,0 +1,242 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "bilininteg.hpp"
+#include "pfespace.hpp"
+#include <algorithm>
+
+namespace mfem
+{
+
+DGDiffusionBR2Integrator::DGDiffusionBR2Integrator(FiniteElementSpace *fes,
+                                                   double e) : eta(e)
+{
+   // Precompute local mass matrix inverses needed for the lifting operators
+   // First compute offsets and total size needed (e.g. for mixed meshes or
+   // p-refinement)
+   int nel = fes->GetNE();
+   Minv_offsets.SetSize(nel+1);
+   ipiv_offsets.SetSize(nel+1);
+   ipiv_offsets[0] = 0;
+   Minv_offsets[0] = 0;
+   for (int i=0; i<nel; ++i)
+   {
+      int dof = fes->GetFE(i)->GetDof();
+      ipiv_offsets[i+1] = ipiv_offsets[i] + dof;
+      Minv_offsets[i+1] = Minv_offsets[i] + dof*dof;
+   }
+
+#ifdef MFEM_USE_MPI
+   // When running in parallel, we also need to compute the local mass matrices
+   // of face neighbor elements
+   ParFiniteElementSpace *pfes = dynamic_cast<ParFiniteElementSpace *>(fes);
+   if (pfes != NULL)
+   {
+      ParMesh *pmesh = pfes->GetParMesh();
+      pfes->ExchangeFaceNbrData();
+      int nel_nbr = pmesh->GetNFaceNeighborElements();
+      Minv_offsets.SetSize(nel+nel_nbr+1);
+      ipiv_offsets.SetSize(nel+nel_nbr+1);
+      for (int i=0; i<nel_nbr; ++i)
+      {
+         int dof = pfes->GetFaceNbrFE(i)->GetDof();
+         ipiv_offsets[nel+i+1] = ipiv_offsets[nel+i] + dof;
+         Minv_offsets[nel+i+1] = Minv_offsets[nel+i] + dof*dof;
+      }
+      nel += nel_nbr;
+   }
+#endif
+   // The final "offset" is the total size of all the blocks
+   Minv.SetSize(Minv_offsets[nel]);
+   ipiv.SetSize(ipiv_offsets[nel]);
+
+   // Assemble the local mass matrices and compute LU factorization
+   MassIntegrator mi;
+   for (int i=0; i<nel; ++i)
+   {
+      const FiniteElement *fe = NULL;
+      ElementTransformation *tr = NULL;
+      if (i < fes->GetNE())
+      {
+         fe = fes->GetFE(i);
+         tr = fes->GetElementTransformation(i);
+      }
+      else
+      {
+#ifdef MFEM_USE_MPI
+         int inbr = i - fes->GetNE();
+         fe = pfes->GetFaceNbrFE(inbr);
+         tr = pfes->GetParMesh()->GetFaceNbrElementTransformation(inbr);
+#endif
+      }
+      int dof = fe->GetDof();
+      double *Minv_el = &Minv[Minv_offsets[i]];
+      int *ipiv_el = &ipiv[ipiv_offsets[i]];
+      DenseMatrix Me(Minv_el, dof, dof);
+      mi.AssembleElementMatrix(*fe, *tr, Me);
+      LUFactors lu(Minv_el, ipiv_el);
+      lu.Factor(dof);
+   }
+}
+
+void DGDiffusionBR2Integrator::AssembleFaceMatrix(
+   const FiniteElement &el1, const FiniteElement &el2,
+   FaceElementTransformations &Trans, DenseMatrix &elmat)
+{
+   int ndof1 = el1.GetDof();
+   shape1.SetSize(ndof1);
+
+   R11.SetSize(ndof1, ndof1);
+   R11 = 0.0;
+   LUFactors M1inv(&Minv[Minv_offsets[Trans.Elem1No]],
+                   &ipiv[ipiv_offsets[Trans.Elem1No]]);
+   LUFactors M2inv;
+
+   double factor = Geometries.NumBdr(Trans.Elem1->GetGeometryType());
+
+   int ndof2;
+   if (Trans.Elem2No >= 0)
+   {
+      ndof2 = el2.GetDof();
+      shape2.SetSize(ndof2);
+      R12.SetSize(ndof1, ndof2);
+      R21.SetSize(ndof2, ndof1);
+      R22.SetSize(ndof2, ndof2);
+      M2inv.data = &Minv[Minv_offsets[Trans.Elem2No]];
+      M2inv.ipiv = &ipiv[ipiv_offsets[Trans.Elem2No]];
+
+      R12 = 0.0;
+      R21 = 0.0;
+      R22 = 0.0;
+
+      Geometry::Type geom2 = Trans.Elem2->GetGeometryType();
+      factor = std::max(factor, double(Geometries.NumBdr(geom2)));
+   }
+   else
+   {
+      ndof2 = 0;
+   }
+
+   int ndofs = ndof1 + ndof2;
+
+   Re.SetSize(ndofs, ndofs);
+   MinvRe.SetSize(ndofs, ndofs);
+
+   elmat.SetSize(ndofs);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (ndof2)
+      {
+         order = 2*std::max(el1.GetOrder(), el2.GetOrder());
+      }
+      else
+      {
+         order = 2*el1.GetOrder();
+      }
+      ir = &IntRules.Get(Trans.FaceGeom, order);
+   }
+
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      IntegrationPoint eip1, eip2;
+
+      Trans.Loc1.Transform(ip, eip1);
+      el1.CalcShape(eip1, shape1);
+      if (ndof2)
+      {
+         Trans.Loc2.Transform(ip, eip2);
+         el2.CalcShape(eip2, shape2);
+      }
+
+      double w = factor*sqrt(eta)*ip.weight*Trans.Face->Weight();
+      if (ndof2)
+      {
+         w /= 2;
+      }
+
+      for (int i = 0; i < ndof1; i++)
+      {
+         const double wsi = w*shape1(i);
+         for (int j = 0; j < ndof1; j++)
+         {
+            R11(i, j) += wsi*shape1(j);
+         }
+      }
+
+      if (ndof2)
+      {
+         for (int i = 0; i < ndof2; i++)
+         {
+            const double wsi = w*shape2(i);
+            for (int j = 0; j < ndof1; j++)
+            {
+               R21(i, j) += wsi*shape1(j);
+               R12(j, i) -= wsi*shape1(j);
+            }
+            for (int j = 0; j < ndof2; j++)
+            {
+               R22(i, j) -= wsi*shape2(j);
+            }
+         }
+      }
+   }
+
+   MinvR11 = R11;
+   M1inv.Solve(ndof1, ndof1, MinvR11.Data());
+   for (int i = 0; i < ndof1; i++)
+   {
+      for (int j = 0; j < ndof1; j++)
+      {
+         Re(i, j) = R11(i, j);
+         MinvRe(i, j) = MinvR11(i, j);
+      }
+   }
+
+   if (ndof2)
+   {
+      MinvR12 = R12;
+      MinvR21 = R21;
+      MinvR22 = R22;
+      M1inv.Solve(ndof1, ndof2, MinvR12.Data());
+      M2inv.Solve(ndof2, ndof1, MinvR21.Data());
+      M2inv.Solve(ndof2, ndof2, MinvR22.Data());
+
+      for (int i = 0; i < ndof2; i++)
+      {
+         for (int j = 0; j < ndof1; j++)
+         {
+            Re(ndof1 + i, j) = R21(i, j);
+            MinvRe(ndof1 + i, j) = MinvR21(i, j);
+
+            Re(j, ndof1 + i) = R12(j, i);
+            MinvRe(j, ndof1 + i) = MinvR12(j, i);
+         }
+         for (int j = 0; j < ndof2; j++)
+         {
+            Re(ndof1 + i, ndof1 + j) = R22(i, j);
+            MinvRe(ndof1 + i, ndof1 + j) = MinvR22(i, j);
+         }
+      }
+   }
+
+   // Compute the matrix associated with (r_e([u]), r_e([u])).
+   // The matrix for r_e([u]) is `MinvRe`, and so we need to form the product
+   // `(MinvRe)^T M MinvRe`. Using `Minv^T M = Minv M = I`, we obtain
+   // `Re^T MinvRe`.
+   MultAtB(Re, MinvRe, elmat);
+}
+
+}
@@ -379,51 +379,54 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   const int MQfullDim = MQ ? MQ->GetHeight() * MQ->GetWidth() : 0;
   if (MQ)
   {
+      symmetric = false;
      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
-      const int MQsymmDim = MQ->GetWidth() * (MQ->GetWidth() + 1) / 2;

-      const int MQdim = MQ->IsSymmetric() ? MQsymmDim : MQfullDim;
-      coeffDim = MQdim;
+      coeffDim = MQfullDim;

-      coeff.SetSize(MQdim * nq * ne);
-      symmetric = MQ ? MQ->IsSymmetric() : true;
+      coeff.SetSize(MQfullDim * nq * ne);

      DenseMatrix M;
-      Vector Msymm;
-      if (symmetric)
-      {
-         Msymm.SetSize(MQsymmDim);
-      }
-      else
-      {
-         M.SetSize(dim);
-      }
+      M.SetSize(dim);

-      auto C = Reshape(coeff.HostWrite(), MQdim, nq, ne);
+      auto C = Reshape(coeff.HostWrite(), MQfullDim, nq, ne);
      for (int e=0; e<ne; ++e)
      {
         ElementTransformation *tr = mesh->GetElementTransformation(e);
         for (int p=0; p<nq; ++p)
         {
-            if (MQ->IsSymmetric())
-            {
-               MQ->EvalSymmetric(Msymm, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<MQsymmDim; ++i)
+            MQ->Eval(M, *tr, ir->IntPoint(p));
+            for (int i=0; i<dim; ++i)
+               for (int j=0; j<dim; ++j)
               {
-                  C(i, p, e) = Msymm[i];
+                  C(j+(i*dim), p, e) = M(i,j);
               }
-            }
-            else
-            {
-               MQ->Eval(M, *tr, ir->IntPoint(p));
+         }
+      }
+   }
+   else if (SMQ)
+   {
+      MFEM_VERIFY(SMQ->GetSize() == dim, "");
+      coeffDim = symmDims;
+      coeff.SetSize(symmDims * nq * ne);

-               for (int i=0; i<dim; ++i)
-                  for (int j=0; j<dim; ++j)
-                  {
-                     C(j+(i*dim), p, e) = M(i,j);
-                  }
-            }
+      DenseSymmetricMatrix M;
+      M.SetSize(dim);
+
+      auto C = Reshape(coeff.HostWrite(), symmDims, nq, ne);
+
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            SMQ->Eval(M, *tr, ir->IntPoint(p));
+            int cnt = 0;
+            for (int i=0; i<dim; ++i)
+               for (int j=i; j<dim; ++j, ++cnt)
+               {
+                  C(cnt, p, e) = M(i,j);
+               }
         }
      }
   }
@@ -1901,4 +1904,17 @@ void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
   }
 }

+void DiffusionIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
+{
+   if (symmetric)
+   {
+      AddMultPA(x, y);
+   }
+   else
+   {
+      MFEM_ABORT("DiffusionIntegrator::AddMultTransposePA only implemented in "
+                 "the symmetric case.")
+   }
+}
+
 } // namespace mfem
@@ -334,7 +334,7 @@ static void PAGradientApplyTranspose2D(const int NE,
                                       const int q1d = 0)
 {
   // TODO
-   MFEM_ASSERT(false, "GradientPAApplyTranspose 3D not implemented.");
+   MFEM_ASSERT(false, "PAGradientApplyTranspose2D not implemented.");
 }

 // PA Gradient Apply 3D kernel
@@ -1226,4 +1226,10 @@ void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
   }
 }

+void MassIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
+{
+   // Mass integrator is symmetric
+   AddMultPA(x, y);
+}
+
 } // namespace mfem
@@ -761,12 +761,12 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   trial_fetype = trial_el->GetDerivType();
   test_fetype = test_el->GetDerivType();

-   const int MQsymmDim = MQ ? (MQ->GetWidth() * (MQ->GetWidth() + 1)) / 2 : 0;
+   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = MQ ? (MQ->IsSymmetric() ? MQsymmDim : MQfullDim) : 0;
-   const int coeffDim = MQ ? MQdim : (VQ ? VQ->GetVDim() : 1);
+   const int MQdim = MQ ? MQfullDim : MQsymmDim;
+   const int coeffDim = (MQ || SMQ) ? MQdim : (DQ ? DQ->GetVDim() : 1);

-   symmetric = MQ ? MQ->IsSymmetric() : true;
+   symmetric = (MQ == NULL);

   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
@@ -783,24 +783,13 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   Vector coeff(coeffDim * ne * nq);
   coeff = 1.0;
   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || VQ || MQ)
+   if (Q || DQ || MQ || SMQ)
   {
-      Vector D(VQ ? coeffDim : 0);
+      Vector D(DQ ? coeffDim : 0);
      DenseMatrix M;
-      Vector Msymm;
-      if (MQ)
-      {
-         if (symmetric)
-         {
-            Msymm.SetSize(MQsymmDim);
-         }
-         else
-         {
-            M.SetSize(dim);
-         }
-      }
+      DenseSymmetricMatrix SM;

-      if (VQ)
+      if (DQ)
      {
         MFEM_VERIFY(coeffDim == dim, "");
      }
@@ -808,6 +797,12 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
      {
         MFEM_VERIFY(coeffDim == MQdim, "");
         MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+         M.SetSize(dim);
+      }
+      if (SMQ)
+      {
+         MFEM_VERIFY(SMQ->GetSize() == dim, "");
+         SM.SetSize(dim);
      }

      for (int e=0; e<ne; ++e)
@@ -817,29 +812,27 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
         {
            if (MQ)
            {
-               if (MQ->IsSymmetric())
-               {
-                  MQ->EvalSymmetric(Msymm, *tr, ir->IntPoint(p));
+               MQ->Eval(M, *tr, ir->IntPoint(p));

-                  for (int i=0; i<MQsymmDim; ++i)
+               for (int i=0; i<dim; ++i)
+                  for (int j=0; j<dim; ++j)
                  {
-                     coeffh(i, p, e) = Msymm[i];
+                     coeffh(j+(i*dim), p, e) = M(i,j);
                  }
-               }
-               else
-               {
-                  MQ->Eval(M, *tr, ir->IntPoint(p));
-
-                  for (int i=0; i<dim; ++i)
-                     for (int j=0; j<dim; ++j)
-                     {
-                        coeffh(j+(i*dim), p, e) = M(i,j);
-                     }
-               }
            }
-            else if (VQ)
+            else if (SMQ)
            {
-               VQ->Eval(D, *tr, ir->IntPoint(p));
+               SMQ->Eval(SM, *tr, ir->IntPoint(p));
+               int cnt = 0;
+               for (int i=0; i<dim; ++i)
+                  for (int j=i; j<dim; ++j, ++cnt)
+                  {
+                     coeffh(cnt, p, e) = SM(i,j);
+                  }
+            }
+            else if (DQ)
+            {
+               DQ->Eval(D, *tr, ir->IntPoint(p));
               for (int i=0; i<coeffDim; ++i)
               {
                  coeffh(i, p, e) = D[i];
@@ -1007,14 +1000,14 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
      }
      else if (trial_curl && test_div)
      {
-         const bool scalarCoeff = !(VQ || MQ);
+         const bool scalarCoeff = !(DQ || MQ || SMQ);
         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
                                true, mapsO->B, mapsC->B, mapsOtest->Bt,
                                mapsCtest->Bt, pa_data, x, y);
      }
      else if (trial_div && test_curl)
      {
-         const bool scalarCoeff = !(VQ || MQ);
+         const bool scalarCoeff = !(DQ || MQ || SMQ);
         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
                                false, mapsO->B, mapsC->B, mapsOtest->Bt,
                                mapsCtest->Bt, pa_data, x, y);
@@ -1038,7 +1031,7 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
      }
      else if ((trial_curl && test_div) || (trial_div && test_curl))
      {
-         const bool scalarCoeff = !(VQ || MQ);
+         const bool scalarCoeff = !(DQ || MQ || SMQ);
         PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
                                trial_curl, mapsO->B, mapsC->B, mapsOtest->Bt,
                                mapsCtest->Bt, pa_data, x, y);
@@ -301,7 +301,7 @@ void MatrixFunctionCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,

   K.SetSize(height, width);

-   if (symmetric) // Use SymmFunction
+   if (symmetric) // Use SymmFunction (deprecated version)
   {
      MFEM_VERIFY(height == width && SymmFunction,
                  "MatrixFunctionCoefficient is not symmetric");
@@ -371,6 +371,36 @@ void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
   }
 }

+void SymmetricMatrixFunctionCoefficient::Eval(DenseSymmetricMatrix &K,
+                                              ElementTransformation &T,
+                                              const IntegrationPoint &ip)
+{
+   double x[3];
+   Vector transip(x, 3);
+
+   T.Transform(ip, transip);
+
+   K.SetSize(dim);
+
+   if (Function)
+   {
+      Function(transip, K);
+   }
+   else if (TDFunction)
+   {
+      TDFunction(transip, GetTime(), K);
+   }
+   else
+   {
+      K = mat;
+   }
+
+   if (Q)
+   {
+      K *= Q->Eval(T, ip, GetTime());
+   }
+}
+
 MatrixArrayCoefficient::MatrixArrayCoefficient (int dim)
   : MatrixCoefficient (dim)
 {
@@ -595,6 +625,7 @@ void MatrixVectorProductCoefficient::Eval(Vector &V, ElementTransformation &T,
 {
   a->Eval(ma, T, ip);
   b->Eval(vb, T, ip);
+   V.SetSize(vdim);
   ma.Mult(vb, V);
 }

@@ -688,6 +688,7 @@ public:
                     const IntegrationRule &ir);
 };

+typedef VectorCoefficient DiagonalMatrixCoefficient;

 /// Base class for Matrix Coefficients that optionally depend on time and space.
 class MatrixCoefficient
@@ -695,7 +696,7 @@ class MatrixCoefficient
 protected:
   int height, width;
   double time;
-   bool symmetric;
+   bool symmetric;  // deprecated

 public:
   /// Construct a dim x dim matrix coefficient.
@@ -721,6 +722,7 @@ public:
   /// For backward compatibility get the width of the matrix.
   int GetVDim() const { return width; }

+   /** @deprecated Use SymmetricMatrixCoefficient instead */
   bool IsSymmetric() const { return symmetric; }

   /** @brief Evaluate the matrix coefficient in the element described by @a T
@@ -731,11 +733,13 @@ public:
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip) = 0;

+   /// (DEPRECATED) Evaluate a symmetric matrix coefficient.
   /** @brief Evaluate the upper triangular entries of the matrix coefficient
       in the symmetric case, similarly to Eval. Matrix entry (i,j) is stored
       in K[j - i + os_i] for 0 <= i <= j < width, os_0 = 0,
       os_{i+1} = os_i + width - i. That is, K = {M(0,0), ..., M(0,w-1),
-       M(1,1), ..., M(1,w-1), ..., M(w-1,w-1) with w = width. */
+       M(1,1), ..., M(1,w-1), ..., M(w-1,w-1) with w = width.
+       @deprecated Use Eval() instead. */
   virtual void EvalSymmetric(Vector &K, ElementTransformation &T,
                              const IntegrationPoint &ip)
   { mfem_error("MatrixCoefficient::EvalSymmetric"); }
@@ -767,7 +771,7 @@ class MatrixFunctionCoefficient : public MatrixCoefficient
 {
 private:
   std::function<void(const Vector &, DenseMatrix &)> Function;
-   std::function<void(const Vector &, Vector &)> SymmFunction;
+   std::function<void(const Vector &, Vector &)> SymmFunction;  // deprecated
   std::function<void(const Vector &, double, DenseMatrix &)> TDFunction;

   Coefficient *Q;
@@ -791,6 +795,18 @@ public:
      : MatrixCoefficient(m.Height(), m.Width()), Q(&q), mat(m)
   { }

+   /** @brief Define a time-independent symmetric square matrix coefficient from
+       a std function */
+   /** \param dim - the size of the matrix
+       \param SymmF - function used in EvalSymmetric
+       \param q - optional scalar Coefficient to scale the matrix coefficient
+       @deprecated Use another constructor without setting SymmFunction. */
+   MatrixFunctionCoefficient(int dim,
+                             std::function<void(const Vector &, Vector &)> SymmF,
+                             Coefficient *q = NULL)
+      : MatrixCoefficient(dim, true), SymmFunction(std::move(SymmF)), Q(q), mat(0)
+   { }
+
   /// Define a time-dependent square matrix coefficient from a std function
   /** \param dim - the size of the matrix
       \param TDF - time-dependent function
@@ -801,22 +817,12 @@ public:
      : MatrixCoefficient(dim), TDFunction(std::move(TDF)), Q(q)
   { }

-   /** @brief Define a time-independent symmetric square matrix coefficient from
-       a std function */
-   /** \param dim - the size of the matrix
-       \param SymmF - function used in EvalSymmetric
-       \param q - optional scalar Coefficient to scale the matrix coefficient */
-   MatrixFunctionCoefficient(int dim,
-                             std::function<void(const Vector &, Vector &)> SymmF,
-                             Coefficient *q = NULL)
-      : MatrixCoefficient(dim, true), SymmFunction(std::move(SymmF)), Q(q), mat(0)
-   { }
-
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip);

-   /// Evaluate the symmetric matrix coefficient at @a ip.
+   /// (DEPRECATED) Evaluate the symmetric matrix coefficient at @a ip.
+   /** @deprecated Use Eval() instead. */
   virtual void EvalSymmetric(Vector &K, ElementTransformation &T,
                              const IntegrationPoint &ip);

@@ -824,7 +830,6 @@ public:
 };


-
 /** @brief Matrix coefficient defined by a matrix of scalar coefficients.
    Coefficients that are not set will evaluate to zero in the vector. The
    coefficient is stored as a flat Array with indexing (i,j) -> i*width+j. */
@@ -940,6 +945,106 @@ public:
   }
 };

+
+/// Base class for symmetric matrix coefficients that optionally depend on time and space.
+class SymmetricMatrixCoefficient
+{
+protected:
+   int dim;
+   double time;
+
+public:
+   /// Construct a dim x dim matrix coefficient.
+   explicit SymmetricMatrixCoefficient(int dimension)
+   { dim = dimension; time = 0.; }
+
+   /// Set the time for time dependent coefficients
+   void SetTime(double t) { time = t; }
+
+   /// Get the time for time dependent coefficients
+   double GetTime() { return time; }
+
+   /// Get the size of the matrix.
+   int GetSize() const { return dim; }
+
+   /** @brief Evaluate the matrix coefficient in the element described by @a T
+       at the point @a ip, storing the result in @a K. */
+   /** @note When this method is called, the caller must make sure that the
+       IntegrationPoint associated with @a T is the same as @a ip. This can be
+       achieved by calling T.SetIntPoint(&ip). */
+   virtual void Eval(DenseSymmetricMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip) = 0;
+
+   virtual ~SymmetricMatrixCoefficient() { }
+};
+
+
+/// A matrix coefficient that is constant in space and time.
+class SymmetricMatrixConstantCoefficient : public SymmetricMatrixCoefficient
+{
+private:
+   DenseSymmetricMatrix mat;
+
+public:
+   ///Construct using matrix @a m for the constant.
+   SymmetricMatrixConstantCoefficient(const DenseSymmetricMatrix &m)
+      : SymmetricMatrixCoefficient(m.Height()), mat(m) { }
+   using SymmetricMatrixCoefficient::Eval;
+   /// Evaluate the matrix coefficient at @a ip.
+   virtual void Eval(DenseSymmetricMatrix &M, ElementTransformation &T,
+                     const IntegrationPoint &ip) { M = mat; }
+};
+
+
+/** @brief A matrix coefficient with an optional scalar coefficient multiplier
+    \a q.  The matrix function can either be represented by a std function or
+    a constant matrix provided when constructing this object.  */
+class SymmetricMatrixFunctionCoefficient : public SymmetricMatrixCoefficient
+{
+private:
+   std::function<void(const Vector &, DenseSymmetricMatrix &)> Function;
+   std::function<void(const Vector &, double, DenseSymmetricMatrix &)> TDFunction;
+
+   Coefficient *Q;
+   DenseSymmetricMatrix mat;
+
+public:
+   /// Define a time-independent symmetric matrix coefficient from a std function
+   /** \param dim - the size of the matrix
+       \param F - time-independent function
+       \param q - optional scalar Coefficient to scale the matrix coefficient */
+   SymmetricMatrixFunctionCoefficient(int dim,
+                                      std::function<void(const Vector &, DenseSymmetricMatrix &)> F,
+                                      Coefficient *q = nullptr)
+      : SymmetricMatrixCoefficient(dim), Function(std::move(F)), Q(q), mat(0)
+   { }
+
+   /// Define a constant matrix coefficient times a scalar Coefficient
+   /** \param m - constant matrix
+       \param q - optional scalar Coefficient to scale the matrix coefficient */
+   SymmetricMatrixFunctionCoefficient(const DenseSymmetricMatrix &m,
+                                      Coefficient &q)
+      : SymmetricMatrixCoefficient(m.Height()), Q(&q), mat(m)
+   { }
+
+   /// Define a time-dependent square matrix coefficient from a std function
+   /** \param dim - the size of the matrix
+       \param TDF - time-dependent function
+       \param q - optional scalar Coefficient to scale the matrix coefficient */
+   SymmetricMatrixFunctionCoefficient(int dim,
+                                      std::function<void(const Vector &, double, DenseSymmetricMatrix &)> TDF,
+                                      Coefficient *q = nullptr)
+      : SymmetricMatrixCoefficient(dim), TDFunction(std::move(TDF)), Q(q)
+   { }
+
+   /// Evaluate the matrix coefficient at @a ip.
+   virtual void Eval(DenseSymmetricMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+
+   virtual ~SymmetricMatrixFunctionCoefficient() { }
+};
+
+
 /** @brief Scalar coefficient defined as the product of two scalar coefficients
    or a scalar and a scalar coefficient. */
 class ProductCoefficient : public Coefficient
@@ -1204,6 +1204,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
      });
      // Modify offdiagonal blocks (imaginary parts of the matrix) to conform
      // with standard essential BC treatment
+      ess_tdof_list.HostRead();
      if (A_i.Type() == Operator::Hypre_ParCSR)
      {
         HypreParMatrix * Ah;
@@ -48,8 +48,373 @@ void L2ZienkiewiczZhuEstimator::ComputeEstimates()
   current_sequence = solution->FESpace()->GetMesh()->GetSequence();
 }

+
+KellyErrorEstimator::KellyErrorEstimator(BilinearFormIntegrator& di_,
+                                         ParGridFunction& sol_,
+                                         ParFiniteElementSpace& flux_fespace_,
+                                         const Array<int> &attributes_)
+   : attributes(attributes_)
+   , flux_integrator(&di_)
+   , solution(&sol_)
+   , flux_space(&flux_fespace_)
+   , own_flux_fespace(false)
+{
+   ResetCoefficientFunctions();
+}
+
+KellyErrorEstimator::KellyErrorEstimator(BilinearFormIntegrator& di_,
+                                         ParGridFunction& sol_,
+                                         ParFiniteElementSpace* flux_fespace_,
+                                         const Array<int> &attributes_)
+   : attributes(attributes_)
+   , flux_integrator(&di_)
+   , solution(&sol_)
+   , flux_space(flux_fespace_)
+   , own_flux_fespace(true)
+{
+   ResetCoefficientFunctions();
+}
+
+KellyErrorEstimator::~KellyErrorEstimator()
+{
+   if (own_flux_fespace)
+   {
+      delete flux_space;
+   }
+}
+
+void KellyErrorEstimator::ResetCoefficientFunctions()
+{
+   compute_element_coefficient = [](ParMesh* pmesh, const int e)
+   {
+      return 1.0;
+   };
+
+   compute_face_coefficient = [](ParMesh* pmesh, const int f,
+                                 const bool shared_face)
+   {
+      auto FT = [&]()
+      {
+         if (shared_face)
+         {
+            return pmesh->GetSharedFaceTransformations(f);
+         }
+         return pmesh->GetFaceElementTransformations(f);
+      }();
+      const auto order = FT->GetFE()->GetOrder();
+
+      // Poor man's face diameter.
+      double diameter = 0.0;
+
+      Vector p1(pmesh->SpaceDimension());
+      Vector p2(pmesh->SpaceDimension());
+      // NOTE: We have no direct access to vertices for shared faces,
+      // so we fall back to compute the positions from the element.
+      // This can also be modified to compute the diameter for non-linear
+      // geometries by sampling along geometry-specific lines.
+      auto vtx_intrule = Geometries.GetVertices(FT->GetGeometryType());
+      const auto nip = vtx_intrule->GetNPoints();
+      for (int i = 0; i < nip; i++)
+      {
+         // Evaluate flux vector at integration point
+         auto fip1 = vtx_intrule->IntPoint(i);
+         FT->Transform(fip1, p1);
+
+         for (int j = 0; j < nip; j++)
+         {
+            auto fip2 = vtx_intrule->IntPoint(j);
+            FT->Transform(fip2, p2);
+
+            diameter = std::max<double>(diameter, p2.DistanceTo(p1));
+         }
+      }
+      return diameter/(2.0*order);
+   };
+}
+
+void KellyErrorEstimator::ComputeEstimates()
+{
+   // Remarks:
+   // For some context you may have to consult the documentation of
+   // the FaceInfo class [1]. Also, the FaceElementTransformations
+   // documentation [2] may be helpful to grasp what is going on. Note
+   // that the FaceElementTransformations also works in the non-
+   // conforming case to transfer the gauss points from the slave to
+   // the master element.
+   // [1]
+   // https://github.com/mfem/mfem/blob/02d0bfe9c18ce049c3c93a6a4208080fcfc96991/mesh/mesh.hpp#L94
+   // [2]
+   // https://github.com/mfem/mfem/blob/02d0bfe9c18ce049c3c93a6a4208080fcfc96991/fem/eltrans.hpp#L435
+
+   flux_space->Update(false);
+
+   auto xfes      = solution->ParFESpace();
+   MFEM_ASSERT(xfes->GetVDim() == 1,
+               "Estimation for vector-valued problems not implemented yet.");
+   auto pmesh     = xfes->GetParMesh();
+
+   this->error_estimates.SetSize(xfes->GetNE());
+   this->error_estimates = 0.0;
+
+   // 1. Compute fluxes in discontinuous space
+   ParGridFunction flux(flux_space);
+   flux = 0.0;
+
+   // We pre-sort the array to speed up the search in the following loops.
+   if (attributes.Size())
+   {
+      attributes.Sort();
+   }
+
+   Array<int> xdofs, fdofs;
+   Vector el_x, el_f;
+   for (int e = 0; e < xfes->GetNE(); e++)
+   {
+      auto attr = xfes->GetAttribute(e);
+      if (attributes.Size() && attributes.FindSorted(attr) == -1)
+      {
+         continue;
+      }
+
+      xfes->GetElementVDofs(e, xdofs);
+      solution->GetSubVector(xdofs, el_x);
+
+      ElementTransformation* Transf = xfes->GetElementTransformation(e);
+      flux_integrator->ComputeElementFlux(*xfes->GetFE(e), *Transf, el_x,
+                                          *flux_space->GetFE(e), el_f, true);
+
+      flux_space->GetElementVDofs(e, fdofs);
+      flux.AddElementVector(fdofs, el_f);
+   }
+
+   // 2. Add error contribution from local interior faces
+   for (int f = 0; f < pmesh->GetNumFaces(); f++)
+   {
+      auto FT = pmesh->GetFaceElementTransformations(f);
+
+      auto &int_rule = IntRules.Get(FT->FaceGeom, 2 * xfes->GetFaceOrder(f));
+      const auto nip = int_rule.GetNPoints();
+
+      if (pmesh->FaceIsInterior(f))
+      {
+         int Inf1, Inf2, NCFace;
+         pmesh->GetFaceInfos(f, &Inf1, &Inf2, &NCFace);
+
+         // Convention
+         // * Conforming face: Face side with smaller element id handles
+         // the integration
+         // * Non-conforming face: The slave handles the integration.
+         // See FaceInfo documentation for details.
+         bool isNCSlave    = FT->Elem2No >= 0 && NCFace >= 0;
+         bool isConforming = FT->Elem2No >= 0 && NCFace == -1;
+         if ((FT->Elem1No < FT->Elem2No && isConforming) || isNCSlave)
+         {
+            if (attributes.Size() &&
+                (attributes.FindSorted(FT->Elem1->Attribute) == -1
+                 || attributes.FindSorted(FT->Elem2->Attribute) == -1))
+            {
+               continue;
+            }
+
+            IntegrationRule eir;
+            Vector jumps(nip);
+
+            // Integral over local half face on the side of e₁
+            // i.e. the numerical integration of ∫ flux ⋅ n dS₁
+            for (int i = 0; i < nip; i++)
+            {
+               // Evaluate flux at IP
+               auto &fip = int_rule.IntPoint(i);
+               IntegrationPoint ip;
+               FT->Loc1.Transform(fip, ip);
+
+               Vector val(flux_space->GetVDim());
+               flux.GetVectorValue(FT->Elem1No, ip, val);
+
+               // And build scalar product with normal
+               Vector normal(pmesh->SpaceDimension());
+               FT->Face->SetIntPoint(&fip);
+               if (pmesh->Dimension() == pmesh->SpaceDimension())
+               {
+                  CalcOrtho(FT->Face->Jacobian(), normal);
+               }
+               else
+               {
+                  Vector ref_normal(pmesh->Dimension());
+                  FT->Loc1.Transf.SetIntPoint(&fip);
+                  CalcOrtho(FT->Loc1.Transf.Jacobian(), ref_normal);
+                  auto &e1 = FT->GetElement1Transformation();
+                  e1.AdjugateJacobian().MultTranspose(ref_normal, normal);
+                  normal /= e1.Weight();
+               }
+               jumps(i) = val * normal * fip.weight * FT->Face->Weight();
+            }
+
+            // Subtract integral over half face of e₂
+            // i.e. the numerical integration of ∫ flux ⋅ n dS₂
+            for (int i = 0; i < nip; i++)
+            {
+               // Evaluate flux vector at IP
+               auto &fip = int_rule.IntPoint(i);
+               IntegrationPoint ip;
+               FT->Loc2.Transform(fip, ip);
+
+               Vector val(flux_space->GetVDim());
+               flux.GetVectorValue(FT->Elem2No, ip, val);
+
+               // And build scalar product with normal
+               Vector normal(pmesh->SpaceDimension());
+               FT->Face->SetIntPoint(&fip);
+               if (pmesh->Dimension() == pmesh->SpaceDimension())
+               {
+                  CalcOrtho(FT->Face->Jacobian(), normal);
+               }
+               else
+               {
+                  Vector ref_normal(pmesh->Dimension());
+                  FT->Loc1.Transf.SetIntPoint(&fip);
+                  CalcOrtho(FT->Loc1.Transf.Jacobian(), ref_normal);
+                  auto &e1 = FT->GetElement1Transformation();
+                  e1.AdjugateJacobian().MultTranspose(ref_normal, normal);
+                  normal /= e1.Weight();
+               }
+
+               jumps(i) -= val * normal * fip.weight * FT->Face->Weight();
+            }
+
+            // Finalize "local" L₂ contribution
+            for (int i = 0; i < nip; i++)
+            {
+               jumps(i) *= jumps(i);
+            }
+            auto h_k_face = compute_face_coefficient(pmesh, f, false);
+            double jump_integral = h_k_face*jumps.Sum();
+
+            // A local face is shared between two local elements, so we
+            // can get away with integrating the jump only once and add
+            // it to both elements. To minimize communication, the jump
+            // of shared faces is computed locally by each process.
+            error_estimates(FT->Elem1No) += jump_integral;
+            error_estimates(FT->Elem2No) += jump_integral;
+         }
+      }
+   }
+
+   // 3. Add error contribution from shared interior faces
+   // Synchronize face data.
+   flux.ExchangeFaceNbrData();
+
+   for (int sf = 0; sf < pmesh->GetNSharedFaces(); sf++)
+   {
+      auto FT = pmesh->GetSharedFaceTransformations(sf, true);
+      if (attributes.Size() &&
+          (attributes.FindSorted(FT->Elem1->Attribute) == -1
+           || attributes.FindSorted(FT->Elem2->Attribute) == -1))
+      {
+         continue;
+      }
+
+      auto &int_rule = IntRules.Get(FT->FaceGeom, 2 * xfes->GetFaceOrder(0));
+      const auto nip = int_rule.GetNPoints();
+
+      IntegrationRule eir;
+      Vector jumps(nip);
+
+      // Integral over local half face on the side of e₁
+      // i.e. the numerical integration of ∫ flux ⋅ n dS₁
+      for (int i = 0; i < nip; i++)
+      {
+         // Evaluate flux vector at integration point
+         auto &fip = int_rule.IntPoint(i);
+         IntegrationPoint ip;
+         FT->Loc1.Transform(fip, ip);
+
+         Vector val(flux_space->GetVDim());
+         flux.GetVectorValue(FT->Elem1No, ip, val);
+
+         Vector normal(pmesh->SpaceDimension());
+         FT->Face->SetIntPoint(&fip);
+         if (pmesh->Dimension() == pmesh->SpaceDimension())
+         {
+            CalcOrtho(FT->Face->Jacobian(), normal);
+         }
+         else
+         {
+            Vector ref_normal(pmesh->Dimension());
+            FT->Loc1.Transf.SetIntPoint(&fip);
+            CalcOrtho(FT->Loc1.Transf.Jacobian(), ref_normal);
+            auto &e1 = FT->GetElement1Transformation();
+            e1.AdjugateJacobian().MultTranspose(ref_normal, normal);
+            normal /= e1.Weight();
+         }
+
+         jumps(i) = val * normal * fip.weight * FT->Face->Weight();
+      }
+
+      // Subtract integral over non-local half face of e₂
+      // i.e. the numerical integration of ∫ flux ⋅ n dS₂
+      for (int i = 0; i < nip; i++)
+      {
+         // Evaluate flux vector at integration point
+         auto &fip = int_rule.IntPoint(i);
+         IntegrationPoint ip;
+         FT->Loc2.Transform(fip, ip);
+
+         Vector val(flux_space->GetVDim());
+         flux.GetVectorValue(FT->Elem2No, ip, val);
+
+         // Evaluate gauss point
+         Vector normal(pmesh->SpaceDimension());
+         FT->Face->SetIntPoint(&fip);
+         if (pmesh->Dimension() == pmesh->SpaceDimension())
+         {
+            CalcOrtho(FT->Face->Jacobian(), normal);
+         }
+         else
+         {
+            Vector ref_normal(pmesh->Dimension());
+            CalcOrtho(FT->Loc1.Transf.Jacobian(), ref_normal);
+            auto &e1 = FT->GetElement1Transformation();
+            e1.AdjugateJacobian().MultTranspose(ref_normal, normal);
+            normal /= e1.Weight();
+         }
+
+         jumps(i) -= val * normal * fip.weight * FT->Face->Weight();
+      }
+
+      // Finalize "local" L₂ contribution
+      for (int i = 0; i < nip; i++)
+      {
+         jumps(i) *= jumps(i);
+      }
+      auto h_k_face = compute_face_coefficient(pmesh, sf, true);
+      double jump_integral = h_k_face*jumps.Sum();
+
+      error_estimates(FT->Elem1No) += jump_integral;
+      // We skip "error_estimates(FT->Elem2No) += jump_integral"
+      // because the error is stored on the remote process and
+      // recomputed there.
+   }
+
+   // Finalize element errors
+   for (int e = 0; e < xfes->GetNE(); e++)
+   {
+      auto factor = compute_element_coefficient(pmesh, e);
+      // The sqrt belongs to the norm and hₑ to the indicator.
+      error_estimates(e) = sqrt(factor * error_estimates(e));
+   }
+
+   current_sequence = solution->FESpace()->GetMesh()->GetSequence();
+
+   // Finish by computing the global error.
+   double process_local_error = error_estimates.Sum();
+   MPI_Allreduce(&process_local_error, &total_error, 1, MPI_DOUBLE,
+                 MPI_SUM, xfes->GetComm());
+}
+
 #endif // MFEM_USE_MPI

+
 void LpErrorEstimator::ComputeEstimates()
 {
   MFEM_VERIFY(coef != NULL || vcoef != NULL,
@@ -64,6 +429,17 @@ void LpErrorEstimator::ComputeEstimates()
   {
      sol->ComputeElementLpErrors(local_norm_p, *vcoef, error_estimates);
   }
+#ifdef MFEM_USE_MPI
+   total_error = error_estimates.Sum();
+   auto pfes = dynamic_cast<ParFiniteElementSpace*>(sol->FESpace());
+   if (pfes)
+   {
+      auto process_local_error = total_error;
+      MPI_Allreduce(&process_local_error, &total_error, 1, MPI_DOUBLE,
+                    MPI_SUM, pfes->GetComm());
+   }
+#endif // MFEM_USE_MPI
+   total_error = pow(total_error, 1.0/local_norm_p);
   current_sequence = sol->FESpace()->GetMesh()->GetSequence();
 }

@@ -12,6 +12,8 @@
 #ifndef MFEM_ERROR_ESTIMATORS
 #define MFEM_ERROR_ESTIMATORS

+#include <functional>
+
 #include "../config/config.hpp"
 #include "../linalg/vector.hpp"
 #include "bilinearform.hpp"
@@ -39,6 +41,11 @@ public:
 class ErrorEstimator : public AbstractErrorEstimator
 {
 public:
+   /// Return the total error from the last error estimate.
+   /** @note This method is optional for derived classes to override and the
+       base class implementation simply returns 0. */
+   virtual double GetTotalError() const { return 0.0; }
+
   /// Get a Vector with all element errors.
   virtual const Vector &GetLocalErrors() = 0;

@@ -148,8 +155,8 @@ public:
        own_flux_fes(false)
   { }

-   /** @brief Consider the coefficient in BilinearFormIntegrator to calculate the
-       fluxes for the error estimator.*/
+   /** @brief Consider the coefficient in BilinearFormIntegrator to calculate
+       the fluxes for the error estimator.*/
   void SetWithCoeff(bool w_coeff = true) { with_coeff = w_coeff; }

   /** @brief Enable/disable anisotropic estimates. To enable this option, the
@@ -166,10 +173,10 @@ public:
   void SetFluxAveraging(int fa) { flux_averaging = fa; }

   /// Return the total error from the last error estimate.
-   double GetTotalError() const { return total_error; }
+   virtual double GetTotalError() const override { return total_error; }

   /// Get a Vector with all element errors.
-   virtual const Vector &GetLocalErrors()
+   virtual const Vector &GetLocalErrors() override
   {
      if (MeshIsModified()) { ComputeEstimates(); }
      return error_estimates;
@@ -178,14 +185,14 @@ public:
   /** @brief Get an Array<int> with anisotropic flags for all mesh elements.
       Return an empty array when anisotropic estimates are not available or
       enabled. */
-   virtual const Array<int> &GetAnisotropicFlags()
+   virtual const Array<int> &GetAnisotropicFlags() override
   {
      if (MeshIsModified()) { ComputeEstimates(); }
      return aniso_flags;
   }

   /// Reset the error estimator.
-   virtual void Reset() { current_sequence = -1; }
+   virtual void Reset() override { current_sequence = -1; }

   /** @brief Destroy a ZienkiewiczZhuEstimator object. Destroys, if owned, the
       FiniteElementSpace, flux_space. */
@@ -292,17 +299,17 @@ public:
   void SetLocalErrorNormP(int p) { local_norm_p = p; }

   /// Return the total error from the last error estimate.
-   double GetTotalError() const { return total_error; }
+   virtual double GetTotalError() const override { return total_error; }

   /// Get a Vector with all element errors.
-   virtual const Vector &GetLocalErrors()
+   virtual const Vector &GetLocalErrors() override
   {
      if (MeshIsModified()) { ComputeEstimates(); }
      return error_estimates;
   }

   /// Reset the error estimator.
-   virtual void Reset() { current_sequence = -1; }
+   virtual void Reset() override { current_sequence = -1; }

   /** @brief Destroy a L2ZienkiewiczZhuEstimator object. Destroys, if owned,
       the FiniteElementSpace, flux_space. */
@@ -314,6 +321,7 @@ public:

 #endif // MFEM_USE_MPI

+
 /** @brief The LpErrorEstimator class compares the solution to a known
    coefficient.

@@ -332,6 +340,8 @@ protected:
   int local_norm_p;
   Vector error_estimates;

+   double total_error = 0.0;
+
   Coefficient * coef;
   VectorCoefficient * vcoef;
   GridFunction * sol;
@@ -383,10 +393,10 @@ public:
   void SetCoef(VectorCoefficient &A) { vcoef = &A; }

   /// Reset the error estimator.
-   virtual void Reset() { current_sequence = -1; }
+   virtual void Reset() override { current_sequence = -1; }

   /// Get a Vector with all element errors.
-   virtual const Vector &GetLocalErrors()
+   virtual const Vector &GetLocalErrors() override
   {
      if (MeshIsModified()) { ComputeEstimates(); }
      return error_estimates;
@@ -396,6 +406,174 @@ public:
   virtual ~LpErrorEstimator() {}
 };

+
+#ifdef MFEM_USE_MPI
+/** @brief The KellyErrorEstimator class provides a fast error indication
+    strategy for smooth scalar parallel problems.
+
+    The Kelly error indicator is based on the following papers:
+
+    Kelly, D. W., et al. "A posteriori error analysis and adaptive processes in
+    the finite element method: Part I—Error analysis." International journal for
+    numerical methods in engineering 19.11 (1983): 1593-1619.
+
+    De SR Gago, J. P., et al. "A posteriori error analysis and adaptive
+    processes in the finite element method: Part II—Adaptive mesh refinement."
+    International journal for numerical methods in engineering 19.11 (1983):
+    1621-1656.
+
+    It can be roughly described by:
+        ||∇(u-uₕ)||ₑ ≦ √( C hₑ ∑ₖ (hₖ ∫ |J[∇uₕ]|²) dS )
+    where "e" denotes an element, ||⋅||ₑ the corresponding local norm and k the
+    corresponding faces. u is the analytic solution and uₕ the discretized
+    solution. hₖ and hₑ are factors dependend on the face and element geometry.
+    J is the jump function, i.e. the difference between the limits at each point
+    for each side of the face. A custom method to compute hₖ can be provided. It
+    is also possible to estimate the error only on a subspace by feeding this
+    class an attribute array describing the subspace.
+
+    @note This algorithm is only for Poisson problems a proper error esimator.
+    The current implementation does not reflect this, because the "C" factor is
+    not included.
+    It further assumes that the approximation error at the boundary is small
+    enough, as the implementation ignores boundary faces.
+*/
+class KellyErrorEstimator final : public ErrorEstimator
+{
+public:
+   /// Function type to compute the local coefficient hₑ of an element.
+   using ElementCoefficientFunction =
+      std::function<double(ParMesh*, const int)>;
+   /** @brief Function type to compute the local coefficient hₖ of a face. The
+       third argument is true for shared faces and false for local faces. */
+   using FaceCoefficientFunction =
+      std::function<double(ParMesh*, const int, const bool)>;
+
+private:
+   int current_sequence = -1;
+
+   Vector error_estimates;
+
+   double total_error = 0.0;
+
+   Array<int> attributes;
+
+   /** @brief A method to compute hₑ on per-element basis.
+
+       This method weights the error approximation on the element level.
+
+       Defaults to hₑ=1.0.
+   */
+   ElementCoefficientFunction compute_element_coefficient;
+
+   /** @brief A method to compute hₖ on per-face basis.
+
+       This method weights the error approximation on the face level. The
+       background here is that classical Kelly error estimator implementations
+       approximate the geometrical characteristic hₖ with the face diameter,
+       which should be also be a possibility in this implementation.
+
+       Defaults to hₖ=diameter/2p.
+   */
+   FaceCoefficientFunction compute_face_coefficient;
+
+   BilinearFormIntegrator* flux_integrator; ///< Not owned.
+   ParGridFunction* solution;               ///< Not owned.
+
+   ParFiniteElementSpace*
+   flux_space; /**< @brief Ownership based on own_flux_fes. */
+   bool own_flux_fespace; ///< Ownership flag for flux_space.
+
+   /// Check if the mesh of the solution was modified.
+   bool MeshIsModified()
+   {
+      long mesh_sequence = solution->FESpace()->GetMesh()->GetSequence();
+      MFEM_ASSERT(mesh_sequence >= current_sequence,
+                  "improper mesh update sequence");
+      return (mesh_sequence > current_sequence);
+   }
+
+   /** @brief Compute the element error estimates.
+
+       Algorithm outline:
+       1. Compute flux field for each element
+       2. Add error contribution from local interior faces
+       3. Add error contribution from shared interior faces
+       4. Finalize by computing hₖ and scale errors.
+   */
+   void ComputeEstimates();
+
+public:
+   /** @brief Construct a new KellyErrorEstimator object for a scalar field.
+       @param di_         The bilinearform to compute the interface flux.
+       @param sol_        The solution field whose error is to be estimated.
+       @param flux_fes_   The finite element space for the interface flux.
+       @param attributes_ The attributes of the subdomain(s) for which the
+                          error should be estimated. An empty array results in
+                          estimating the error over the complete domain.
+   */
+   KellyErrorEstimator(BilinearFormIntegrator& di_, ParGridFunction& sol_,
+                       ParFiniteElementSpace& flux_fes_,
+                       const Array<int> &attributes_ = Array<int>());
+
+   /** @brief Construct a new KellyErrorEstimator object for a scalar field.
+       @param di_         The bilinearform to compute the interface flux.
+       @param sol_        The solution field whose error is to be estimated.
+       @param flux_fes_   The finite element space for the interface flux.
+       @param attributes_ The attributes of the subdomain(s) for which the
+                          error should be estimated. An empty array results in
+                          estimating the error over the complete domain.
+   */
+   KellyErrorEstimator(BilinearFormIntegrator& di_, ParGridFunction& sol_,
+                       ParFiniteElementSpace* flux_fes_,
+                       const Array<int> &attributes_ = Array<int>());
+
+   ~KellyErrorEstimator();
+
+   /// Get a Vector with all element errors.
+   const Vector& GetLocalErrors() override
+   {
+      if (MeshIsModified())
+      {
+         ComputeEstimates();
+      }
+      return error_estimates;
+   }
+
+   /// Reset the error estimator.
+   void Reset() override { current_sequence = -1; };
+
+   virtual double GetTotalError() const override { return total_error; }
+
+   /** @brief Change the method to compute hₑ on a per-element basis.
+       @param compute_element_coefficient_
+                        A function taking a mesh and an element index to
+                        compute the local hₑ for the element.
+   */
+   void SetElementCoefficientFunction(ElementCoefficientFunction
+                                      compute_element_coefficient_)
+   {
+      compute_element_coefficient = compute_element_coefficient_;
+   }
+
+   /** @brief Change the method to compute hₖ on a per-element basis.
+       @param compute_face_coefficient_
+                        A function taking a mesh and a face index to
+                        compute the local hₖ for the face.
+   */
+   void SetFaceCoefficientFunction(
+      FaceCoefficientFunction
+      compute_face_coefficient_)
+   {
+      compute_face_coefficient = compute_face_coefficient_;
+   }
+
+   /// Change the coefficients back to default as described above.
+   void ResetCoefficientFunctions();
+};
+
+#endif // MFEM_USE_MPI
+
 } // namespace mfem

 #endif // MFEM_ERROR_ESTIMATORS
@@ -435,7 +435,7 @@ void ScalarFiniteElement::ScalarLocalInterpolation(
   IntegrationPoint f_ip;

   const int fs = fine_fe.GetDof(), cs = this->GetDof();
-   I.SetSize(fs, cs );
+   I.SetSize(fs, cs);
   Vector fine_shape(fs), coarse_shape(cs);
   DenseMatrix fine_mass(fs), fine_coarse_mass(fs, cs); // initialized with 0
   const int ir_order = GetOrder() + fine_fe.GetOrder();
@@ -464,6 +464,44 @@ void ScalarFiniteElement::ScalarLocalInterpolation(
   }
 }

+void ScalarFiniteElement::ScalarLocalRestriction(
+   ElementTransformation &Trans, DenseMatrix &R,
+   const ScalarFiniteElement &coarse_fe) const
+{
+   // General "restriction", defined by L2 projection
+   double v[Geometry::MaxDim];
+   Vector vv (v, dim);
+   IntegrationPoint f_ip;
+
+   const int cs = coarse_fe.GetDof(), fs = this->GetDof();
+   R.SetSize(cs, fs);
+   Vector fine_shape(fs), coarse_shape(cs);
+   DenseMatrix coarse_mass(cs), coarse_fine_mass(cs, fs); // initialized with 0
+   const int ir_order = GetOrder() + coarse_fe.GetOrder();
+   const IntegrationRule &ir = IntRules.Get(coarse_fe.GetGeomType(), ir_order);
+
+   for (int i = 0; i < ir.GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir.IntPoint(i);
+      this->CalcShape(ip, fine_shape);
+      Trans.Transform(ip, vv);
+      f_ip.Set(v, dim);
+      coarse_fe.CalcShape(f_ip, coarse_shape);
+
+      AddMult_a_VVt(ip.weight, coarse_shape, coarse_mass);
+      AddMult_a_VWt(ip.weight, coarse_shape, fine_shape, coarse_fine_mass);
+   }
+
+   DenseMatrixInverse coarse_mass_inv(coarse_mass);
+   coarse_mass_inv.Mult(coarse_fine_mass, R);
+
+   if (map_type == INTEGRAL)
+   {
+      // assuming Trans is linear; this should be ok for all refinement types
+      Trans.SetIntPoint(&Geometries.GetCenter(geom_type));
+      R *= 1.0 / Trans.Weight();
+   }
+}
 const DofToQuad &ScalarFiniteElement::GetDofToQuad(const IntegrationRule &ir,
                                                   DofToQuad::Mode mode) const
 {
@@ -558,17 +596,22 @@ void NodalFiniteElement::ProjectCurl_2D(
   const FiniteElement &fe, ElementTransformation &Trans,
   DenseMatrix &curl) const
 {
-   MFEM_ASSERT(GetMapType() == FiniteElement::INTEGRAL, "");
-
   DenseMatrix curl_shape(fe.GetDof(), 1);

   curl.SetSize(dof, fe.GetDof());
   for (int i = 0; i < dof; i++)
   {
      fe.CalcCurlShape(Nodes.IntPoint(i), curl_shape);
+
+      double w = 1.0;
+      if (GetMapType() == FiniteElement::VALUE)
+      {
+         Trans.SetIntPoint(&Nodes.IntPoint(i));
+         w /= Trans.Weight();
+      }
      for (int j = 0; j < fe.GetDof(); j++)
      {
-         curl(i,j) = curl_shape(j,0);
+         curl(i,j) = w * curl_shape(j,0);
      }
   }
 }
@@ -684,17 +727,34 @@ void NodalFiniteElement::Project(
 {
   if (fe.GetRangeType() == SCALAR)
   {
-      MFEM_ASSERT(map_type == fe.GetMapType(), "");
-
      Vector shape(fe.GetDof());

      I.SetSize(dof, fe.GetDof());
-      for (int k = 0; k < dof; k++)
+      if (map_type == fe.GetMapType())
      {
-         fe.CalcShape(Nodes.IntPoint(k), shape);
-         for (int j = 0; j < shape.Size(); j++)
+         for (int k = 0; k < dof; k++)
         {
-            I(k,j) = (fabs(shape(j)) < 1e-12) ? 0.0 : shape(j);
+            fe.CalcShape(Nodes.IntPoint(k), shape);
+            for (int j = 0; j < shape.Size(); j++)
+            {
+               I(k,j) = (fabs(shape(j)) < 1e-12) ? 0.0 : shape(j);
+            }
+         }
+      }
+      else
+      {
+         for (int k = 0; k < dof; k++)
+         {
+            Trans.SetIntPoint(&Nodes.IntPoint(k));
+            fe.CalcPhysShape(Trans, shape);
+            if (map_type == INTEGRAL)
+            {
+               shape *= Trans.Weight();
+            }
+            for (int j = 0; j < shape.Size(); j++)
+            {
+               I(k,j) = (fabs(shape(j)) < 1e-12) ? 0.0 : shape(j);
+            }
         }
      }
   }
@@ -993,6 +1053,8 @@ void VectorFiniteElement::Project_RT(

         fe.CalcShape(ip, shape);
         Trans.SetIntPoint(&ip);
+         // Transform RT face normals from reference to physical space
+         // vk = adj(J)^T nk
         Trans.AdjugateJacobian().MultTranspose(nk + d2n[k]*dim, vk);
         if (fe.GetMapType() == INTEGRAL)
         {
@@ -1010,6 +1072,8 @@ void VectorFiniteElement::Project_RT(
            {
               s = 0.0;
            }
+            // Project scalar basis function multiplied by each coordinate
+            // direction onto the transformed face normals
            for (int d = 0; d < sdim; d++)
            {
               I(k,j+d*shape.Size()) = s*vk[d];
@@ -1019,7 +1083,31 @@ void VectorFiniteElement::Project_RT(
   }
   else
   {
-      mfem_error("VectorFiniteElement::Project_RT (fe version)");
+      int sdim = Trans.GetSpaceDim();
+      double vk[Geometry::MaxDim];
+      DenseMatrix vshape(fe.GetDof(), sdim);
+      Vector vshapenk(fe.GetDof());
+      const bool square_J = (dim == sdim);
+
+      I.SetSize(dof, fe.GetDof());
+      for (int k = 0; k < dof; k++)
+      {
+         const IntegrationPoint &ip = Nodes.IntPoint(k);
+
+         Trans.SetIntPoint(&ip);
+         // Transform RT face normals from reference to physical space
+         // vk = adj(J)^T nk
+         Trans.AdjugateJacobian().MultTranspose(nk + d2n[k]*dim, vk);
+         // Compute fe basis functions in physical space
+         fe.CalcVShape(Trans, vshape);
+         // Project fe basis functions onto transformed face normals
+         vshape.Mult(vk, vshapenk);
+         if (!square_J) { vshapenk /= Trans.Weight(); }
+         for (int j=0; j<vshapenk.Size(); j++)
+         {
+            I(k,j) = vshapenk(j);
+         }
+      }
   }
 }

@@ -1180,6 +1268,8 @@ void VectorFiniteElement::Project_ND(

         fe.CalcShape(ip, shape);
         Trans.SetIntPoint(&ip);
+         // Transform ND edge tengents from reference to physical space
+         // vk = J tk
         Trans.Jacobian().Mult(tk + d2t[k]*dim, vk);
         if (fe.GetMapType() == INTEGRAL)
         {
@@ -1197,6 +1287,8 @@ void VectorFiniteElement::Project_ND(
            {
               s = 0.0;
            }
+            // Project scalar basis function multiplied by each coordinate
+            // direction onto the transformed edge tangents
            for (int d = 0; d < sdim; d++)
            {
               I(k, j + d*shape.Size()) = s*vk[d];
@@ -1206,7 +1298,29 @@ void VectorFiniteElement::Project_ND(
   }
   else
   {
-      mfem_error("VectorFiniteElement::Project_ND (fe version)");
+      int sdim = Trans.GetSpaceDim();
+      double vk[Geometry::MaxDim];
+      DenseMatrix vshape(fe.GetDof(), sdim);
+      Vector vshapetk(fe.GetDof());
+
+      I.SetSize(dof, fe.GetDof());
+      for (int k = 0; k < dof; k++)
+      {
+         const IntegrationPoint &ip = Nodes.IntPoint(k);
+
+         Trans.SetIntPoint(&ip);
+         // Transform ND edge tangents from reference to physical space
+         // vk = J tk
+         Trans.Jacobian().Mult(tk + d2t[k]*dim, vk);
+         // Compute fe basis functions in physical space
+         fe.CalcVShape(Trans, vshape);
+         // Project fe basis functions onto transformed edge tangents
+         vshape.Mult(vk, vshapetk);
+         for (int j=0; j<vshapetk.Size(); j++)
+         {
+            I(k, j) = vshapetk(j);
+         }
+      }
   }
 }

@@ -504,12 +504,18 @@ public:
   /** @brief Given a coefficient and a transformation, compute its projection
       (approximation) in the local finite dimensional space in terms
       of the degrees of freedom. */
+   /** The approximation used to project is usually local interpolation of
+       degrees of freedom. The derived class could use other methods not
+       implemented yet, e.g. local L2 projection. */
   virtual void Project(Coefficient &coeff,
                        ElementTransformation &Trans, Vector &dofs) const;

   /** @brief Given a vector coefficient and a transformation, compute its
       projection (approximation) in the local finite dimensional space
       in terms of the degrees of freedom. (VectorFiniteElements) */
+   /** The approximation used to project is usually local interpolation of
+       degrees of freedom. The derived class could use other methods not
+       implemented yet, e.g. local L2 projection. */
   virtual void Project(VectorCoefficient &vc,
                        ElementTransformation &Trans, Vector &dofs) const;

@@ -666,7 +672,7 @@ public:
                                const ScalarFiniteElement &fine_fe) const;

   /** @brief Get matrix @a I "Interpolation" defined through local
-       L2-projection in the space defined by the @a fine_fe.  */
+       L2-projection in the space defined by the @a fine_fe. */
   /** If the "fine" elements cannot represent all basis functions of the
       "coarse" element, then boundary values from different sub-elements are
       generally different. */
@@ -674,6 +680,15 @@ public:
                                 DenseMatrix &I,
                                 const ScalarFiniteElement &fine_fe) const;

+   /** @brief Get restriction matrix @a R defined through local L2-projection
+        in the space defined by the @a coarse_fe. */
+   /** If the "fine" elements cannot represent all basis functions of the
+       "coarse" element, then boundary values from different sub-elements are
+       generally different. */
+   void ScalarLocalRestriction(ElementTransformation &Trans,
+                               DenseMatrix &R,
+                               const ScalarFiniteElement &coarse_fe) const;
+
   virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
                                         DofToQuad::Mode mode) const;
 };
@@ -754,6 +769,10 @@ public:
                                      DenseMatrix &I) const
   { ScalarLocalInterpolation(Trans, I, *this); }

+   virtual void GetLocalRestriction(ElementTransformation &Trans,
+                                    DenseMatrix &R) const
+   { ScalarLocalRestriction(Trans, R, *this); }
+
   virtual void GetTransferMatrix(const FiniteElement &fe,
                                  ElementTransformation &Trans,
                                  DenseMatrix &I) const
@@ -800,11 +819,25 @@ protected:
   void CalcVShape_ND(ElementTransformation &Trans,
                      DenseMatrix &shape) const;

+   /** @brief Project a vector coefficient onto the RT basis functions
+       @param nk    Face normal vectors for this element type
+       @param d2n   Offset into nk for each degree of freedom
+       @param vc    Vector coefficient to be projected
+       @param Trans Transformation from reference to physical coordinates
+       @param dofs  Expansion coefficients for the approximation of vc
+   */
   void Project_RT(const double *nk, const Array<int> &d2n,
                   VectorCoefficient &vc, ElementTransformation &Trans,
                   Vector &dofs) const;

   /// Projects the vector of values given at FE nodes to RT space
+   /** Project vector values onto the RT basis functions
+       @param nk    Face normal vectors for this element type
+       @param d2n   Offset into nk for each degree of freedom
+       @param vc    Vector values at each interpolation point
+       @param Trans Transformation from reference to physical coordinates
+       @param dofs  Expansion coefficients for the approximation of vc
+   */
   void Project_RT(const double *nk, const Array<int> &d2n,
                   Vector &vc, ElementTransformation &Trans,
                   Vector &dofs) const;
@@ -814,6 +847,19 @@ protected:
      const double *nk, const Array<int> &d2n,
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const;

+   /** @brief Project vector-valued basis functions onto the RT basis functions
+       @param nk    Face normal vectors for this element type
+       @param d2n   Offset into nk for each degree of freedom
+       @param fe    Vector-valued finite element basis
+       @param Trans Transformation from reference to physical coordinates
+       @param I     Expansion coefficients for the approximation of each basis
+                    function
+
+       Note: If the FiniteElement, fe, is scalar-valued the projection will
+             assume that a FiniteElementSpace is being used to define a vector
+             field using the scalar basis functions for each component of the
+             vector field.
+   */
   void Project_RT(const double *nk, const Array<int> &d2n,
                   const FiniteElement &fe, ElementTransformation &Trans,
                   DenseMatrix &I) const;
@@ -833,11 +879,25 @@ protected:
                       const FiniteElement &fe, ElementTransformation &Trans,
                       DenseMatrix &curl) const;

+   /** @brief Project a vector coefficient onto the ND basis functions
+       @param tk    Edge tangent vectors for this element type
+       @param d2t   Offset into tk for each degree of freedom
+       @param vc    Vector coefficient to be projected
+       @param Trans Transformation from reference to physical coordinates
+       @param dofs  Expansion coefficients for the approximation of vc
+   */
   void Project_ND(const double *tk, const Array<int> &d2t,
                   VectorCoefficient &vc, ElementTransformation &Trans,
                   Vector &dofs) const;

   /// Projects the vector of values given at FE nodes to ND space
+   /** Project vector values onto the ND basis functions
+       @param tk    Edge tangent vectors for this element type
+       @param d2t   Offset into tk for each degree of freedom
+       @param vc    Vector values at each interpolation point
+       @param Trans Transformation from reference to physical coordinates
+       @param dofs  Expansion coefficients for the approximation of vc
+   */
   void Project_ND(const double *tk, const Array<int> &d2t,
                   Vector &vc, ElementTransformation &Trans,
                   Vector &dofs) const;
@@ -847,6 +907,19 @@ protected:
      const double *tk, const Array<int> &d2t,
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const;

+   /** @brief Project vector-valued basis functions onto the ND basis functions
+       @param tk    Edge tangent vectors for this element type
+       @param d2t   Offset into tk for each degree of freedom
+       @param fe    Vector-valued finite element basis
+       @param Trans Transformation from reference to physical coordinates
+       @param I     Expansion coefficients for the approximation of each basis
+                    function
+
+       Note: If the FiniteElement, fe, is scalar-valued the projection will
+             assume that a FiniteElementSpace is being used to define a vector
+             field using the scalar basis functions for each component of the
+             vector field.
+   */
   void Project_ND(const double *tk, const Array<int> &d2t,
                   const FiniteElement &fe, ElementTransformation &Trans,
                   DenseMatrix &I) const;
@@ -2608,9 +2608,9 @@ const Operator &GridTransfer::MakeTrueOperator(
   else // Parallel() == true
   {
 #ifdef MFEM_USE_MPI
-      const SparseMatrix *out_R = fes_out.GetRestrictionMatrix();
      if (oper_type == Operator::Hypre_ParCSR)
      {
+         const SparseMatrix *out_R = fes_out.GetRestrictionMatrix();
         const ParFiniteElementSpace *pfes_in =
            dynamic_cast<const ParFiniteElementSpace *>(&fes_in);
         const ParFiniteElementSpace *pfes_out =
@@ -2638,6 +2638,7 @@ const Operator &GridTransfer::MakeTrueOperator(
      }
      else if (oper_type == Operator::ANY_TYPE)
      {
+         const Operator *out_R = fes_out.GetRestrictionOperator();
         t_oper.Reset(new TripleProductOperator(
                         out_R, &oper, fes_in.GetProlongationMatrix(),
                         false, false, false));
@@ -2805,9 +2806,9 @@ L2ProjectionGridTransfer::L2Projection::L2Projection(
   Vector shape_ho(ndof_ho);
   Vector shape_lor(ndof_lor);

-   const Geometry::Type geom = fe_ho->GetGeomType();
-   const DenseTensor &pmats = cf_tr.point_matrices[geom];
-   emb_tr.SetIdentityTransformation(geom);
+   const Geometry::Type lor_geom = fe_lor->GetGeomType();
+   const DenseTensor &pmats = cf_tr.point_matrices[lor_geom];
+   emb_tr.SetIdentityTransformation(lor_geom);

   for (int iho=0; iho<nel_ho; ++iho)
   {
@@ -2832,7 +2833,7 @@ L2ProjectionGridTransfer::L2Projection::L2Projection(
         emb_tr.SetPointMat(pmats(cf_tr.embeddings[ilor].matrix));

         int order = fe_lor->GetOrder() + fe_ho->GetOrder() + el_tr->OrderW();
-         const IntegrationRule *ir = &IntRules.Get(geom, order);
+         const IntegrationRule *ir = &IntRules.Get(lor_geom, order);
         M_mixed_el = 0.0;
         for (int i = 0; i < ir->GetNPoints(); i++)
         {
@@ -330,6 +330,18 @@ public:
   virtual const Operator *GetProlongationMatrix() const
   { return GetConformingProlongation(); }

+   /// Return an operator that performs the transpose of GetRestrictionOperator
+   /** The returned operator is owned by the FiniteElementSpace. In serial this
+       is the same as GetProlongationMatrix() */
+   virtual const Operator *GetRestrictionTransposeOperator() const
+   { return GetConformingProlongation(); }
+
+   /// An abstract operator that performs the same action as GetRestrictionMatrix
+   /** In some cases this is an optimized matrix-free implementation. The
+       returned operator is owned by the FiniteElementSpace. */
+   virtual const Operator *GetRestrictionOperator() const
+   { return GetConformingRestriction(); }
+
   /// The returned SparseMatrix is owned by the FiniteElementSpace.
   virtual const SparseMatrix *GetRestrictionMatrix() const
   { return GetConformingRestriction(); }
@@ -571,7 +583,7 @@ public:

   /** @brief Returns pointer to the FiniteElement in the FiniteElementCollection
        associated with i'th element in the mesh object. */
-   const FiniteElement *GetFE(int i) const;
+   virtual const FiniteElement *GetFE(int i) const;

   /** @brief Returns pointer to the FiniteElement in the FiniteElementCollection
        associated with i'th boundary face in the mesh object. */
@@ -756,6 +768,9 @@ public:
   /// Return the total number of quadrature points.
   int GetSize() const { return size; }

+   /// Return the order of the quadrature rule(s) used by all elements.
+   int GetOrder() const { return order; }
+
   /// Returns the mesh
   inline Mesh *GetMesh() const { return mesh; }

@@ -15,6 +15,10 @@
 #include "../mesh/nurbs.hpp"
 #include "../general/text.hpp"

+#ifdef MFEM_USE_MPI
+#include "pfespace.hpp"
+#endif
+
 #include <limits>
 #include <cstring>
 #include <string>
@@ -22,6 +26,7 @@
 #include <iostream>
 #include <algorithm>

+
 namespace mfem
 {

@@ -462,15 +467,26 @@ const
   fes->GetElementDofs(i, dofs);
   fes->DofsToVDofs(vdim-1, dofs);
   const FiniteElement *FElem = fes->GetFE(i);
-   MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-               "invalid FE map type");
   int dof = FElem->GetDof();
   Vector DofVal(dof), loc_data(dof);
   GetSubVector(dofs, loc_data);
-   for (int k = 0; k < n; k++)
+   if (FElem->GetMapType() == FiniteElement::VALUE)
   {
-      FElem->CalcShape(ir.IntPoint(k), DofVal);
-      vals(k) = DofVal * loc_data;
+      for (int k = 0; k < n; k++)
+      {
+         FElem->CalcShape(ir.IntPoint(k), DofVal);
+         vals(k) = DofVal * loc_data;
+      }
+   }
+   else
+   {
+      ElementTransformation *Tr = fes->GetElementTransformation(i);
+      for (int k = 0; k < n; k++)
+      {
+         Tr->SetIntPoint(&ir.IntPoint(k));
+         FElem->CalcPhysShape(*Tr, DofVal);
+         vals(k) = DofVal * loc_data;
+      }
   }
 }

@@ -984,15 +1000,14 @@ void GridFunction::GetVectorValues(ElementTransformation &T,

   if (FElem->GetRangeType() == FiniteElement::SCALAR)
   {
-      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-                  "invalid FE map type");
      Vector shape(dof);
      int vdim = fes->GetVDim();
      vals.SetSize(vdim, nip);
      for (int j = 0; j < nip; j++)
      {
         const IntegrationPoint &ip = ir.IntPoint(j);
-         FElem->CalcShape(ip, shape);
+         T.SetIntPoint(&ip);
+         FElem->CalcPhysShape(T, shape);

         for (int k = 0; k < vdim; k++)
         {
@@ -1550,18 +1565,16 @@ void GridFunction::GetGradient(ElementTransformation &T, Vector &grad) const
   {
      case ElementTransformation::ELEMENT:
      {
-         const FiniteElement * fe = fes->GetFE(T.ElementNo);
+         const FiniteElement *fe = fes->GetFE(T.ElementNo);
         MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
                     "invalid FE map type");
         int spaceDim = fes->GetMesh()->SpaceDimension();
         int dim = fe->GetDim(), dof = fe->GetDof();
         DenseMatrix dshape(dof, dim);
         Vector lval, gh(dim);
-         Array<int> dofs;

         grad.SetSize(spaceDim);
-         fes->GetElementDofs(T.ElementNo, dofs);
-         GetSubVector(dofs, lval);
+         GetElementDofValues(T.ElementNo, lval);
         fe->CalcDShape(T.GetIntPoint(), dshape);
         dshape.MultTranspose(lval, gh);
         T.InverseJacobian().MultTranspose(gh, grad);
@@ -1731,6 +1744,13 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
   }
 }

+void GridFunction::GetElementDofValues(int el, Vector &dof_vals) const
+{
+   Array<int> dof_idx;
+   fes->GetElementVDofs(el, dof_idx);
+   GetSubVector(dof_idx, dof_vals);
+}
+
 void GridFunction::ProjectGridFunction(const GridFunction &src)
 {
   Mesh *mesh = fes->GetMesh();
@@ -3793,7 +3813,15 @@ double ZZErrorEstimator(BilinearFormIntegrator &blfi,
         }
      }
   }
-
+#ifdef MFEM_USE_MPI
+   auto pfes = dynamic_cast<ParFiniteElementSpace*>(ufes);
+   if (pfes)
+   {
+      auto process_local_error = total_error;
+      MPI_Allreduce(&process_local_error, &total_error, 1, MPI_DOUBLE,
+                    MPI_SUM, pfes->GetComm());
+   }
+#endif // MFEM_USE_MPI
   return std::sqrt(total_error);
 }

@@ -325,6 +325,10 @@ public:
       Both FE spaces should be scalar and on the same mesh. */
   void GetElementAverages(GridFunction &avgs) const;

+   /** Sets the output vector @a dof_vals to the values of the degrees of
+       freedom of element @a el. */
+   virtual void GetElementDofValues(int el, Vector &dof_vals) const;
+
   /** Impose the given bounds on the function's DOFs while preserving its local
    *  integral (described in terms of the given weights) on the i'th element
    *  through SLBPQ optimization.
@@ -345,14 +349,30 @@ public:
       projection matrix. */
   void ProjectGridFunction(const GridFunction &src);

+   /** @brief Project @a coeff Coefficient to @a this GridFunction. The
+       projection computation depends on the choice of the FiniteElementSpace
+       #fes. Note that this is usually interpolation at the degrees of freedom
+       in each element (not L2 projection). */
   virtual void ProjectCoefficient(Coefficient &coeff);

+   /** @brief Project @a coeff Coefficient to @a this GridFunction, using one
+       element for each degree of freedom in @a dofs and nodal interpolation on
+       that element. */
   void ProjectCoefficient(Coefficient &coeff, Array<int> &dofs, int vd = 0);

+   /** @brief Project @a vcoeff VectorCoefficient to @a this GridFunction. The
+       projection computation depends on the choice of the FiniteElementSpace
+       #fes. Note that this is usually interpolation at the degrees of freedom
+       in each element (not L2 projection).*/
   void ProjectCoefficient(VectorCoefficient &vcoeff);

+   /** @brief Project @a vcoeff VectorCoefficient to @a this GridFunction, using
+       one element for each degree of freedom in @a dofs and nodal interpolation
+       on that element. */
   void ProjectCoefficient(VectorCoefficient &vcoeff, Array<int> &dofs);

+   /** @brief Analogous to the version with argument @a vcoeff VectorCoefficient
+       but using an array of scalar coefficients for each component. */
   void ProjectCoefficient(Coefficient *coeff[]);

   /** @brief Project a discontinuous vector coefficient as a grid function on
@@ -462,11 +482,10 @@ public:

   /// Returns the Face Jumps error for L2 elements, with 1/h scaling.
   MFEM_DEPRECATED
-   virtual double ComputeDGFaceJumpError(Coefficient *exsol,
-                                         Coefficient *ell_coeff,
-                                         double Nu,
-                                         const IntegrationRule *irs[] = NULL)
-   const;
+   double ComputeDGFaceJumpError(Coefficient *exsol,
+                                 Coefficient *ell_coeff,
+                                 double Nu,
+                                 const IntegrationRule *irs[] = NULL) const;

   /** This method is kept for backward compatibility.

@@ -406,6 +406,10 @@ private:
   Vector shape;

 public:
+   BoundaryFlowIntegrator(Coefficient &_f, VectorCoefficient &_u,
+                          double a)
+   { f = &_f; u = &_u; alpha = a; beta = 0.5*a; }
+
   BoundaryFlowIntegrator(Coefficient &_f, VectorCoefficient &_u,
                          double a, double b)
   { f = &_f; u = &_u; alpha = a; beta = b; }
@@ -14,11 +14,40 @@
 namespace mfem
 {

-Multigrid::Multigrid(const FiniteElementSpaceHierarchy& fespaces_)
-   : fespaces(fespaces_), cycleType(CycleType::VCYCLE), preSmoothingSteps(1),
-     postSmoothingSteps(1)
+Multigrid::Multigrid()
+   : cycleType(CycleType::VCYCLE), preSmoothingSteps(1), postSmoothingSteps(1)
 {}

+Multigrid::Multigrid(const Array<Operator*>& operators_,
+                     const Array<Solver*>& smoothers_,
+                     const Array<Operator*>& prolongations_,
+                     const Array<bool>& ownedOperators_,
+                     const Array<bool>& ownedSmoothers_,
+                     const Array<bool>& ownedProlongations_)
+   : Solver(operators_.Last()->NumRows()), cycleType(CycleType::VCYCLE),
+     preSmoothingSteps(1), postSmoothingSteps(1),
+     X(operators_.Size()), Y(X.Size()), R(X.Size()), Z(X.Size())
+{
+   operators_.Copy(operators);
+   smoothers_.Copy(smoothers);
+   prolongations_.Copy(prolongations);
+   ownedOperators_.Copy(ownedOperators);
+   ownedSmoothers_.Copy(ownedSmoothers);
+   ownedProlongations_.Copy(ownedProlongations);
+
+   for (int level = 0; level < operators.Size(); ++level)
+   {
+      X[level] = new Vector(operators[level]->NumRows());
+      *X[level] = 0.0;
+      Y[level] = new Vector(operators[level]->NumRows());
+      *Y[level] = 0.0;
+      R[level] = new Vector(operators[level]->NumRows());
+      *R[level] = 0.0;
+      Z[level] = new Vector(operators[level]->NumRows());
+      *Z[level] = 0.0;
+   }
+}
+
 Multigrid::~Multigrid()
 {
   for (int i = 0; i < operators.Size(); ++i)
@@ -37,26 +66,21 @@ Multigrid::~Multigrid()
      delete Z[i];
   }

+   for (int i = 0; i < prolongations.Size(); ++i)
+   {
+      if (ownedProlongations[i])
+      {
+         delete prolongations[i];
+      }
+   }
+
   operators.DeleteAll();
   smoothers.DeleteAll();
+   prolongations.DeleteAll();
   X.DeleteAll();
   Y.DeleteAll();
   R.DeleteAll();
   Z.DeleteAll();
-
-   for (int i = 0; i < bfs.Size(); ++i)
-   {
-      delete bfs[i];
-   }
-
-   bfs.DeleteAll();
-
-   for (int i = 0; i < essentialTrueDofs.Size(); ++i)
-   {
-      delete essentialTrueDofs[i];
-   }
-
-   essentialTrueDofs.DeleteAll();
 }

 void Multigrid::AddLevel(Operator* opr, Solver* smoother, bool ownOperator,
@@ -168,8 +192,7 @@ void Multigrid::Cycle(int level) const
   subtract(*X[level], *R[level], *R[level]);

   // Restrict residual
-   fespaces.GetProlongationAtLevel(level - 1)->MultTranspose(*R[level],
-                                                             *X[level - 1]);
+   GetProlongationAtLevel(level - 1)->MultTranspose(*R[level], *X[level - 1]);

   // Init zeros
   *Y[level - 1] = 0.0;
@@ -186,7 +209,7 @@ void Multigrid::Cycle(int level) const
   }

   // Prolongate
-   fespaces.GetProlongationAtLevel(level - 1)->Mult(*Y[level - 1], *R[level]);
+   GetProlongationAtLevel(level - 1)->Mult(*Y[level - 1], *R[level]);

   // Add update
   *Y[level] += *R[level];
@@ -198,16 +221,44 @@ void Multigrid::Cycle(int level) const
   }
 }

-void Multigrid::FormFineLinearSystem(Vector& x, Vector& b, OperatorHandle& A,
-                                     Vector& X, Vector& B)
+const Operator* Multigrid::GetProlongationAtLevel(int level) const
+{
+   return prolongations[level];
+}
+
+GeometricMultigrid::~GeometricMultigrid()
+{
+   for (int i = 0; i < bfs.Size(); ++i)
+   {
+      delete bfs[i];
+   }
+
+   bfs.DeleteAll();
+
+   for (int i = 0; i < essentialTrueDofs.Size(); ++i)
+   {
+      delete essentialTrueDofs[i];
+   }
+
+   essentialTrueDofs.DeleteAll();
+}
+
+void GeometricMultigrid::FormFineLinearSystem(Vector& x, Vector& b,
+                                              OperatorHandle& A,
+                                              Vector& X, Vector& B)
 {
   bfs.Last()->FormLinearSystem(*essentialTrueDofs.Last(), x, b, A, X, B);
 }

-void Multigrid::RecoverFineFEMSolution(const Vector& X, const Vector& b,
-                                       Vector& x)
+void GeometricMultigrid::RecoverFineFEMSolution(const Vector& X,
+                                                const Vector& b, Vector& x)
 {
   bfs.Last()->RecoverFEMSolution(X, b, x);
 }

+const Operator* GeometricMultigrid::GetProlongationAtLevel(int level) const
+{
+   return fespaces.GetProlongationAtLevel(level);
+}
+
 } // namespace mfem
@@ -32,16 +32,13 @@ public:
   };

 protected:
-   const FiniteElementSpaceHierarchy& fespaces;
-   Array<Array<int>*> essentialTrueDofs;
-   Array<BilinearForm*> bfs;
-
-private:
   Array<Operator*> operators;
   Array<Solver*> smoothers;
+   Array<Operator*> prolongations;

   Array<bool> ownedOperators;
   Array<bool> ownedSmoothers;
+   Array<bool> ownedProlongations;

   CycleType cycleType;
   int preSmoothingSteps;
@@ -53,8 +50,16 @@ private:
   mutable Array<Vector*> Z;

 public:
-   /// Constructs an empty multigrid for the given FiniteElementSpaceHierarchy
-   Multigrid(const FiniteElementSpaceHierarchy& fespaces_);
+   /// Constructs an empty multigrid hierarchy.
+   Multigrid();
+
+   /// Constructs a multigrid hierarchy from the given inputs.
+   /** Inputs include operators and smoothers on all levels, prolongation
+       operators that go from coarser to finer levels, and ownership of the
+       given operators, smoothers, and prolongations. */
+   Multigrid(const Array<Operator*>& operators_, const Array<Solver*>& smoothers_,
+             const Array<Operator*>& prolongations_, const Array<bool>& ownedOperators_,
+             const Array<bool>& ownedSmoothers_, const Array<bool>& ownedProlongations_);

   /// Destructor
   virtual ~Multigrid();
@@ -89,7 +94,7 @@ public:
   /// Returns smoother at given level
   Solver* GetSmootherAtLevel(int level);

-   /// Set the cycle type and number of pre- and post-smoothing steps used by Mult
+   /// Set cycle type and number of pre- and post-smoothing steps used by Mult
   void SetCycleType(CycleType cycleType_, int preSmoothingSteps_,
                     int postSmoothingSteps_);

@@ -99,7 +104,36 @@ public:
   /// Not supported for multigrid
   virtual void SetOperator(const Operator& op) override;

-   /// Form the linear system A X = B, corresponding to the operator on the finest level
+private:
+   /// Application of a smoothing step at particular level
+   void SmoothingStep(int level, bool transpose) const;
+
+   /// Application of a multigrid cycle at particular level
+   void Cycle(int level) const;
+
+   /// Returns prolongation operator at given level
+   virtual const Operator* GetProlongationAtLevel(int level) const;
+};
+
+/// Geometric multigrid associated with a hierarchy of finite element spaces
+class GeometricMultigrid : public Multigrid
+{
+protected:
+   const FiniteElementSpaceHierarchy& fespaces;
+   Array<Array<int>*> essentialTrueDofs;
+   Array<BilinearForm*> bfs;
+
+public:
+   /** Construct an empty multigrid object for the given finite element space
+       hierarchy @a fespaces_ */
+   GeometricMultigrid(const FiniteElementSpaceHierarchy& fespaces_)
+      : Multigrid(), fespaces(fespaces_) { }
+
+   /// Destructor
+   virtual ~GeometricMultigrid();
+
+   /** Form the linear system A X = B, corresponding to the operator on the
+       finest level of the geometric multigrid hierarchy */
   void FormFineLinearSystem(Vector& x, Vector& b, OperatorHandle& A, Vector& X,
                             Vector& B);

@@ -107,11 +141,8 @@ public:
   void RecoverFineFEMSolution(const Vector& X, const Vector& b, Vector& x);

 private:
-   /// Application of a smoothing step at particular level
-   void SmoothingStep(int level, bool transpose) const;
-
-   /// Application of a cycle at particular level
-   void Cycle(int level) const;
+   /// Returns prolongation operator at given level
+   virtual const Operator* GetProlongationAtLevel(int level) const override;
 };

 } // namespace mfem
@@ -565,6 +565,38 @@ HypreParMatrix* ParDiscreteLinearOperator::ParallelAssemble() const
   return RAP;
 }

+void ParDiscreteLinearOperator::ParallelAssemble(OperatorHandle &A)
+{
+   // construct the rectangular block-diagonal matrix dA
+   OperatorHandle dA(A.Type());
+   dA.MakeRectangularBlockDiag(domain_fes->GetComm(),
+                               range_fes->GlobalVSize(),
+                               domain_fes->GlobalVSize(),
+                               range_fes->GetDofOffsets(),
+                               domain_fes->GetDofOffsets(),
+                               mat);
+
+   OperatorHandle R_test_transpose(A.Type()), P_trial(A.Type());
+
+   // TODO - construct the Dof_TrueDof_Matrix directly in the required format.
+   R_test_transpose.ConvertFrom(range_fes->Dof_TrueDof_Matrix());
+   P_trial.ConvertFrom(domain_fes->Dof_TrueDof_Matrix());
+
+   A.MakeRAP(R_test_transpose, dA, P_trial);
+}
+
+void ParDiscreteLinearOperator::FormRectangularSystemMatrix(OperatorHandle &A)
+{
+   if (ext)
+   {
+      Array<int> empty;
+      ext->FormRectangularSystemOperator(empty, empty, A);
+      return;
+   }
+
+   mfem_error("not implemented!");
+}
+
 void ParDiscreteLinearOperator::GetParBlocks(Array2D<HypreParMatrix *> &blocks)
 const
 {
@@ -160,6 +160,9 @@ public:
   /// Get the parallel finite element space prolongation matrix
   virtual const Operator *GetProlongation() const
   { return pfes->GetProlongationMatrix(); }
+   /// Get the transpose of GetRestriction, useful for matrix-free RAP
+   virtual const Operator *GetRestrictionTranspose() const
+   { return pfes->GetRestrictionTransposeOperator(); }
   /// Get the parallel finite element space restriction matrix
   virtual const Operator *GetRestriction() const
   { return pfes->GetRestrictionMatrix(); }
@@ -301,10 +304,18 @@ public:
   /// Returns the matrix "assembled" on the true dofs
   HypreParMatrix *ParallelAssemble() const;

+   /** @brief Returns the matrix assembled on the true dofs, i.e.
+       @a A = R_test A_local P_trial, in the format (type id) specified by
+       @a A. */
+   void ParallelAssemble(OperatorHandle &A);
+
   /** Extract the parallel blocks corresponding to the vector dimensions of the
       domain and range parallel finite element spaces */
   void GetParBlocks(Array2D<HypreParMatrix *> &blocks) const;

+   /** @brief Return in @a A a parallel (on truedofs) version of this operator. */
+   virtual void FormRectangularSystemMatrix(OperatorHandle &A);
+
   virtual ~ParDiscreteLinearOperator() { }
 };

@@ -101,6 +101,8 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)

   P = NULL;
   Pconf = NULL;
+   Rconf = NULL;
+   R_transpose = NULL;
   R = NULL;

   num_face_nbr_dofs = -1;
@@ -499,6 +501,12 @@ void ParFiniteElementSpace::GetFaceDofs(int i, Array<int> &dofs) const
   }
 }

+const FiniteElement *ParFiniteElementSpace::GetFE(int i) const
+{
+   int ne = mesh->GetNE();
+   if (i >= ne) { return GetFaceNbrFE(i - ne); }
+   else { return FiniteElementSpace::GetFE(i); }
+}

 const Operator *ParFiniteElementSpace::GetFaceRestriction(
   ElementDofOrdering e_ordering, FaceType type, L2FaceValues mul) const
@@ -921,6 +929,45 @@ const Operator *ParFiniteElementSpace::GetProlongationMatrix() const
   }
 }

+const Operator *ParFiniteElementSpace::GetRestrictionOperator() const
+{
+   if (Conforming())
+   {
+      if (Rconf) { return Rconf; }
+
+      if (NRanks == 1)
+      {
+         R_transpose = new IdentityOperator(GetTrueVSize());
+      }
+      else
+      {
+         if (!Device::Allows(Backend::DEVICE_MASK))
+         {
+            R_transpose = new ConformingProlongationOperator(*this, true);
+         }
+         else
+         {
+            R_transpose =
+               new DeviceConformingProlongationOperator(*this, true);
+         }
+      }
+      Rconf = new TransposeOperator(R_transpose);
+      return Rconf;
+   }
+   else
+   {
+      Dof_TrueDof_Matrix();
+      R_transpose = new TransposeOperator(R);
+      return R;
+   }
+}
+
+const Operator *ParFiniteElementSpace::GetRestrictionTransposeOperator() const
+{
+   GetRestrictionOperator();
+   return R_transpose;
+}
+
 void ParFiniteElementSpace::ExchangeFaceNbrData()
 {
   if (num_face_nbr_dofs >= 0) { return; }
@@ -2834,6 +2881,8 @@ void ParFiniteElementSpace::Destroy()

   delete P; P = NULL;
   delete Pconf; Pconf = NULL;
+   delete Rconf; Rconf = NULL;
+   delete R_transpose; R_transpose = NULL;
   delete R; R = NULL;

   delete gcomm; gcomm = NULL;
@@ -2959,12 +3008,12 @@ void ParFiniteElementSpace::Update(bool want_transform)
   }
 }

-
 ConformingProlongationOperator::ConformingProlongationOperator(
-   const ParFiniteElementSpace &pfes)
+   const ParFiniteElementSpace &pfes, bool local_)
   : Operator(pfes.GetVSize(), pfes.GetTrueVSize()),
     external_ldofs(),
-     gc(pfes.GroupComm())
+     gc(pfes.GroupComm()),
+     local(local_)
 {
   MFEM_VERIFY(pfes.Conforming(), "");
   const Table &group_ldof = gc.GroupLDofTable();
@@ -3013,7 +3062,14 @@ void ConformingProlongationOperator::Mult(const Vector &x, Vector &y) const
   const int m = external_ldofs.Size();

   const int in_layout = 2; // 2 - input is ltdofs array
-   gc.BcastBegin(const_cast<double*>(xdata), in_layout);
+   if (local)
+   {
+      y = 0.0;
+   }
+   else
+   {
+      gc.BcastBegin(const_cast<double*>(xdata), in_layout);
+   }

   int j = 0;
   for (int i = 0; i < m; i++)
@@ -3025,7 +3081,10 @@ void ConformingProlongationOperator::Mult(const Vector &x, Vector &y) const
   std::copy(xdata+j-m, xdata+Width(), ydata+j);

   const int out_layout = 0; // 0 - output is ldofs array
-   gc.BcastEnd(ydata, out_layout);
+   if (!local)
+   {
+      gc.BcastEnd(ydata, out_layout);
+   }
 }

 void ConformingProlongationOperator::MultTranspose(
@@ -3038,7 +3097,10 @@ void ConformingProlongationOperator::MultTranspose(
   double *ydata = y.HostWrite();
   const int m = external_ldofs.Size();

-   gc.ReduceBegin(xdata);
+   if (!local)
+   {
+      gc.ReduceBegin(xdata);
+   }

   int j = 0;
   for (int i = 0; i < m; i++)
@@ -3050,13 +3112,18 @@ void ConformingProlongationOperator::MultTranspose(
   std::copy(xdata+j, xdata+Height(), ydata+j-m);

   const int out_layout = 2; // 2 - output is an array on all ltdofs
-   gc.ReduceEnd<double>(ydata, out_layout, GroupCommunicator::Sum);
+   if (!local)
+   {
+      gc.ReduceEnd<double>(ydata, out_layout, GroupCommunicator::Sum);
+   }
 }

 DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
-   const ParFiniteElementSpace &pfes) :
+   const ParFiniteElementSpace &pfes,
+   bool local_) :
   ConformingProlongationOperator(pfes),
-   mpi_gpu_aware(Device::GetGPUAwareMPI())
+   mpi_gpu_aware(Device::GetGPUAwareMPI()),
+   local(local_)
 {
   MFEM_ASSERT(pfes.Conforming(), "internal error");
   const SparseMatrix *R = pfes.GetRestrictionMatrix();
@@ -3173,32 +3240,42 @@ void DeviceConformingProlongationOperator::Mult(const Vector &x,
                                                Vector &y) const
 {
   const GroupTopology &gtopo = gc.GetGroupTopology();
-   BcastBeginCopy(x); // copy to 'shr_buf'
   int req_counter = 0;
-   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+   if (local)
   {
-      const int send_offset = shr_buf_offsets[nbr];
-      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
-      if (send_size > 0)
+      y = 0.0;
+   }
+   else
+   {
+      BcastBeginCopy(x); // copy to 'shr_buf'
+      for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
      {
-         auto send_buf = mpi_gpu_aware ? shr_buf.Read() : shr_buf.HostRead();
-         MPI_Isend(send_buf + send_offset, send_size, MPI_DOUBLE,
-                   gtopo.GetNeighborRank(nbr), 41822,
-                   gtopo.GetComm(), &requests[req_counter++]);
-      }
-      const int recv_offset = ext_buf_offsets[nbr];
-      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
-      if (recv_size > 0)
-      {
-         auto recv_buf = mpi_gpu_aware ? ext_buf.Write() : ext_buf.HostWrite();
-         MPI_Irecv(recv_buf + recv_offset, recv_size, MPI_DOUBLE,
-                   gtopo.GetNeighborRank(nbr), 41822,
-                   gtopo.GetComm(), &requests[req_counter++]);
+         const int send_offset = shr_buf_offsets[nbr];
+         const int send_size = shr_buf_offsets[nbr+1] - send_offset;
+         if (send_size > 0)
+         {
+            auto send_buf = mpi_gpu_aware ? shr_buf.Read() : shr_buf.HostRead();
+            MPI_Isend(send_buf + send_offset, send_size, MPI_DOUBLE,
+                      gtopo.GetNeighborRank(nbr), 41822,
+                      gtopo.GetComm(), &requests[req_counter++]);
+         }
+         const int recv_offset = ext_buf_offsets[nbr];
+         const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
+         if (recv_size > 0)
+         {
+            auto recv_buf = mpi_gpu_aware ? ext_buf.Write() : ext_buf.HostWrite();
+            MPI_Irecv(recv_buf + recv_offset, recv_size, MPI_DOUBLE,
+                      gtopo.GetNeighborRank(nbr), 41822,
+                      gtopo.GetComm(), &requests[req_counter++]);
+         }
      }
   }
   BcastLocalCopy(x, y);
-   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
-   BcastEndCopy(y); // copy from 'ext_buf'
+   if (!local)
+   {
+      MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
+      BcastEndCopy(y); // copy from 'ext_buf'
+   }
 }

 DeviceConformingProlongationOperator::~DeviceConformingProlongationOperator()
@@ -3261,32 +3338,38 @@ void DeviceConformingProlongationOperator::MultTranspose(const Vector &x,
                                                         Vector &y) const
 {
   const GroupTopology &gtopo = gc.GetGroupTopology();
-   ReduceBeginCopy(x); // copy to 'ext_buf'
   int req_counter = 0;
-   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+   if (!local)
   {
-      const int send_offset = ext_buf_offsets[nbr];
-      const int send_size = ext_buf_offsets[nbr+1] - send_offset;
-      if (send_size > 0)
+      ReduceBeginCopy(x); // copy to 'ext_buf'
+      for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
      {
-         auto send_buf = mpi_gpu_aware ? ext_buf.Read() : ext_buf.HostRead();
-         MPI_Isend(send_buf + send_offset, send_size, MPI_DOUBLE,
-                   gtopo.GetNeighborRank(nbr), 41823,
-                   gtopo.GetComm(), &requests[req_counter++]);
-      }
-      const int recv_offset = shr_buf_offsets[nbr];
-      const int recv_size = shr_buf_offsets[nbr+1] - recv_offset;
-      if (recv_size > 0)
-      {
-         auto recv_buf = mpi_gpu_aware ? shr_buf.Write() : shr_buf.HostWrite();
-         MPI_Irecv(recv_buf + recv_offset, recv_size, MPI_DOUBLE,
-                   gtopo.GetNeighborRank(nbr), 41823,
-                   gtopo.GetComm(), &requests[req_counter++]);
+         const int send_offset = ext_buf_offsets[nbr];
+         const int send_size = ext_buf_offsets[nbr+1] - send_offset;
+         if (send_size > 0)
+         {
+            auto send_buf = mpi_gpu_aware ? ext_buf.Read() : ext_buf.HostRead();
+            MPI_Isend(send_buf + send_offset, send_size, MPI_DOUBLE,
+                      gtopo.GetNeighborRank(nbr), 41823,
+                      gtopo.GetComm(), &requests[req_counter++]);
+         }
+         const int recv_offset = shr_buf_offsets[nbr];
+         const int recv_size = shr_buf_offsets[nbr+1] - recv_offset;
+         if (recv_size > 0)
+         {
+            auto recv_buf = mpi_gpu_aware ? shr_buf.Write() : shr_buf.HostWrite();
+            MPI_Irecv(recv_buf + recv_offset, recv_size, MPI_DOUBLE,
+                      gtopo.GetNeighborRank(nbr), 41823,
+                      gtopo.GetComm(), &requests[req_counter++]);
+         }
      }
   }
   ReduceLocalCopy(x, y);
-   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
-   ReduceEndAssemble(y); // assemble from 'shr_buf'
+   if (!local)
+   {
+      MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
+      ReduceEndAssemble(y); // assemble from 'shr_buf'
+   }
 }

 } // namespace mfem
@@ -75,6 +75,12 @@ private:

   /// The (block-diagonal) matrix R (restriction of dof to true dof). Owned.
   mutable SparseMatrix *R;
+   /// Optimized action-only restriction operator for conforming meshes. Owned.
+   mutable Operator *Rconf;
+   /** Transpose of R or Rconf. For conforming mesh, this is a matrix-free
+       (Device)ConformingProlongationOperator, for a non-conforming mesh
+       this is a TransposeOperator wrapping R. */
+   mutable Operator *R_transpose;

   ParNURBSExtension *pNURBSext() const
   { return dynamic_cast<ParNURBSExtension *>(NURBSext); }
@@ -264,6 +270,12 @@ public:
       including the dofs for the edges and the vertices of the face. */
   virtual void GetFaceDofs(int i, Array<int> &dofs) const;

+   /** Returns pointer to the FiniteElement in the FiniteElementCollection
+       associated with i'th element in the mesh object. If @a i is greater than
+       or equal to the number of local mesh elements, @a i will be interpreted
+       as a shifted index of a face neigbor element. */
+   virtual const FiniteElement *GetFE(int i) const;
+
   /** Returns an Operator that converts L-vectors to E-vectors on each face.
       The parallel version is different from the serial one because of the
       presence of shared faces. Shared faces are treated as interior faces,
@@ -335,6 +347,16 @@ public:
   HYPRE_Int GetMyTDofOffset() const;

   virtual const Operator *GetProlongationMatrix() const;
+   /** @brief Return logical transpose of restriction matrix, but in
+       non-assembled optimized matrix-free form.
+
+       The implementation is like GetProlongationMatrix, but it sets local
+       DOFs to the true DOF values if owned locally, otherwise zero. */
+   virtual const Operator *GetRestrictionTransposeOperator() const;
+   /** Get an Operator that performs the action of GetRestrictionMatrix(),
+       but potentially with a non-assembled optimized matrix-free
+       implementation. */
+   virtual const Operator *GetRestrictionOperator() const;
   /// Get the R matrix which restricts a local dof vector to true dof vector.
   virtual const SparseMatrix *GetRestrictionMatrix() const
   { Dof_TrueDof_Matrix(); return R; }
@@ -389,9 +411,11 @@ class ConformingProlongationOperator : public Operator
 protected:
   Array<int> external_ldofs;
   const GroupCommunicator &gc;
+   bool local;

 public:
-   ConformingProlongationOperator(const ParFiniteElementSpace &pfes);
+   ConformingProlongationOperator(const ParFiniteElementSpace &pfes,
+                                  bool local_=false);

   virtual void Mult(const Vector &x, Vector &y) const;

@@ -410,6 +434,8 @@ protected:
   Array<int> ltdof_ldof, unq_ltdof;
   Array<int> unq_shr_i, unq_shr_j;
   MPI_Request *requests;
+   bool local;
+
   // Kernel: copy ltdofs from 'src' to 'shr_buf' - prepare for send.
   //         shr_buf[i] = src[shr_ltdof[i]]
   void BcastBeginCopy(const Vector &src) const;
@@ -435,7 +461,8 @@ protected:
   void ReduceEndAssemble(Vector &dst) const;

 public:
-   DeviceConformingProlongationOperator(const ParFiniteElementSpace &pfes);
+   DeviceConformingProlongationOperator(const ParFiniteElementSpace &pfes,
+                                        bool local_=false);

   virtual ~DeviceConformingProlongationOperator();

@@ -471,6 +471,25 @@ void ParGridFunction::GetVectorValue(ElementTransformation &T,
   }
 }

+void ParGridFunction::GetElementDofValues(int el, Vector &dof_vals) const
+{
+   int ne = fes->GetNE();
+   if (el >= ne)
+   {
+      MFEM_ASSERT(face_nbr_data.Size() > 0,
+                  "ParGridFunction::GetElementDofValues: ExchangeFaceNbrData "
+                  "must be called before accessing face neighbor elements.");
+      // Face neighbor element
+      Array<int> dof_idx;
+      pfes->GetFaceNbrElementVDofs(el - ne, dof_idx);
+      face_nbr_data.GetSubVector(dof_idx, dof_vals);
+   }
+   else
+   {
+      GridFunction::GetElementDofValues(el, dof_vals);
+   }
+}
+
 void ParGridFunction::ProjectCoefficient(Coefficient &coeff)
 {
   DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
@@ -221,6 +221,12 @@ public:
                               const IntegrationPoint &ip,
                               Vector &val, Vector *tr = NULL) const;

+   /** Sets the output vector @a dof_vals to the values of the degrees of
+       freedom of element @a el. If @a el is greater than or equal to the number
+       of local elements, it will be interpreted as a shifted index of a face
+       neighbor element. */
+   virtual void GetElementDofValues(int el, Vector &dof_vals) const;
+
   using GridFunction::ProjectCoefficient;
   virtual void ProjectCoefficient(Coefficient &coeff);

@@ -420,6 +420,35 @@ void QuadratureInterpolator::MultTranspose(
   MFEM_ABORT("this method is not implemented yet");
 }

+static void D2QValues1D(const int NE,
+                        const Array<double> &b_,
+                        const Vector &x_,
+                        Vector &y_,
+                        const int vdim = 1,
+                        const int d1d = 0,
+                        const int q1d = 0)
+{
+   auto b = Reshape(b_.Read(), q1d, d1d);
+   auto x = Reshape(x_.Read(), d1d, vdim, NE);
+   auto y = Reshape(y_.Write(), vdim, q1d, NE);
+
+   MFEM_FORALL(e, NE,
+   {
+      for (int c = 0; c < vdim; c++)
+      {
+         for (int q = 0; q < q1d; ++q)
+         {
+            double val = 0.0;
+            for (int d = 0; d < d1d; ++d)
+            {
+               val += b(q, d) * x(d, c, e);
+            }
+            y(c, q, e) = val;
+         }
+      }
+   });
+}
+

 template<int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 static void D2QValues2D(const int NE,
@@ -631,6 +660,15 @@ static void D2QValues(const FiniteElementSpace &fes,
   const int Q1D = maps->nqpt;
   const int id = (vdim<<8) | (D1D<<4) | Q1D;

+   if (dim == 1)
+   {
+      MFEM_VERIFY(D1D <= MAX_D1D, "Orders higher than " << MAX_D1D-1
+                  << " are not supported!");
+      MFEM_VERIFY(Q1D <= MAX_Q1D, "Quadrature rules with more than "
+                  << MAX_Q1D << " 1D points are not supported!");
+      D2QValues1D(NE, maps->B, e_vec, q_val, vdim, D1D, Q1D);
+      return;
+   }
   if (dim == 2)
   {
      switch (id)
@@ -195,6 +195,31 @@ void ElementRestriction::MultTransposeUnsigned(const Vector& x, Vector& y) const
   });
 }

+void ElementRestriction::MultLeftInverse(const Vector& x, Vector& y) const
+{
+   // Assumes all elements have the same number of dofs
+   const int nd = dof;
+   const int vd = vdim;
+   const bool t = byvdim;
+   auto d_offsets = offsets.Read();
+   auto d_indices = indices.Read();
+   auto d_x = Reshape(x.Read(), nd, vd, ne);
+   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+   MFEM_FORALL(i, ndofs,
+   {
+      const int nextOffset = d_offsets[i + 1];
+      for (int c = 0; c < vd; ++c)
+      {
+         double dofValue = 0;
+         const int j = nextOffset - 1;
+         const int idx_j = (d_indices[j] >= 0) ? d_indices[j] : -1 - d_indices[j];
+         dofValue = (d_indices[j] >= 0) ? d_x(idx_j % nd, c, idx_j / nd) :
+         -d_x(idx_j % nd, c, idx_j / nd);
+         d_y(t?c:i,t?i:c) = dofValue;
+      }
+   });
+}
+
 void ElementRestriction::BooleanMask(Vector& y) const
 {
   // Assumes all elements have the same number of dofs
@@ -57,6 +57,10 @@ public:
   /// Compute MultTranspose without applying signs based on DOF orientations.
   void MultTransposeUnsigned(const Vector &x, Vector &y) const;

+   /// Compute MultTranspose by setting (rather than adding) element
+   /// contributions; this is a left inverse of the Mult() operation
+   void MultLeftInverse(const Vector &x, Vector &y) const;
+
   /// @brief Fills the E-vector y with `boolean` values 0.0 and 1.0 such that each
   /// each entry of the L-vector is uniquely represented in `y`.
   /** This means, the sum of the E-vector `y` is equal to the sum of the
@@ -19,6 +19,42 @@ namespace mfem

 // Target-matrix optimization paradigm (TMOP) mesh quality metrics.

+double TMOP_Combo_QualityMetric::EvalW(const DenseMatrix &Jpt) const
+{
+   double metric = 0.;
+   for (int i = 0; i < tmop_q_arr.Size(); i++)
+   {
+      metric += wt_arr[i]*tmop_q_arr[i]->EvalW(Jpt);
+   }
+   return metric;
+}
+
+void TMOP_Combo_QualityMetric::EvalP(const DenseMatrix &Jpt,
+                                     DenseMatrix &P) const
+{
+   DenseMatrix Pt(P.Size());
+   for (int i = 0; i < tmop_q_arr.Size(); i++)
+   {
+      tmop_q_arr[i]->EvalP(Jpt, Pt);
+      Pt *= wt_arr[i];
+      P += Pt;
+   }
+}
+
+void TMOP_Combo_QualityMetric::AssembleH(const DenseMatrix &Jpt,
+                                         const DenseMatrix &DS,
+                                         const double weight,
+                                         DenseMatrix &A) const
+{
+   DenseMatrix At(A.Size());
+   for (int i = 0; i < tmop_q_arr.Size(); i++)
+   {
+      tmop_q_arr[i]->AssembleH(Jpt, DS, weight, At);
+      At *= wt_arr[i];
+      A += At;
+   }
+}
+
 double TMOP_Metric_001::EvalW(const DenseMatrix &Jpt) const
 {
   ie.SetJacobian(Jpt.GetData());
@@ -160,23 +196,6 @@ double TMOP_Metric_aspratio3D::EvalW(const DenseMatrix &Jpt) const
          ) / 3.0;
 }

-// mu_14 = |T-I|^2
-double TMOP_Metric_SSA2D::EvalW(const DenseMatrix &Jpt) const
-{
-   MFEM_VERIFY(Jtr != NULL,
-               "Requires a target Jacobian, use SetTargetJacobian().");
-
-   DenseMatrix Id(2,2);
-
-   Id(0,0) = 1; Id(0,1) = 0;
-   Id(1,0) = 0; Id(1,1) = 1;
-
-   DenseMatrix Mat(2,2);
-   Mat = Jpt;
-   Mat.Add(-1,Id);
-   return Mat.FNorm2();
-}
-
 double TMOP_Metric_002::EvalW(const DenseMatrix &Jpt) const
 {
   ie.SetJacobian(Jpt.GetData());
@@ -272,13 +291,41 @@ void TMOP_Metric_009::AssembleH(const DenseMatrix &Jpt,
   ie.Assemble_ddI1b(weight, A.GetData());
 }

+// mu_14 = |T-I|^2
+double TMOP_Metric_014::EvalW(const DenseMatrix &Jpt) const
+{
+   MFEM_VERIFY(Jtr != NULL,
+               "Requires a target Jacobian, use SetTargetJacobian().");
+
+   DenseMatrix Id(2,2);
+
+   Id(0,0) = 1; Id(0,1) = 0;
+   Id(1,0) = 0; Id(1,1) = 1;
+
+   DenseMatrix Mat(2,2);
+   Mat = Jpt;
+   Mat.Add(-1,Id);
+   return Mat.FNorm2();
+}
+
 double TMOP_Metric_022::EvalW(const DenseMatrix &Jpt) const
 {
   // mu_22 = (0.5*|J|^2 - det(J)) / (det(J) - tau0)
   //       = (0.5*I1 - I2b) / (I2b - tau0)
   ie.SetJacobian(Jpt.GetData());
   const double I2b = ie.Get_I2b();
-   return (0.5*ie.Get_I1() - I2b) / (I2b - tau0);
+
+   double d = I2b - min_detT;
+   if (d < 0.0 && min_detT == 0.0)
+   {
+      // The mesh has been untangled, but it's still possible to get negative
+      // detJ in FD calculations, as they move the nodes around with some small
+      // increments and can produce negative determinants. Thus we put a small
+      // value in the denominator. Note that here I2b < 0.
+      d = - I2b * 0.1;
+   }
+
+   return (0.5*ie.Get_I1() - I2b) / d;
 }

 void TMOP_Metric_022::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
@@ -287,8 +334,8 @@ void TMOP_Metric_022::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
   // P = 1/(I2b - tau0)*(0.5*dI1 - dI2b) - (0.5*I1 - I2b)/(I2b - tau0)^2*dI2b
   //   = 0.5/(I2b - tau0)*dI1 + (tau0 - 0.5*I1)/(I2b - tau0)^2*dI2b
   ie.SetJacobian(Jpt.GetData());
-   const double c1 = 1.0/(ie.Get_I2b() - tau0);
-   Add(c1/2, ie.Get_dI1(), (tau0 - ie.Get_I1()/2)*c1*c1, ie.Get_dI2b(), P);
+   const double c1 = 1.0/(ie.Get_I2b() - min_detT);
+   Add(c1/2, ie.Get_dI1(), (min_detT - ie.Get_I1()/2)*c1*c1, ie.Get_dI2b(), P);
 }

 void TMOP_Metric_022::AssembleH(const DenseMatrix &Jpt,
@@ -308,10 +355,10 @@ void TMOP_Metric_022::AssembleH(const DenseMatrix &Jpt,
   //      +0.5/(I2b - tau0)*ddI1 + z*ddI2b
   ie.SetJacobian(Jpt.GetData());
   ie.SetDerivativeMatrix(DS.Height(), DS.GetData());
-   const double c1 = 1.0/(ie.Get_I2b() - tau0);
+   const double c1 = 1.0/(ie.Get_I2b() - min_detT);
   const double c2 = weight*c1/2;
   const double c3 = c1*c2;
-   const double c4 = (2*tau0 - ie.Get_I1())*c3; // weight*z
+   const double c4 = (2*min_detT - ie.Get_I1())*c3; // weight*z
   ie.Assemble_TProd(-c3, ie.Get_dI1(), ie.Get_dI2b(), A.GetData());
   ie.Assemble_TProd(-2*c1*c4, ie.Get_dI2b(), A.GetData());
   ie.Assemble_ddI1(c2, A.GetData());
@@ -484,6 +531,23 @@ double TMOP_Metric_085::EvalW(const DenseMatrix &Jpt) const
   return Mat.FNorm2();
 }

+// mu_98 = 1/(tau)|T-I|^2
+double TMOP_Metric_098::EvalW(const DenseMatrix &Jpt) const
+{
+   MFEM_VERIFY(Jtr != NULL,
+               "Requires a target Jacobian, use SetTargetJacobian().");
+
+   DenseMatrix Id(2,2);
+
+   Id(0,0) = 1; Id(0,1) = 0;
+   Id(1,0) = 0; Id(1,1) = 1;
+
+   DenseMatrix Mat(2,2);
+   Mat = Jpt;
+   Mat.Add(-1,Id);
+   return Mat.FNorm2()/Jtr->Det();
+}
+
 double TMOP_Metric_211::EvalW(const DenseMatrix &Jpt) const
 {
   // mu_211 = (det(J) - 1)^2 - det(J) + (det(J)^2 + eps)^{1/2}
@@ -650,6 +714,71 @@ void TMOP_Metric_303::AssembleH(const DenseMatrix &Jpt,
   ie.Assemble_ddI1b(weight/3., A.GetData());
 }

+double TMOP_Metric_311::EvalW(const DenseMatrix &Jpt) const
+{
+   // mu_311 = (det(J) - 1)^2 - det(J) + (det(J)^2 + eps)^{1/2}
+   //        = (I3b - 1)^2 - I3b + sqrt(I3b^2 + eps)
+   ie.SetJacobian(Jpt.GetData());
+   const double I3b = ie.Get_I3b();
+   return (I3b - 1.0)*(I3b - 1.0) - I3b + std::sqrt(I3b*I3b + eps);
+}
+
+void TMOP_Metric_311::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+{
+   ie.SetJacobian(Jpt.GetData());
+   const double I3b = ie.Get_I3b();
+   const double c = 2*I3b-3+(I3b)/(std::pow((I3b*I3b+eps),0.5));
+   P.Set(c, ie.Get_dI3b());
+}
+
+void TMOP_Metric_311::AssembleH(const DenseMatrix &Jpt,
+                                const DenseMatrix &DS,
+                                const double weight,
+                                DenseMatrix &A) const
+{
+   ie.SetJacobian(Jpt.GetData());
+   ie.SetDerivativeMatrix(DS.Height(), DS.GetData());
+   const double I3b = ie.Get_I3b();
+   const double c0 = I3b*I3b+eps;
+   const double c1 = 2 + 1/(pow(c0,0.5)) - I3b*I3b/(pow(c0,1.5));
+   const double c2 = 2*I3b - 3 + I3b/(pow(c0,0.5));
+   ie.Assemble_TProd(weight*c1, ie.Get_dI3b(), A.GetData());
+   ie.Assemble_ddI3b(c2*weight, A.GetData());
+}
+
+double TMOP_Metric_313::EvalW(const DenseMatrix &Jpt) const
+{
+   ie.SetJacobian(Jpt.GetData());
+
+   const double I3b = ie.Get_I3b();
+   double d = I3b - min_detT;
+   if (d < 0.0 && min_detT == 0.0)
+   {
+      // The mesh has been untangled, but it's still possible to get negative
+      // detJ in FD calculations, as they move the nodes around with some small
+      // increments and can produce negative determinants. Thus we put a small
+      // value in the denominator. Note that here I3b < 0.
+      d = - I3b * 0.1;
+   }
+
+   const double c = std::pow(d, -2.0/3.0);
+
+   return ie.Get_I1() * c / 3.0;
+}
+
+void TMOP_Metric_313::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+{
+   MFEM_ABORT("Metric not implemented yet.");
+}
+
+void TMOP_Metric_313::AssembleH(const DenseMatrix &Jpt,
+                                const DenseMatrix &DS,
+                                const double weight,
+                                DenseMatrix &A) const
+{
+   MFEM_ABORT("Metric not implemented yet.");
+}
+
 double TMOP_Metric_315::EvalW(const DenseMatrix &Jpt) const
 {
   // mu_315 = mu_15_3D = (det(J) - 1)^2
@@ -800,6 +929,84 @@ void TMOP_Metric_352::AssembleH(const DenseMatrix &Jpt,
   ie.Assemble_ddI3b(weight*(c - 0.5*c*c), A.GetData());
 }

+double TMOP_AMetric_011::EvalW(const DenseMatrix &Jpt) const
+{
+   MFEM_VERIFY(Jtr != NULL,
+               "Requires a target Jacobian, use SetTargetJacobian().");
+
+   int dim = Jpt.Size();
+
+   DenseMatrix Jpr(dim, dim);
+   Mult(Jpt, *Jtr, Jpr);
+
+   double alpha = Jpr.Det(),
+          omega = Jtr->Det();
+
+   DenseMatrix AdjAt(dim), WtW(dim), WRK(dim), Jtrt(dim);
+   CalcAdjugateTranspose(Jpr, AdjAt);
+   Jtrt.Transpose(*Jtr);
+   MultAAt(Jtrt, WtW);
+   WtW *= 1./omega;
+   Mult(AdjAt, WtW, WRK);
+
+   WRK -= Jpr;
+   WRK *= -1.;
+
+   return (0.25/alpha)*WRK.FNorm2();
+}
+
+double TMOP_AMetric_014a::EvalW(const DenseMatrix &Jpt) const
+{
+   MFEM_VERIFY(Jtr != NULL,
+               "Requires a target Jacobian, use SetTargetJacobian().");
+
+   int dim = Jpt.Size();
+
+   DenseMatrix Jpr(dim, dim);
+   Mult(Jpt, *Jtr, Jpr);
+
+   double sqalpha = pow(Jpr.Det(), 0.5),
+          sqomega = pow(Jtr->Det(), 0.5);
+
+   return 0.5*pow(sqalpha/sqomega - sqomega/sqalpha, 2.);
+}
+
+double TMOP_AMetric_036::EvalW(const DenseMatrix &Jpt) const
+{
+   MFEM_VERIFY(Jtr != NULL,
+               "Requires a target Jacobian, use SetTargetJacobian().");
+
+   int dim = Jpt.Size();
+
+   DenseMatrix Jpr(dim, dim);
+   Mult(Jpt, *Jtr, Jpr); // T*W = A
+
+   double alpha = Jpr.Det(); // det(A)
+   Jpr -= *Jtr; // A-W
+
+   return (1./alpha)*(Jpr.FNorm2()); //(1/alpha)*(|A-W|^2)
+}
+
+double TMOP_AMetric_107a::EvalW(const DenseMatrix &Jpt) const
+{
+   MFEM_VERIFY(Jtr != NULL,
+               "Requires a target Jacobian, use SetTargetJacobian().");
+
+   int dim = Jpt.Size();
+
+   DenseMatrix Jpr(dim, dim);
+   Mult(Jpt, *Jtr, Jpr);
+
+   double alpha = Jpr.Det(),
+          aw = Jpr.FNorm()/Jtr->FNorm();
+
+   DenseMatrix W = *Jtr;
+   W *= aw;
+   Jpr -= W;
+
+   return (0.5/alpha)*Jpr.FNorm2();
+}
+

 void TargetConstructor::ComputeAvgVolume() const
 {
@@ -40,7 +40,7 @@ public:
       The specified Jacobian matrix, #Jtr, can be used by metrics that cannot
       be written just as a function of the target->physical Jacobian matrix,
       Jpt. */
-   void SetTargetJacobian(const DenseMatrix &_Jtr) { Jtr = &_Jtr; }
+   virtual void SetTargetJacobian(const DenseMatrix &_Jtr) { Jtr = &_Jtr; }

   /** @brief Evaluate the strain energy density function, W = W(Jpt).
       @param[in] Jpt  Represents the target->physical transformation
@@ -70,8 +70,37 @@ public:
                          const double weight, DenseMatrix &A) const = 0;
 };

+/// Abstract class used to define combination of metrics with constant coefficients.
+class TMOP_Combo_QualityMetric : public TMOP_QualityMetric
+{
+protected:
+   Array<TMOP_QualityMetric *> tmop_q_arr; //not owned
+   Array<double> wt_arr;

-/// Metric without a type, 2D
+public:
+   virtual void AddQualityMetric(TMOP_QualityMetric *tq, double wt = 1.0)
+   {
+      tmop_q_arr.Append(tq);
+      wt_arr.Append(wt);
+   }
+
+   virtual void SetTargetJacobian(const DenseMatrix &_Jtr)
+   {
+      for (int i = 0; i < tmop_q_arr.Size(); i++)
+      {
+         tmop_q_arr[i]->SetTargetJacobian(_Jtr);
+      }
+   }
+
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const;
+};
+
+/// 2D non-barrier metric without a type.
 class TMOP_Metric_001 : public TMOP_QualityMetric
 {
 protected:
@@ -87,7 +116,7 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Skew metric, 2D.
+/// 2D non-barrier Skew metric.
 class TMOP_Metric_skew2D : public TMOP_QualityMetric
 {
 public:
@@ -102,7 +131,7 @@ public:
   { MFEM_ABORT("Not implemented"); }
 };

-/// Skew metric, 3D.
+/// 3D non-barrier Skew metric.
 class TMOP_Metric_skew3D : public TMOP_QualityMetric
 {
 public:
@@ -117,7 +146,7 @@ public:
   { MFEM_ABORT("Not implemented"); }
 };

-/// Aspect ratio metric, 2D.
+/// 2D non-barrier Aspect ratio metric.
 class TMOP_Metric_aspratio2D : public TMOP_QualityMetric
 {
 public:
@@ -132,7 +161,7 @@ public:
   { MFEM_ABORT("Not implemented"); }
 };

-/// Aspect ratio metric, 3D.
+/// 3D non-barrier Aspect ratio metric.
 class TMOP_Metric_aspratio3D : public TMOP_QualityMetric
 {
 public:
@@ -147,22 +176,7 @@ public:
   { MFEM_ABORT("Not implemented"); }
 };

-/// Shape+Size+Orientation metric, 2D.
-class TMOP_Metric_SSA2D : public TMOP_QualityMetric
-{
-public:
-   // W = 0.5 (1 - cos(theta_Jpr - theta_Jtr)).
-   virtual double EvalW(const DenseMatrix &Jpt) const;
-
-   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
-   { MFEM_ABORT("Not implemented"); }
-
-   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
-                          const double weight, DenseMatrix &A) const
-   { MFEM_ABORT("Not implemented"); }
-};
-
-/// Shape, ideal barrier metric, 2D
+/// 2D barrier shape (S) metric (polyconvex).
 class TMOP_Metric_002 : public TMOP_QualityMetric
 {
 protected:
@@ -178,7 +192,7 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Shape & area, ideal barrier metric, 2D
+/// 2D barrier Shape+Size (VS) metric (not polyconvex).
 class TMOP_Metric_007 : public TMOP_QualityMetric
 {
 protected:
@@ -194,7 +208,7 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Shape & area metric, 2D
+/// 2D barrier Shape+Size (VS) metric (not polyconvex).
 class TMOP_Metric_009 : public TMOP_QualityMetric
 {
 protected:
@@ -210,15 +224,30 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Shifted barrier form of metric 2 (shape, ideal barrier metric), 2D
+/// 2D non-barrier Shape+Size+Orientation (VOS) metric (polyconvex).
+class TMOP_Metric_014 : public TMOP_QualityMetric
+{
+public:
+   // W = |T-I|^2.
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+   { MFEM_ABORT("Not implemented"); }
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const
+   { MFEM_ABORT("Not implemented"); }
+};
+
+/// 2D Shifted barrier form of shape metric (mu_2).
 class TMOP_Metric_022 : public TMOP_QualityMetric
 {
 protected:
-   double &tau0;
+   double &min_detT;
   mutable InvariantsEvaluator2D<double> ie;

 public:
-   TMOP_Metric_022(double &t0): tau0(t0) {}
+   TMOP_Metric_022(double &t0): min_detT(t0) {}

   // W = 0.5(|J|^2 - 2det(J)) / (det(J) - tau0).
   virtual double EvalW(const DenseMatrix &Jpt) const;
@@ -229,7 +258,7 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Shape, ideal barrier metric, 2D
+/// 2D barrier (not a shape) metric (polyconvex).
 class TMOP_Metric_050 : public TMOP_QualityMetric
 {
 protected:
@@ -245,7 +274,7 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Area metric, 2D
+/// 2D non-barrier size (V) metric (not polyconvex).
 class TMOP_Metric_055 : public TMOP_QualityMetric
 {
 protected:
@@ -262,7 +291,7 @@ public:

 };

-/// Area, ideal barrier metric, 2D
+/// 2D barrier size (V) metric (polyconvex).
 class TMOP_Metric_056 : public TMOP_QualityMetric
 {
 protected:
@@ -281,7 +310,7 @@ public:

 };

-/// Shape, ideal barrier metric, 2D
+/// 2D barrier shape (S) metric (not polyconvex).
 class TMOP_Metric_058 : public TMOP_QualityMetric
 {
 protected:
@@ -299,7 +328,7 @@ public:

 };

-/// Area, ideal barrier metric, 2D
+/// 2D barrier size (V) metric (polyconvex).
 class TMOP_Metric_077 : public TMOP_QualityMetric
 {
 protected:
@@ -316,7 +345,28 @@ public:

 };

-/// Shape & orientation metric, 2D.
+/// 2D barrier Shape+Size (VS) metric (polyconvex).
+class TMOP_Metric_080 : public TMOP_Combo_QualityMetric
+{
+protected:
+   mutable InvariantsEvaluator2D<double> ie;
+   double gamma;
+   TMOP_QualityMetric *sh_metric, *sz_metric;
+
+public:
+   TMOP_Metric_080(double gamma_) : gamma(gamma_),
+      sh_metric(new TMOP_Metric_002),
+      sz_metric(new TMOP_Metric_077)
+   {
+      // (1-gamma) mu_2 + gamma mu_77
+      AddQualityMetric(sh_metric, 1.-gamma_);
+      AddQualityMetric(sz_metric, gamma_);
+   }
+
+   virtual ~TMOP_Metric_080() { delete sh_metric; delete sz_metric; }
+};
+
+/// 2D barrier Shape+Orientation (OS) metric (polyconvex).
 class TMOP_Metric_085 : public TMOP_QualityMetric
 {
 public:
@@ -331,7 +381,22 @@ public:
   { MFEM_ABORT("Not implemented"); }
 };

-/// Untangling metric, 2D
+/// 2D barrier Shape+Size+Orientation (VOS) metric (polyconvex).
+class TMOP_Metric_098 : public TMOP_QualityMetric
+{
+public:
+   // W = 1/tau |T-I|^2.
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+   { MFEM_ABORT("Not implemented"); }
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const
+   { MFEM_ABORT("Not implemented"); }
+};
+
+/// 2D untangling metric.
 class TMOP_Metric_211 : public TMOP_QualityMetric
 {
 protected:
@@ -370,7 +435,7 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Shape, ideal barrier metric, 3D
+/// 3D barrier Shape (S) metric.
 class TMOP_Metric_301 : public TMOP_QualityMetric
 {
 protected:
@@ -386,7 +451,7 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Shape, ideal barrier metric, 3D
+/// 3D barrier Shape (S) metric.
 class TMOP_Metric_302 : public TMOP_QualityMetric
 {
 protected:
@@ -402,14 +467,14 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Shape, ideal barrier metric, 3D
+/// 3D barrier Shape (S) metric.
 class TMOP_Metric_303 : public TMOP_QualityMetric
 {
 protected:
   mutable InvariantsEvaluator3D<double> ie;

 public:
-   // W = |J|^2 / 3 * det(J)^(2/3) - 1.
+   // W = |J|^2 / 3 * det(J)^(-2/3) - 1.
   virtual double EvalW(const DenseMatrix &Jpt) const;

   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
@@ -418,7 +483,45 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Volume metric, 3D
+/// 3D Size (V) untangling metric.
+class TMOP_Metric_311 : public TMOP_QualityMetric
+{
+protected:
+   const double eps;
+   mutable InvariantsEvaluator3D<double> ie;
+
+public:
+   TMOP_Metric_311(double epsilon = 1e-4) : eps(epsilon) { }
+
+   // W = (det(J) - 1)^2 - det(J)  + (det(J)^2 + eps)^(1/2).
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const;
+};
+
+/// 3D Shape (S) metric, untangling version of 303.
+class TMOP_Metric_313 : public TMOP_QualityMetric
+{
+protected:
+   double &min_detT;
+   mutable InvariantsEvaluator3D<double> ie;
+
+public:
+   TMOP_Metric_313(double &mindet) : min_detT(mindet) { }
+
+   // W = 1/3 |J|^2 / [det(J)-tau0]^(-2/3).
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const;
+};
+
+/// 3D non-barrier Size (V) metric.
 class TMOP_Metric_315 : public TMOP_QualityMetric
 {
 protected:
@@ -434,7 +537,7 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Volume, ideal barrier metric, 3D
+/// 3D barrier Size (V) metric.
 class TMOP_Metric_316 : public TMOP_QualityMetric
 {
 protected:
@@ -452,7 +555,7 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

-/// Shape & volume, ideal barrier metric, 3D
+/// 3D barrier Shape+Size (VS) metric.
 class TMOP_Metric_321 : public TMOP_QualityMetric
 {
 protected:
@@ -487,6 +590,99 @@ public:
                          const double weight, DenseMatrix &A) const;
 };

+/// A-metrics
+/// 2D barrier Shape (S) metric (polyconvex).
+class TMOP_AMetric_011 : public TMOP_QualityMetric
+{
+protected:
+   mutable InvariantsEvaluator3D<double> ie;
+
+public:
+   // (1/4 alpha) | A - (adj A)^t W^t W / omega |^2
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+   { MFEM_ABORT("Not implemented"); }
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const
+   { MFEM_ABORT("Not implemented"); }
+};
+
+/// 2D barrier Size (V) metric (polyconvex).
+class TMOP_AMetric_014a : public TMOP_QualityMetric
+{
+protected:
+   mutable InvariantsEvaluator3D<double> ie;
+
+public:
+   // 0.5 * ( sqrt(alpha/omega) - sqrt(omega/alpha) )^2
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+   { MFEM_ABORT("Not implemented"); }
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const
+   { MFEM_ABORT("Not implemented"); }
+};
+
+/// 2D barrier Shape+Size+Orientation (VOS) metric (polyconvex).
+class TMOP_AMetric_036 : public TMOP_QualityMetric
+{
+protected:
+   mutable InvariantsEvaluator3D<double> ie;
+
+public:
+   // (1/alpha) | A - W |^2
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+   { MFEM_ABORT("Not implemented"); }
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const
+   { MFEM_ABORT("Not implemented"); }
+};
+
+/// 2D barrier Shape+Orientation (OS) metric (polyconvex).
+class TMOP_AMetric_107a : public TMOP_QualityMetric
+{
+protected:
+   mutable InvariantsEvaluator3D<double> ie;
+
+public:
+   // (1/2 alpha) | A - (|A|/|W|) W |^2
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+   { MFEM_ABORT("Not implemented"); }
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const
+   { MFEM_ABORT("Not implemented"); }
+};
+
+/// 2D barrier Shape+Size (VS) metric (polyconvex).
+class TMOP_AMetric_126 : public TMOP_Combo_QualityMetric
+{
+protected:
+   mutable InvariantsEvaluator2D<double> ie;
+   double gamma;
+   TMOP_QualityMetric *sh_metric, *sz_metric;
+
+public:
+   TMOP_AMetric_126(double gamma_) : gamma(gamma_),
+      sh_metric(new TMOP_AMetric_011),
+      sz_metric(new TMOP_AMetric_014a)
+   {
+      // (1-gamma) nu_11 + gamma nu_14
+      AddQualityMetric(sh_metric, 1.-gamma_);
+      AddQualityMetric(sz_metric, gamma_);
+   }
+
+   virtual ~TMOP_AMetric_126() { delete sh_metric; delete sz_metric; }
+};

 /// Base class for limiting functions to be used in class TMOP_Integrator.
 /** This class represents a scalar function f(x, x0, d), where x and x0 are
@@ -331,10 +331,6 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
      energy_in = nlf->GetEnergy(x);
   }

-   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetMesh()->Dimension();
-   Array<int> xdofs;
-   DenseMatrix Jpr(dim);
-
   // Get the local prolongation of the solution vector.
   Vector x_out_loc(fes->GetVSize());
   if (serial)
@@ -350,51 +346,32 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
   }
 #endif

-   // Check if the starting mesh (given by x) is inverted.
-   // Note that x hasn't been modified by the Newton update yet.
-   double min_detJ = infinity();
-   for (int i = 0; i < NE; i++)
+   // Check if the starting mesh (given by x) is inverted. Note that x hasn't
+   // been modified by the Newton update yet.
+   const double min_detT_in = ComputeMinDet(x_out_loc, *fes);
+   const bool untangling = (min_detT_in <= 0.0) ? true : false;
+   const double untangle_factor = 1.5;
+   if (untangling)
   {
-      const int dof = fes->GetFE(i)->GetDof();
-      DenseMatrix dshape(dof, dim), pos(dof, dim);
-      Vector posV(pos.Data(), dof * dim);
-
-      fes->GetElementVDofs(i, xdofs);
-      x_out_loc.GetSubVector(xdofs, posV);
-
-      const IntegrationRule &irule = GetIntegrationRule(*fes->GetFE(i));
-      const int nsp = irule.GetNPoints();
-      for (int j = 0; j < nsp; j++)
-      {
-         fes->GetFE(i)->CalcDShape(irule.IntPoint(j), dshape);
-         MultAtB(pos, dshape, Jpr);
-         min_detJ = std::min(min_detJ, Jpr.Det());
-      }
+      // Needed for the line search below. The untangling metrics see this
+      // reference to detect deteriorations.
+      *min_det_ptr = untangle_factor * min_detT_in;
   }
-   double min_detJ_all = min_detJ;
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      MPI_Allreduce(&min_detJ, &min_detJ_all, 1, MPI_DOUBLE, MPI_MIN,
-                    p_nlf->ParFESpace()->GetComm());
-   }
-#endif
-   const bool untangling = (min_detJ_all <= 0) ? true : false;

   const bool have_b = (b.Size() == Height());

   Vector x_out(x.Size());
   bool x_out_ok = false;
-   double scale = 1.0, energy_out = 0.0;
-   const double norm0 = Norm(r);
+   double scale = 1.0, energy_out = 0.0, min_detT_out;
+   const double norm_in = Norm(r);

   const double detJ_factor = (solver_type == 1) ? 0.25 : 0.5;

   // Perform the line search.
   for (int i = 0; i < 12; i++)
   {
+      // Update the mesh and get the L-vector in x_out_loc.
      add(x, -scale, c, x_out);
-
      if (serial)
      {
         const SparseMatrix *cP = fes->GetConformingProlongation();
@@ -408,47 +385,29 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
      }
 #endif

-      // Check det(Jpr) > 0.
-      if (!untangling)
+      // Check the changes in detJ.
+      min_detT_out = ComputeMinDet(x_out_loc, *fes);
+      if (untangling == false && min_detT_out < 0.0)
      {
-         int jac_ok = 1;
-         for (int i = 0; i < NE; i++)
-         {
-            const int dof = fes->GetFE(i)->GetDof();
-            DenseMatrix dshape(dof, dim), pos(dof, dim);
-            Vector posV(pos.Data(), dof * dim);
+         // No untangling, and detJ got negative -- no good.
+         if (print_level >= 0)
+         { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
+         scale *= detJ_factor; continue;
+      }
+      if (untangling == true && min_detT_out < *min_det_ptr)
+      {
+         // Untangling, and detJ got even more negative -- no good.
+         if (print_level >= 0)
+         { mfem::out << "Scale = " << scale << " Neg det(J) decreased.\n"; }
+         scale *= detJ_factor; continue;
+      }

-            fes->GetElementVDofs(i, xdofs);
-            x_out_loc.GetSubVector(xdofs, posV);
-
-            const IntegrationRule &irule = GetIntegrationRule(*fes->GetFE(i));
-            const int nsp = irule.GetNPoints();
-            for (int j = 0; j < nsp; j++)
-            {
-               fes->GetFE(i)->CalcDShape(irule.IntPoint(j), dshape);
-               MultAtB(pos, dshape, Jpr);
-               if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
-            }
-         }
-
-      break2:
-         int jac_ok_all = jac_ok;
-#ifdef MFEM_USE_MPI
-         if (parallel)
-         {
-            MPI_Allreduce(&jac_ok, &jac_ok_all, 1, MPI_INT, MPI_LAND,
-                          p_nlf->ParFESpace()->GetComm());
-         }
-#endif
-
-         if (jac_ok_all == 0)
-         {
-            if (print_level >= 0)
-            { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
-            scale *= detJ_factor; continue;
-         }
-      } // endif(!untangling)
+      // Skip the energy and residual checks when we're untangling. The
+      // untangling metrics change their denominators, which can affect the
+      // energy and residual, so their increase/decrease is not relevant.
+      if (untangling) { x_out_ok = true; break; }

+      // Check the changes in total energy.
      ProcessNewState(x_out);
      if (serial)
      {
@@ -460,43 +419,55 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
         energy_out = p_nlf->GetParGridFunctionEnergy(x_out_loc);
      }
 #endif
-
-      if (untangling)
+      if (energy_out > 1.2*energy_in || std::isnan(energy_out) != 0)
      {
-         if (energy_out > energy_in || std::isnan(energy_out) != 0)
+         if (print_level >= 0)
         {
-            scale *= 0.5;
+            mfem::out << "Scale = " << scale << " Increasing energy.\n";
         }
-         else { x_out_ok = true; break; }
+         scale *= 0.5; continue;
      }
-      else
+
+      // Check the changes in the Newton residual.
+      oper->Mult(x_out, r);
+      if (have_b) { r -= b; }
+      double norm_out = Norm(r);
+
+      if (norm_out > 1.2*norm_in)
      {
-         if (energy_out > 1.2*energy_in || std::isnan(energy_out) != 0)
-         {
-            if (print_level >= 0)
-            { mfem::out << "Scale = " << scale << " Increasing energy.\n"; }
-            scale *= 0.5; continue;
-         }
+         if (print_level >= 0)
+         { mfem::out << "Scale = " << scale << " Norm increased.\n"; }
+         scale *= 0.5; continue;
+      }
+      else { x_out_ok = true; break; }
+   } // end line search

-         oper->Mult(x_out, r);
-         if (have_b) { r -= b; }
-         double norm = Norm(r);
-
-         if (norm > 1.2*norm0)
-         {
-            if (print_level >= 0)
-            { mfem::out << "Scale = " << scale << " Norm increased.\n"; }
-            scale *= 0.5; continue;
-         }
-         else { x_out_ok = true; break; }
-      } // endif (untangling)
-   } // enddo (i)
+   if (untangling)
+   {
+      // Update the global min detJ. Untangling metrics see this min_det_ptr.
+      if (min_detT_out > 0.0)
+      {
+         *min_det_ptr = 0.0;
+         if (print_level >= 0)
+         { mfem::out << "The mesh has been untangled at the used points!\n"; }
+      }
+      else { *min_det_ptr = untangle_factor * min_detT_out; }
+   }

   if (print_level >= 0)
   {
-      mfem::out << "Energy decrease: "
-                << (energy_in - energy_out) / energy_in * 100.0
-                << "% with " << scale << " scaling.\n";
+      if (untangling)
+      {
+         mfem::out << "Min det(T) change: "
+                   << min_detT_in << " -> " << min_detT_out
+                   << " with " << scale << " scaling.\n";
+      }
+      else
+      {
+         mfem::out << "Energy decrease: "
+                   << (energy_in - energy_out) / energy_in * 100.0
+                   << "% with " << scale << " scaling.\n";
+      }
   }

   if (x_out_ok == false) { scale = 0.0; }
@@ -508,8 +479,8 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
   const Array<NonlinearFormIntegrator*> &integs = *nlf->GetDNFI();

-   // Reset the update flags of all TargetConstructors.
-   // This is done to avoid repeated updates of shared TargetConstructors.
+   // Reset the update flags of all TargetConstructors. This is done to avoid
+   // repeated updates of shared TargetConstructors.
   TMOP_Integrator *ti  = NULL;
   TMOPComboIntegrator *co = NULL;
   DiscreteAdaptTC *dtc = NULL;
@@ -617,6 +588,47 @@ void TMOPNewtonSolver::UpdateDiscreteTC(const TMOP_Integrator &ti,
   }
 }

+double TMOPNewtonSolver::ComputeMinDet(const Vector &x_loc,
+                                       const FiniteElementSpace &fes) const
+{
+   double min_detJ = infinity();
+   const int NE = fes.GetNE(), dim = fes.GetMesh()->Dimension();
+   Array<int> xdofs;
+   DenseMatrix Jpr(dim);
+   for (int i = 0; i < NE; i++)
+   {
+      const int dof = fes.GetFE(i)->GetDof();
+      DenseMatrix dshape(dof, dim), pos(dof, dim);
+      Vector posV(pos.Data(), dof * dim);
+
+      fes.GetElementVDofs(i, xdofs);
+      x_loc.GetSubVector(xdofs, posV);
+
+      const IntegrationRule &irule = GetIntegrationRule(*fes.GetFE(i));
+      const int nsp = irule.GetNPoints();
+      for (int j = 0; j < nsp; j++)
+      {
+         fes.GetFE(i)->CalcDShape(irule.IntPoint(j), dshape);
+         MultAtB(pos, dshape, Jpr);
+         min_detJ = std::min(min_detJ, Jpr.Det());
+      }
+   }
+   double min_detT_all = min_detJ;
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      auto p_nlf = dynamic_cast<const ParNonlinearForm *>(oper);
+      MPI_Allreduce(&min_detJ, &min_detT_all, 1, MPI_DOUBLE, MPI_MIN,
+                    p_nlf->ParFESpace()->GetComm());
+   }
+#endif
+   const DenseMatrix &Wideal =
+      Geometries.GetGeomToPerfGeomJac(fes.GetFE(0)->GetGeomType());
+   min_detT_all /= Wideal.Det();
+
+   return min_detT_all;
+}
+
 #ifdef MFEM_USE_MPI
 // Metric values are visualized by creating an L2 finite element functions and
 // computing the metric values at the nodes.
@@ -114,6 +114,9 @@ protected:
   int solver_type;
   bool parallel;

+   // Minimum determinant over the whole mesh. Used for mesh untangling.
+   double *min_det_ptr = nullptr;
+
   // Quadrature points that are checked for negative Jacobians etc.
   const IntegrationRule &ir;
   // These fields are relevant for mixed meshes.
@@ -131,6 +134,9 @@ protected:

   void UpdateDiscreteTC(const TMOP_Integrator &ti, const Vector &x_new) const;

+   double ComputeMinDet(const Vector &x_loc,
+                        const FiniteElementSpace &fes) const;
+
 public:
 #ifdef MFEM_USE_MPI
   TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule, int type = 0)
@@ -150,6 +156,8 @@ public:
      integ_order = order;
   }

+   void SetMinDetPtr(double *md_ptr) { min_det_ptr = md_ptr; }
+
   virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;

   virtual void ProcessNewState(const Vector &x) const;
@@ -43,7 +43,7 @@ CeedRestrMap ceed_restr_map;
 static const Backend::Id backend_list[Backend::NUM_BACKENDS] =
 {
   Backend::CEED_CUDA, Backend::OCCA_CUDA, Backend::RAJA_CUDA, Backend::CUDA,
-   Backend::CEED_HIP, Backend::HIP, Backend::DEBUG_DEVICE,
+   Backend::CEED_HIP, Backend::RAJA_HIP, Backend::HIP, Backend::DEBUG_DEVICE,
   Backend::OCCA_OMP, Backend::RAJA_OMP, Backend::OMP,
   Backend::CEED_CPU, Backend::OCCA_CPU, Backend::RAJA_CPU, Backend::CPU
 };
@@ -52,7 +52,7 @@ static const Backend::Id backend_list[Backend::NUM_BACKENDS] =
 static const char *backend_name[Backend::NUM_BACKENDS] =
 {
   "ceed-cuda", "occa-cuda", "raja-cuda", "cuda",
-   "ceed-hip", "hip", "debug",
+   "ceed-hip", "raja-hip", "hip", "debug",
   "occa-omp", "raja-omp", "omp",
   "ceed-cpu", "occa-cpu", "raja-cpu", "cpu"
 };
@@ -394,6 +394,8 @@ static void RajaDeviceSetup(const int dev, int &ngpu)
 {
 #ifdef MFEM_USE_CUDA
   if (ngpu <= 0) { DeviceSetup(dev, ngpu); }
+#elif defined(MFEM_USE_HIP)
+   HipDeviceSetup(dev, ngpu);
 #else
   MFEM_CONTRACT_VAR(dev);
   MFEM_CONTRACT_VAR(ngpu);
@@ -507,7 +509,8 @@ void Device::Setup(const int device)
 #endif
   if (Allows(Backend::CUDA)) { CudaDeviceSetup(dev, ngpu); }
   if (Allows(Backend::HIP)) { HipDeviceSetup(dev, ngpu); }
-   if (Allows(Backend::RAJA_CUDA)) { RajaDeviceSetup(dev, ngpu); }
+   if (Allows(Backend::RAJA_CUDA) || Allows(Backend::RAJA_HIP))
+   { RajaDeviceSetup(dev, ngpu); }
   // The check for MFEM_USE_OCCA is in the function OccaDeviceSetup().
   if (Allows(Backend::OCCA_MASK)) { OccaDeviceSetup(dev); }
   if (Allows(Backend::CEED_CPU))
@@ -46,30 +46,33 @@ struct Backend
      /** @brief [device] RAJA CUDA backend. Enabled when MFEM_USE_RAJA = YES
          and MFEM_USE_CUDA = YES. */
      RAJA_CUDA = 1 << 6,
+      /** @brief [device] RAJA HIP backend. Enabled when MFEM_USE_RAJA = YES
+          and MFEM_USE_HIP = YES. */
+      RAJA_HIP = 1 << 7,
      /** @brief [host] OCCA CPU backend: sequential execution on each MPI rank.
          Enabled when MFEM_USE_OCCA = YES. */
-      OCCA_CPU = 1 << 7,
+      OCCA_CPU = 1 << 8,
      /// [host] OCCA OpenMP backend. Enabled when MFEM_USE_OCCA = YES.
-      OCCA_OMP = 1 << 8,
+      OCCA_OMP = 1 << 9,
      /** @brief [device] OCCA CUDA backend. Enabled when MFEM_USE_OCCA = YES
          and MFEM_USE_CUDA = YES. */
-      OCCA_CUDA = 1 << 9,
+      OCCA_CUDA = 1 << 10,
      /** @brief [host] CEED CPU backend. GPU backends can still be used, but
          with expensive memory transfers. Enabled when MFEM_USE_CEED = YES. */
-      CEED_CPU  = 1 << 10,
+      CEED_CPU  = 1 << 11,
      /** @brief [device] CEED CUDA backend working together with the CUDA
          backend. Enabled when MFEM_USE_CEED = YES and MFEM_USE_CUDA = YES.
          NOTE: The current default libCEED CUDA backend is non-deterministic! */
-      CEED_CUDA = 1 << 11,
+      CEED_CUDA = 1 << 12,
      /** @brief [device] CEED HIP backend working together with the HIP
          backend. Enabled when MFEM_USE_CEED = YES and MFEM_USE_HIP = YES. */
-      CEED_HIP = 1 << 12,
+      CEED_HIP = 1 << 13,
      /** @brief [device] Debug backend: host memory is READ/WRITE protected
          while a device is in use. It allows to test the "device" code-path
          (using separate host/device memory pools and host <-> device
          transfers) without any GPU hardware. As 'DEBUG' is sometimes used
          as a macro, `_DEVICE` has been added to avoid conflicts. */
-      DEBUG_DEVICE = 1 << 13
+      DEBUG_DEVICE = 1 << 14
   };

   /** @brief Additional useful constants. For example, the *_MASK constants can
@@ -77,14 +80,14 @@ struct Backend
   enum
   {
      /// Number of backends: from (1 << 0) to (1 << (NUM_BACKENDS-1)).
-      NUM_BACKENDS = 14,
+      NUM_BACKENDS = 15,

      /// Biwise-OR of all CPU backends
      CPU_MASK = CPU | RAJA_CPU | OCCA_CPU | CEED_CPU,
      /// Biwise-OR of all CUDA backends
      CUDA_MASK = CUDA | RAJA_CUDA | OCCA_CUDA | CEED_CUDA,
      /// Biwise-OR of all HIP backends
-      HIP_MASK = HIP | CEED_HIP,
+      HIP_MASK = HIP | RAJA_HIP | CEED_HIP,
      /// Biwise-OR of all OpenMP backends
      OMP_MASK = OMP | RAJA_OMP | OCCA_OMP,
      /// Bitwise-OR of all CEED backends
@@ -93,7 +96,7 @@ struct Backend
      DEVICE_MASK = CUDA_MASK | HIP_MASK | DEBUG_DEVICE,

      /// Biwise-OR of all RAJA backends
-      RAJA_MASK = RAJA_CPU | RAJA_OMP | RAJA_CUDA,
+      RAJA_MASK = RAJA_CPU | RAJA_OMP | RAJA_CUDA | RAJA_HIP,
      /// Biwise-OR of all OCCA backends
      OCCA_MASK = OCCA_CPU | OCCA_OMP | OCCA_CUDA
   };
@@ -86,77 +86,157 @@ void OmpWrap(const int N, HBODY &&h_body)
 }


-/// RAJA Cuda backend
+/// RAJA Cuda and Hip backends
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA)
-
-#if RAJA_VERSION_MAJOR == 0 && RAJA_VERSION_MINOR < 12
-using RAJA::statement::Segs;
-#else
-using RAJA::Segs;
+using cuda_launch_policy =
+   RAJA::expt::LaunchPolicy<RAJA::expt::null_launch_t, RAJA::expt::cuda_launch_t<false>>;
+using cuda_teams_x =
+   RAJA::expt::LoopPolicy<RAJA::loop_exec,RAJA::cuda_block_x_direct>;
+using cuda_threads_z =
+   RAJA::expt::LoopPolicy<RAJA::loop_exec,RAJA::cuda_thread_z_direct>;
 #endif

+#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_HIP)
+using hip_launch_policy =
+   RAJA::expt::LaunchPolicy<RAJA::expt::null_launch_t, RAJA::expt::hip_launch_t<false>>;
+using hip_teams_x =
+   RAJA::expt::LoopPolicy<RAJA::loop_exec,RAJA::hip_block_x_direct>;
+using hip_threads_z =
+   RAJA::expt::LoopPolicy<RAJA::loop_exec,RAJA::hip_thread_z_direct>;
+#endif
+
+#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA)
 template <const int BLOCKS = MFEM_CUDA_BLOCKS, typename DBODY>
-void RajaCudaWrap1D(const int N, DBODY &&d_body)
+void RajaCuWrap1D(const int N, DBODY &&d_body)
 {
-   // true denotes asynchronous kernel
+   //true denotes asynchronous kernel
   RAJA::forall<RAJA::cuda_exec<BLOCKS,true>>(RAJA::RangeSegment(0,N),d_body);
 }

 template <typename DBODY>
-void RajaCudaWrap2D(const int N, DBODY &&d_body,
-                    const int X, const int Y, const int BZ)
+void RajaCuWrap2D(const int N, DBODY &&d_body,
+                  const int X, const int Y, const int BZ)
 {
   MFEM_VERIFY(N>0, "");
   MFEM_VERIFY(BZ>0, "");
   const int G = (N+BZ-1)/BZ;
-   RAJA::kernel<RAJA::KernelPolicy<
-   RAJA::statement::CudaKernelAsync<
-   RAJA::statement::For<0, RAJA::cuda_block_x_direct,
-        RAJA::statement::For<1, RAJA::cuda_thread_x_direct,
-        RAJA::statement::For<2, RAJA::cuda_thread_y_direct,
-        RAJA::statement::For<3, RAJA::cuda_thread_z_direct,
-        RAJA::statement::Lambda<0, Segs<0>>>>>>>>>
-        (RAJA::make_tuple(RAJA::RangeSegment(0,G), RAJA::RangeSegment(0,X),
-                          RAJA::RangeSegment(0,Y), RAJA::RangeSegment(0,BZ)),
-         [=] RAJA_DEVICE (const int n)
+
+   using namespace RAJA::expt;
+   using RAJA::RangeSegment;
+
+   launch<cuda_launch_policy>
+   (DEVICE, Resources(Teams(G), Threads(X, Y, BZ)),
+    [=] RAJA_DEVICE (LaunchContext ctx)
   {
-      const int k = n*BZ + threadIdx.z;
-      if (k >= N) { return; }
-      d_body(k);
+
+      loop<cuda_teams_x>(ctx, RangeSegment(0, G), [&] (const int n)
+      {
+
+         loop<cuda_threads_z>(ctx, RangeSegment(0, BZ), [&] (const int tz)
+         {
+
+            const int k = n*BZ + tz;
+            if (k >= N) { return; }
+            d_body(k);
+
+         });
+
+      });
+
   });
+
   MFEM_GPU_CHECK(cudaGetLastError());
 }

 template <typename DBODY>
-void RajaCudaWrap3D(const int N, DBODY &&d_body,
-                    const int X, const int Y, const int Z)
+void RajaCuWrap3D(const int N, DBODY &&d_body,
+                  const int X, const int Y, const int Z)
 {
   MFEM_VERIFY(N>0, "");
-   RAJA::kernel<RAJA::KernelPolicy<
-   RAJA::statement::CudaKernelAsync<
-   RAJA::statement::For<0, RAJA::cuda_block_x_direct,
-        RAJA::statement::For<1, RAJA::cuda_thread_x_direct,
-        RAJA::statement::For<2, RAJA::cuda_thread_y_direct,
-        RAJA::statement::For<3, RAJA::cuda_thread_z_direct,
-        RAJA::statement::Lambda<0, Segs<0>>>>>>>>>
-        (RAJA::make_tuple(RAJA::RangeSegment(0,N), RAJA::RangeSegment(0,X),
-                          RAJA::RangeSegment(0,Y), RAJA::RangeSegment(0,Z)),
-   [=] RAJA_DEVICE (const int k) { d_body(k); });
+   using namespace RAJA::expt;
+   using RAJA::RangeSegment;
+
+   launch<cuda_launch_policy>
+   (DEVICE, Resources(Teams(N), Threads(X, Y, Z)),
+    [=] RAJA_DEVICE (LaunchContext ctx)
+   {
+
+      loop<cuda_teams_x>(ctx, RangeSegment(0, N), d_body);
+
+   });
+
   MFEM_GPU_CHECK(cudaGetLastError());
 }

 #endif

+#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_HIP)
+template <const int BLOCKS = MFEM_HIP_BLOCKS, typename DBODY>
+void RajaHipWrap1D(const int N, DBODY &&d_body)
+{
+   //true denotes asynchronous kernel
+   RAJA::forall<RAJA::hip_exec<BLOCKS,true>>(RAJA::RangeSegment(0,N),d_body);
+}
+
+template <typename DBODY>
+void RajaHipWrap2D(const int N, DBODY &&d_body,
+                   const int X, const int Y, const int BZ)
+{
+   MFEM_VERIFY(N>0, "");
+   MFEM_VERIFY(BZ>0, "");
+   const int G = (N+BZ-1)/BZ;
+
+   using namespace RAJA::expt;
+   using RAJA::RangeSegment;
+
+   launch<hip_launch_policy>
+   (DEVICE, Resources(Teams(G), Threads(X, Y, BZ)),
+    [=] RAJA_DEVICE (LaunchContext ctx)
+   {
+
+      loop<hip_teams_x>(ctx, RangeSegment(0, G), [&] (const int n)
+      {
+
+         loop<hip_threads_z>(ctx, RangeSegment(0, BZ), [&] (const int tz)
+         {
+
+            const int k = n*BZ + tz;
+            if (k >= N) { return; }
+            d_body(k);
+
+         });
+
+      });
+
+   });
+
+   MFEM_GPU_CHECK(hipGetLastError());
+}
+
+template <typename DBODY>
+void RajaHipWrap3D(const int N, DBODY &&d_body,
+                   const int X, const int Y, const int Z)
+{
+   MFEM_VERIFY(N>0, "");
+   using namespace RAJA::expt;
+   using RAJA::RangeSegment;
+
+   launch<hip_launch_policy>
+   (DEVICE, Resources(Teams(N), Threads(X, Y, Z)),
+    [=] RAJA_DEVICE (LaunchContext ctx)
+   {
+
+      loop<hip_teams_x>(ctx, RangeSegment(0, N), d_body);
+
+   });
+
+   MFEM_GPU_CHECK(hipGetLastError());
+}
+#endif

 /// RAJA OpenMP backend
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP)

-#if RAJA_VERSION_MAJOR == 0 && RAJA_VERSION_MINOR < 12
-using RAJA::statement::Segs;
-#else
-using RAJA::Segs;
-#endif
-
 template <typename HBODY>
 void RajaOmpWrap(const int N, HBODY &&h_body)
 {
@@ -319,9 +399,19 @@ inline void ForallWrap(const bool use_dev, const int N,
   // If Backend::RAJA_CUDA is allowed, use it
   if (Device::Allows(Backend::RAJA_CUDA))
   {
-      if (DIM == 1) { return RajaCudaWrap1D(N, d_body); }
-      if (DIM == 2) { return RajaCudaWrap2D(N, d_body, X, Y, Z); }
-      if (DIM == 3) { return RajaCudaWrap3D(N, d_body, X, Y, Z); }
+      if (DIM == 1) { return RajaCuWrap1D(N, d_body); }
+      if (DIM == 2) { return RajaCuWrap2D(N, d_body, X, Y, Z); }
+      if (DIM == 3) { return RajaCuWrap3D(N, d_body, X, Y, Z); }
+   }
+#endif
+
+#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_HIP)
+   // If Backend::RAJA_HIP is allowed, use it
+   if (Device::Allows(Backend::RAJA_HIP))
+   {
+      if (DIM == 1) { return RajaHipWrap1D(N, d_body); }
+      if (DIM == 2) { return RajaHipWrap2D(N, d_body, X, Y, Z); }
+      if (DIM == 3) { return RajaHipWrap3D(N, d_body, X, Y, Z); }
   }
 #endif

@@ -125,7 +125,7 @@ void* HipMemcpyDtoDAsync(void* dst, const void *src, size_t bytes)
 void* HipMemcpyDtoH(void *dst, const void *src, size_t bytes)
 {
 #ifdef MFEM_USE_HIP
-#ifdef MFEM_TRACK_HPI_MEM
+#ifdef MFEM_TRACK_HIP_MEM
   mfem::out << "HipMemcpyDtoH(): copying " << bytes << " bytes from "
             << src << " to " << dst << " ... " << std::flush;
 #endif
@@ -71,7 +71,7 @@ static std::string strerror()
   }
 #elif (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && ! _GNU_SOURCE || \
      defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \
-      defined(__NetBSD__) || defined(__DragonFly__)
+      defined(__NetBSD__) || defined(__DragonFly__) || defined(__EMSCRIPTEN__)
   // XSI-compliant strerror_r()
   if (strerror_r(errno, &buff[0], buff.size()) != 0)
   {
@@ -15,6 +15,7 @@ list(APPEND SRCS
  blockvector.cpp
  complex_operator.cpp
  densemat.cpp
+  symmat.cpp
  handle.cpp
  matrix.cpp
  ode.cpp
@@ -31,6 +32,7 @@ list(APPEND HDRS
  blockvector.hpp
  complex_operator.hpp
  densemat.hpp
+  symmat.hpp
  dtensor.hpp
  handle.hpp
  invariants.hpp
@@ -30,10 +30,16 @@ int AmgXSolver::count = 0;

 AMGX_resources_handle AmgXSolver::rsrc = nullptr;

+AmgXSolver::AmgXSolver()
+   : ConvergenceCheck(false) {};
+
 AmgXSolver::AmgXSolver(const AMGX_MODE amgxMode_, const bool verbose)
 {
   amgxMode = amgxMode_;

+   if (amgxMode == AmgXSolver::SOLVER) { ConvergenceCheck = true;}
+   else { ConvergenceCheck = false;}
+
   DefaultParameters(amgxMode, verbose);

   InitSerial();
@@ -47,6 +53,9 @@ AmgXSolver::AmgXSolver(const MPI_Comm &comm,
   std::string config;
   amgxMode = amgxMode_;

+   if (amgxMode == AmgXSolver::SOLVER) { ConvergenceCheck = true;}
+   else { ConvergenceCheck = false;}
+
   DefaultParameters(amgxMode, verbose);

   InitExclusiveGPU(comm);
@@ -58,6 +67,9 @@ AmgXSolver::AmgXSolver(const MPI_Comm &comm, const int nDevs,
   std::string config;
   amgxMode = amgxMode_;

+   if (amgxMode == AmgXSolver::SOLVER) { ConvergenceCheck = true;}
+   else { ConvergenceCheck = false;}
+
   DefaultParameters(amgxMode_, verbose);

   InitMPITeams(comm, nDevs);
@@ -178,6 +190,11 @@ void AmgXSolver::ReadParameters(const std::string config,
   configSrc = source;
 }

+void AmgXSolver::SetConvergenceCheck(bool setConvergenceCheck_)
+{
+   ConvergenceCheck = setConvergenceCheck_;
+}
+
 void AmgXSolver::DefaultParameters(const AMGX_MODE amgxMode_,
                                   const bool verbose)
 {
@@ -201,8 +218,8 @@ void AmgXSolver::DefaultParameters(const AMGX_MODE amgxMode_,
      {
         amgx_config = amgx_config + ",\n"
                       "   \"obtain_timings\": 1, \n"
-                       "   \"monitor_residual\": 1, \n"
                       "   \"print_grid_stats\": 1, \n"
+                       "   \"monitor_residual\": 1, \n"
                       "   \"print_solve_stats\": 1 \n";
      }
      else
@@ -238,12 +255,12 @@ void AmgXSolver::DefaultParameters(const AMGX_MODE amgxMode_,
                    "  \"convergence\": \"RELATIVE_MAX\", \n"
                    "  \"scope\": \"main\", \n"
                    "  \"tolerance\": 1e-12, \n"
+                    "  \"monitor_residual\": 1, \n"
                    "  \"norm\": \"L2\" ";
      if (verbose)
      {
         amgx_config = amgx_config + ", \n"
                       "        \"obtain_timings\": 1, \n"
-                       "        \"monitor_residual\": 1, \n"
                       "        \"print_grid_stats\": 1, \n"
                       "        \"print_solve_stats\": 1 \n";
      }
@@ -884,7 +901,7 @@ void AmgXSolver::Mult(const Vector& B, Vector& X) const

      AMGX_SOLVE_STATUS   status;
      AMGX_solver_get_status(solver, &status);
-      if (status != AMGX_SOLVE_SUCCESS && amgxMode == SOLVER)
+      if (status != AMGX_SOLVE_SUCCESS && ConvergenceCheck)
      {
         if (status == AMGX_SOLVE_DIVERGED)
         {
@@ -73,13 +73,16 @@ public:
   /// Flags to configure AmgXSolver as a solver or preconditioner
   enum AMGX_MODE {SOLVER, PRECONDITIONER};

+   /// Flag to check for convergence
+   bool ConvergenceCheck;
+
   /**
      Flags to determine whether user solver settings are defined internally in
      the source code or will be read through an external JSON file.
   */
   enum CONFIG_SRC {INTERNAL, EXTERNAL, UNDEFINED};

-   AmgXSolver() = default;
+   AmgXSolver();

   /**
      Configures AmgX with a default configuration based on the AmgX mode, and
@@ -162,6 +165,9 @@ public:
   */
   void DefaultParameters(const AMGX_MODE amgxMode_, const bool verbose);

+   /// Add a check for convergence after applying Mult.
+   void SetConvergenceCheck(bool setConvergenceCheck_=true);
+
   ~AmgXSolver();

   void Finalize();
@@ -101,6 +101,7 @@ void BlockVector::Update(Vector & data, const Array<int> & bOffsets)
   {
      blocks[i].MakeRef(data, blockOffsets[i], BlockSize(i));
   }
+   MakeRef(data, 0, blockOffsets[numBlocks]);
 }

 void BlockVector::Update(const Array<int> &bOffsets)
@@ -287,10 +287,7 @@ void ComplexUMFPackSolver::Init()

 void ComplexUMFPackSolver::SetOperator(const Operator &op)
 {
-   int *Ap, *Ai;
   void *Symbolic;
-   double *Ax;
-   double *Az;

   if (Numeric)
   {
@@ -322,10 +319,11 @@ void ComplexUMFPackSolver::SetOperator(const Operator &op)
   width = mat->real().Width();
   MFEM_VERIFY(width == height, "not a square matrix");

-   Ap = mat->real().GetI(); // assuming real and imag have the same sparsity
-   Ai = mat->real().GetJ();
-   Ax = mat->real().GetData();
-   Az = mat->imag().GetData();
+   const int * Ap =
+      mat->real().HostReadI(); // assuming real and imag have the same sparsity
+   const int * Ai = mat->real().HostReadJ();
+   const double * Ax = mat->real().HostReadData();
+   const double * Az = mat->imag().HostReadData();

   if (!use_long_ints)
   {
@@ -395,6 +393,10 @@ void ComplexUMFPackSolver::Mult(const Vector &b, Vector &x) const
   if (mat == NULL)
      mfem_error("ComplexUMFPackSolver::Mult : matrix is not set!"
                 " Call SetOperator first!");
+
+   b.HostRead();
+   x.HostReadWrite();
+
   int n = b.Size()/2;
   double * datax = x.GetData();
   double * datab = b.GetData();
@@ -413,8 +415,8 @@ void ComplexUMFPackSolver::Mult(const Vector &b, Vector &x) const
   if (!use_long_ints)
   {
      int status =
-         umfpack_zi_solve(UMFPACK_Aat, mat->real().GetI(), mat->real().GetJ(),
-                          mat->real().GetData(), mat->imag().GetData(),
+         umfpack_zi_solve(UMFPACK_Aat, mat->real().HostReadI(), mat->real().HostReadJ(),
+                          mat->real().HostReadData(), mat->imag().HostReadData(),
                          datax, &datax[n], datab, &datab[n], Numeric, Control, Info);
      umfpack_zi_report_info(Control, Info);
      if (status < 0)
@@ -426,8 +428,8 @@ void ComplexUMFPackSolver::Mult(const Vector &b, Vector &x) const
   else
   {
      SuiteSparse_long status =
-         umfpack_zl_solve(UMFPACK_Aat,AI,AJ,mat->real().GetData(),
-                          mat->imag().GetData(),
+         umfpack_zl_solve(UMFPACK_Aat,AI,AJ,mat->real().HostReadData(),
+                          mat->imag().HostReadData(),
                          datax,&datax[n],datab,&datab[n],Numeric,Control,Info);

      umfpack_zl_report_info(Control, Info);
@@ -448,6 +450,8 @@ void ComplexUMFPackSolver::MultTranspose(const Vector &b, Vector &x) const
   if (mat == NULL)
      mfem_error("ComplexUMFPackSolver::Mult : matrix is not set!"
                 " Call SetOperator first!");
+   b.HostRead();
+   x.HostReadWrite();
   int n = b.Size()/2;
   double * datax = x.GetData();
   double * datab = b.GetData();
@@ -467,8 +471,8 @@ void ComplexUMFPackSolver::MultTranspose(const Vector &b, Vector &x) const
   if (!use_long_ints)
   {
      int status =
-         umfpack_zi_solve(UMFPACK_A, mat->real().GetI(), mat->real().GetJ(),
-                          mat->real().GetData(), mat->imag().GetData(),
+         umfpack_zi_solve(UMFPACK_A, mat->real().HostReadI(), mat->real().HostReadJ(),
+                          mat->real().HostReadData(), mat->imag().HostReadData(),
                          datax, &datax[n], datab, &datab[n], Numeric, Control, Info);
      umfpack_zi_report_info(Control, Info);
      if (status < 0)
@@ -480,8 +484,8 @@ void ComplexUMFPackSolver::MultTranspose(const Vector &b, Vector &x) const
   else
   {
      SuiteSparse_long status =
-         umfpack_zl_solve(UMFPACK_A,AI,AJ,mat->real().GetData(),
-                          mat->imag().GetData(),
+         umfpack_zl_solve(UMFPACK_A,AI,AJ,mat->real().HostReadData(),
+                          mat->imag().HostReadData(),
                          datax,&datax[n],datab,&datab[n],Numeric,Control,Info);

      umfpack_zl_report_info(Control, Info);
@@ -276,8 +276,8 @@ CGSolver::CGSolver(
                               ATOLERANCE)
 {
   using cg = gko::solver::Cg<double>;
-   this->solver_gen =
-      cg::build().with_criteria(this->combined_factory).on(this->executor);
+   //   this->solver_gen =
+   //      cg::build().with_criteria(this->combined_factory).on(this->executor);
 }

 CGSolver::CGSolver(
@@ -292,10 +292,10 @@ CGSolver::CGSolver(
                               ATOLERANCE)
 {
   using cg         = gko::solver::Cg<double>;
-   this->solver_gen = cg::build()
-                      .with_criteria(this->combined_factory)
-                      .with_preconditioner(preconditioner)
-                      .on(this->executor);
+   //   this->solver_gen = cg::build()
+   //                      .with_criteria(this->combined_factory)
+   //                      .with_preconditioner(preconditioner)
+   //                      .on(this->executor);
 }


@@ -311,9 +311,9 @@ BICGSTABSolver::BICGSTABSolver(
                               ATOLERANCE)
 {
   using bicgstab   = gko::solver::Bicgstab<double>;
-   this->solver_gen = bicgstab::build()
-                      .with_criteria(this->combined_factory)
-                      .on(this->executor);
+   //   this->solver_gen = bicgstab::build()
+   //                      .with_criteria(this->combined_factory)
+   //                      .on(this->executor);
 }

 BICGSTABSolver::BICGSTABSolver(
@@ -328,10 +328,10 @@ BICGSTABSolver::BICGSTABSolver(
                               ATOLERANCE)
 {
   using bicgstab   = gko::solver::Bicgstab<double>;
-   this->solver_gen = bicgstab::build()
-                      .with_criteria(this->combined_factory)
-                      .with_preconditioner(preconditioner)
-                      .on(this->executor);
+   //   this->solver_gen = bicgstab::build()
+   //                      .with_criteria(this->combined_factory)
+   //                      .with_preconditioner(preconditioner)
+   //                      .on(this->executor);
 }


@@ -347,8 +347,8 @@ CGSSolver::CGSSolver(
                               ATOLERANCE)
 {
   using cgs = gko::solver::Cgs<double>;
-   this->solver_gen =
-      cgs::build().with_criteria(this->combined_factory).on(this->executor);
+   //   this->solver_gen =
+   //      cgs::build().with_criteria(this->combined_factory).on(this->executor);
 }

 CGSSolver::CGSSolver(
@@ -363,10 +363,10 @@ CGSSolver::CGSSolver(
                               ATOLERANCE)
 {
   using cgs        = gko::solver::Cgs<double>;
-   this->solver_gen = cgs::build()
-                      .with_criteria(this->combined_factory)
-                      .with_preconditioner(preconditioner)
-                      .on(this->executor);
+   //   this->solver_gen = cgs::build()
+   //                      .with_criteria(this->combined_factory)
+   //                      .with_preconditioner(preconditioner)
+   //                      .on(this->executor);
 }


@@ -382,8 +382,8 @@ FCGSolver::FCGSolver(
                               ATOLERANCE)
 {
   using fcg = gko::solver::Fcg<double>;
-   this->solver_gen =
-      fcg::build().with_criteria(this->combined_factory).on(this->executor);
+   //   this->solver_gen =
+   //      fcg::build().with_criteria(this->combined_factory).on(this->executor);
 }

 FCGSolver::FCGSolver(
@@ -398,10 +398,10 @@ FCGSolver::FCGSolver(
                               ATOLERANCE)
 {
   using fcg        = gko::solver::Fcg<double>;
-   this->solver_gen = fcg::build()
-                      .with_criteria(this->combined_factory)
-                      .with_preconditioner(preconditioner)
-                      .on(this->executor);
+   //   this->solver_gen = fcg::build()
+   //                      .with_criteria(this->combined_factory)
+   //                      .with_preconditioner(preconditioner)
+   //                      .on(this->executor);
 }


@@ -417,10 +417,10 @@ GMRESSolver::GMRESSolver(
                               ATOLERANCE)
 {
   using gmres      = gko::solver::Gmres<double>;
-   this->solver_gen = gmres::build()
-                      .with_krylov_dim(m)
-                      .with_criteria(this->combined_factory)
-                      .on(this->executor);
+   //   this->solver_gen = gmres::build()
+   //                      .with_krylov_dim(m)
+   //                      .with_criteria(this->combined_factory)
+   //                      .on(this->executor);
 }

 GMRESSolver::GMRESSolver(
@@ -435,11 +435,11 @@ GMRESSolver::GMRESSolver(
                               ATOLERANCE)
 {
   using gmres      = gko::solver::Gmres<double>;
-   this->solver_gen = gmres::build()
-                      .with_krylov_dim(m)
-                      .with_criteria(this->combined_factory)
-                      .with_preconditioner(preconditioner)
-                      .on(this->executor);
+   //   this->solver_gen = gmres::build()
+   //                      .with_krylov_dim(m)
+   //                      .with_criteria(this->combined_factory)
+   //                      .with_preconditioner(preconditioner)
+   //                      .on(this->executor);
 }


@@ -455,8 +455,8 @@ IRSolver::IRSolver(
                               ATOLERANCE)
 {
   using ir = gko::solver::Ir<double>;
-   this->solver_gen =
-      ir::build().with_criteria(this->combined_factory).on(this->executor);
+   //   this->solver_gen =
+   //      ir::build().with_criteria(this->combined_factory).on(this->executor);
 }

 IRSolver::IRSolver(
@@ -471,10 +471,105 @@ IRSolver::IRSolver(
                               ATOLERANCE)
 {
   using ir         = gko::solver::Ir<double>;
-   this->solver_gen = ir::build()
-                      .with_criteria(this->combined_factory)
-                      .with_solver(inner_solver)
-                      .on(this->executor);
+   //   this->solver_gen = ir::build()
+   //                      .with_criteria(this->combined_factory)
+   //                      .with_solver(inner_solver)
+   //                      .on(this->executor);
+}
+
+
+/* ---------------------- GinkgoPreconditioner ------------------------ */
+void GinkgoPreconditionerBase::Mult(const Vector &x, Vector &y) const
+{
+
+   if (!iterative_mode)
+   {
+      y = 0.0;
+   }
+
+   //Create Ginkgo wrapped-vectors
+   bool on_device = false;
+   if (exec_ != exec_->get_master())
+   {
+      on_device = true;
+   }
+   using vec = gko::matrix::Dense<double>;
+   auto gko_x = vec::create(exec_, gko::dim<2> {x.Size(), 1},
+                            gko::Array<double>::view(exec_,
+                                                     x.Size(), const_cast<double *>(
+                                                        x.Read(on_device))), 1);
+   auto gko_y = vec::create(exec_, gko::dim<2> {y.Size(), 1},
+                            gko::Array<double>::view(exec_,
+                                                     y.Size(), y.ReadWrite(on_device)), 1);
+
+   if (permute_)
+   {
+
+      auto gko_x_perm = vec::create(exec_, gko::dim<2> {x.Size(), 1});
+      auto gko_y_perm = vec::create(exec_, gko::dim<2> {y.Size(), 1});
+
+      vec_permute_->apply(gko::lend(gko_x), gko::lend(gko_x_perm));
+      vec_permute_->apply(gko::lend(gko_y), gko::lend(gko_y_perm));
+
+      gko_precond_.get()->apply(gko::lend(gko_x_perm), gko::lend(gko_y_perm));
+
+      vec_inv_permute_->apply(gko::lend(gko_y_perm), gko::lend(gko_y));
+
+      // Reset MFEM Vector to use Ginkgo output ?
+      //   y = gko_y->get_values();
+
+   }
+   else
+   {
+
+      gko_precond_.get()->apply(gko::lend(gko_x), gko::lend(gko_y));
+
+   }
+}
+
+// TEMP!  This only works for symmetric operators
+void GinkgoPreconditionerBase::MultTranspose(const Vector &x, Vector &y) const
+{
+  this->Mult(x, y);
+}
+
+void GinkgoPreconditionerBase::SetOperator(const Operator &op)
+{
+
+   // Only accept SparseMatrix for this type (see SparseSmoother::SetOperator)
+   SparseMatrix *op_mat = const_cast<SparseMatrix*>(
+                             dynamic_cast<const SparseMatrix*>(&op));
+   if (op_mat == NULL)
+   {
+      mfem_error("GinkgoPreconditioner::SetOperator : not a SparseMatrix!");
+   }
+   height = op_mat->Height();
+   width = op_mat->Width();
+
+   // Release current preconditioner
+   gko_precond_.release();
+
+   bool on_device = false;
+   if (exec_ != exec_->get_master())
+   {
+      on_device = true;
+   }
+
+   using mtx = gko::matrix::Csr<double, int>;
+   auto gko_sparse = mtx::create(
+                        exec_, gko::dim<2>(op_mat->Height(), op_mat->Width()),
+                        gko::Array<double>::view(exec_,
+                                                 op_mat->NumNonZeroElems(),
+                                                 op_mat->ReadWriteData(on_device)),
+                        gko::Array<int>::view(exec_,
+                                              op_mat->NumNonZeroElems(),
+                                              op_mat->ReadWriteJ(on_device)),
+                        gko::Array<int>::view(exec_,
+                                              op_mat->Height() + 1,
+                                              op_mat->ReadWriteI(on_device)));
+   gko_precond_ = gko_precond_factory_.get()->generate(
+                     gko::give(gko_sparse));
+
 }


@@ -640,6 +640,725 @@ public:
 };


+class GinkgoPreconditionerBase : public Solver
+{
+protected:
+   GinkgoPreconditionerBase(std::shared_ptr<const gko::Executor> exec,
+                            SparseMatrix &a,
+                            bool iter_mode=false)
+      : Solver(a.Height(), a.Width(), iter_mode)
+   {
+      exec_ = std::move(exec);
+      permute_ = false;
+   }
+
+   GinkgoPreconditionerBase(std::shared_ptr<const gko::Executor> exec,
+                            SparseMatrix &a, Array<int> &inv_permutation_indices,
+                            bool iter_mode=false)
+      : Solver(a.Height(), a.Width(), iter_mode)
+   {
+      exec_ = std::move(exec);
+
+      bool on_device = false;
+      if (exec_->get_master() != exec_)
+      {
+         on_device = true;
+      }
+      permute_ = true;
+      auto gko_inv_perm_ind = gko::Array<int>::view(
+                                 exec_,
+                                 inv_permutation_indices.Size(),
+                                 inv_permutation_indices.ReadWrite(
+                                    on_device));
+      // Note the "forward" permutation uses the inverse flag because
+      //  the indices are for the inverse permutation as defined by Ginkgo
+      vec_permute_ = gko::matrix::Permutation<int>::create(
+                        exec_, gko::dim<2> {inv_permutation_indices.Size()},
+                        gko_inv_perm_ind,
+                        gko::matrix::row_permute | gko::matrix::inverse_permute);
+      vec_inv_permute_ = gko::matrix::Permutation<int>::create(
+                            exec_, gko::dim<2> {inv_permutation_indices.Size()},
+                            gko_inv_perm_ind,
+                            gko::matrix::row_permute);
+   }
+
+public:
+   std::shared_ptr<const gko::Executor> get_exec() {return this->exec_; }
+   const gko::LinOpFactory* get_gko_precond_factory()
+   {
+      return this->gko_precond_factory_.get();
+   }
+   gko::LinOp* get_gko_precond() {return this->gko_precond_.get(); }
+
+   virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void MultTranspose(const Vector &x, Vector &y) const;
+   virtual void SetOperator(const Operator &op);
+
+protected:
+   std::shared_ptr<const gko::Executor> exec_;
+   std::unique_ptr<const gko::LinOpFactory> gko_precond_factory_;
+   std::unique_ptr<gko::LinOp> gko_precond_;
+   bool permute_;
+   std::unique_ptr<gko::matrix::Permutation<int>> vec_permute_;
+   std::unique_ptr<gko::matrix::Permutation<int>> vec_inv_permute_;
+
+};
+
+
+class GinkgoJacobiPreconditioner : public GinkgoPreconditionerBase
+{
+public:
+   GinkgoJacobiPreconditioner(std::shared_ptr<const gko::Executor> exec,
+                              SparseMatrix &a,
+                              const char *storage_opt="none",
+                              const double accuracy=1.e-1,
+                              const int max_block_size=32,
+                              bool iter_mode=false)
+      : GinkgoPreconditionerBase(exec, a, iter_mode)
+   {
+
+      bool on_device = false;
+      if (exec->get_master() != exec)
+      {
+         on_device = true;
+      }
+
+      using mtx = gko::matrix::Csr<double, int>;
+      const int nnz =  a.GetMemoryData().Capacity();
+      auto gko_sparse = mtx::create(
+                           exec, gko::dim<2>(a.Height(), a.Width()),
+                           gko::Array<double>::view(exec,
+                                                    nnz,
+                                                    a.ReadWriteData(on_device)),
+                           gko::Array<int>::view(exec,
+                                                 nnz,
+                                                 a.ReadWriteJ(on_device)),
+                           gko::Array<int>::view(exec, a.Height() + 1,
+                                                 a.ReadWriteI(on_device)));
+
+      gko_sparse->sort_by_column_index();
+
+      if (storage_opt == "auto")
+      {
+         gko_precond_factory_ = gko::preconditioner::Jacobi<double, int>::build()
+                                .with_storage_optimization(
+                                   gko::precision_reduction::autodetect())
+                                .with_accuracy(accuracy)
+                                .with_max_block_size(max_block_size)
+                                .on(exec);
+      }
+      else
+      {
+         gko_precond_factory_ = gko::preconditioner::Jacobi<double, int>::build()
+                                .with_storage_optimization(
+                                   gko::precision_reduction(0, 0))
+                                .with_accuracy(accuracy)
+                                .with_max_block_size(max_block_size)
+                                .on(exec);
+      }
+
+      gko_precond_ = gko_precond_factory_.get()->generate(
+                        gko::give(gko_sparse));
+   }
+
+   GinkgoJacobiPreconditioner(std::shared_ptr<const gko::Executor> exec,
+                              SparseMatrix &a,
+                              Array<int> &inv_permutation_indices,
+                              const char *storage_opt="none",
+                              const double accuracy=1.e-1,
+                              const int max_block_size=32,
+                              bool iter_mode=false)
+      : GinkgoPreconditionerBase(exec, a, inv_permutation_indices, iter_mode)
+   {
+
+      bool on_device = false;
+      if (exec->get_master() != exec)
+      {
+         on_device = true;
+      }
+
+      using mtx = gko::matrix::Csr<double, int>;
+      const int nnz =  a.GetMemoryData().Capacity();
+      auto gko_sparse = mtx::create(
+                           exec, gko::dim<2>(a.Height(), a.Width()),
+                           gko::Array<double>::view(exec,
+                                                    nnz,
+                                                    a.ReadWriteData(on_device)),
+                           gko::Array<int>::view(exec,
+                                                 nnz,
+                                                 a.ReadWriteJ(on_device)),
+                           gko::Array<int>::view(exec, a.Height() + 1,
+                                                 a.ReadWriteI(on_device)));
+
+
+      gko_sparse->sort_by_column_index();
+
+      if (storage_opt == "auto")
+      {
+         gko_precond_factory_ = gko::preconditioner::Jacobi<double, int>::build()
+                                .with_storage_optimization(
+                                   gko::precision_reduction::autodetect())
+                                .with_accuracy(accuracy)
+                                .with_max_block_size(max_block_size)
+                                .on(exec);
+      }
+      else
+      {
+         gko_precond_factory_ = gko::preconditioner::Jacobi<double, int>::build()
+                                .with_storage_optimization(
+                                   gko::precision_reduction(0, 0))
+                                .with_accuracy(accuracy)
+                                .with_max_block_size(max_block_size)
+                                .on(exec);
+      }
+
+      gko_precond_ = gko_precond_factory_.get()->generate(
+                        gko::give(gko_sparse));
+   }
+};
+
+class GinkgoIcPreconditioner : public GinkgoPreconditionerBase
+{
+public:
+   GinkgoIcPreconditioner(std::shared_ptr<const gko::Executor> exec,
+                           SparseMatrix &a, const char *trisolve_type = "exact",
+                           int sparsity_power=1,
+                           int par_ic_its=0,
+                           bool iter_mode=false)
+      : GinkgoPreconditionerBase(exec, a, iter_mode)
+   {
+
+      bool on_device = false;
+      if (exec->get_master() != exec)
+      {
+         on_device = true;
+      }
+
+      using mtx = gko::matrix::Csr<double, int>;
+      const int nnz =  a.GetMemoryData().Capacity();
+      auto gko_sparse = mtx::create(
+                           exec, gko::dim<2>(a.Height(), a.Width()),
+                           gko::Array<double>::view(exec,
+                                                    nnz,
+                                                    a.ReadWriteData(on_device)),
+                           gko::Array<int>::view(exec,
+                                                 nnz,
+                                                 a.ReadWriteJ(on_device)),
+                           gko::Array<int>::view(exec, a.Height() + 1,
+                                                 a.ReadWriteI(on_device)));
+
+      using ic_fact_type = gko::factorization::ParIc<double, int>;
+      std::shared_ptr<ic_fact_type::Factory> fact_factory = std::move(
+                                                                ic_fact_type::build()
+                                                                .with_iterations(par_ic_its)
+                                                                .with_both_factors(false)
+                                                                .on(exec));
+
+
+      if (trisolve_type == "isai")
+      {
+
+         using l_solver_type = gko::preconditioner::LowerIsai<>;
+
+         std::shared_ptr<l_solver_type::Factory> l_solver_factory = std::move(
+                                                                       l_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+
+         gko_precond_factory_ = gko::preconditioner::Ic<l_solver_type>::build()
+         .with_factorization_factory(fact_factory)
+         .with_l_solver_factory(l_solver_factory)
+         .on(exec);
+
+      }
+      else
+      {
+
+         gko_precond_factory_ = gko::preconditioner::Ic<>::build()
+                                .with_factorization_factory(fact_factory)
+                                .on(exec);
+      }
+
+      gko_precond_ = gko_precond_factory_.get()->generate(
+                        gko::give(gko_sparse));
+   }
+
+   GinkgoIcPreconditioner(std::shared_ptr<const gko::Executor> exec,
+                           SparseMatrix &a, Array<int> &inv_permutation_indices,
+                           const char *trisolve_type = "exact",
+                           int sparsity_power=1,
+                           int par_ic_its=0,
+                           bool iter_mode=false)
+      : GinkgoPreconditionerBase(exec, a, inv_permutation_indices, iter_mode)
+   {
+
+      bool on_device = false;
+      if (exec->get_master() != exec)
+      {
+         on_device = true;
+      }
+
+      using mtx = gko::matrix::Csr<double, int>;
+      const int nnz =  a.GetMemoryData().Capacity();
+      auto gko_sparse = mtx::create(
+                           exec, gko::dim<2>(a.Height(), a.Width()),
+                           gko::Array<double>::view(exec,
+                                                    nnz,
+                                                    a.ReadWriteData(on_device)),
+                           gko::Array<int>::view(exec,
+                                                 nnz,
+                                                 a.ReadWriteJ(on_device)),
+                           gko::Array<int>::view(exec, a.Height() + 1,
+                                                 a.ReadWriteI(on_device)));
+
+      using ic_fact_type = gko::factorization::ParIc<double, int>;
+      std::shared_ptr<ic_fact_type::Factory> fact_factory = std::move(
+                                                                ic_fact_type::build()
+                                                                .with_iterations(par_ic_its)
+                                                                .with_both_factors(false)
+                                                                .on(exec));
+
+      if (trisolve_type == "isai")
+      {
+
+         using l_solver_type = gko::preconditioner::LowerIsai<>;
+
+         std::shared_ptr<l_solver_type::Factory> l_solver_factory = std::move(
+                                                                       l_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+
+         gko_precond_factory_ = gko::preconditioner::Ic<l_solver_type>::build()
+         .with_factorization_factory(fact_factory)
+         .with_l_solver_factory(l_solver_factory)
+         .on(exec);
+      }
+      else
+      {
+
+         gko_precond_factory_ = gko::preconditioner::Ic<>::build()
+                                .with_factorization_factory(fact_factory)
+                                .on(exec);
+      }
+
+      gko_precond_ = gko_precond_factory_.get()->generate(
+                        gko::give(gko_sparse));
+   }
+};
+
+class GinkgoIluPreconditioner : public GinkgoPreconditionerBase
+{
+public:
+   GinkgoIluPreconditioner(std::shared_ptr<const gko::Executor> exec,
+                           SparseMatrix &a, const char *trisolve_type = "exact",
+                           int sparsity_power=1,
+                           int par_ilu_its=0,
+                           bool iter_mode=false)
+      : GinkgoPreconditionerBase(exec, a, iter_mode)
+   {
+
+      bool on_device = false;
+      if (exec->get_master() != exec)
+      {
+         on_device = true;
+      }
+
+      using mtx = gko::matrix::Csr<double, int>;
+      const int nnz =  a.GetMemoryData().Capacity();
+      auto gko_sparse = mtx::create(
+                           exec, gko::dim<2>(a.Height(), a.Width()),
+                           gko::Array<double>::view(exec,
+                                                    nnz,
+                                                    a.ReadWriteData(on_device)),
+                           gko::Array<int>::view(exec,
+                                                 nnz,
+                                                 a.ReadWriteJ(on_device)),
+                           gko::Array<int>::view(exec, a.Height() + 1,
+                                                 a.ReadWriteI(on_device)));
+
+      using ilu_fact_type = gko::factorization::ParIlu<double, int>;
+      std::shared_ptr<ilu_fact_type::Factory> fact_factory = std::move(
+                                                                ilu_fact_type::build()
+                                                                .with_iterations(par_ilu_its)
+                                                                .on(exec));
+
+
+      if (trisolve_type == "isai")
+      {
+
+         using l_solver_type = gko::preconditioner::LowerIsai<>;
+         using u_solver_type = gko::preconditioner::UpperIsai<>;
+
+         std::shared_ptr<l_solver_type::Factory> l_solver_factory = std::move(
+                                                                       l_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+         std::shared_ptr<u_solver_type::Factory> u_solver_factory = std::move(
+                                                                       u_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+
+         gko_precond_factory_ = gko::preconditioner::Ilu<l_solver_type,
+         u_solver_type>::build()
+         .with_factorization_factory(fact_factory)
+         .with_l_solver_factory(l_solver_factory)
+         .with_u_solver_factory(u_solver_factory)
+         .on(exec);
+
+      }
+      else
+      {
+
+         gko_precond_factory_ = gko::preconditioner::Ilu<>::build()
+                                .with_factorization_factory(fact_factory)
+                                .on(exec);
+      }
+
+      gko_precond_ = gko_precond_factory_.get()->generate(
+                        gko::give(gko_sparse));
+   }
+
+   GinkgoIluPreconditioner(std::shared_ptr<const gko::Executor> exec,
+                           SparseMatrix &a, Array<int> &inv_permutation_indices,
+                           const char *trisolve_type = "exact",
+                           int sparsity_power=1,
+                           int par_ilu_its=0,
+                           bool iter_mode=false)
+      : GinkgoPreconditionerBase(exec, a, inv_permutation_indices, iter_mode)
+   {
+
+      bool on_device = false;
+      if (exec->get_master() != exec)
+      {
+         on_device = true;
+      }
+
+      using mtx = gko::matrix::Csr<double, int>;
+      const int nnz =  a.GetMemoryData().Capacity();
+      auto gko_sparse = mtx::create(
+                           exec, gko::dim<2>(a.Height(), a.Width()),
+                           gko::Array<double>::view(exec,
+                                                    nnz,
+                                                    a.ReadWriteData(on_device)),
+                           gko::Array<int>::view(exec,
+                                                 nnz,
+                                                 a.ReadWriteJ(on_device)),
+                           gko::Array<int>::view(exec, a.Height() + 1,
+                                                 a.ReadWriteI(on_device)));
+
+      using ilu_fact_type = gko::factorization::ParIlu<double, int>;
+      std::shared_ptr<ilu_fact_type::Factory> fact_factory = std::move(
+                                                                ilu_fact_type::build()
+                                                                .with_iterations(par_ilu_its)
+                                                                .on(exec));
+
+      if (trisolve_type == "isai")
+      {
+
+         using l_solver_type = gko::preconditioner::LowerIsai<>;
+         using u_solver_type = gko::preconditioner::UpperIsai<>;
+
+         std::shared_ptr<l_solver_type::Factory> l_solver_factory = std::move(
+                                                                       l_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+         std::shared_ptr<u_solver_type::Factory> u_solver_factory = std::move(
+                                                                       u_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+
+         gko_precond_factory_ = gko::preconditioner::Ilu<l_solver_type,
+         u_solver_type>::build()
+         .with_factorization_factory(fact_factory)
+         .with_l_solver_factory(l_solver_factory)
+         .with_u_solver_factory(u_solver_factory)
+         .on(exec);
+      }
+      else
+      {
+
+         gko_precond_factory_ = gko::preconditioner::Ilu<>::build()
+                                .with_factorization_factory(fact_factory)
+                                .on(exec);
+      }
+
+      gko_precond_ = gko_precond_factory_.get()->generate(
+                        gko::give(gko_sparse));
+   }
+};
+
+class GinkgoCuIcPreconditioner : public GinkgoPreconditionerBase
+{
+public:
+   GinkgoCuIcPreconditioner(std::shared_ptr<const gko::Executor> exec,
+                             SparseMatrix &a, const char *trisolve_type = "exact",
+                             int sparsity_power=1,
+                             bool iter_mode=false)
+      : GinkgoPreconditionerBase(exec, a, iter_mode)
+   {
+
+      bool on_device = false;
+      if (exec->get_master() != exec)
+      {
+         on_device = true;
+      }
+
+      using mtx = gko::matrix::Csr<double, int>;
+      const int nnz =  a.GetMemoryData().Capacity();
+      auto gko_sparse = mtx::create(
+                           exec, gko::dim<2>(a.Height(), a.Width()),
+                           gko::Array<double>::view(exec,
+                                                    nnz,
+                                                    a.ReadWriteData(on_device)),
+                           gko::Array<int>::view(exec,
+                                                 nnz,
+                                                 a.ReadWriteJ(on_device)),
+                           gko::Array<int>::view(exec, a.Height() + 1,
+                                                 a.ReadWriteI(on_device)));
+
+      // TEST
+      gko_sparse->sort_by_column_index();
+
+      using ic_fact_type = gko::factorization::Ic<double, int>;
+      std::shared_ptr<ic_fact_type::Factory> fact_factory = std::move(
+                                                                ic_fact_type::build()
+                                                                .with_both_factors(false)
+                                                                .on(exec));
+
+
+      if (trisolve_type == "isai")
+      {
+
+         using l_solver_type = gko::preconditioner::LowerIsai<>;
+
+         std::shared_ptr<l_solver_type::Factory> l_solver_factory = std::move(
+                                                                       l_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+
+         gko_precond_factory_ = gko::preconditioner::Ic<l_solver_type>::build()
+         .with_factorization_factory(fact_factory)
+         .with_l_solver_factory(l_solver_factory)
+         .on(exec);
+
+      }
+      else
+      {
+
+         gko_precond_factory_ = gko::preconditioner::Ic<>::build()
+                                .with_factorization_factory(fact_factory)
+                                .on(exec);
+      }
+
+      gko_precond_ = gko_precond_factory_.get()->generate(
+                        gko::give(gko_sparse));
+   }
+
+   GinkgoCuIcPreconditioner(std::shared_ptr<const gko::Executor> exec,
+                             SparseMatrix &a, Array<int> &inv_permutation_indices,
+                             const char *trisolve_type = "exact",
+                             int sparsity_power=1,
+                             bool iter_mode=false)
+      : GinkgoPreconditionerBase(exec, a, inv_permutation_indices, iter_mode)
+   {
+
+      bool on_device = false;
+      if (exec->get_master() != exec)
+      {
+         on_device = true;
+      }
+
+      using mtx = gko::matrix::Csr<double, int>;
+      const int nnz =  a.GetMemoryData().Capacity();
+      auto gko_sparse = mtx::create(
+                           exec, gko::dim<2>(a.Height(), a.Width()),
+                           gko::Array<double>::view(exec,
+                                                    nnz,
+                                                    a.ReadWriteData(on_device)),
+                           gko::Array<int>::view(exec,
+                                                 nnz,
+                                                 a.ReadWriteJ(on_device)),
+                           gko::Array<int>::view(exec, a.Height() + 1,
+                                                 a.ReadWriteI(on_device)));
+
+      // TEST
+      gko_sparse->sort_by_column_index();
+
+      using ic_fact_type = gko::factorization::Ic<double, int>;
+      std::shared_ptr<ic_fact_type::Factory> fact_factory = std::move(
+                                                                ic_fact_type::build()
+                                                                .with_both_factors(false)
+                                                                .on(exec));
+      if (trisolve_type == "isai")
+      {
+
+         using l_solver_type = gko::preconditioner::LowerIsai<>;
+
+         std::shared_ptr<l_solver_type::Factory> l_solver_factory = std::move(
+                                                                       l_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+
+         gko_precond_factory_ = gko::preconditioner::Ic<l_solver_type>::build()
+         .with_factorization_factory(fact_factory)
+         .with_l_solver_factory(l_solver_factory)
+         .on(exec);
+      }
+      else
+      {
+
+         gko_precond_factory_ = gko::preconditioner::Ic<>::build()
+                                .with_factorization_factory(fact_factory)
+                                .on(exec);
+      }
+
+      gko_precond_ = gko_precond_factory_.get()->generate(
+                        gko::give(gko_sparse));
+   }
+};
+
+class GinkgoCuIluPreconditioner : public GinkgoPreconditionerBase
+{
+public:
+   GinkgoCuIluPreconditioner(std::shared_ptr<const gko::Executor> exec,
+                             SparseMatrix &a, const char *trisolve_type = "exact",
+                             int sparsity_power=1,
+                             bool iter_mode=false)
+      : GinkgoPreconditionerBase(exec, a, iter_mode)
+   {
+
+      bool on_device = false;
+      if (exec->get_master() != exec)
+      {
+         on_device = true;
+      }
+
+      using mtx = gko::matrix::Csr<double, int>;
+      const int nnz =  a.GetMemoryData().Capacity();
+      auto gko_sparse = mtx::create(
+                           exec, gko::dim<2>(a.Height(), a.Width()),
+                           gko::Array<double>::view(exec,
+                                                    nnz,
+                                                    a.ReadWriteData(on_device)),
+                           gko::Array<int>::view(exec,
+                                                 nnz,
+                                                 a.ReadWriteJ(on_device)),
+                           gko::Array<int>::view(exec, a.Height() + 1,
+                                                 a.ReadWriteI(on_device)));
+
+      // TEST
+      gko_sparse->sort_by_column_index();
+
+      using ilu_fact_type = gko::factorization::Ilu<double, int>;
+      std::shared_ptr<ilu_fact_type::Factory> fact_factory = std::move(
+                                                                ilu_fact_type::build()
+                                                                .on(exec));
+
+
+      if (trisolve_type == "isai")
+      {
+
+         using l_solver_type = gko::preconditioner::LowerIsai<>;
+         using u_solver_type = gko::preconditioner::UpperIsai<>;
+
+         std::shared_ptr<l_solver_type::Factory> l_solver_factory = std::move(
+                                                                       l_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+         std::shared_ptr<u_solver_type::Factory> u_solver_factory = std::move(
+                                                                       u_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+
+         gko_precond_factory_ = gko::preconditioner::Ilu<l_solver_type,
+         u_solver_type>::build()
+         .with_factorization_factory(fact_factory)
+         .with_l_solver_factory(l_solver_factory)
+         .with_u_solver_factory(u_solver_factory)
+         .on(exec);
+
+      }
+      else
+      {
+
+         gko_precond_factory_ = gko::preconditioner::Ilu<>::build()
+                                .with_factorization_factory(fact_factory)
+                                .on(exec);
+      }
+
+      gko_precond_ = gko_precond_factory_.get()->generate(
+                        gko::give(gko_sparse));
+   }
+
+   GinkgoCuIluPreconditioner(std::shared_ptr<const gko::Executor> exec,
+                             SparseMatrix &a, Array<int> &inv_permutation_indices,
+                             const char *trisolve_type = "exact",
+                             int sparsity_power=1,
+                             bool iter_mode=false)
+      : GinkgoPreconditionerBase(exec, a, inv_permutation_indices, iter_mode)
+   {
+
+      bool on_device = false;
+      if (exec->get_master() != exec)
+      {
+         on_device = true;
+      }
+
+      using mtx = gko::matrix::Csr<double, int>;
+      const int nnz =  a.GetMemoryData().Capacity();
+      auto gko_sparse = mtx::create(
+                           exec, gko::dim<2>(a.Height(), a.Width()),
+                           gko::Array<double>::view(exec,
+                                                    nnz,
+                                                    a.ReadWriteData(on_device)),
+                           gko::Array<int>::view(exec,
+                                                 nnz,
+                                                 a.ReadWriteJ(on_device)),
+                           gko::Array<int>::view(exec, a.Height() + 1,
+                                                 a.ReadWriteI(on_device)));
+
+      // TEST
+      gko_sparse->sort_by_column_index();
+
+      using ilu_fact_type = gko::factorization::Ilu<double, int>;
+      std::shared_ptr<ilu_fact_type::Factory> fact_factory = std::move(
+                                                                ilu_fact_type::build()
+                                                                .on(exec));
+      if (trisolve_type == "isai")
+      {
+
+         using l_solver_type = gko::preconditioner::LowerIsai<>;
+         using u_solver_type = gko::preconditioner::UpperIsai<>;
+
+         std::shared_ptr<l_solver_type::Factory> l_solver_factory = std::move(
+                                                                       l_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+         std::shared_ptr<u_solver_type::Factory> u_solver_factory = std::move(
+                                                                       u_solver_type::build()
+                                                                       .with_sparsity_power(sparsity_power)
+                                                                       .on(exec));
+
+         gko_precond_factory_ = gko::preconditioner::Ilu<l_solver_type,
+         u_solver_type>::build()
+         .with_factorization_factory(fact_factory)
+         .with_l_solver_factory(l_solver_factory)
+         .with_u_solver_factory(u_solver_factory)
+         .on(exec);
+      }
+      else
+      {
+
+         gko_precond_factory_ = gko::preconditioner::Ilu<>::build()
+                                .with_factorization_factory(fact_factory)
+                                .on(exec);
+      }
+
+      gko_precond_ = gko_precond_factory_.get()->generate(
+                        gko::give(gko_sparse));
+   }
+};
+
 } // namespace GinkgoWrappers

 }
@@ -128,6 +128,15 @@ HypreParVector::HypreParVector(ParFiniteElementSpace *pfes)
   own_ParVector = 1;
 }

+void HypreParVector::WrapHypreParVector(hypre_ParVector *y, bool owner)
+{
+   if (own_ParVector) { hypre_ParVectorDestroy(x); }
+   Destroy();
+   x = y;
+   _SetDataAndSize_();
+   own_ParVector = owner;
+}
+
 Vector * HypreParVector::GlobalVector() const
 {
   hypre_Vector *hv = hypre_ParVectorToVectorAll(*this);
@@ -926,6 +935,15 @@ void HypreParMatrix::GetOffd(SparseMatrix &offd, HYPRE_Int* &cmap) const
   cmap = A->col_map_offd;
 }

+void HypreParMatrix::MergeDiagAndOffd(SparseMatrix &merged)
+{
+   SparseMatrix tmp_wrapper;
+   hypre_CSRMatrix *hypre_merged = hypre_MergeDiagAndOffd(A);
+   MakeWrapper(hypre_merged, tmp_wrapper);
+   merged = tmp_wrapper;
+   hypre_CSRMatrixDestroy(hypre_merged);
+}
+
 void HypreParMatrix::GetBlocks(Array2D<HypreParMatrix*> &blocks,
                               bool interleaved_rows,
                               bool interleaved_cols) const
@@ -966,6 +984,46 @@ HypreParMatrix * HypreParMatrix::Transpose() const
   return new HypreParMatrix(At);
 }

+#if MFEM_HYPRE_VERSION >= 21800
+HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array<int> &indices,
+                                                 double threshhold) const
+{
+   if (!(A->comm))
+   {
+      hypre_MatvecCommPkgCreate(A);
+   }
+
+   hypre_ParCSRMatrix *submat;
+
+   // Get number of rows stored on this processor
+   int local_num_vars = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A));
+
+   // Form hypre CF-splitting array designating submatrix as F-points (-1)
+   Array<int> CF_marker(local_num_vars);
+   CF_marker = 1;
+   for (int j=0; j<indices.Size(); j++)
+   {
+      if (indices[j] > local_num_vars)
+      {
+         MFEM_WARNING("WARNING : " << indices[j] << " > " << local_num_vars);
+      }
+      CF_marker[indices[j]] = -1;
+   }
+
+   // Construct cpts_global array on hypre matrix structure
+   HYPRE_BigInt *cpts_global;
+   hypre_BoomerAMGCoarseParms(MPI_COMM_WORLD, local_num_vars, 1, NULL,
+                              CF_marker, NULL, &cpts_global);
+
+   // Extract submatrix into *submat
+   hypre_ParCSRMatrixExtractSubmatrixFC(A, CF_marker, cpts_global,
+                                        "FF", &submat, threshhold);
+
+   mfem_hypre_TFree(cpts_global);
+   return new HypreParMatrix(submat);
+}
+#endif
+
 HYPRE_Int HypreParMatrix::Mult(HypreParVector &x, HypreParVector &y,
                               double a, double b)
 {
@@ -1279,6 +1337,7 @@ void HypreParMatrix::operator*=(double s)
 static void get_sorted_rows_cols(const Array<int> &rows_cols,
                                 Array<HYPRE_Int> &hypre_sorted)
 {
+   rows_cols.HostRead();
   hypre_sorted.SetSize(rows_cols.Size());
   bool sorted = true;
   for (int i = 0; i < rows_cols.Size(); i++)
@@ -1568,6 +1627,36 @@ void HypreParMatrix::Destroy()
   }
 }

+#if MFEM_HYPRE_VERSION >= 21800
+
+void BlockInverseScale(const HypreParMatrix *A, HypreParMatrix *C,
+                       const Vector *b, HypreParVector *d,
+                       int blocksize, BlockInverseScaleJob job)
+{
+   if (job == BlockInverseScaleJob::MATRIX_ONLY ||
+       job == BlockInverseScaleJob::MATRIX_AND_RHS)
+   {
+      hypre_ParCSRMatrix *C_hypre;
+      hypre_ParcsrBdiagInvScal(*A, blocksize, &C_hypre);
+      hypre_ParCSRMatrixDropSmallEntries(C_hypre, 1e-15, 1);
+      C->WrapHypreParCSRMatrix(C_hypre);
+   }
+
+   if (job == BlockInverseScaleJob::RHS_ONLY ||
+       job == BlockInverseScaleJob::MATRIX_AND_RHS)
+   {
+      HypreParVector b_Hypre(A->GetComm(),
+                             A->GetGlobalNumRows(),
+                             b->GetData(), A->GetRowStarts());
+      hypre_ParVector *d_hypre;
+      hypre_ParvecBdiagInvScal(b_Hypre, blocksize, &d_hypre, *A);
+
+      d->WrapHypreParVector(d_hypre, true);
+   }
+}
+
+#endif
+
 #if MFEM_HYPRE_VERSION < 21400

 HypreParMatrix *Add(double alpha, const HypreParMatrix &A,
@@ -1995,6 +2084,8 @@ void EliminateBC(HypreParMatrix &A, HypreParMatrix &Ae,
   double *data_offd = hypre_CSRMatrixData(A_offd);
 #endif

+   ess_dof_list.HostRead();
+
   for (int i = 0; i < ess_dof_list.Size(); i++)
   {
      int r = ess_dof_list[i];
@@ -2161,6 +2252,7 @@ HypreSmoother::HypreSmoother() : Solver()
   B = X = V = Z = NULL;
   X0 = X1 = NULL;
   fir_coeffs = NULL;
+   A_is_symmetric = false;
 }

 HypreSmoother::HypreSmoother(HypreParMatrix &_A, int _type,
@@ -2180,6 +2272,7 @@ HypreSmoother::HypreSmoother(HypreParMatrix &_A, int _type,
   B = X = V = Z = NULL;
   X0 = X1 = NULL;
   fir_coeffs = NULL;
+   A_is_symmetric = false;

   SetOperator(_A);
 }
@@ -2467,6 +2560,16 @@ void HypreSmoother::Mult(const Vector &b, Vector &x) const
   Mult(*B, *X);
 }

+void HypreSmoother::MultTranspose(const Vector &b, Vector &x) const
+{
+   if (A_is_symmetric || type == 0 || type == 1 || type == 5)
+   {
+      Mult(b, x);
+      return;
+   }
+   mfem_error("HypreSmoother::MultTranspose (...) : undefined!\n");
+}
+
 HypreSmoother::~HypreSmoother()
 {
   if (B) { delete B; }
@@ -2511,6 +2614,14 @@ void HypreSolver::Mult(const HypreParVector &b, HypreParVector &x) const
      mfem_error("HypreSolver::Mult (...) : HypreParMatrix A is missing");
      return;
   }
+
+   if (!iterative_mode)
+   {
+      x = 0.0;
+   }
+
+   b.HostRead();
+   x.HostReadWrite();
   if (!setup_called)
   {
      err = SetupFcn()(*this, *A, b, x);
@@ -2526,10 +2637,6 @@ void HypreSolver::Mult(const HypreParVector &b, HypreParVector &x) const
      setup_called = 1;
   }

-   if (!iterative_mode)
-   {
-      x = 0.0;
-   }
   err = SolveFcn()(*this, *A, b, x);
   if (error_mode == WARN_HYPRE_ERRORS)
   {
@@ -2550,7 +2657,7 @@ void HypreSolver::Mult(const Vector &b, Vector &x) const
      return;
   }
   auto b_data = b.HostRead();
-   auto x_data = x.HostWrite();
+   auto x_data = iterative_mode ? x.HostReadWrite() : x.HostWrite();
   if (B == NULL)
   {
      B = new HypreParVector(A->GetComm(),
@@ -2673,6 +2780,11 @@ void HyprePCG::Mult(const HypreParVector &b, HypreParVector &x) const

   HYPRE_ParCSRMatrixGetComm(*A, &comm);

+   if (!iterative_mode)
+   {
+      x = 0.0;
+   }
+
   if (!setup_called)
   {
      if (print_level > 0 && print_level < 3)
@@ -2681,6 +2793,8 @@ void HyprePCG::Mult(const HypreParVector &b, HypreParVector &x) const
         hypre_BeginTiming(time_index);
      }

+      b.HostRead();
+      x.HostReadWrite();
      HYPRE_ParCSRPCGSetup(pcg_solver, *A, b, x);
      setup_called = 1;

@@ -2699,14 +2813,8 @@ void HyprePCG::Mult(const HypreParVector &b, HypreParVector &x) const
      hypre_BeginTiming(time_index);
   }

-   if (!iterative_mode)
-   {
-      x = 0.0;
-   }
-
   b.HostRead();
   x.HostReadWrite();
-
   HYPRE_ParCSRPCGSolve(pcg_solver, *A, b, x);

   if (print_level > 0)
@@ -2797,6 +2905,11 @@ void HypreGMRES::SetTol(double tol)
   HYPRE_GMRESSetTol(gmres_solver, tol);
 }

+void HypreGMRES::SetAbsTol(double tol)
+{
+   HYPRE_GMRESSetAbsoluteTol(gmres_solver, tol);
+}
+
 void HypreGMRES::SetMaxIter(int max_iter)
 {
   HYPRE_GMRESSetMaxIter(gmres_solver, max_iter);
@@ -3600,6 +3713,116 @@ void HypreBoomerAMG::SetElasticityOptions(ParFiniteElementSpace *fespace)
   error_mode = IGNORE_HYPRE_ERRORS;
 }

+#if MFEM_HYPRE_VERSION >= 21800
+
+void HypreBoomerAMG::SetAdvectiveOptions(int distanceR,
+                                         const std::string &prerelax,
+                                         const std::string &postrelax)
+{
+   // Hypre parameters
+   int Sabs = 0;
+   int interp_type = 100;
+   int relax_type = 10;
+   int coarsen_type = 6;
+   double strength_tolC = 0.1;
+   double strength_tolR = 0.01;
+   double filter_tolR = 0.0;
+   double filterA_tol = 0.0;
+
+   // Set relaxation on specified grid points
+   int ns_down, ns_up, ns_coarse;
+   if (distanceR > 0)
+   {
+      ns_down = prerelax.length();
+      ns_up = postrelax.length();
+      ns_coarse = 1;
+
+      // Array to store relaxation scheme and pass to Hypre
+      HYPRE_Int **grid_relax_points = mfem_hypre_TAlloc(HYPRE_Int*, 4);
+      grid_relax_points[0] = NULL;
+      grid_relax_points[1] = mfem_hypre_TAlloc(HYPRE_Int, ns_down);
+      grid_relax_points[2] = mfem_hypre_TAlloc(HYPRE_Int, ns_up);
+      grid_relax_points[3] = mfem_hypre_TAlloc(HYPRE_Int, 1);
+      grid_relax_points[3][0] = 0;
+
+      // set down relax scheme
+      for (int i = 0; i<ns_down; i++)
+      {
+         if (prerelax[i] == 'F')
+         {
+            grid_relax_points[1][i] = -1;
+         }
+         else if (prerelax[i] == 'C')
+         {
+            grid_relax_points[1][i] = 1;
+         }
+         else if (prerelax[i] == 'A')
+         {
+            grid_relax_points[1][i] = 0;
+         }
+      }
+
+      // set up relax scheme
+      for (int i = 0; i<ns_up; i++)
+      {
+         if (postrelax[i] == 'F')
+         {
+            grid_relax_points[2][i] = -1;
+         }
+         else if (postrelax[i] == 'C')
+         {
+            grid_relax_points[2][i] = 1;
+         }
+         else if (postrelax[i] == 'A')
+         {
+            grid_relax_points[2][i] = 0;
+         }
+      }
+
+      HYPRE_BoomerAMGSetRestriction(amg_precond, distanceR);
+
+      HYPRE_BoomerAMGSetGridRelaxPoints(amg_precond, grid_relax_points);
+
+      HYPRE_BoomerAMGSetInterpType(amg_precond, interp_type);
+   }
+
+   if (Sabs)
+   {
+      HYPRE_BoomerAMGSetSabs(amg_precond, Sabs);
+   }
+
+   HYPRE_BoomerAMGSetCoarsenType(amg_precond, coarsen_type);
+
+   // does not support aggressive coarsening
+   HYPRE_BoomerAMGSetAggNumLevels(amg_precond, 0);
+
+   HYPRE_BoomerAMGSetStrongThreshold(amg_precond, strength_tolC);
+
+   if (distanceR > 0)
+   {
+      HYPRE_BoomerAMGSetStrongThresholdR(amg_precond, strength_tolR);
+      HYPRE_BoomerAMGSetFilterThresholdR(amg_precond, filter_tolR);
+   }
+
+   if (relax_type > -1)
+   {
+      HYPRE_BoomerAMGSetRelaxType(amg_precond, relax_type);
+   }
+
+   if (distanceR > 0)
+   {
+      HYPRE_BoomerAMGSetCycleNumSweeps(amg_precond, ns_coarse, 3);
+      HYPRE_BoomerAMGSetCycleNumSweeps(amg_precond, ns_down,   1);
+      HYPRE_BoomerAMGSetCycleNumSweeps(amg_precond, ns_up,     2);
+
+      HYPRE_BoomerAMGSetADropTol(amg_precond, filterA_tol);
+      // type = -1: drop based on row inf-norm
+      HYPRE_BoomerAMGSetADropType(amg_precond, -1);
+   }
+}
+
+#endif
+
 HypreBoomerAMG::~HypreBoomerAMG()
 {
   for (int i = 0; i < rbms.Size(); i++)
@@ -3703,6 +3926,9 @@ void HypreAMS::Init(ParFiniteElementSpace *edge_fespace)
      }
      x = x_coord.ParallelProject();
      y = y_coord.ParallelProject();
+
+      x->HostReadWrite();
+      y->HostReadWrite();
      if (sdim == 2)
      {
         z = NULL;
@@ -3711,6 +3937,7 @@ void HypreAMS::Init(ParFiniteElementSpace *edge_fespace)
      else
      {
         z = z_coord.ParallelProject();
+         z->HostReadWrite();
         HYPRE_AMSSetCoordinateVectors(ams, *x, *y, *z);
      }
   }
@@ -81,6 +81,14 @@ private:
   inline void _SetDataAndSize_();

 public:
+
+   /// Default constructor, no underlying @a hypre_ParVector is created.
+   HypreParVector()
+   {
+      own_ParVector = false;
+      x = NULL;
+   }
+
   /** @brief Creates vector with given global size and parallel partitioning of
       the rows/columns given by @a col. */
   /** @anchor hypre_partitioning_descr
@@ -113,6 +121,9 @@ public:
   /// MPI communicator
   MPI_Comm GetComm() { return x->comm; }

+   /// Converts hypre's format to HypreParVector
+   void WrapHypreParVector(hypre_ParVector *y, bool owner=true);
+
   /// Returns the parallel row/column partitioning
   /** See @ref hypre_partitioning_descr "here" for a description of the
       partitioning array. */
@@ -228,15 +239,24 @@ public:
   /// An empty matrix to be used as a reference to an existing matrix
   HypreParMatrix();

+   /// Converts hypre's format to HypreParMatrix
+   /** If @a owner is false, ownership of @a a is not transferred */
+   void WrapHypreParCSRMatrix(hypre_ParCSRMatrix *a, bool owner = true)
+   {
+      Destroy();
+      Init();
+      A = a;
+      ParCSROwner = owner;
+      height = GetNumRows();
+      width = GetNumCols();
+   }
+
   /// Converts hypre's format to HypreParMatrix
   /** If @a owner is false, ownership of @a a is not transferred */
   explicit HypreParMatrix(hypre_ParCSRMatrix *a, bool owner = true)
   {
      Init();
-      A = a;
-      if (!owner) { ParCSROwner = 0; }
-      height = GetNumRows();
-      width = GetNumCols();
+      WrapHypreParCSRMatrix(a, owner);
   }

   /// Creates block-diagonal square parallel matrix.
@@ -387,6 +407,13 @@ public:
   void GetDiag(SparseMatrix &diag) const;
   /// Get the local off-diagonal block. NOTE: 'offd' will not own any data.
   void GetOffd(SparseMatrix &offd, HYPRE_Int* &cmap) const;
+   /** @brief Get a single SparseMatrix containing all rows from this processor,
+       merged from the diagonal and off-diagonal blocks stored by the
+       HypreParMatrix. */
+   /** @note The number of columns in the SparseMatrix will be the global number
+       of columns in the parallel matrix, so using this method may result in an
+       integer overflow in the column indices. */
+   void MergeDiagAndOffd(SparseMatrix &merged);

   /** Split the matrix into M x N equally sized blocks of parallel matrices.
       The size of 'blocks' must already be set to M x N. */
@@ -397,6 +424,13 @@ public:
   /// Returns the transpose of *this
   HypreParMatrix * Transpose() const;

+   /** Returns principle submatrix given by array of indices of connections
+       with relative size > @a threshold in *this. */
+#if MFEM_HYPRE_VERSION >= 21800
+   HypreParMatrix *ExtractSubmatrix(const Array<int> &indices,
+                                    double threshhold=0.0) const;
+#endif
+
   /// Returns the number of rows in the diagonal block of the ParCSRMatrix
   int GetNumRows() const
   {
@@ -549,6 +583,23 @@ public:
   Type GetType() const { return Hypre_ParCSR; }
 };

+#if MFEM_HYPRE_VERSION >= 21800
+
+enum class BlockInverseScaleJob
+{
+   MATRIX_ONLY,
+   RHS_ONLY,
+   MATRIX_AND_RHS
+};
+
+/** Constructs and applies block diagonal inverse of HypreParMatrix.
+    The enum @a job specifies whether the matrix or the RHS should be
+    scaled (or both). */
+void BlockInverseScale(const HypreParMatrix *A, HypreParMatrix *C,
+                       const Vector *b, HypreParVector *d,
+                       int blocksize, BlockInverseScaleJob job);
+#endif
+
 /** @brief Return a new matrix `C = alpha*A + beta*B`, assuming that both `A`
    and `B` use the same row and column partitions and the same `col_map_offd`
    arrays. */
@@ -638,6 +689,9 @@ protected:
   /// Combined coefficients for windowing and Chebyshev polynomials.
   double* fir_coeffs;

+   /// A flag that indicates whether the linear system matrix A is symmetric
+   bool A_is_symmetric;
+
 public:
   /** Hypre smoother types:
       0    = Jacobi
@@ -646,11 +700,12 @@ public:
       4    = truncated l1-scaled block Gauss-Seidel/SSOR
       5    = lumped Jacobi
       6    = Gauss-Seidel
+       10   = On-processor forward solve for matrix w/ triangular structure
       16   = Chebyshev
       1001 = Taubin polynomial smoother
       1002 = FIR polynomial smoother. */
   enum Type { Jacobi = 0, l1Jacobi = 1, l1GS = 2, l1GStr = 4, lumpedJacobi = 5,
-               GS = 6, Chebyshev = 16, Taubin = 1001, FIR = 1002
+               GS = 6, OPFS = 10, Chebyshev = 16, Taubin = 1001, FIR = 1002
             };

   HypreSmoother();
@@ -684,6 +739,12 @@ public:
       entries in the associated matrix. */
   void SetPositiveDiagonal(bool pos = true) { pos_l1_norms = pos; }

+   /** Explicitly indicate whether the linear system matrix A is symmetric. If A
+       is symmetric, the smoother will also be symmetric. In this case, calling
+       MultTranspose will be redirected to Mult. (This is also done if the
+       smoother is diagonal.) By default, A is assumed to be nonsymmetric. */
+   void SetOperatorSymmetry(bool is_sym) { A_is_symmetric = is_sym; }
+
   /** Set/update the associated operator. Must be called after setting the
       HypreSmoother type and options. */
   virtual void SetOperator(const Operator &op);
@@ -692,6 +753,9 @@ public:
   virtual void Mult(const HypreParVector &b, HypreParVector &x) const;
   virtual void Mult(const Vector &b, Vector &x) const;

+   /// Apply transpose of the smoother to relax the linear system Ax=b
+   virtual void MultTranspose(const Vector &b, Vector &x) const;
+
   virtual ~HypreSmoother();
 };

@@ -755,6 +819,28 @@ public:
   virtual ~HypreSolver();
 };

+
+#if MFEM_HYPRE_VERSION >= 21800
+/** Preconditioner for HypreParMatrices that are triangular in some ordering.
+   Finds correct ordering and performs forward substitution on processor
+   as approximate inverse. Exact on one processor. */
+class HypreTriSolve : public HypreSolver
+{
+public:
+   HypreTriSolve() : HypreSolver() { }
+   explicit HypreTriSolve(HypreParMatrix &A) : HypreSolver(&A) { }
+   virtual operator HYPRE_Solver() const { return NULL; }
+
+   virtual HYPRE_PtrToParSolverFcn SetupFcn() const
+   { return (HYPRE_PtrToParSolverFcn) HYPRE_ParCSROnProcTriSetup; }
+   virtual HYPRE_PtrToParSolverFcn SolveFcn() const
+   { return (HYPRE_PtrToParSolverFcn) HYPRE_ParCSROnProcTriSolve; }
+
+   HypreParMatrix* GetData() { return A; }
+   virtual ~HypreTriSolve() { }
+};
+#endif
+
 /// PCG solver in hypre
 class HyprePCG : public HypreSolver
 {
@@ -832,6 +918,7 @@ public:
   virtual void SetOperator(const Operator &op);

   void SetTol(double tol);
+   void SetAbsTol(double tol);
   void SetMaxIter(int max_iter);
   void SetKDim(int dim);
   void SetLogging(int logging);
@@ -1116,9 +1203,94 @@ public:
       construct A. */
   void SetElasticityOptions(ParFiniteElementSpace *fespace);

+#if MFEM_HYPRE_VERSION >= 21800
+   /** Hypre parameters to use AIR AMG solve for advection-dominated problems.
+       See "Nonsymmetric Algebraic Multigrid Based on Local Approximate Ideal
+       Restriction (AIR)," Manteuffel, Ruge, Southworth, SISC (2018),
+       DOI:/10.1137/17M1144350. Options: "distanceR" -> distance of neighbor
+       DOFs to buld restriction operator; options include 1, 2, and 15 (1.5).
+       Strings "prerelax" and "postrelax" indicate points to relax on:
+       F = F-points, C = C-points, A = all points. E.g., FFC -> relax on
+       F-points, relax again on F-points, then relax on C-points. */
+   void SetAdvectiveOptions(int distance=15,  const std::string &prerelax="",
+                            const std::string &postrelax="FFC");
+
+   /// Expert option - consult hypre documentation/team
+   void SetStrongThresholdR(double strengthR)
+   { HYPRE_BoomerAMGSetStrongThresholdR(amg_precond, strengthR); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetFilterThresholdR(double filterR)
+   { HYPRE_BoomerAMGSetFilterThresholdR(amg_precond, filterR); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetRestriction(int restrict_type)
+   { HYPRE_BoomerAMGSetRestriction(amg_precond, restrict_type); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetIsTriangular()
+   { HYPRE_BoomerAMGSetIsTriangular(amg_precond, 1); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetGMRESSwitchR(int gmres_switch)
+   { HYPRE_BoomerAMGSetGMRESSwitchR(amg_precond, gmres_switch); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetCycleNumSweeps(int prerelax, int postrelax)
+   {
+      HYPRE_BoomerAMGSetCycleNumSweeps(amg_precond, prerelax,  1);
+      HYPRE_BoomerAMGSetCycleNumSweeps(amg_precond, postrelax, 2);
+   }
+#endif
+
   void SetPrintLevel(int print_level)
   { HYPRE_BoomerAMGSetPrintLevel(amg_precond, print_level); }

+   void SetMaxIter(int max_iter)
+   { HYPRE_BoomerAMGSetMaxIter(amg_precond, max_iter); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetMaxLevels(int max_levels)
+   { HYPRE_BoomerAMGSetMaxLevels(amg_precond, max_levels); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetTol(double tol)
+   { HYPRE_BoomerAMGSetTol(amg_precond, tol); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetStrengthThresh(double strength)
+   { HYPRE_BoomerAMGSetStrongThreshold(amg_precond, strength); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetInterpolation(int interp_type)
+   { HYPRE_BoomerAMGSetInterpType(amg_precond, interp_type); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetCoarsening(int coarsen_type)
+   { HYPRE_BoomerAMGSetCoarsenType(amg_precond, coarsen_type); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetRelaxType(int relax_type)
+   { HYPRE_BoomerAMGSetRelaxType(amg_precond, relax_type); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetCycleType(int cycle_type)
+   { HYPRE_BoomerAMGSetCycleType(amg_precond, cycle_type); }
+
+   void GetNumIterations(int &num_it)
+   { HYPRE_BoomerAMGGetNumIterations(amg_precond, &num_it); }
+
+   /// Expert option - consult hypre documentation/team
+   void SetNodal(int blocksize)
+   {
+      HYPRE_BoomerAMGSetNumFunctions(amg_precond, blocksize);
+      HYPRE_BoomerAMGSetNodal(amg_precond, 1);
+   }
+
+   /// Expert option - consult hypre documentation/team
+   void SetAggressiveCoarsening(int num_levels)
+   { HYPRE_BoomerAMGSetAggNumLevels(amg_precond, num_levels); }
+
   /// The typecast to HYPRE_Solver returns the internal amg_precond
   virtual operator HYPRE_Solver() const { return amg_precond; }

@@ -1127,6 +1299,8 @@ public:
   virtual HYPRE_PtrToParSolverFcn SolveFcn() const
   { return (HYPRE_PtrToParSolverFcn) HYPRE_BoomerAMGSolve; }

+   using HypreSolver::Mult;
+
   virtual ~HypreBoomerAMG();
 };

@@ -62,8 +62,6 @@ protected:
   const scalar_t *D; // Always points to external data or is empty
   scalar_t *DaJ, *DJt, *DXt, *DYt;

-   scalar_t sign_detJ;
-
   enum EvalMasks
   {
      HAVE_I1   = 1,
@@ -96,8 +94,7 @@ protected:
   {
      eval_state |= HAVE_I2b;
      const scalar_t det = J[0]*J[3] - J[1]*J[2];
-      sign_detJ = scalar_ops::sign(det);
-      I2b = sign_detJ*det;
+      I2b = det;
   }
   void Eval_dI1()
   {
@@ -136,10 +133,10 @@ protected:
      // I2b = det(J)
      // dI2b = adj(J)^T
      Get_I2b();
-      dI2b[0] =  sign_detJ*J[3];
-      dI2b[1] = -sign_detJ*J[2];
-      dI2b[2] = -sign_detJ*J[1];
-      dI2b[3] =  sign_detJ*J[0];
+      dI2b[0] =  J[3];
+      dI2b[1] = -J[2];
+      dI2b[2] = -J[1];
+      dI2b[3] =  J[0];
   }
   void Eval_DaJ() // D adj(J) = D dI2b^t
   {
@@ -516,8 +513,6 @@ protected:
   const scalar_t *D; // Always points to external data or is empty
   scalar_t *DaJ, *DJt, *DdI2t, *DXt, *DYt;

-   scalar_t sign_detJ;
-
   enum EvalMasks
   {
      HAVE_I1     = 1,
@@ -585,8 +580,6 @@ protected:
      eval_state |= HAVE_I3b;
      I3b = J[0]*(J[4]*J[8] - J[7]*J[5]) - J[1]*(J[3]*J[8] - J[5]*J[6]) +
            J[2]*(J[3]*J[7] - J[4]*J[6]);
-      sign_detJ = scalar_ops::sign(I3b);
-      I3b = sign_detJ*I3b;
   }
   scalar_t Get_I3b_p()  // I3b^{-2/3}
   {
@@ -594,7 +587,7 @@ protected:
      {
         eval_state |= HAVE_I3b_p;
         const scalar_t i3b = Get_I3b();
-         I3b_p = sign_detJ*scalar_ops::pow(i3b, -2, 3);
+         I3b_p = scalar_ops::pow(i3b, -2, 3);
      }
      return I3b_p;
   }
@@ -680,15 +673,15 @@ protected:
      eval_state |= HAVE_dI3b;
      // I3b = det(J)
      // dI3b = adj(J)^T
-      dI3b[0] = sign_detJ*(J[4]*J[8] - J[5]*J[7]);  // 0  3  6
-      dI3b[1] = sign_detJ*(J[5]*J[6] - J[3]*J[8]);  // 1  4  7
-      dI3b[2] = sign_detJ*(J[3]*J[7] - J[4]*J[6]);  // 2  5  8
-      dI3b[3] = sign_detJ*(J[2]*J[7] - J[1]*J[8]);
-      dI3b[4] = sign_detJ*(J[0]*J[8] - J[2]*J[6]);
-      dI3b[5] = sign_detJ*(J[1]*J[6] - J[0]*J[7]);
-      dI3b[6] = sign_detJ*(J[1]*J[5] - J[2]*J[4]);
-      dI3b[7] = sign_detJ*(J[2]*J[3] - J[0]*J[5]);
-      dI3b[8] = sign_detJ*(J[0]*J[4] - J[1]*J[3]);
+      dI3b[0] = J[4]*J[8] - J[5]*J[7];  // 0  3  6
+      dI3b[1] = J[5]*J[6] - J[3]*J[8];  // 1  4  7
+      dI3b[2] = J[3]*J[7] - J[4]*J[6];  // 2  5  8
+      dI3b[3] = J[2]*J[7] - J[1]*J[8];
+      dI3b[4] = J[0]*J[8] - J[2]*J[6];
+      dI3b[5] = J[1]*J[6] - J[0]*J[7];
+      dI3b[6] = J[1]*J[5] - J[2]*J[4];
+      dI3b[7] = J[2]*J[3] - J[0]*J[5];
+      dI3b[8] = J[0]*J[4] - J[1]*J[3];
   }
   void Eval_DZt(const scalar_t *Z, scalar_t **DZt_ptr)
   {
@@ -24,6 +24,7 @@
 #include "blockoperator.hpp"
 #include "sparsesmoothers.hpp"
 #include "densemat.hpp"
+#include "symmat.hpp"
 #include "ode.hpp"
 #include "solvers.hpp"
 #include "handle.hpp"
@@ -196,6 +196,8 @@ void MUMPSSolver::SetOperator(const Operator &op)

 void MUMPSSolver::Mult(const Vector &x, Vector &y) const
 {
+   x.HostRead();
+   y.HostReadWrite();
 #if MFEM_MUMPS_VERSION >= 530

   id->nloc_rhs = x.Size();
@@ -37,7 +37,8 @@ protected:
      const Array<int> &test_tdof_list,
      RectangularConstrainedOperator* &Aout);

-   /// Returns RAP Operator of this, taking in input/output Prolongation matrices
+   /** @brief Returns RAP Operator of this, using input/output Prolongation matrices
+       @a Pi corresponds to "P", @a Po corresponds to "Rt" */
   Operator *SetupRAP(const Operator *Pi, const Operator *Po);

 public:
@@ -112,6 +113,11 @@ public:
   {
      return GetProlongation(); // Assume square unless specialized
   }
+   /** @brief Transpose of GetOutputRestriction, directly available in this
+       form to facilitate matrix-free RAP-type operators.
+
+       `NULL` means identity. */
+   virtual const Operator *GetOutputRestrictionTranspose() const { return NULL; }
   /** @brief Restriction operator from output vectors for the operator to linear
       algebra (linear system) vectors. `NULL` means identity. */
   virtual const Operator *GetOutputRestriction() const
@@ -606,23 +612,22 @@ public:

   using TimeDependentOperator::ImplicitSolve;
   /** @brief Solve the equation:
-       @a k = f(@a x + 1/2 @a dt0^2 @a k, @a dxdt + @a dt1 @a k, t), for the
+       @a k = f(@a x + @a fac0 @a k, @a dxdt + @a fac1 @a k, t), for the
       unknown @a k at the current time t.

       For general F and G, the equation for @a k becomes:
-       F(@a x + 1/2 @a dt0^2 @a k, @a dxdt + @a dt1 @a k, t)
-                        = G(@a x + 1/2 @a dt0^2 @a k, @a dxdt + @a dt1 @a k, t).
+       F(@a x +  @a fac0 @a k, @a dxdt + @a fac1 @a k, t)
+                        = G(@a x +  @a fac0 @a k, @a dxdt + @a fac1 @a k, t).

-       The input vector @a x corresponds to time index (or cycle) n, while the
+       The input vectors @a x and @a dxdt corresponds to time index (or cycle) n, while the
       currently set time, #t, and the result vector @a k correspond to time
-       index n+1. The time step @a dt corresponds to the time interval between
-       cycles n and n+1.
+       index n+1.

       This method allows for the abstract implementation of some time
       integration methods.

       If not re-implemented, this method simply generates an error. */
-   virtual void ImplicitSolve(const double dt0, const double dt1,
+   virtual void ImplicitSolve(const double fac0, const double fac1,
                              const Vector &x, const Vector &dxdt, Vector &k);


@@ -1556,6 +1556,7 @@ void NewtonSolver::SetOperator(const Operator &op)
   width = op.Width();
   MFEM_ASSERT(height == width, "square Operator is required.");

+   xcur.SetSize(width);
   r.SetSize(width);
   c.SetSize(width);
 }
@@ -1615,9 +1616,20 @@ void NewtonSolver::Mult(const Vector &b, Vector &x) const
         break;
      }

-      prec->SetOperator(oper->GetGradient(x));
+      grad = &oper->GetGradient(x);
+      prec->SetOperator(*grad);

-      prec->Mult(r, c);  // c = [DF(x_i)]^{-1} [F(x_i)-b]
+      if (lin_rtol_type)
+      {
+         AdaptiveLinRtolPreSolve(x, it, norm);
+      }
+
+      prec->Mult(r, c); // c = [DF(x_i)]^{-1} [F(x_i)-b]
+
+      if (lin_rtol_type)
+      {
+         AdaptiveLinRtolPostSolve(c, r, it, norm);
+      }

      const double c_scale = ComputeScalingFactor(x, b);
      if (c_scale == 0.0)
@@ -1641,6 +1653,86 @@ void NewtonSolver::Mult(const Vector &b, Vector &x) const
   final_norm = norm;
 }

+void NewtonSolver::SetAdaptiveLinRtol(const int type,
+                                      const double rtol0,
+                                      const double rtol_max,
+                                      const double alpha,
+                                      const double gamma)
+{
+   lin_rtol_type = type;
+   lin_rtol0 = rtol0;
+   lin_rtol_max = rtol_max;
+   this->alpha = alpha;
+   this->gamma = gamma;
+}
+
+void NewtonSolver::AdaptiveLinRtolPreSolve(const Vector &x,
+                                           const int it,
+                                           const double fnorm) const
+{
+   // Assume that when adaptive linear solver relative tolerance is activated,
+   // we are working with an iterative solver.
+   auto iterative_solver = static_cast<IterativeSolver *>(prec);
+   // Adaptive linear solver relative tolerance
+   double eta;
+   // Safeguard threshold
+   double sg_threshold = 0.1;
+
+   if (it == 0)
+   {
+      eta = lin_rtol0;
+   }
+   else
+   {
+      if (lin_rtol_type == 1)
+      {
+         // eta = gamma * abs(||F(x1)|| - ||F(x0) + DF(x0) s0||) / ||F(x0)||
+         eta = gamma * abs(fnorm - lnorm_last) / fnorm_last;
+      }
+      else if (lin_rtol_type == 2)
+      {
+         // eta = gamma * (||F(x1)|| / ||F(x0)||)^alpha
+         eta = gamma * pow(fnorm / fnorm_last, alpha);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown adaptive linear solver rtol version");
+      }
+
+      // Safeguard rtol from "oversolving" ?!
+      const double sg_eta = gamma * pow(eta_last, alpha);
+      if (sg_eta > sg_threshold) { eta = std::max(eta, sg_eta); }
+   }
+
+   eta = std::min(eta, lin_rtol_max);
+   iterative_solver->SetRelTol(eta);
+   eta_last = eta;
+   if (print_level >= 0)
+   {
+      mfem::out << "Eisenstat-Walker rtol = " << eta << "\n";
+   }
+}
+
+void NewtonSolver::AdaptiveLinRtolPostSolve(const Vector &x,
+                                            const Vector &b,
+                                            const int it,
+                                            const double fnorm) const
+{
+   fnorm_last = fnorm;
+
+   // If version 1 is chosen, the true linear residual norm has to be computed
+   // and in most cases we can only retrieve the preconditioned linear residual
+   // norm.
+   if (lin_rtol_type == 1)
+   {
+      // lnorm_last = ||F(x0) + DF(x0) s0||
+      Vector linres(x.Size());
+      grad->Mult(x, linres);
+      linres -= b;
+      lnorm_last = Norm(linres);
+   }
+}
+
 void LBFGSSolver::Mult(const Vector &b, Vector &x) const
 {
   MFEM_VERIFY(oper != NULL, "the Operator is not set (use SetOperator).");
@@ -2720,9 +2812,7 @@ void UMFPackSolver::Init()

 void UMFPackSolver::SetOperator(const Operator &op)
 {
-   int *Ap, *Ai;
   void *Symbolic;
-   double *Ax;

   if (Numeric)
   {
@@ -2748,9 +2838,9 @@ void UMFPackSolver::SetOperator(const Operator &op)
   width = mat->Width();
   MFEM_VERIFY(width == height, "not a square matrix");

-   Ap = mat->GetI();
-   Ai = mat->GetJ();
-   Ax = mat->GetData();
+   const int * Ap = mat->HostReadI();
+   const int * Ai = mat->HostReadJ();
+   const double * Ax = mat->HostReadData();

   if (!use_long_ints)
   {
@@ -2820,12 +2910,13 @@ void UMFPackSolver::Mult(const Vector &b, Vector &x) const
   if (mat == NULL)
      mfem_error("UMFPackSolver::Mult : matrix is not set!"
                 " Call SetOperator first!");
-
+   b.HostRead();
+   x.HostReadWrite();
   if (!use_long_ints)
   {
      int status =
-         umfpack_di_solve(UMFPACK_At, mat->GetI(), mat->GetJ(),
-                          mat->GetData(), x, b, Numeric, Control, Info);
+         umfpack_di_solve(UMFPACK_At, mat->HostReadI(), mat->HostReadJ(),
+                          mat->HostReadData(), x, b, Numeric, Control, Info);
      umfpack_di_report_info(Control, Info);
      if (status < 0)
      {
@@ -2836,7 +2927,7 @@ void UMFPackSolver::Mult(const Vector &b, Vector &x) const
   else
   {
      SuiteSparse_long status =
-         umfpack_dl_solve(UMFPACK_At, AI, AJ, mat->GetData(), x, b,
+         umfpack_dl_solve(UMFPACK_At, AI, AJ, mat->HostReadData(), x, b,
                          Numeric, Control, Info);
      umfpack_dl_report_info(Control, Info);
      if (status < 0)
@@ -2852,12 +2943,13 @@ void UMFPackSolver::MultTranspose(const Vector &b, Vector &x) const
   if (mat == NULL)
      mfem_error("UMFPackSolver::MultTranspose : matrix is not set!"
                 " Call SetOperator first!");
-
+   b.HostRead();
+   x.HostReadWrite();
   if (!use_long_ints)
   {
      int status =
-         umfpack_di_solve(UMFPACK_A, mat->GetI(), mat->GetJ(),
-                          mat->GetData(), x, b, Numeric, Control, Info);
+         umfpack_di_solve(UMFPACK_A, mat->HostReadI(), mat->HostReadJ(),
+                          mat->HostReadData(), x, b, Numeric, Control, Info);
      umfpack_di_report_info(Control, Info);
      if (status < 0)
      {
@@ -2869,7 +2961,7 @@ void UMFPackSolver::MultTranspose(const Vector &b, Vector &x) const
   else
   {
      SuiteSparse_long status =
-         umfpack_dl_solve(UMFPACK_A, AI, AJ, mat->GetData(), x, b,
+         umfpack_dl_solve(UMFPACK_A, AI, AJ, mat->HostReadData(), x, b,
                          Numeric, Control, Info);
      umfpack_dl_report_info(Control, Info);
      if (status < 0)
@@ -28,6 +28,8 @@
 namespace mfem
 {

+void MinimumDiscardedFillOrdering(SparseMatrix &C, Array<int> &p);
+
 class BilinearForm;

 /// Abstract base class for an iterative solver monitor
@@ -406,7 +408,40 @@ void MINRES(const Operator &A, Solver &B, const Vector &b, Vector &x,
 class NewtonSolver : public IterativeSolver
 {
 protected:
-   mutable Vector r, c;
+   mutable Vector xcur, r, c;
+   mutable Operator *grad;
+
+   // Adaptive linear solver rtol variables
+
+   // Method to determine rtol, 0 means the adaptive algorithm is deactivated.
+   int lin_rtol_type = 0;
+   // rtol to use in first iteration
+   double lin_rtol0;
+   // Maximum rtol
+   double lin_rtol_max;
+   // Function norm ||F(x)|| of the previous iterate
+   mutable double fnorm_last = 0.0;
+   // Linear residual norm of the previous iterate
+   mutable double lnorm_last = 0.0;
+   // Forcing term (linear residual rtol) from the previous iterate
+   mutable double eta_last = 0.0;
+   // Eisenstat-Walker factor gamma
+   double gamma;
+   // Eisenstat-Walker factor alpha
+   double alpha;
+
+   /** @brief Method for the adaptive linear solver rtol invoked before the
+       linear solve. */
+   void AdaptiveLinRtolPreSolve(const Vector &x,
+                                const int it,
+                                const double fnorm) const;
+
+   /** @brief Method for the adaptive linear solver rtol invoked after the
+       linear solve. */
+   void AdaptiveLinRtolPostSolve(const Vector &x,
+                                 const Vector &b,
+                                 const int it,
+                                 const double fnorm) const;

 public:
   NewtonSolver() { }
@@ -434,6 +469,26 @@ public:
   /** @brief This method can be overloaded in derived classes to perform
       computations that need knowledge of the newest Newton state. */
   virtual void ProcessNewState(const Vector &x) const { }
+
+   const Vector &GetCurrentResidual() const { return r; }
+   const Vector &GetCurrentIterate() const { return xcur; }
+
+   /// Enable adaptive linear solver relative tolerance algorithm.
+   /** Compute a relative tolerance for the Krylov method after each nonlinear
+    iteration, based on the algorithm presented in [1].
+
+    The maximum linear solver relative tolerance @a rtol_max should be < 1. For
+    @a type 1 the parameters @a alpha and @a gamma are ignored. For @a type 2
+    @a alpha has to be between 0 and 1 and @a gamma between 1 and 2.
+
+    [1] Eisenstat, Stanley C., and Homer F. Walker. "Choosing the forcing terms
+    in an inexact Newton method."
+    */
+   void SetAdaptiveLinRtol(const int type = 2,
+                           const double rtol0 = 0.5,
+                           const double rtol_max = 0.9,
+                           const double alpha = 0.5 * (1.0 + sqrt(5.0)),
+                           const double gamma = 1.0);
 };

 /** L-BFGS method for solving F(x)=b for a given operator F, by minimizing
@@ -415,10 +415,14 @@ void SparseMatrix::SortColumnIndices()
      return;
   }

+   const int * Ip=HostReadI();
+   HostReadWriteJ();
+   HostReadWriteData();
+
   Array<Pair<int,double> > row;
   for (int j = 0, i = 0; i < height; i++)
   {
-      int end = I[i+1];
+      int end = Ip[i+1];
      row.SetSize(end - j);
      for (int k = 0; k < row.Size(); k++)
      {
@@ -3358,12 +3362,12 @@ SparseMatrix *Mult (const SparseMatrix &A, const SparseMatrix &B,
               "number of columns of A (" << ncolsA
               << ") must equal number of rows of B (" << nrowsB << ")");

-   A_i    = A.GetI();
-   A_j    = A.GetJ();
-   A_data = A.GetData();
-   B_i    = B.GetI();
-   B_j    = B.GetJ();
-   B_data = B.GetData();
+   A_i    = A.HostReadI();
+   A_j    = A.HostReadJ();
+   A_data = A.HostReadData();
+   B_i    = B.HostReadI();
+   B_j    = B.HostReadJ();
+   B_data = B.HostReadData();

   B_marker = new int[ncolsB];

@@ -3409,16 +3413,16 @@ SparseMatrix *Mult (const SparseMatrix &A, const SparseMatrix &B,
   {
      C = OAB;

-      MFEM_VERIFY(nrowsA == C -> Height() && ncolsB == C -> Width(),
+      MFEM_VERIFY(nrowsA == C->Height() && ncolsB == C->Width(),
                  "Input matrix sizes do not match output sizes"
                  << " nrowsA = " << nrowsA
                  << ", C->Height() = " << C->Height()
                  << " ncolsB = " << ncolsB
                  << ", C->Width() = " << C->Width());

-      // C_i    = C -> GetI(); // not used
-      C_j    = C -> GetJ();
-      C_data = C -> GetData();
+      // C_i    = C->HostReadI(); // not used
+      C_j    = C->HostWriteJ();
+      C_data = C->HostWriteData();
   }

   counter = 0;
@@ -523,9 +523,10 @@ void SuperLUSolver::Mult( const Vector & x, Vector & y ) const
   // SuperLU overwrites x with y, so copy x to y and pass that to the solve
   // routine.

-   y = x;
+   const double *xPtr = x.HostRead();
+   y = xPtr;
+   double * yPtr = y.HostReadWrite();

-   double*  yPtr = (double*)y;
   int      info = -1, locSize = y.Size();

   // Solve the system
@@ -0,0 +1,109 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+
+// Implementation of data type DenseSymmetricMatrix
+
+#include "symmat.hpp"
+
+namespace mfem
+{
+
+DenseSymmetricMatrix::DenseSymmetricMatrix() : Matrix(0)
+{
+   data.Reset();
+}
+
+DenseSymmetricMatrix::DenseSymmetricMatrix(int s) : Matrix(s)
+{
+   MFEM_ASSERT(s >= 0, "invalid DenseSymmetricMatrix size: " << s);
+   if (s > 0)
+   {
+      data.New((s*(s+1))/2);
+      *this = 0.0; // init with zeroes
+   }
+   else
+   {
+      data.Reset();
+   }
+}
+
+void DenseSymmetricMatrix::SetSize(int s)
+{
+   MFEM_ASSERT(s >= 0,
+               "invalid DenseSymmetricMatrix size: " << s);
+   if (Height() == s)
+   {
+      return;
+   }
+   height = s;
+   width = s;
+   const int s2 = (s*(s+1))/2;
+   if (s2 > data.Capacity())
+   {
+      data.Delete();
+      data.New(s2);
+      *this = 0.0; // init with zeroes
+   }
+}
+
+DenseSymmetricMatrix &DenseSymmetricMatrix::operator=(double c)
+{
+   const int s = (Height()*(Height()+1))/2;
+   for (int i = 0; i < s; i++)
+   {
+      data[i] = c;
+   }
+   return *this;
+}
+
+double &DenseSymmetricMatrix::Elem(int i, int j)
+{
+   return (*this)(i,j);
+}
+
+const double &DenseSymmetricMatrix::Elem(int i, int j) const
+{
+   return (*this)(i,j);
+}
+
+DenseSymmetricMatrix &DenseSymmetricMatrix::operator*=(double c)
+{
+   int s = Height()*(Height()+1)/2;
+   for (int i = 0; i < s; i++)
+   {
+      data[i] *= c;
+   }
+   return *this;
+}
+
+void DenseSymmetricMatrix::Mult(const Vector &x, Vector &y) const
+{
+   mfem_error("DenseSymmetricMatrix::Mult() not implemented!");
+}
+
+MatrixInverse *DenseSymmetricMatrix::Inverse() const
+{
+   mfem_error("DenseSymmetricMatrix::Inverse() not implemented!");
+   return nullptr;
+}
+
+void DenseSymmetricMatrix::Print (std::ostream & out, int width_) const
+{
+   mfem_error("DenseSymmetricMatrix::Print() not implemented!");
+}
+
+DenseSymmetricMatrix::~DenseSymmetricMatrix()
+{
+   data.Delete();
+}
+
+}
@@ -0,0 +1,175 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_SYMMETRICMAT
+#define MFEM_SYMMETRICMAT
+
+#include "../config/config.hpp"
+#include "../general/globals.hpp"
+#include "matrix.hpp"
+
+namespace mfem
+{
+
+/// Dense symmetric matrix storing the upper triangular part. This class so far
+/// has little functionality beyond storage.
+class DenseSymmetricMatrix : public Matrix
+{
+private:
+   Memory<double> data;
+
+public:
+
+   /** Default constructor for DenseSymmetricMatrix.
+       Sets data = NULL and height = width = 0. */
+   DenseSymmetricMatrix();
+
+   /// Creates square matrix of size s.
+   explicit DenseSymmetricMatrix(int s);
+
+   /// Construct a DenseSymmetricMatrix using an existing data array.
+   /** The DenseSymmetricMatrix does not assume ownership of the data array, i.e. it will
+       not delete the array. */
+   DenseSymmetricMatrix(double *d, int s)
+      : Matrix(s, s) { UseExternalData(d, s); }
+
+   /// Change the data array and the size of the DenseSymmetricMatrix.
+   /** The DenseSymmetricMatrix does not assume ownership of the data array, i.e. it will
+       not delete the data array @a d. This method should not be used with
+       DenseSymmetricMatrix that owns its current data array. */
+   void UseExternalData(double *d, int s)
+   {
+      data.Wrap(d, (s*(s+1))/2, false);
+      height = s; width = s;
+   }
+
+   /// Change the data array and the size of the DenseSymmetricMatrix.
+   /** The DenseSymmetricMatrix does not assume ownership of the data array, i.e. it will
+       not delete the new array @a d. This method will delete the current data
+       array, if owned. */
+   void Reset(double *d, int s)
+   { if (OwnsData()) { data.Delete(); } UseExternalData(d, s); }
+
+   /** Clear the data array and the dimensions of the DenseSymmetricMatrix. This method
+       should not be used with DenseSymmetricMatrix that owns its current data array. */
+   void ClearExternalData() { data.Reset(); height = width = 0; }
+
+   /// Delete the matrix data array (if owned) and reset the matrix state.
+   void Clear()
+   { if (OwnsData()) { data.Delete(); } ClearExternalData(); }
+
+   /// Change the size of the DenseSymmetricMatrix to s x s.
+   void SetSize(int s);
+
+   /// Returns the matrix data array.
+   inline double *Data() const
+   { return const_cast<double*>((const double*)data);}
+
+   /// Returns the matrix data array.
+   inline double *GetData() const { return Data(); }
+
+   Memory<double> &GetMemory() { return data; }
+   const Memory<double> &GetMemory() const { return data; }
+
+   /// Return the DenseSymmetricMatrix data (host pointer) ownership flag.
+   inline bool OwnsData() const { return data.OwnsHostPtr(); }
+
+   /// Returns reference to a_{ij}.
+   inline double &operator()(int i, int j);
+
+   /// Returns constant reference to a_{ij}.
+   inline const double &operator()(int i, int j) const;
+
+   /// Returns reference to a_{ij}.
+   virtual double &Elem(int i, int j);
+
+   /// Returns constant reference to a_{ij}.
+   virtual const double &Elem(int i, int j) const;
+
+   /// Sets the matrix elements equal to constant c
+   DenseSymmetricMatrix &operator=(double c);
+
+   DenseSymmetricMatrix &operator*=(double c);
+
+   long MemoryUsage() const { return data.Capacity() * sizeof(double); }
+
+   /// Shortcut for mfem::Read( GetMemory(), TotalSize(), on_dev).
+   const double *Read(bool on_dev = true) const
+   { return mfem::Read(data, Height()*Width(), on_dev); }
+
+   /// Shortcut for mfem::Read(GetMemory(), TotalSize(), false).
+   const double *HostRead() const
+   { return mfem::Read(data, Height()*Width(), false); }
+
+   /// Shortcut for mfem::Write(GetMemory(), TotalSize(), on_dev).
+   double *Write(bool on_dev = true)
+   { return mfem::Write(data, Height()*Width(), on_dev); }
+
+   /// Shortcut for mfem::Write(GetMemory(), TotalSize(), false).
+   double *HostWrite()
+   { return mfem::Write(data, Height()*Width(), false); }
+
+   /// Shortcut for mfem::ReadWrite(GetMemory(), TotalSize(), on_dev).
+   double *ReadWrite(bool on_dev = true)
+   { return mfem::ReadWrite(data, Height()*Width(), on_dev); }
+
+   /// Shortcut for mfem::ReadWrite(GetMemory(), TotalSize(), false).
+   double *HostReadWrite()
+   { return mfem::ReadWrite(data, Height()*Width(), false); }
+
+   /// Matrix vector multiplication.
+   virtual void Mult(const Vector &x, Vector &y) const;
+
+   /// Returns a pointer to (an approximation) of the matrix inverse.
+   virtual MatrixInverse *Inverse() const;
+
+   /// Prints matrix to stream out.
+   virtual void Print (std::ostream & out = mfem::out, int width_ = 4) const;
+
+   /// Destroys the symmetric matrix.
+   virtual ~DenseSymmetricMatrix();
+};
+
+// Inline methods
+
+// The number of entries stored in rows 1,...,k is
+// n + n-1 + n-2 + ... + n-k+1, where there are k terms. This equals
+// kn - sum_{i=1}^{k-1} i = kn - (k-1)k/2
+// This formula is used for the offset for each row.
+inline double &DenseSymmetricMatrix::operator()(int i, int j)
+{
+   MFEM_ASSERT(data && i >= 0 && i < height && j >= 0 && j < width, "");
+   if (i > j)  // reverse i and j
+   {
+      return data[(j*height) - (((j-1)*j)/2) + i - j];
+   }
+   else
+   {
+      return data[(i*height) - (((i-1)*i)/2) + j - i];
+   }
+}
+
+inline const double &DenseSymmetricMatrix::operator()(int i, int j) const
+{
+   MFEM_ASSERT(data && i >= 0 && i < height && j >= 0 && j < width, "");
+   if (i > j)  // reverse i and j
+   {
+      return data[(j*height) - (((j-1)*j)/2) + i - j];
+   }
+   else
+   {
+      return data[(i*height) - (((i-1)*i)/2) + j - i];
+   }
+}
+
+} // namespace mfem
+
+#endif
@@ -536,7 +536,7 @@ clean: $(addsuffix /clean,$(EM_DIRS) $(TEST_DIRS))
 distclean: clean config/clean doc/clean
 	rm -rf mfem/

-INSTALL_SHARED_LIB = $(MFEM_CXX) $(MFEM_BUILD_FLAGS) $(INSTALL_SOFLAGS)\
+INSTALL_SHARED_LIB = $(MFEM_CXX) $(MFEM_LINK_FLAGS) $(INSTALL_SOFLAGS)\
   $(OBJECT_FILES) $(EXT_LIBS) -o $(PREFIX_LIB)/libmfem.$(SO_VER) && \
   cd $(PREFIX_LIB) && ln -sf libmfem.$(SO_VER) libmfem.$(SO_EXT)

@@ -1049,6 +1049,13 @@ void Mesh::GetFaceInfos(int Face, int *Inf1, int *Inf2) const
   *Inf2 = faces_info[Face].Elem2Inf;
 }

+void Mesh::GetFaceInfos(int Face, int *Inf1, int *Inf2, int *NCFace) const
+{
+   *Inf1   = faces_info[Face].Elem1Inf;
+   *Inf2   = faces_info[Face].Elem2Inf;
+   *NCFace = faces_info[Face].NCFace;
+}
+
 Geometry::Type Mesh::GetFaceGeometryType(int Face) const
 {
   switch (Dim)
@@ -3836,34 +3843,18 @@ Mesh::Mesh(Mesh *orig_mesh, int ref_factor, int ref_type)

      rfes.GetBdrElementDofs(el, rdofs);
      MFEM_ASSERT(rdofs.Size() == RG.RefPts.Size(), "");
-      if (Dim == 1)
+      const int *c2h_map = (Dim == 1) ? NULL : rfec.GetDofMap(geom);
+      for (int j = 0; j < RG.RefGeoms.Size()/nvert; j++)
      {
-         // Dim == 1 is a special case because the boundary elements are
-         // zero-dimensional points, and therefore don't have a DofMap
-         for (int j = 0; j < RG.RefGeoms.Size()/nvert; j++)
+         Element *elem = NewElement(geom);
+         elem->SetAttribute(attrib);
+         int *v = elem->GetVertices();
+         for (int k = 0; k < nvert; k++)
         {
-            Element *elem = NewElement(geom);
-            elem->SetAttribute(attrib);
-            int *v = elem->GetVertices();
-            v[0] = rdofs[RG.RefGeoms[nvert*j]];
-            AddBdrElement(elem);
-         }
-      }
-      else
-      {
-         const int *c2h_map = rfec.GetDofMap(geom);
-         for (int j = 0; j < RG.RefGeoms.Size()/nvert; j++)
-         {
-            Element *elem = NewElement(geom);
-            elem->SetAttribute(attrib);
-            int *v = elem->GetVertices();
-            for (int k = 0; k < nvert; k++)
-            {
-               int cid = RG.RefGeoms[k+nvert*j]; // local Cartesian index
-               v[k] = rdofs[c2h_map[cid]];
-            }
-            AddBdrElement(elem);
+            int cid = RG.RefGeoms[k+nvert*j]; // local Cartesian index
+            v[k] = rdofs[c2h_map ? c2h_map[cid] : cid];
         }
+         AddBdrElement(elem);
      }
   }

@@ -3904,6 +3895,326 @@ Mesh::Mesh(Mesh *orig_mesh, int ref_factor, int ref_type)
   MFEM_ASSERT(CheckBdrElementOrientation(false) == 0, "");
 }

+void Mesh::MakeSimplicial(Mesh &orig_mesh, int *vglobal)
+{
+   MFEM_VERIFY(orig_mesh.CheckElementOrientation(false) == 0,
+               "Mesh::MakeSimplicial requires a properly oriented input mesh");
+
+   int dim = orig_mesh.Dimension();
+   int sdim = orig_mesh.SpaceDimension();
+
+   if (dim == 1)
+   {
+      Mesh copy(orig_mesh);
+      Swap(copy, true);
+      return;
+   }
+
+   int nv = orig_mesh.GetNV();
+   int ne = orig_mesh.GetNE();
+   int nbe = orig_mesh.GetNBE();
+
+   static int num_subdivisions[Geometry::NUM_GEOMETRIES];
+   num_subdivisions[Geometry::POINT] = 1;
+   num_subdivisions[Geometry::SEGMENT] = 1;
+   num_subdivisions[Geometry::TRIANGLE] = 1;
+   num_subdivisions[Geometry::TETRAHEDRON] = 1;
+   num_subdivisions[Geometry::SQUARE] = 2;
+   num_subdivisions[Geometry::PRISM] = 3;
+   num_subdivisions[Geometry::CUBE] = 6;
+   // NOTE: some hexes may be subdivided into only 5 tets, so this is an
+   // estimate only. The actual number of created tets may be less, so the
+   // elements array will need to be shrunk after mesh creation.
+   int new_ne = 0, new_nbe = 0;
+   for (int i=0; i<ne; ++i)
+   {
+      new_ne += num_subdivisions[orig_mesh.GetElementBaseGeometry(i)];
+   }
+   for (int i=0; i<nbe; ++i)
+   {
+      new_nbe += num_subdivisions[orig_mesh.GetBdrElementBaseGeometry(i)];
+   }
+
+   InitMesh(dim, sdim, nv, new_ne, new_nbe);
+
+   // Vertices of the new mesh are same as the original mesh
+   NumOfVertices = nv;
+   for (int i=0; i<nv; ++i)
+   {
+      vertices[i].SetCoords(dim, orig_mesh.vertices[i]());
+   }
+
+   // We need a global vertex numbering to identify which diagonals to split
+   // (quad faces are split using the diagonal originating from the smallest
+   // global vertex number). Use the supplied global numbering, if it is
+   // non-NULL, otherwise use the local numbering.
+   Array<int> vglobal_id;
+   if (vglobal == NULL)
+   {
+      vglobal_id.SetSize(nv);
+      for (int i=0; i<nv; ++i) { vglobal_id[i] = i; }
+      vglobal = vglobal_id.GetData();
+   }
+
+   constexpr int nv_tri = 3, nv_quad = 4, nv_tet = 4, nv_prism = 6, nv_hex = 8;
+   constexpr int quad_ntris = 2, prism_ntets = 3;
+   static const int quad_trimap[2][nv_tri*quad_ntris] =
+   {
+      {
+         0, 0,
+         1, 2,
+         2, 3
+      },{
+         0, 1,
+         1, 2,
+         3, 3
+      }
+   };
+   static const int prism_rot[nv_prism*nv_prism] =
+   {
+      0, 1, 2, 3, 4, 5,
+      1, 2, 0, 4, 5, 3,
+      2, 0, 1, 5, 3, 4,
+      3, 5, 4, 0, 2, 1,
+      4, 3, 5, 1, 0, 2,
+      5, 4, 3, 2, 1, 0
+   };
+   static const int prism_f[nv_quad] = {1, 2, 5, 4};
+   static const int prism_tetmaps[2][nv_prism*prism_ntets] =
+   {
+      {
+         0, 0, 0,
+         1, 1, 4,
+         2, 5, 5,
+         5, 4, 3
+      },{
+         0, 0, 0,
+         1, 4, 4,
+         2, 2, 5,
+         4, 5, 3
+      }
+   };
+   static const int hex_rot[nv_hex*nv_hex] =
+   {
+      0, 1, 2, 3, 4, 5, 6, 7,
+      1, 0, 4, 5, 2, 3, 7, 6,
+      2, 1, 5, 6, 3, 0, 4, 7,
+      3, 0, 1, 2, 7, 4, 5, 6,
+      4, 0, 3, 7, 5, 1, 2, 6,
+      5, 1, 0, 4, 6, 2, 3, 7,
+      6, 2, 1, 5, 7, 3, 0, 4,
+      7, 3, 2, 6, 4, 0, 1, 5
+   };
+   static const int hex_f0[nv_quad] = {1, 2, 6, 5};
+   static const int hex_f1[nv_quad] = {2, 3, 7, 6};
+   static const int hex_f2[nv_quad] = {4, 5, 6, 7};
+   static const int num_rot[8] = {0, 1, 2, 0, 0, 2, 1, 0};
+   static const int hex_tetmap0[nv_tet*5] =
+   {
+      0, 0, 0, 0, 2,
+      1, 2, 2, 5, 7,
+      2, 7, 3, 7, 5,
+      5, 5, 7, 4, 6
+   };
+   static const int hex_tetmap1[nv_tet*6] =
+   {
+      0, 0, 1, 0, 0, 1,
+      5, 1, 6, 7, 7, 7,
+      7, 7, 7, 2, 1, 6,
+      4, 5, 5, 3, 2, 2
+   };
+   static const int hex_tetmap2[nv_tet*6] =
+   {
+      0, 0, 0, 0, 0, 0,
+      4, 3, 7, 1, 3, 6,
+      5, 7, 4, 2, 6, 5,
+      6, 6, 6, 5, 2, 2
+   };
+   static const int hex_tetmap3[nv_tet*6] =
+   {
+      0, 0, 0, 0, 1, 1,
+      2, 3, 7, 5, 5, 6,
+      3, 7, 4, 6, 6, 2,
+      6, 6, 6, 4, 0, 0
+   };
+   static const int *hex_tetmaps[4] =
+   {
+      hex_tetmap0, hex_tetmap1, hex_tetmap2, hex_tetmap3
+   };
+
+   auto find_min = [](const int*a, int n) { return std::min_element(a,a+n)-a; };
+
+   for (int i=0; i<ne; ++i)
+   {
+      const int *v = orig_mesh.elements[i]->GetVertices();
+      const int attrib = orig_mesh.GetAttribute(i);
+      const Geometry::Type orig_geom = orig_mesh.GetElementBaseGeometry(i);
+
+      if (num_subdivisions[orig_geom] == 1)
+      {
+         // (num_subdivisions[orig_geom] == 1) implies that the element does
+         // not need to be further split (it is either a segment, triangle,
+         // or tetrahedron), and so it is left unchanged.
+         Element *e = NewElement(orig_geom);
+         e->SetAttribute(attrib);
+         e->SetVertices(v);
+         AddElement(e);
+      }
+      else if (orig_geom == Geometry::SQUARE)
+      {
+         for (int itri=0; itri<quad_ntris; ++itri)
+         {
+            Element *e = NewElement(Geometry::TRIANGLE);
+            e->SetAttribute(attrib);
+            int *v2 = e->GetVertices();
+            for (int iv=0; iv<nv_tri; ++iv)
+            {
+               v2[iv] = v[quad_trimap[0][itri + iv*quad_ntris]];
+            }
+            AddElement(e);
+         }
+      }
+      else if (orig_geom == Geometry::PRISM)
+      {
+         int vg[nv_prism];
+         for (int iv=0; iv<nv_prism; ++iv) { vg[iv] = vglobal[v[iv]]; }
+         // Rotate the vertices of the prism so that the smallest vertex index
+         // is in the first place
+         int irot = find_min(vg, nv_prism);
+         for (int iv=0; iv<nv_prism; ++iv)
+         {
+            int jv = prism_rot[iv + irot*nv_prism];
+            vg[iv] = v[jv];
+         }
+         // Two cases according to which diagonal splits third quad face
+         int q[nv_quad];
+         for (int iv=0; iv<nv_quad; ++iv) { q[iv] = vglobal[vg[prism_f[iv]]]; }
+         int j = find_min(q, nv_quad);
+         const int *tetmap = (j == 0 || j == 2) ? prism_tetmaps[0] : prism_tetmaps[1];
+         for (int itet=0; itet<prism_ntets; ++itet)
+         {
+            Element *e = NewElement(Geometry::TETRAHEDRON);
+            e->SetAttribute(attrib);
+            int *v2 = e->GetVertices();
+            for (int iv=0; iv<nv_tet; ++iv)
+            {
+               v2[iv] = vg[tetmap[itet + iv*prism_ntets]];
+            }
+            AddElement(e);
+         }
+      }
+      else if (orig_geom == Geometry::CUBE)
+      {
+         int vg[nv_hex];
+         for (int iv=0; iv<nv_hex; ++iv) { vg[iv] = vglobal[v[iv]]; }
+
+         // Rotate the vertices of the hex so that the smallest vertex index is
+         // in the first place
+         int irot = find_min(vg, nv_hex);
+         for (int iv=0; iv<nv_hex; ++iv)
+         {
+            int jv = hex_rot[iv + irot*nv_hex];
+            vg[iv] = v[jv];
+         }
+
+         int q[nv_quad];
+         // Bitmask is three binary digits, each digit is 1 if the diagonal of
+         // the corresponding face goes through the 7th vertex, and 0 if not.
+         int bitmask = 0;
+         int j;
+         // First quad face
+         for (int iv=0; iv<nv_quad; ++iv) { q[iv] = vglobal[vg[hex_f0[iv]]]; }
+         j = find_min(q, nv_quad);
+         if (j == 0 || j == 2) { bitmask += 4; }
+         // Second quad face
+         for (int iv=0; iv<nv_quad; ++iv) { q[iv] = vglobal[vg[hex_f1[iv]]]; }
+         j = find_min(q, nv_quad);
+         if (j == 1 || j == 3) { bitmask += 2; }
+         // Third quad face
+         for (int iv=0; iv<nv_quad; ++iv) { q[iv] = vglobal[vg[hex_f2[iv]]]; }
+         j = find_min(q, nv_quad);
+         if (j == 0 || j == 2) { bitmask += 1; }
+
+         // Apply rotations
+         int nrot = num_rot[bitmask];
+         for (int irot=0; irot<nrot; ++irot)
+         {
+            int vtemp;
+            vtemp = vg[1];
+            vg[1] = vg[4];
+            vg[4] = vg[3];
+            vg[3] = vtemp;
+            vtemp = vg[5];
+            vg[5] = vg[7];
+            vg[7] = vg[2];
+            vg[2] = vtemp;
+         }
+
+         // Sum up nonzero bits in bitmask
+         int ndiags = ((bitmask&4) >> 2) + ((bitmask&2) >> 1) + (bitmask&1);
+         int ntets = (ndiags == 0) ? 5 : 6;
+         const int *tetmap = hex_tetmaps[ndiags];
+         for (int itet=0; itet<ntets; ++itet)
+         {
+            Element *e = NewElement(Geometry::TETRAHEDRON);
+            e->SetAttribute(attrib);
+            int *v2 = e->GetVertices();
+            for (int iv=0; iv<nv_tet; ++iv)
+            {
+               v2[iv] = vg[tetmap[itet + iv*ntets]];
+            }
+            AddElement(e);
+         }
+      }
+   }
+   // In 3D, shrink the element array because some hexes have only 5 tets
+   if (dim == 3) { elements.SetSize(NumOfElements); }
+
+   for (int i=0; i<nbe; ++i)
+   {
+      const int *v = orig_mesh.boundary[i]->GetVertices();
+      const int attrib = orig_mesh.GetBdrAttribute(i);
+      const Geometry::Type orig_geom = orig_mesh.GetBdrElementBaseGeometry(i);
+      if (num_subdivisions[orig_geom] == 1)
+      {
+         Element *be = NewElement(orig_geom);
+         be->SetAttribute(attrib);
+         be->SetVertices(v);
+         AddBdrElement(be);
+      }
+      else if (orig_geom == Geometry::SQUARE)
+      {
+         int vg[nv_quad];
+         for (int iv=0; iv<nv_quad; ++iv) { vg[iv] = vglobal[v[iv]]; }
+         // Split quad according the smallest (global) vertex
+         int iv_min = find_min(vg, nv_quad);
+         int isplit = (iv_min == 0 || iv_min == 2) ? 0 : 1;
+         for (int itri=0; itri<quad_ntris; ++itri)
+         {
+            Element *be = NewElement(Geometry::TRIANGLE);
+            be->SetAttribute(attrib);
+            int *v2 = be->GetVertices();
+            for (int iv=0; iv<nv_tri; ++iv)
+            {
+               v2[iv] = v[quad_trimap[isplit][itri + iv*quad_ntris]];
+            }
+            AddBdrElement(be);
+         }
+      }
+      else
+      {
+         MFEM_ABORT("Unreachable");
+      }
+   }
+
+   FinalizeTopology(false);
+   sequence = orig_mesh.GetSequence();
+   last_operation = orig_mesh.last_operation;
+
+   MFEM_ASSERT(CheckElementOrientation(false) == 0, "");
+   MFEM_ASSERT(CheckBdrElementOrientation(false) == 0, "");
+}
+
 void Mesh::KnotInsert(Array<KnotVector *> &kv)
 {
   if (NURBSext == NULL)
@@ -4167,6 +4478,25 @@ void Mesh::EnsureNodes()
   }
 }

+void Mesh::SetVerticesFromNodes()
+{
+   if (!Nodes) { return; }
+   for (int iel=0; iel<GetNE(); ++iel)
+   {
+      Geometry::Type geom = GetElementBaseGeometry(iel);
+      const IntegrationRule *ref_verts = Geometries.GetVertices(geom);
+      DenseMatrix node_coords;
+      ElementTransformation *T = GetElementTransformation(iel);
+      Nodes->GetVectorValues(*T, *ref_verts, node_coords);
+      Element *el = GetElement(iel);
+      for (int iv=0; iv<el->GetNVertices(); ++iv)
+      {
+         int v = el->GetVertices()[iv];
+         vertices[v].SetCoords(&node_coords(0,iv));
+      }
+   }
+}
+
 void Mesh::SetNodalGridFunction(GridFunction *nodes, bool make_owner)
 {
   GetNodes(*nodes);
@@ -5983,6 +6313,7 @@ int *Mesh::GeneratePartitioning(int nparts, int part_method)
   el_to_el = NULL;

   // Check for empty partitionings (a "feature" in METIS)
+   if (nparts > 1 && NumOfElements > nparts)
   {
      Array< Pair<int,int> > psize(nparts);
      int empty_parts;
@@ -8929,9 +9260,11 @@ void Mesh::PrintVTK(std::ostream &out)
         const int *v = elements[i]->GetVertices();
         const int nv = elements[i]->GetNVertices();
         out << nv;
+         Geometry::Type geom = elements[i]->GetGeometryType();
+         const int *perm = (geom == Geometry::PRISM) ? vtk_prism_perm : NULL;
         for (int j = 0; j < nv; j++)
         {
-            out << ' ' << v[j];
+            out << ' ' << v[perm ? perm[j] : j];
         }
         out << '\n';
      }
@@ -9259,9 +9592,10 @@ void Mesh::PrintVTU(std::ostream &out, int ref, VTKFormat format,
         {
            coff = coff+nv;
            offset.push_back(coff);
+            const int *p = (geom == Geometry::PRISM) ? vtk_prism_perm : NULL;
            for (int k = 0; k < nv; k++, j++)
            {
-               WriteBinaryOrASCII(out, buf, np + RG[j], " ", format);
+               WriteBinaryOrASCII(out, buf, np + RG[p ? p[j] : j], " ", format);
            }
            if (format == VTKFormat::ASCII) { out << '\n'; }
         }
@@ -701,11 +701,11 @@ public:
                             BasisType::GaussLobatto.

       The refinement data which can be accessed with GetRefinementTransforms()
-       is set to reflect the performed refinements.
-
-       @note The constructed Mesh is linear, i.e. it does not have nodes. */
+       is set to reflect the performed refinements. */
   Mesh(Mesh *orig_mesh, int ref_factor, int ref_type);

+   void MakeSimplicial(Mesh &orig_mesh, int *vglobal=NULL);
+
   /** This is similar to the mesh constructor with the same arguments, but here
       the current mesh is destroyed and another one created based on the data
       stream again given in MFEM, Netgen, or VTK format. If generate_edges = 0
@@ -1036,6 +1036,7 @@ public:
   }
   void GetFaceElements (int Face, int *Elem1, int *Elem2) const;
   void GetFaceInfos (int Face, int *Inf1, int *Inf2) const;
+   void GetFaceInfos (int Face, int *Inf1, int *Inf2, int *NCFace) const;

   Geometry::Type GetFaceGeometryType(int Face) const;
   Element::Type  GetFaceElementType(int Face) const;
@@ -1136,6 +1137,9 @@ public:
       with straight edges). */
   void EnsureNodes();

+    /** Updates the coordinates of the vertices from the node locations. */
+   void SetVerticesFromNodes();
+
   /** Set the curvature of the mesh nodes using the given polynomial degree,
       'order', and optionally: discontinuous or continuous FE space, 'discont',
       new space dimension, 'space_dim' (if != -1), and 'ordering'. */
@@ -1142,7 +1142,7 @@ ParMesh::ParMesh(ParMesh *orig_mesh, int ref_factor, int ref_type)
         group_sedge.AddColumnsInRow(gr-1, orig_nq*(RG.RefEdges.Size()/2-
                                                    RG.NumBdrEdges));
         // count refined faces
-         group_squad.AddColumnsInRow(gr-1, orig_nq*(RG.RefGeoms.Size()/nvert));
+         group_squad.AddColumnsInRow(gr-1, orig_nq*RG.RefGeoms.Size()/nvert);
      }
   }

@@ -1307,6 +1307,144 @@ ParMesh::ParMesh(ParMesh *orig_mesh, int ref_factor, int ref_type)
   }
 }

+void ParMesh::MakeSimplicial(ParMesh &orig_mesh)
+{
+   MyComm = orig_mesh.GetComm();
+   NRanks = orig_mesh.GetNRanks();
+   MyRank = orig_mesh.GetMyRank();
+   glob_elem_offset = -1;
+   glob_offset_sequence = -1;
+   gtopo = orig_mesh.gtopo;
+   have_face_nbr_data = false;
+   pncmesh = NULL;
+   meshgen = orig_mesh.meshgen;
+
+   H1_FECollection fec(1, orig_mesh.Dimension());
+   ParFiniteElementSpace fes(&orig_mesh, &fec);
+
+   Array<int> vglobal(orig_mesh.GetNV());
+   for (int iv=0; iv<orig_mesh.GetNV(); ++iv)
+   {
+      vglobal[iv] = fes.GetGlobalTDofNumber(iv);
+   }
+   Mesh::MakeSimplicial(orig_mesh, vglobal);
+
+   // count the number of entries in each row of group_s{vert,edge,face}
+   group_svert.MakeI(GetNGroups()-1); // exclude the local group 0
+   group_sedge.MakeI(GetNGroups()-1);
+   group_stria.MakeI(GetNGroups()-1);
+   group_squad.MakeI(GetNGroups()-1);
+   for (int gr = 1; gr < GetNGroups(); gr++)
+   {
+      group_svert.AddColumnsInRow(gr-1, orig_mesh.GroupNVertices(gr));
+      group_sedge.AddColumnsInRow(gr-1, orig_mesh.GroupNEdges(gr));
+      // Every quad gives an extra edge
+      const int orig_nq = orig_mesh.GroupNQuadrilaterals(gr);
+      group_sedge.AddColumnsInRow(gr-1, orig_nq);
+      // Every quad is subdivided into two triangles
+      group_stria.AddColumnsInRow(gr-1, 2*orig_nq);
+      // Existing triangles remain unchanged
+      const int orig_nt = orig_mesh.GroupNTriangles(gr);
+      group_stria.AddColumnsInRow(gr-1, orig_nt);
+   }
+   group_svert.MakeJ();
+   svert_lvert.Reserve(group_svert.Size_of_connections());
+
+   group_sedge.MakeJ();
+   shared_edges.Reserve(group_sedge.Size_of_connections());
+   sedge_ledge.SetSize(group_sedge.Size_of_connections());
+
+   group_stria.MakeJ();
+   shared_trias.Reserve(group_stria.Size_of_connections());
+   sface_lface.SetSize(shared_trias.Size());
+
+   group_squad.MakeJ();
+
+   constexpr int ntris = 2, nv_tri = 3, nv_quad = 4;
+
+   Array<int> dofs;
+   for (int gr = 1; gr < GetNGroups(); gr++)
+   {
+      // add shared vertices from original shared vertices
+      const int orig_n_verts = orig_mesh.GroupNVertices(gr);
+      for (int j = 0; j < orig_n_verts; j++)
+      {
+         fes.GetVertexDofs(orig_mesh.GroupVertex(gr, j), dofs);
+         group_svert.AddConnection(gr-1, svert_lvert.Append(dofs[0])-1);
+      }
+
+      // add original shared edges
+      const int orig_n_edges = orig_mesh.GroupNEdges(gr);
+      for (int e = 0; e < orig_n_edges; e++)
+      {
+         int iedge, o;
+         orig_mesh.GroupEdge(gr, e, iedge, o);
+         Element *elem = NewElement(Geometry::SEGMENT);
+         Array<int> edge_verts;
+         orig_mesh.GetEdgeVertices(iedge, edge_verts);
+         elem->SetVertices(edge_verts);
+         group_sedge.AddConnection(gr-1, shared_edges.Append(elem)-1);
+      }
+      // add original shared triangles
+      const int orig_nt = orig_mesh.GroupNTriangles(gr);
+      for (int e = 0; e < orig_nt; e++)
+      {
+         int itri, o;
+         orig_mesh.GroupTriangle(gr, e, itri, o);
+         const int *v = orig_mesh.GetFace(itri)->GetVertices();
+         shared_trias.SetSize(shared_trias.Size()+1);
+         int *v2 = shared_trias.Last().v;
+         for (int iv=0; iv<nv_tri; ++iv) { v2[iv] = v[iv]; }
+         group_stria.AddConnection(gr-1, shared_trias.Size()-1);
+      }
+      // add triangles from split quads and add resulting diagonal edge
+      const int orig_nq = orig_mesh.GroupNQuadrilaterals(gr);
+      if (orig_nq > 0)
+      {
+         static const int trimap[12] =
+         {
+            0, 0, 0, 1,
+            1, 2, 1, 2,
+            2, 3, 3, 3
+         };
+         static const int diagmap[4] = { 0, 2, 1, 3 };
+         for (int f = 0; f < orig_nq; ++f)
+         {
+            int iquad, o;
+            orig_mesh.GroupQuadrilateral(gr, f, iquad, o);
+            const int *v = orig_mesh.GetFace(iquad)->GetVertices();
+            // Split quad according the smallest (global) vertex
+            int vg[nv_quad];
+            for (int iv=0; iv<nv_quad; ++iv) { vg[iv] = vglobal[v[iv]]; }
+            int iv_min = std::min_element(vg, vg+nv_quad) - vg;
+            int isplit = (iv_min == 0 || iv_min == 2) ? 0 : 1;
+            // Add diagonal
+            Element *diag = NewElement(Geometry::SEGMENT);
+            int *v_diag = diag->GetVertices();
+            v_diag[0] = v[diagmap[0 + isplit*2]];
+            v_diag[1] = v[diagmap[1 + isplit*2]];
+            group_sedge.AddConnection(gr-1, shared_edges.Append(diag)-1);
+            // Add two new triangles
+            for (int itri=0; itri<ntris; ++itri)
+            {
+               shared_trias.SetSize(shared_trias.Size()+1);
+               int *v2 = shared_trias.Last().v;
+               for (int iv=0; iv<nv_tri; ++iv)
+               {
+                  v2[iv] = v[trimap[itri + isplit*2 + iv*ntris*2]];
+               }
+               group_stria.AddConnection(gr-1, shared_trias.Size()-1);
+            }
+         }
+      }
+   }
+   group_svert.ShiftUpI();
+   group_sedge.ShiftUpI();
+   group_stria.ShiftUpI();
+
+   FinalizeParTopo();
+}
+
 void ParMesh::Finalize(bool refine, bool fix_orientation)
 {
   const int meshgen_save = meshgen; // Mesh::Finalize() may call SetMeshGen()
@@ -31,10 +31,10 @@ class ParPumiMesh;
 /// Class for parallel meshes
 class ParMesh : public Mesh
 {
-protected:
+public:
   ParMesh() : MyComm(0), NRanks(0), MyRank(-1),
      have_face_nbr_data(false), pncmesh(NULL) {}
-
+protected:
   MPI_Comm MyComm;
   int NRanks, MyRank;

@@ -224,11 +224,11 @@ public:
                             BasisType::GaussLobatto.

       The refinement data which can be accessed with GetRefinementTransforms()
-       is set to reflect the performed refinements.
-
-       @note The constructed ParMesh is linear, i.e. it does not have nodes. */
+       is set to reflect the performed refinements. */
   ParMesh(ParMesh *orig_mesh, int ref_factor, int ref_type);

+   void MakeSimplicial(ParMesh &orig_mesh);
+
   virtual void Finalize(bool refine = false, bool fix_orientation = false);

   virtual void SetAttributes();
@@ -284,6 +284,7 @@ public:
                             int ordering = 1);

   int GetNFaceNeighbors() const { return face_nbr_group.Size(); }
+   int GetNFaceNeighborElements() const { return face_nbr_elements.Size(); }
   int GetFaceNbrGroup(int fn) const { return face_nbr_group[fn]; }
   int GetFaceNbrRank(int fn) const;

@@ -18,6 +18,8 @@
 namespace mfem
 {

+const int vtk_prism_perm[6] = {0, 2, 1, 3, 5, 4};
+
 int BarycentricToVTKTriangle(int *b, int ref)
 {
   // Cf. https://git.io/JvW8f
@@ -26,6 +26,8 @@ enum class VTKFormat
   BINARY32
 };

+extern const int vtk_prism_perm[6];
+
 /// Create the VTK element connectivity array for a given element geometry and
 /// refinement level. Converts node numbers from MFEM to VTK ordering.
 void CreateVTKElementConnectivity(Array<int> &con, Geometry::Type geom,
@@ -19,6 +19,10 @@ if (MFEM_USE_GSLIB)
    MAIN findpts.cpp
    LIBRARIES mfem)

+  add_mfem_miniapp(field-interp
+    MAIN field-interp.cpp
+    LIBRARIES mfem)
+
  # Parallel apps.
  if (MFEM_USE_MPI)
    add_mfem_miniapp(pfindpts
--- a/Show More
+++ b/Show More